From e814d5dd8e91a716377f64413e4f699498ce20a9 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Mon, 10 Feb 2020 15:16:19 -0800 Subject: [PATCH 1/7] Storage Market integration test (#119) - Uses mock libp2p network, and actually transfers payloads - Doesn't use real signatures Co-authored-by: Ingar Shu --- shared_testutil/mocknet.go | 20 +- storagemarket/fixtures/payload.txt | 49 ++++ storagemarket/impl/client.go | 13 +- storagemarket/impl/client_storagemarket.go | 11 +- storagemarket/impl/client_utils.go | 4 +- storagemarket/impl/provider.go | 2 +- storagemarket/impl/provider_states.go | 2 +- storagemarket/impl/provider_utils.go | 5 +- storagemarket/integration_test.go | 322 +++++++++++++++++++++ storagemarket/network/libp2p_impl.go | 2 +- storagemarket/network/types.go | 4 +- storagemarket/network/types_cbor_gen.go | 4 +- storagemarket/types.go | 10 +- 13 files changed, 426 insertions(+), 22 deletions(-) create mode 100644 storagemarket/fixtures/payload.txt create mode 100644 storagemarket/integration_test.go diff --git a/shared_testutil/mocknet.go b/shared_testutil/mocknet.go index 9f2ae4d5..1bf7edeb 100644 --- a/shared_testutil/mocknet.go +++ b/shared_testutil/mocknet.go @@ -3,11 +3,20 @@ package shared_testutil import ( "bytes" "errors" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" dss "github.com/ipfs/go-datastore/sync" + "github.com/ipfs/go-graphsync" + graphsyncimpl "github.com/ipfs/go-graphsync/impl" "github.com/ipfs/go-graphsync/ipldbridge" + "github.com/ipfs/go-graphsync/network" bstore "github.com/ipfs/go-ipfs-blockstore" chunk "github.com/ipfs/go-ipfs-chunker" offline "github.com/ipfs/go-ipfs-exchange-offline" @@ -26,11 +35,6 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" "golang.org/x/net/context" - "io" - "io/ioutil" - "os" - "path/filepath" - "testing" ) type Libp2pTestData struct { @@ -39,6 +43,8 @@ type Libp2pTestData struct { Bs2 bstore.Blockstore DagService1 ipldformat.DAGService DagService2 ipldformat.DAGService + GraphSync1 graphsync.GraphExchange + GraphSync2 graphsync.GraphExchange Loader1 ipld.Loader Loader2 ipld.Loader Storer1 ipld.Storer @@ -117,6 +123,9 @@ func NewLibp2pTestData(ctx context.Context, t *testing.T) *Libp2pTestData { testData.Bridge1 = ipldbridge.NewIPLDBridge() testData.Bridge2 = ipldbridge.NewIPLDBridge() + testData.GraphSync1 = graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host1), testData.Bridge1, testData.Loader1, testData.Storer1) + testData.GraphSync2 = graphsyncimpl.New(ctx, network.NewFromLibp2pHost(testData.Host2), testData.Bridge2, testData.Loader2, testData.Storer2) + // create a selector for the whole UnixFS dag ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) @@ -128,6 +137,7 @@ func NewLibp2pTestData(ctx context.Context, t *testing.T) *Libp2pTestData { const unixfsChunkSize uint64 = 1 << 10 const unixfsLinksPerLevel = 1024 + // LoadUnixFSFile injects the fixture `filename` into the given blockstore from the // fixtures directory. If useSecondNode is true, fixture is injected to the second node; // otherwise the first node gets it diff --git a/storagemarket/fixtures/payload.txt b/storagemarket/fixtures/payload.txt new file mode 100644 index 00000000..fd4a2f3c --- /dev/null +++ b/storagemarket/fixtures/payload.txt @@ -0,0 +1,49 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vitae semper quis lectus nulla at volutpat diam ut venenatis. Ac tortor dignissim convallis aenean et tortor at. Faucibus ornare suspendisse sed nisi lacus sed. Commodo ullamcorper a lacus vestibulum sed arcu non. Est pellentesque elit ullamcorper dignissim. Quam quisque id diam vel quam. Pretium aenean pharetra magna ac. In nulla posuere sollicitudin aliquam ultrices. Sed arcu non odio euismod lacinia at. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla pellentesque. Feugiat vivamus at augue eget arcu. + +Pellentesque nec nam aliquam sem et tortor. Vitae tortor condimentum lacinia quis vel. Cras pulvinar mattis nunc sed. In massa tempor nec feugiat. Ornare arcu odio ut sem nulla. Diam maecenas sed enim ut sem. Pretium vulputate sapien nec sagittis. Bibendum arcu vitae elementum curabitur vitae nunc sed velit dignissim. Duis ut diam quam nulla porttitor massa. Viverra mauris in aliquam sem fringilla ut morbi. Ullamcorper eget nulla facilisi etiam dignissim. Vulputate mi sit amet mauris commodo quis imperdiet massa tincidunt. Nunc consequat interdum varius sit. Nunc mi ipsum faucibus vitae aliquet nec ullamcorper. Nunc sed augue lacus viverra. Lobortis scelerisque fermentum dui faucibus in ornare quam. Urna neque viverra justo nec ultrices. Varius vel pharetra vel turpis nunc eget lorem dolor sed. + +Feugiat nisl pretium fusce id velit ut tortor pretium. Lorem dolor sed viverra ipsum nunc aliquet bibendum. Ultrices vitae auctor eu augue ut lectus. Pharetra massa massa ultricies mi quis. Nibh cras pulvinar mattis nunc sed blandit libero. Ac felis donec et odio pellentesque diam volutpat. Lectus proin nibh nisl condimentum id venenatis. Quis vel eros donec ac odio. Commodo sed egestas egestas fringilla phasellus faucibus scelerisque eleifend donec. Adipiscing diam donec adipiscing tristique. + +Tempus imperdiet nulla malesuada pellentesque elit eget gravida cum sociis. Libero nunc consequat interdum varius sit. Et pharetra pharetra massa massa. Feugiat pretium nibh ipsum consequat. Amet commodo nulla facilisi nullam vehicula. Ornare arcu dui vivamus arcu felis bibendum ut tristique. At erat pellentesque adipiscing commodo elit at imperdiet dui. Auctor neque vitae tempus quam pellentesque nec nam aliquam sem. Eget velit aliquet sagittis id consectetur. Enim diam vulputate ut pharetra sit amet aliquam id diam. Eget velit aliquet sagittis id consectetur purus ut faucibus pulvinar. Amet porttitor eget dolor morbi. Felis eget velit aliquet sagittis id. Facilisis magna etiam tempor orci eu. Lacus suspendisse faucibus interdum posuere lorem. Pharetra et ultrices neque ornare aenean euismod. Platea dictumst quisque sagittis purus. + +Quis varius quam quisque id diam vel quam elementum. Augue mauris augue neque gravida in fermentum et sollicitudin. Sapien nec sagittis aliquam malesuada bibendum arcu. Urna duis convallis convallis tellus id interdum velit. Tellus in hac habitasse platea dictumst vestibulum. Fames ac turpis egestas maecenas pharetra convallis. Diam volutpat commodo sed egestas egestas fringilla phasellus faucibus. Placerat orci nulla pellentesque dignissim enim sit amet venenatis. Sed adipiscing diam donec adipiscing. Praesent elementum facilisis leo vel fringilla est. Sed enim ut sem viverra aliquet eget sit amet tellus. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra. Turpis egestas pretium aenean pharetra magna ac placerat vestibulum. Massa id neque aliquam vestibulum morbi blandit cursus risus. Vitae congue eu consequat ac. Egestas erat imperdiet sed euismod nisi porta lorem mollis aliquam. Dolor purus non enim praesent elementum facilisis. Ultrices mi tempus imperdiet nulla malesuada pellentesque elit. In est ante in nibh. + +Facilisis gravida neque convallis a. Urna nunc id cursus metus aliquam eleifend mi. Lacus luctus accumsan tortor posuere ac. Molestie nunc non blandit massa. Iaculis urna id volutpat lacus laoreet non. Cursus vitae congue mauris rhoncus aenean. Nunc vel risus commodo viverra maecenas. A pellentesque sit amet porttitor eget dolor morbi. Leo vel orci porta non pulvinar neque laoreet suspendisse. Sit amet facilisis magna etiam tempor. Consectetur a erat nam at lectus urna duis convallis convallis. Vestibulum morbi blandit cursus risus at ultrices. Dolor purus non enim praesent elementum. Adipiscing elit pellentesque habitant morbi tristique senectus et netus et. Et odio pellentesque diam volutpat commodo sed egestas egestas fringilla. Leo vel fringilla est ullamcorper eget nulla. Dui ut ornare lectus sit amet. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan sit. + +Tristique senectus et netus et. Pellentesque diam volutpat commodo sed egestas egestas fringilla. Mauris pharetra et ultrices neque ornare aenean. Amet tellus cras adipiscing enim. Convallis aenean et tortor at risus viverra adipiscing at. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. Dictumst vestibulum rhoncus est pellentesque elit. Fringilla ut morbi tincidunt augue interdum velit euismod in pellentesque. Dictum at tempor commodo ullamcorper a lacus vestibulum. Sed viverra tellus in hac habitasse platea. Sed id semper risus in hendrerit. In hendrerit gravida rutrum quisque non tellus orci ac. Sit amet risus nullam eget. Sit amet est placerat in egestas erat imperdiet sed. In nisl nisi scelerisque eu ultrices. Sit amet mattis vulputate enim nulla aliquet. + +Dignissim suspendisse in est ante in nibh mauris cursus. Vitae proin sagittis nisl rhoncus. Id leo in vitae turpis massa sed elementum. Lobortis elementum nibh tellus molestie nunc non blandit massa enim. Arcu dictum varius duis at consectetur. Suspendisse faucibus interdum posuere lorem ipsum dolor sit amet consectetur. Imperdiet nulla malesuada pellentesque elit eget gravida cum sociis. Sed adipiscing diam donec adipiscing. Purus sit amet volutpat consequat mauris nunc congue nisi vitae. Elementum nisi quis eleifend quam adipiscing vitae proin sagittis nisl. Mattis ullamcorper velit sed ullamcorper morbi tincidunt ornare massa. Sit amet nisl purus in mollis nunc sed. Turpis tincidunt id aliquet risus feugiat in ante. Id diam maecenas ultricies mi eget mauris pharetra et ultrices. + +Aliquam purus sit amet luctus venenatis lectus magna fringilla urna. Id diam vel quam elementum pulvinar. Elementum sagittis vitae et leo duis. Viverra aliquet eget sit amet tellus cras adipiscing enim eu. Et tortor at risus viverra adipiscing at in tellus integer. Purus in massa tempor nec feugiat. Augue neque gravida in fermentum et sollicitudin ac orci. Sodales ut eu sem integer vitae justo eget magna fermentum. Netus et malesuada fames ac. Augue interdum velit euismod in. Sed elementum tempus egestas sed sed risus pretium. Mattis vulputate enim nulla aliquet porttitor lacus luctus. Dui vivamus arcu felis bibendum ut tristique et egestas quis. + +Viverra justo nec ultrices dui sapien. Quisque egestas diam in arcu cursus euismod quis viverra nibh. Nam libero justo laoreet sit amet cursus sit amet. Lacus sed viverra tellus in hac habitasse. Blandit aliquam etiam erat velit scelerisque in. Ut sem nulla pharetra diam sit amet nisl suscipit adipiscing. Diam sollicitudin tempor id eu nisl nunc. Eget duis at tellus at urna condimentum mattis. Urna porttitor rhoncus dolor purus non enim praesent elementum facilisis. Sed turpis tincidunt id aliquet risus feugiat. Est velit egestas dui id ornare arcu odio ut sem. Nibh sit amet commodo nulla facilisi nullam vehicula. Sit amet consectetur adipiscing elit duis tristique sollicitudin. Eu facilisis sed odio morbi. Massa id neque aliquam vestibulum morbi. In eu mi bibendum neque egestas congue quisque egestas. Massa sed elementum tempus egestas sed sed risus. Quam elementum pulvinar etiam non. At augue eget arcu dictum varius duis at consectetur lorem. + +Penatibus et magnis dis parturient montes nascetur ridiculus. Dictumst quisque sagittis purus sit amet volutpat consequat. Bibendum at varius vel pharetra. Sed adipiscing diam donec adipiscing tristique risus nec feugiat in. Phasellus faucibus scelerisque eleifend donec pretium. Vitae tortor condimentum lacinia quis vel eros. Ac tincidunt vitae semper quis lectus nulla at volutpat diam. Eget sit amet tellus cras adipiscing. Morbi tristique senectus et netus. Nullam vehicula ipsum a arcu cursus vitae congue mauris rhoncus. Auctor urna nunc id cursus metus aliquam eleifend. Ultrices vitae auctor eu augue. Eu non diam phasellus vestibulum lorem sed risus ultricies. Fames ac turpis egestas sed tempus. Volutpat blandit aliquam etiam erat. Dictum varius duis at consectetur lorem. Sit amet volutpat consequat mauris nunc congue. Volutpat sed cras ornare arcu dui vivamus arcu felis. + +Scelerisque fermentum dui faucibus in ornare quam viverra. Interdum velit laoreet id donec ultrices tincidunt arcu. Netus et malesuada fames ac. Netus et malesuada fames ac turpis. Suscipit tellus mauris a diam maecenas sed enim ut sem. Id velit ut tortor pretium. Neque aliquam vestibulum morbi blandit cursus risus at. Cum sociis natoque penatibus et magnis dis parturient. Lobortis elementum nibh tellus molestie nunc non blandit. Ipsum dolor sit amet consectetur adipiscing elit duis tristique. Amet nisl purus in mollis. Amet massa vitae tortor condimentum lacinia quis vel eros donec. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. + +Nullam ac tortor vitae purus faucibus. Dis parturient montes nascetur ridiculus mus mauris. Molestie at elementum eu facilisis sed odio morbi. Scelerisque felis imperdiet proin fermentum leo vel orci porta. Lectus proin nibh nisl condimentum id venenatis a. Eget nullam non nisi est sit amet facilisis. Hendrerit gravida rutrum quisque non tellus orci ac auctor. Ut faucibus pulvinar elementum integer enim. Rhoncus dolor purus non enim praesent elementum facilisis. Enim sed faucibus turpis in eu mi bibendum. Faucibus nisl tincidunt eget nullam. + +Cursus risus at ultrices mi tempus imperdiet nulla malesuada pellentesque. Pretium nibh ipsum consequat nisl vel pretium lectus quam. Semper viverra nam libero justo laoreet sit amet cursus sit. Augue eget arcu dictum varius duis at consectetur lorem donec. Et malesuada fames ac turpis. Erat nam at lectus urna duis convallis convallis. Dictum sit amet justo donec enim. Urna condimentum mattis pellentesque id nibh tortor id. Morbi tempus iaculis urna id. Lectus proin nibh nisl condimentum id venenatis a condimentum. Nibh sit amet commodo nulla facilisi nullam vehicula. Dui faucibus in ornare quam. Gravida arcu ac tortor dignissim convallis aenean. Consectetur adipiscing elit pellentesque habitant morbi tristique. Pulvinar elementum integer enim neque volutpat ac tincidunt vitae. Pharetra pharetra massa massa ultricies mi quis hendrerit. Dictum at tempor commodo ullamcorper a lacus vestibulum sed. Mattis pellentesque id nibh tortor id. Ultricies integer quis auctor elit sed vulputate. Pretium vulputate sapien nec sagittis aliquam malesuada. + +Auctor augue mauris augue neque gravida. Porttitor lacus luctus accumsan tortor posuere ac ut. Urna neque viverra justo nec ultrices dui. Sit amet est placerat in egestas. Urna nec tincidunt praesent semper feugiat nibh sed pulvinar. Tincidunt eget nullam non nisi est sit amet facilisis magna. Elementum tempus egestas sed sed risus pretium quam vulputate dignissim. Fermentum posuere urna nec tincidunt praesent semper feugiat nibh sed. Porttitor eget dolor morbi non arcu risus quis. Non quam lacus suspendisse faucibus interdum. Venenatis cras sed felis eget velit aliquet sagittis id. Arcu ac tortor dignissim convallis aenean et. Morbi tincidunt ornare massa eget egestas purus. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed ullamcorper. Vestibulum morbi blandit cursus risus at ultrices. Volutpat blandit aliquam etiam erat velit scelerisque. + +Et egestas quis ipsum suspendisse. Amet consectetur adipiscing elit duis. Purus ut faucibus pulvinar elementum integer enim neque. Cursus vitae congue mauris rhoncus aenean vel elit scelerisque mauris. Tincidunt eget nullam non nisi est. Aliquam purus sit amet luctus. Dui ut ornare lectus sit amet est placerat in. Fringilla ut morbi tincidunt augue interdum velit euismod in. Felis eget nunc lobortis mattis aliquam faucibus purus in. Suspendisse interdum consectetur libero id faucibus nisl. + +Scelerisque fermentum dui faucibus in ornare quam. Lectus proin nibh nisl condimentum id venenatis a condimentum vitae. Fames ac turpis egestas integer eget aliquet nibh praesent tristique. Arcu non sodales neque sodales ut etiam sit. Pharetra convallis posuere morbi leo urna. Nec dui nunc mattis enim ut tellus. Nunc sed augue lacus viverra vitae. Consequat id porta nibh venenatis cras sed felis. Dolor sit amet consectetur adipiscing. Tellus rutrum tellus pellentesque eu tincidunt tortor aliquam nulla. + +Metus aliquam eleifend mi in nulla posuere. Blandit massa enim nec dui nunc mattis enim. Aliquet nibh praesent tristique magna. In aliquam sem fringilla ut. Magna fermentum iaculis eu non. Eget aliquet nibh praesent tristique magna sit amet purus. Ultrices gravida dictum fusce ut placerat orci. Fermentum posuere urna nec tincidunt praesent. Enim tortor at auctor urna nunc. Ridiculus mus mauris vitae ultricies leo integer malesuada nunc vel. Sed id semper risus in hendrerit gravida rutrum. Vestibulum lectus mauris ultrices eros in cursus turpis. Et sollicitudin ac orci phasellus egestas tellus rutrum. Pellentesque elit ullamcorper dignissim cras tincidunt lobortis feugiat vivamus at. Metus vulputate eu scelerisque felis imperdiet proin fermentum leo. Porta non pulvinar neque laoreet suspendisse. Suscipit adipiscing bibendum est ultricies integer quis auctor elit sed. Euismod in pellentesque massa placerat duis ultricies lacus sed. Pellentesque adipiscing commodo elit at imperdiet dui accumsan sit amet. + +Pellentesque eu tincidunt tortor aliquam nulla facilisi. Commodo nulla facilisi nullam vehicula ipsum a arcu. Commodo quis imperdiet massa tincidunt nunc pulvinar sapien et. Faucibus purus in massa tempor. Purus semper eget duis at tellus at urna condimentum. Vivamus at augue eget arcu dictum. Lacus vel facilisis volutpat est velit egestas dui id. Malesuada fames ac turpis egestas maecenas pharetra. Nunc faucibus a pellentesque sit amet porttitor eget dolor. Ultricies tristique nulla aliquet enim. Vel risus commodo viverra maecenas accumsan lacus vel facilisis volutpat. Dignissim diam quis enim lobortis scelerisque. Donec ultrices tincidunt arcu non sodales neque sodales ut etiam. + +Vitae proin sagittis nisl rhoncus mattis rhoncus urna neque. Fermentum leo vel orci porta non. At elementum eu facilisis sed. Quis enim lobortis scelerisque fermentum. Fermentum odio eu feugiat pretium nibh ipsum consequat. Habitant morbi tristique senectus et netus et. Enim praesent elementum facilisis leo vel fringilla est ullamcorper. Egestas quis ipsum suspendisse ultrices gravida dictum. Nam libero justo laoreet sit amet cursus sit amet. Viverra tellus in hac habitasse platea dictumst vestibulum. Varius vel pharetra vel turpis nunc eget. Nullam non nisi est sit amet facilisis magna. Ullamcorper eget nulla facilisi etiam dignissim diam. Ante metus dictum at tempor commodo ullamcorper a lacus. + +Etiam non quam lacus suspendisse. Ut venenatis tellus in metus vulputate eu scelerisque felis. Pulvinar sapien et ligula ullamcorper malesuada proin libero. Consequat interdum varius sit amet mattis. Nunc eget lorem dolor sed viverra ipsum nunc aliquet. Potenti nullam ac tortor vitae purus faucibus ornare. Urna et pharetra pharetra massa massa ultricies mi quis hendrerit. Purus in mollis nunc sed id. Pharetra vel turpis nunc eget lorem dolor sed viverra. Et netus et malesuada fames ac turpis. Libero id faucibus nisl tincidunt eget nullam non nisi. Cursus sit amet dictum sit amet. Porttitor lacus luctus accumsan tortor. + +Volutpat diam ut venenatis tellus in metus vulputate eu scelerisque. Sed viverra tellus in hac habitasse. Aliquam sem et tortor consequat id. Pellentesque habitant morbi tristique senectus et netus et. Consectetur purus ut faucibus pulvinar elementum. Aliquam malesuada bibendum arcu vitae elementum curabitur vitae nunc sed. Malesuada bibendum arcu vitae elementum curabitur vitae nunc sed. Sollicitudin tempor id eu nisl nunc mi ipsum. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate sapien nec. Quis eleifend quam adipiscing vitae proin sagittis nisl rhoncus. Bibendum neque egestas congue quisque egestas. A iaculis at erat pellentesque adipiscing commodo elit at imperdiet. Pulvinar etiam non quam lacus. Adipiscing commodo elit at imperdiet. Scelerisque eu ultrices vitae auctor. Sed cras ornare arcu dui vivamus arcu felis bibendum ut. Ornare lectus sit amet est. + +Consequat semper viverra nam libero justo laoreet sit. Imperdiet sed euismod nisi porta lorem mollis aliquam ut porttitor. Cras sed felis eget velit aliquet sagittis id consectetur. Dolor morbi non arcu risus quis. Adipiscing tristique risus nec feugiat in fermentum posuere urna. Dolor magna eget est lorem ipsum dolor. Mauris pharetra et ultrices neque ornare aenean euismod. Nulla facilisi etiam dignissim diam quis. Ultrices tincidunt arcu non sodales. Fames ac turpis egestas maecenas pharetra convallis posuere morbi leo. Interdum varius sit amet mattis vulputate. Tincidunt praesent semper feugiat nibh sed pulvinar. Quisque sagittis purus sit amet volutpat. + +Sed vulputate odio ut enim blandit. Vitae auctor eu augue ut lectus arcu bibendum. Consectetur adipiscing elit pellentesque habitant morbi tristique senectus et. Scelerisque eu ultrices vitae auctor eu augue. Etiam dignissim diam quis enim lobortis scelerisque fermentum dui faucibus. Tellus integer feugiat scelerisque varius. Vulputate enim nulla aliquet porttitor lacus luctus accumsan tortor. Amet nisl purus in mollis. Scelerisque viverra mauris in aliquam sem fringilla ut morbi tincidunt. Semper eget duis at tellus at. Erat velit scelerisque in dictum non consectetur a erat nam. Gravida rutrum quisque non tellus orci. Morbi blandit cursus risus at. Mauris sit amet massa vitae. Non odio euismod lacinia at quis risus sed vulputate. Fermentum posuere urna nec tincidunt praesent. Ut eu sem integer vitae justo eget magna fermentum iaculis. Ullamcorper velit sed ullamcorper morbi tincidunt ornare massa. Arcu cursus euismod quis viverra nibh. Arcu dui vivamus arcu felis bibendum. + +Eros in cursus turpis massa tincidunt dui ut. Urna condimentum mattis pellentesque id nibh tortor id aliquet lectus. Nibh venenatis cras sed felis. Ac felis donec et odio pellentesque diam. Ultricies lacus sed turpis tincidunt id aliquet risus. Diam volutpat commodo sed egestas. Dignissim sodales ut eu sem integer vitae. Pellentesque eu tincidunt tortor aliquam nulla facilisi. Et tortor consequat id porta nibh venenatis cras sed. \ No newline at end of file diff --git a/storagemarket/impl/client.go b/storagemarket/impl/client.go index 2c9258a0..6735a13b 100644 --- a/storagemarket/impl/client.go +++ b/storagemarket/impl/client.go @@ -3,8 +3,8 @@ package storageimpl import ( "context" - "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-data-transfer" + "github.com/filecoin-project/go-fil-markets/storagemarket/network" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/ipfs/go-cid" @@ -27,7 +27,7 @@ import ( //go:generate cbor-gen-for ClientDeal ClientDealProposal -var log = logging.Logger("deals") +var log = logging.Logger("storagemarket_impl") type ClientDeal struct { storagemarket.ClientDeal @@ -68,7 +68,14 @@ type clientDealUpdate struct { mut func(*ClientDeal) } -func NewClient(net network.StorageMarketNetwork, bs blockstore.Blockstore, dataTransfer datatransfer.Manager, discovery *discovery.Local, deals *statestore.StateStore, scn storagemarket.StorageClientNode) *Client { +func NewClient( + net network.StorageMarketNetwork, + bs blockstore.Blockstore, + dataTransfer datatransfer.Manager, + discovery *discovery.Local, + deals *statestore.StateStore, + scn storagemarket.StorageClientNode, +) *Client { carIO := cario.NewCarIO() pio := pieceio.NewPieceIO(carIO, bs) diff --git a/storagemarket/impl/client_storagemarket.go b/storagemarket/impl/client_storagemarket.go index 45d3b6af..cac3a724 100644 --- a/storagemarket/impl/client_storagemarket.go +++ b/storagemarket/impl/client_storagemarket.go @@ -82,7 +82,16 @@ func (c *Client) GetAsk(ctx context.Context, info storagemarket.StorageProviderI return c.QueryAsk(ctx, info.PeerID, info.Address) } -func (c *Client) ProposeStorageDeal(ctx context.Context, addr address.Address, info *storagemarket.StorageProviderInfo, payloadCid cid.Cid, proposalExpiration storagemarket.Epoch, duration storagemarket.Epoch, price tokenamount.TokenAmount, collateral tokenamount.TokenAmount) (*storagemarket.ProposeStorageDealResult, error) { +func (c *Client) ProposeStorageDeal( + ctx context.Context, + addr address.Address, + info *storagemarket.StorageProviderInfo, + payloadCid cid.Cid, + proposalExpiration storagemarket.Epoch, + duration storagemarket.Epoch, + price tokenamount.TokenAmount, + collateral tokenamount.TokenAmount, +) (*storagemarket.ProposeStorageDealResult, error) { proposal := ClientDealProposal{ Data: payloadCid, diff --git a/storagemarket/impl/client_utils.go b/storagemarket/impl/client_utils.go index 9b8a6f43..f2aa65a3 100644 --- a/storagemarket/impl/client_utils.go +++ b/storagemarket/impl/client_utils.go @@ -13,9 +13,9 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/go-data-transfer" ) func (c *Client) failDeal(id cid.Cid, cerr error) { @@ -62,7 +62,7 @@ func (c *Client) readStorageDealResp(deal ClientDeal) (*network.Response, error) return nil, err } - if err := resp.Verify(deal.MinerWorker); err != nil { + if err := resp.Verify(deal.MinerWorker, c.node.VerifySignature); err != nil { return nil, xerrors.Errorf("verifying response signature failed", err) } diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index fc7e276e..eb39231c 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/pieceio/cario" @@ -22,7 +23,6 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/go-data-transfer" ) var ProviderDsPrefix = "/deals/provider" diff --git a/storagemarket/impl/provider_states.go b/storagemarket/impl/provider_states.go index c3051047..6f5e9c51 100644 --- a/storagemarket/impl/provider_states.go +++ b/storagemarket/impl/provider_states.go @@ -85,7 +85,7 @@ func (p *Provider) transferring(ctx context.Context, deal MinerDeal) (func(*Mine allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() - log.Infof("fetching data for a deal %d", deal.ProposalCid) + log.Infof("fetching data for a deal %s", deal.ProposalCid) // initiate a pull data transfer. This will complete asynchronously and the // completion of the data transfer will trigger a change in deal state diff --git a/storagemarket/impl/provider_utils.go b/storagemarket/impl/provider_utils.go index a7e8c2b0..cae201c6 100644 --- a/storagemarket/impl/provider_utils.go +++ b/storagemarket/impl/provider_utils.go @@ -6,9 +6,10 @@ import ( "github.com/ipld/go-ipld-prime" + "github.com/filecoin-project/go-data-transfer" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" - "github.com/filecoin-project/go-data-transfer" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-statestore" @@ -58,7 +59,7 @@ func (p *Provider) readProposal(s network.StorageDealStream) (proposal network.P return proposal, xerrors.Errorf("incoming deal proposal has no signature") } - if err := proposal.DealProposal.Verify(); err != nil { + if err := proposal.DealProposal.Verify(p.spn.VerifySignature); err != nil { return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) } diff --git a/storagemarket/integration_test.go b/storagemarket/integration_test.go new file mode 100644 index 00000000..a2261a75 --- /dev/null +++ b/storagemarket/integration_test.go @@ -0,0 +1,322 @@ +package storagemarket_test + +import ( + "context" + "io" + "io/ioutil" + "reflect" + "testing" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-data-transfer" + graphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" + "github.com/filecoin-project/go-statestore" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-fil-markets/filestore" + "github.com/filecoin-project/go-fil-markets/piecestore" + "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" + "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/shared_testutil" + "github.com/filecoin-project/go-fil-markets/storagemarket" + storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + "github.com/filecoin-project/go-fil-markets/storagemarket/network" +) + +func TestMakeDeal(t *testing.T) { + ctx := context.Background() + epoch := uint64(100) + nodeCommon := fakeCommon{newStorageMarketState()} + ds1 := datastore.NewMapDatastore() + td := shared_testutil.NewLibp2pTestData(ctx, t) + rootLink := td.LoadUnixFSFile(t, "payload.txt", false) + payloadCid := rootLink.(cidlink.Link).Cid + + clientNode := fakeClientNode{ + fakeCommon: nodeCommon, + ClientAddr: address.TestAddress, + } + + providerAddr := address.TestAddress2 + ds2 := datastore.NewMapDatastore() + tempPath, err := ioutil.TempDir("", "storagemarket_test") + assert.NoError(t, err) + ps := piecestore.NewPieceStore(ds2) + providerNode := fakeProviderNode{ + fakeCommon: nodeCommon, + MinerAddr: providerAddr, + } + fs, err := filestore.NewLocalFileStore(filestore.OsPath(tempPath)) + assert.NoError(t, err) + + // create provider and client + dt1 := graphsync.NewGraphSyncDataTransfer(td.Host1, td.GraphSync1) + require.NoError(t, dt1.RegisterVoucherType(reflect.TypeOf(&storageimpl.StorageDataTransferVoucher{}), &fakeDTValidator{})) + + client := storageimpl.NewClient( + network.NewFromLibp2pHost(td.Host1), + td.Bs1, + dt1, + discovery.NewLocal(ds1), + statestore.New(ds1), + &clientNode, + ) + + dt2 := graphsync.NewGraphSyncDataTransfer(td.Host2, td.GraphSync2) + provider, err := storageimpl.NewProvider( + network.NewFromLibp2pHost(td.Host2), + ds2, + td.Bs2, + fs, + ps, + dt2, + &providerNode, + providerAddr, + ) + assert.NoError(t, err) + + // set ask price where we'll accept any price + err = provider.AddAsk(tokenamount.FromInt(0), 50_000) + assert.NoError(t, err) + + err = provider.Start(ctx) + assert.NoError(t, err) + + // Closely follows the MinerInfo struct in the spec + providerInfo := storagemarket.StorageProviderInfo{ + Address: providerAddr, + Owner: providerAddr, + Worker: providerAddr, + SectorSize: 32, + PeerID: td.Host2.ID(), + } + + var proposalCid cid.Cid + + // make a deal + go func() { + client.Run(ctx) + result, err := client.ProposeStorageDeal(ctx, providerAddr, &providerInfo, payloadCid, storagemarket.Epoch(epoch+100), 20000, tokenamount.FromInt(1), tokenamount.FromInt(0)) + assert.NoError(t, err) + + proposalCid = result.ProposalCid + }() + + time.Sleep(time.Millisecond * 100) + + cd, err := client.GetInProgressDeal(ctx, proposalCid) + assert.NoError(t, err) + assert.Equal(t, cd.State, storagemarket.StorageDealActive) + + providerDeals, err := provider.ListIncompleteDeals() + assert.NoError(t, err) + + pd := providerDeals[0] + assert.True(t, pd.ProposalCid.Equals(proposalCid)) + assert.Equal(t, pd.State, storagemarket.StorageDealActive) +} + +type fakeDTValidator struct{} + +func (v *fakeDTValidator) ValidatePush(sender peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, selector ipld.Node) error { + return nil +} + +func (v *fakeDTValidator) ValidatePull(receiver peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, selector ipld.Node) error { + return nil +} + +var _ datatransfer.RequestValidator = (*fakeDTValidator)(nil) + +// Below fake node implementations +type testStateKey struct{ Epoch uint64 } + +func (k *testStateKey) Height() uint64 { + return k.Epoch +} + +type storageMarketState struct { + Epoch uint64 + DealId uint64 + Balances map[address.Address]tokenamount.TokenAmount + StorageDeals map[address.Address][]storagemarket.StorageDeal + Providers []*storagemarket.StorageProviderInfo +} + +func newStorageMarketState() *storageMarketState { + return &storageMarketState{ + Epoch: 0, + DealId: 0, + Balances: map[address.Address]tokenamount.TokenAmount{}, + StorageDeals: map[address.Address][]storagemarket.StorageDeal{}, + Providers: nil, + } +} + +func (sma *storageMarketState) AddFunds(addr address.Address, amount tokenamount.TokenAmount) { + if existing, ok := sma.Balances[addr]; ok { + sma.Balances[addr] = tokenamount.Add(existing, amount) + } else { + sma.Balances[addr] = amount + } +} + +func (sma *storageMarketState) Balance(addr address.Address) storagemarket.Balance { + if existing, ok := sma.Balances[addr]; ok { + return storagemarket.Balance{tokenamount.FromInt(0), existing} + } + return storagemarket.Balance{tokenamount.FromInt(0), tokenamount.FromInt(0)} +} + +func (sma *storageMarketState) Deals(addr address.Address) []storagemarket.StorageDeal { + if existing, ok := sma.StorageDeals[addr]; ok { + return existing + } + return nil +} + +func (sma *storageMarketState) StateKey() storagemarket.StateKey { + return &testStateKey{sma.Epoch} +} + +func (sma *storageMarketState) AddDeal(deal storagemarket.StorageDeal) storagemarket.StateKey { + for _, addr := range []address.Address{deal.Client, deal.Provider} { + if existing, ok := sma.StorageDeals[addr]; ok { + sma.StorageDeals[addr] = append(existing, deal) + } else { + sma.StorageDeals[addr] = []storagemarket.StorageDeal{deal} + } + } + return sma.StateKey() +} + +type fakeCommon struct { + SMState *storageMarketState +} + +func (n *fakeCommon) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { + return n.SMState.StateKey(), nil +} + +func (n *fakeCommon) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { + n.SMState.AddFunds(addr, amount) + return nil +} + +func (n *fakeCommon) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { + balance := n.SMState.Balance(addr) + if balance.Available.LessThan(amount) { + n.SMState.AddFunds(addr, tokenamount.Sub(amount, balance.Available)) + } + return nil +} + +func (n *fakeCommon) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { + return n.SMState.Balance(addr), nil +} + +func (n *fakeCommon) VerifySignature(addr address.Address, data []byte) error { + return nil +} + +type fakeClientNode struct { + fakeCommon + ClientAddr address.Address + ValidationError error +} + +func (n *fakeClientNode) ListClientDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) { + return n.SMState.Deals(addr), nil +} + +func (n *fakeClientNode) ListStorageProviders(ctx context.Context) ([]*storagemarket.StorageProviderInfo, error) { + return n.SMState.Providers, nil +} + +func (n *fakeClientNode) ValidatePublishedDeal(ctx context.Context, deal storagemarket.ClientDeal) (uint64, error) { + return 0, nil +} + +func (n *fakeClientNode) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error { + proposal.ProposerSignature = shared_testutil.MakeTestSignature() + return nil +} + +func (n *fakeClientNode) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { + return n.ClientAddr, nil +} + +func (n *fakeClientNode) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb storagemarket.DealSectorCommittedCallback) error { + cb(nil) + return nil +} + +func (n *fakeClientNode) ValidateAskSignature(ask *types.SignedStorageAsk) error { + return n.ValidationError +} + +var _ storagemarket.StorageClientNode = (*fakeClientNode)(nil) + +type fakeProviderNode struct { + fakeCommon + MinerAddr address.Address + Epoch uint64 + PieceLength uint64 + PieceSectorID uint64 + CompletedDeal storagemarket.MinerDeal + PublishDealID storagemarket.DealID +} + +func (n *fakeProviderNode) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) { + p := deal.Proposal + + sd := storagemarket.StorageDeal{ + PieceRef: p.PieceRef, + PieceSize: p.PieceSize, + Client: p.Client, + Provider: p.Provider, + ProposalExpiration: p.ProposalExpiration, + Duration: p.Duration, + StoragePricePerEpoch: p.StoragePricePerEpoch, + StorageCollateral: p.StorageCollateral, + } + + n.SMState.AddDeal(sd) + + return n.PublishDealID, shared_testutil.GenerateCids(1)[0], nil +} + +func (n *fakeProviderNode) ListProviderDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) { + return n.SMState.Deals(addr), nil +} + +func (n *fakeProviderNode) OnDealComplete(ctx context.Context, deal storagemarket.MinerDeal, pieceSize uint64, pieceReader io.Reader) error { + return nil +} + +func (n *fakeProviderNode) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) { + return n.MinerAddr, nil +} + +func (n *fakeProviderNode) SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) { + return shared_testutil.MakeTestSignature(), nil +} + +func (n *fakeProviderNode) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealID uint64, cb storagemarket.DealSectorCommittedCallback) error { + cb(nil) + return nil +} + +func (n *fakeProviderNode) LocatePieceForDealWithinSector(ctx context.Context, dealID uint64) (sectorID uint64, offset uint64, length uint64, err error) { + return n.PieceSectorID, 0, n.PieceLength, nil +} + +var _ storagemarket.StorageProviderNode = (*fakeProviderNode)(nil) diff --git a/storagemarket/network/libp2p_impl.go b/storagemarket/network/libp2p_impl.go index 30e3d136..a2b86a2f 100644 --- a/storagemarket/network/libp2p_impl.go +++ b/storagemarket/network/libp2p_impl.go @@ -11,7 +11,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) -var log = logging.Logger("retrieval_network") +var log = logging.Logger("storagemarket_network") // NewFromLibp2pHost builds a storage market network on top of libp2p func NewFromLibp2pHost(h host.Host) StorageMarketNetwork { diff --git a/storagemarket/network/types.go b/storagemarket/network/types.go index f28597e7..5b54dbd4 100644 --- a/storagemarket/network/types.go +++ b/storagemarket/network/types.go @@ -43,13 +43,13 @@ type SignedResponse struct { var SignedResponseUndefined = SignedResponse{} // Verify verifies that a proposal was signed by the given provider -func (r *SignedResponse) Verify(addr address.Address) error { +func (r *SignedResponse) Verify(addr address.Address, verifier storagemarket.SignatureVerifier) error { b, err := cborutil.Dump(&r.Response) if err != nil { return err } - return r.Signature.Verify(addr, b) + return verifier(addr, b) } // AskRequest is a request for current ask parameters for a given miner diff --git a/storagemarket/network/types_cbor_gen.go b/storagemarket/network/types_cbor_gen.go index a807d1ed..895bc2a2 100644 --- a/storagemarket/network/types_cbor_gen.go +++ b/storagemarket/network/types_cbor_gen.go @@ -1,3 +1,5 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + package network import ( @@ -10,8 +12,6 @@ import ( xerrors "golang.org/x/xerrors" ) -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - var _ = xerrors.Errorf func (t *AskRequest) MarshalCBOR(w io.Writer) error { diff --git a/storagemarket/types.go b/storagemarket/types.go index 3efe28f8..8db72b69 100644 --- a/storagemarket/types.go +++ b/storagemarket/types.go @@ -127,7 +127,9 @@ func (sdp *StorageDealProposal) Cid() (cid.Cid, error) { return nd.Cid(), nil } -func (sdp *StorageDealProposal) Verify() error { +type SignatureVerifier func(address.Address, []byte) error + +func (sdp *StorageDealProposal) Verify(verifier SignatureVerifier) error { unsigned := *sdp unsigned.ProposerSignature = nil var buf bytes.Buffer @@ -135,7 +137,7 @@ func (sdp *StorageDealProposal) Verify() error { return err } - return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()) + return verifier(sdp.Client, buf.Bytes()) } type StorageDeal struct { @@ -240,6 +242,8 @@ type StorageProviderNode interface { OnDealSectorCommitted(ctx context.Context, provider address.Address, dealID uint64, cb DealSectorCommittedCallback) error LocatePieceForDealWithinSector(ctx context.Context, dealID uint64) (sectorID uint64, offset uint64, length uint64, err error) + + VerifySignature(addr address.Address, data []byte) error } type DealSectorCommittedCallback func(err error) @@ -281,6 +285,8 @@ type StorageClientNode interface { OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb DealSectorCommittedCallback) error ValidateAskSignature(ask *types.SignedStorageAsk) error + + VerifySignature(addr address.Address, data []byte) error } type StorageClientProofs interface { From e6b87a89a788c8b9d1c72e8b6cf9a433da0d2fed Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 10 Feb 2020 18:42:04 -0800 Subject: [PATCH 2/7] chore(cborgen): update cborgen --- storagemarket/impl/client_cbor_gen.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/storagemarket/impl/client_cbor_gen.go b/storagemarket/impl/client_cbor_gen.go index 0d2e939d..cfe21713 100644 --- a/storagemarket/impl/client_cbor_gen.go +++ b/storagemarket/impl/client_cbor_gen.go @@ -6,6 +6,7 @@ import ( "fmt" "io" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" @@ -65,7 +66,10 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.Data (cid.Cid) (struct) + // t.Data (storagemarket.DataRef) (struct) + if err := t.Data.MarshalCBOR(w); err != nil { + return err + } // t.PricePerEpoch (tokenamount.TokenAmount) (struct) if err := t.PricePerEpoch.MarshalCBOR(w); err != nil { @@ -126,10 +130,26 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Data (cid.Cid) (struct) + // t.Data (storagemarket.DataRef) (struct) { + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Data = new(storagemarket.DataRef) + if err := t.Data.UnmarshalCBOR(br); err != nil { + return err + } + } + } // t.PricePerEpoch (tokenamount.TokenAmount) (struct) From 7137a9dcd313a0a83e28f11887609408236363ab Mon Sep 17 00:00:00 2001 From: Mosh <1306020+mishmosh@users.noreply.github.com> Date: Tue, 11 Feb 2020 15:44:18 -0500 Subject: [PATCH 3/7] Update README (#120) * Update README - list & add one-liner description for each component - stub a Usage section, for a future PR * Install instructions --- README.md | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0c4eb087..86591335 100644 --- a/README.md +++ b/README.md @@ -3,21 +3,48 @@ [![CircleCI](https://circleci.com/gh/filecoin-project/go-fil-markets.svg?style=svg)](https://circleci.com/gh/filecoin-project/go-fil-markets) [![codecov](https://codecov.io/gh/filecoin-project/go-fil-markets/branch/master/graph/badge.svg)](https://codecov.io/gh/filecoin-project/go-fil-markets) -This repository contains modular implementations of the storage and retrieval market subsystems of Filecoin. These modules are guided by the [v1.0 and 1.1 Filecoin specification updates](https://filecoin-project.github.io/specs/#intro__changelog). +This repository contains modular implementations of the [storage and retrieval market subsystems](https://filecoin-project.github.io/specs/#systems__filecoin_markets) of Filecoin. They are guided by the [v1.0 and 1.1 Filecoin specification updates](https://filecoin-project.github.io/specs/#intro__changelog). -Separating an implementation into a blockchain component and one or more mining and market components presents an opportunity to encourage implementation diversity while re-using non-security-critical components, and also greatly ease miner-operator customisations. +Separating implementations into a blockchain component and one or more mining and market components presents an opportunity to encourage implementation diversity while reusing non-security-critical components. ## Components -* [filestore](./filestore), ... +* **[filestore](./filestore)**: a submodule branch that is a side effect of using the ffi to generate commP +* **[pieceio](./pieceio)**: utilities that take IPLD graphs and turn them into pieces +* **[piecestore](/.piecestore)**: +* **[storage market](./storagemarket)**: for finding, negotiating, and consummating deals to store data between clients and providers (storage miners) +* **[retrieval market](./retrievalmarket)**: for finding, negotiating, and consummating deals to retrieve data between clients and providers (retrieval miners) + +Related components in other repos: +* **[data transfer](https://github.com/filecoin-project/go-data-transfer)**: for exchanging piece data between clients and miners, used by storage & retrieval market modules. + +### Background reading +* The [Markets in Filecoin](https://filecoin-project.github.io/specs/#systems__filecoin_markets) section of the Filecoin Specification contains the canonical spec +* The [Storage Market Module design doc](https://docs.google.com/document/d/1FfMUpW8vanR9FrXsybxBBbba7DzeyuCIN2uAXgE7J8U) is a more specific overview of these component implementations + +## Usage + +**Requires go 1.13** + +Install the go-fil-markets module(s) that you want to use. Available modules are: +* `filestore` +* `pieceio` +* `piecestore` +* `retrievalmarket` +* `storagemarket` + +Install with: +`go get "github.com/filecoin-project/go-fil-markets/"` + +TODO: usage for each module (maybe in subdirectories) ## Contributing -PRs are welcome! Please first read the design docs and look over the current code. PRs against -master require approval of at least two maintainers. For the rest, please see our -[CONTRIBUTING](.go-fil-markets/CONTRIBUTING.md) guide. +Issues and PRs are welcome! Please first read the [background reading](#background-reading) and [CONTRIBUTING](.go-fil-markets/CONTRIBUTING.md) guide, and look over the current code. PRs against master require approval of at least two maintainers. + +Day-to-day discussion takes place in the #fil-components channel of the [Filecoin project chat](https://github.com/filecoin-project/community#chat). Usage or design questions are welcome. ## Project-level documentation -The filecoin-project has a [community repo](https://github.com/filecoin-project/community) that documents in more detail our policies and guidelines, such as discussion forums and chat rooms and [Code of Conduct](https://github.com/filecoin-project/community/blob/master/CODE_OF_CONDUCT.md). +The filecoin-project has a [community repo](https://github.com/filecoin-project/community) with more detail about our resources and policies, such as the [Code of Conduct](https://github.com/filecoin-project/community/blob/master/CODE_OF_CONDUCT.md). ## License This repository is dual-licensed under Apache 2.0 and MIT terms. From a7bad902c9a8602360e6854a78d5c4ecde89644a Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Wed, 12 Feb 2020 12:05:13 -0800 Subject: [PATCH 4/7] Import spec actor types (#118) * feat(types): switch to spec actor big.Int Switch to the tokenamount in the spec-actor repo and remove tokenamount.TokenAmount * refactor(types): remove shared types move ask to storage market, import voucher & signature from specs-actors * refactor(types): remove shared folder entirely Just remove 2 references to a single constant that does not have the same meaning any more * feat(storagemarket): import chain deal types use chains deal proposal and storage deal proposal, switch to using PieceCIDs, switch storageAsk to chain epoch based, and have the piece store be keyed by CIDs as well. * refactor(storagemarket): simplify deal access use embedded struct to simplify property access * test(storagemarket): integration test offline deals add an integration test to verify the offline deal flow is working * fix(lint): fix lint errors --- go.mod | 18 +- go.sum | 33 ++ piecestore/piecestore.go | 35 +- piecestore/piecestore_test.go | 8 +- piecestore/types.go | 10 +- piecestore/types_cbor_gen.go | 71 +-- .../blockunsealing/blockunsealing_test.go | 4 +- retrievalmarket/impl/client.go | 10 +- retrievalmarket/impl/client_test.go | 4 +- .../impl/clientstates/client_states.go | 17 +- .../impl/clientstates/client_states_test.go | 45 +- retrievalmarket/impl/integration_test.go | 44 +- retrievalmarket/impl/provider.go | 27 +- retrievalmarket/impl/provider_test.go | 6 +- .../impl/providerstates/provider_states.go | 17 +- .../providerstates/provider_states_test.go | 27 +- .../testnodes/test_retrieval_client_node.go | 26 +- .../testnodes/test_retrieval_provider_node.go | 28 +- retrievalmarket/network/libp2p_impl_test.go | 4 +- retrievalmarket/types.go | 54 +-- retrievalmarket/types_cbor_gen.go | 75 +-- shared/params/params_shared.go | 46 -- shared/tokenamount/tokenamount.go | 296 ------------ shared/tokenamount/tokenamount_test.go | 197 -------- shared/types/ask.go | 30 -- shared/types/ask_cbor_gen.go | 211 -------- shared/types/signature.go | 123 ----- shared/types/signature_cgo.go | 55 --- shared/types/signature_test.go | 27 -- shared/types/voucher.go | 92 ---- shared/types/voucher_cbor_gen.go | 384 --------------- shared_testutil/test_piecestore.go | 36 +- shared_testutil/test_types.go | 87 ++-- shared_testutil/testutil.go | 13 + storagemarket/impl/client.go | 47 +- storagemarket/impl/client_cbor_gen.go | 91 +++- storagemarket/impl/client_states.go | 3 +- storagemarket/impl/client_storagemarket.go | 45 +- storagemarket/impl/client_utils.go | 20 +- storagemarket/impl/provider.go | 35 +- storagemarket/impl/provider_asks.go | 28 +- storagemarket/impl/provider_states.go | 42 +- storagemarket/impl/provider_storagemarket.go | 25 +- storagemarket/impl/provider_utils.go | 25 +- storagemarket/impl/request_validation_test.go | 51 +- storagemarket/integration_test.go | 182 +++++-- storagemarket/network/types.go | 20 +- storagemarket/network/types_cbor_gen.go | 21 +- storagemarket/types.go | 149 ++---- storagemarket/types_cbor_gen.go | 455 ++++++++---------- 50 files changed, 1022 insertions(+), 2377 deletions(-) delete mode 100644 shared/params/params_shared.go delete mode 100644 shared/tokenamount/tokenamount.go delete mode 100644 shared/tokenamount/tokenamount_test.go delete mode 100644 shared/types/ask.go delete mode 100644 shared/types/ask_cbor_gen.go delete mode 100644 shared/types/signature.go delete mode 100644 shared/types/signature_cgo.go delete mode 100644 shared/types/signature_test.go delete mode 100644 shared/types/voucher.go delete mode 100644 shared/types/voucher_cbor_gen.go diff --git a/go.mod b/go.mod index b27020c2..f2c66db1 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,19 @@ module github.com/filecoin-project/go-fil-markets go 1.13 require ( - github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590 - github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f + github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 - github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce + github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 github.com/filecoin-project/go-padreader v0.0.0-20200130212543-892867c4edf9 github.com/filecoin-project/go-sectorbuilder v0.0.1 github.com/filecoin-project/go-statestore v0.1.0 + github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c github.com/ipfs/go-car v0.0.3-0.20200131220434-3f68f6ebd093 - github.com/ipfs/go-cid v0.0.4 + github.com/ipfs/go-cid v0.0.5 github.com/ipfs/go-datastore v0.1.1 github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-ipfs-blockstore v0.1.0 @@ -24,7 +24,7 @@ require ( github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.4 - github.com/ipfs/go-ipld-cbor v0.0.3 + github.com/ipfs/go-ipld-cbor v0.0.4 github.com/ipfs/go-ipld-format v0.0.2 github.com/ipfs/go-log/v2 v2.0.1 github.com/ipfs/go-merkledag v0.2.4 @@ -33,12 +33,10 @@ require ( github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/libp2p/go-libp2p v0.3.0 - github.com/libp2p/go-libp2p-core v0.2.4 - github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 - github.com/multiformats/go-multihash v0.0.10 - github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a + github.com/libp2p/go-libp2p-core v0.3.0 + github.com/multiformats/go-multihash v0.0.13 github.com/stretchr/testify v1.4.0 - github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 + github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 ) diff --git a/go.sum b/go.sum index 9aa065a7..1814305c 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcug github.com/btcsuite/btcd v0.0.0-20190629003639-c26ffa870fd8/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -60,12 +62,18 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2jaVU8TvWTx7iZPhlYvUE8vkoOnj778XuKavz8W36g= github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms= +github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5 h1:/MmWluswvDIbuPvBct4q6HeQgVm62O2DzWYTB38kt4A= +github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e h1:IOoff6yAZSJ5zHCPY2jzGNwQYQU6ygsRVe/cSnJrY+o= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= +github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= +github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= github.com/filecoin-project/go-padreader v0.0.0-20200130212543-892867c4edf9 h1:CQsjS+oWG96rk5YbeKpPw84fhbgc5H6/BGvrlPgd63A= github.com/filecoin-project/go-padreader v0.0.0-20200130212543-892867c4edf9/go.mod h1:r0gyD7zvnqyRKSY8stil5G/LF0kXFgNzW/yR4vjga+Y= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= @@ -74,6 +82,8 @@ github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCst github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= +github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf h1:fbxBG12yrxilPFV1EG2lYqpUyAlRZWkvtqjk2svSeXY= +github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= @@ -153,6 +163,8 @@ github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10 h1:5mRf2p8Bv2iKiuPsG github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= +github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= @@ -167,6 +179,8 @@ github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaH github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-graphsync v0.0.4 h1:iF98+J8pcqvEb48IM0TemqeGARsCDtwQ73P9ejMZIuU= github.com/ipfs/go-graphsync v0.0.4/go.mod h1:6UACBjfOXEa8rQL3Q/JpZpWS0nZDCLx134WUkjrmFpQ= +github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e h1:bUtmeXx6JpjxRPlMdlKfPXC5kKhLHuueXKgs1Txb9ZU= +github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= @@ -198,6 +212,8 @@ github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyB github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= +github.com/ipfs/go-ipld-cbor v0.0.4 h1:Aw3KPOKXjvrm6VjwJvFf1F1ekR/BH3jdof3Bk7OTiSA= +github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= @@ -272,6 +288,8 @@ github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvM github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-flow-metrics v0.0.3 h1:8tAs/hSdNvUiLgtlSy3mxwxWP4I9y/jlkPFT7epKdeM= +github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= github.com/libp2p/go-libp2p v0.2.1/go.mod h1:HZbtEOrgZN4F1fGZVvkV+930Wx3DkqlpBlO8dIoZWds= @@ -294,6 +312,8 @@ github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= github.com/libp2p/go-libp2p-core v0.2.4 h1:Et6ykkTwI6PU44tr8qUF9k43vP0aduMNniShAbUJJw8= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= +github.com/libp2p/go-libp2p-core v0.3.0 h1:F7PqduvrztDtFsAa/bcheQ3azmNo+Nq7m8hQY5GiUW8= +github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= @@ -364,6 +384,8 @@ github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36Gchpc github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-openssl v0.0.3 h1:wjlG7HvQkt4Fq4cfH33Ivpwp0omaElYEi9z26qaIkIk= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= +github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= @@ -420,6 +442,8 @@ github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1 h1:rVAztJYMhCQ7vEFr8FvxW3mS+HF2eY/oPbOMeS0ZDnE= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= +github.com/multiformats/go-multiaddr v0.2.0 h1:lR52sFwcTCuQb6bTfnXF6zA2XfyYvyd+5a9qECv/J90= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= @@ -442,11 +466,16 @@ github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHxszqlPTA= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -471,6 +500,8 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= @@ -534,6 +565,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 h1:efb/4Cnr github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66 h1:LolR9FiEfQNn5U031bAhn/46po2JgWHKadYbcWFIJ+0= +github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= diff --git a/piecestore/piecestore.go b/piecestore/piecestore.go index d2fed68e..df83e187 100644 --- a/piecestore/piecestore.go +++ b/piecestore/piecestore.go @@ -1,9 +1,6 @@ package piecestore import ( - "bytes" - "fmt" - "github.com/filecoin-project/go-statestore" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -29,7 +26,7 @@ type pieceStore struct { cidInfos *statestore.StateStore } -func (ps *pieceStore) AddDealForPiece(pieceCID []byte, dealInfo DealInfo) error { +func (ps *pieceStore) AddDealForPiece(pieceCID cid.Cid, dealInfo DealInfo) error { return ps.mutatePieceInfo(pieceCID, func(pi *PieceInfo) error { for _, di := range pi.Deals { if di == dealInfo { @@ -41,11 +38,11 @@ func (ps *pieceStore) AddDealForPiece(pieceCID []byte, dealInfo DealInfo) error }) } -func (ps *pieceStore) AddPieceBlockLocations(pieceCID []byte, blockLocations map[cid.Cid]BlockLocation) error { +func (ps *pieceStore) AddPieceBlockLocations(pieceCID cid.Cid, blockLocations map[cid.Cid]BlockLocation) error { for c, blockLocation := range blockLocations { err := ps.mutateCIDInfo(c, func(ci *CIDInfo) error { for _, pbl := range ci.PieceBlockLocations { - if bytes.Equal(pbl.PieceCID, pieceCID) && pbl.BlockLocation == blockLocation { + if pbl.PieceCID.Equals(pieceCID) && pbl.BlockLocation == blockLocation { return nil } } @@ -59,9 +56,9 @@ func (ps *pieceStore) AddPieceBlockLocations(pieceCID []byte, blockLocations map return nil } -func (ps *pieceStore) GetPieceInfo(pieceCID []byte) (PieceInfo, error) { +func (ps *pieceStore) GetPieceInfo(pieceCID cid.Cid) (PieceInfo, error) { var out PieceInfo - if err := ps.pieces.Get(newKey(pieceCID)).Get(&out); err != nil { + if err := ps.pieces.Get(pieceCID).Get(&out); err != nil { return PieceInfo{}, err } return out, nil @@ -75,8 +72,8 @@ func (ps *pieceStore) GetCIDInfo(payloadCID cid.Cid) (CIDInfo, error) { return out, nil } -func (ps *pieceStore) ensurePieceInfo(pieceCID []byte) error { - has, err := ps.pieces.Has(newKey(pieceCID)) +func (ps *pieceStore) ensurePieceInfo(pieceCID cid.Cid) error { + has, err := ps.pieces.Has(pieceCID) if err != nil { return err @@ -86,7 +83,7 @@ func (ps *pieceStore) ensurePieceInfo(pieceCID []byte) error { } pieceInfo := PieceInfo{PieceCID: pieceCID} - return ps.pieces.Begin(newKey(pieceCID), &pieceInfo) + return ps.pieces.Begin(pieceCID, &pieceInfo) } func (ps *pieceStore) ensureCIDInfo(c cid.Cid) error { @@ -104,13 +101,13 @@ func (ps *pieceStore) ensureCIDInfo(c cid.Cid) error { return ps.cidInfos.Begin(c, &cidInfo) } -func (ps *pieceStore) mutatePieceInfo(pieceCID []byte, mutator interface{}) error { +func (ps *pieceStore) mutatePieceInfo(pieceCID cid.Cid, mutator interface{}) error { err := ps.ensurePieceInfo(pieceCID) if err != nil { return err } - return ps.pieces.Get(newKey(pieceCID)).Mutate(mutator) + return ps.pieces.Get(pieceCID).Mutate(mutator) } func (ps *pieceStore) mutateCIDInfo(c cid.Cid, mutator interface{}) error { @@ -121,15 +118,3 @@ func (ps *pieceStore) mutateCIDInfo(c cid.Cid, mutator interface{}) error { return ps.cidInfos.Get(c).Mutate(mutator) } - -func newKey(pieceCID []byte) fmt.Stringer { - return &pieceStoreKey{pieceCID} -} - -type pieceStoreKey struct { - cid []byte -} - -func (k *pieceStoreKey) String() string { - return string(k.cid) -} diff --git a/piecestore/piecestore_test.go b/piecestore/piecestore_test.go index e758b224..4014a584 100644 --- a/piecestore/piecestore_test.go +++ b/piecestore/piecestore_test.go @@ -14,7 +14,7 @@ import ( func TestStorePieceInfo(t *testing.T) { - pieceCid := []byte{1, 2, 3, 4} + pieceCid := shared_testutil.GenerateCids(1)[0] initializePieceStore := func(t *testing.T) piecestore.PieceStore { ps := piecestore.NewPieceStore(datastore.NewMapDatastore()) _, err := ps.GetPieceInfo(pieceCid) @@ -67,9 +67,9 @@ func TestStorePieceInfo(t *testing.T) { } func TestStoreCIDInfo(t *testing.T) { - - pieceCid1 := []byte{1, 2, 3, 4} - pieceCid2 := []byte{5, 6, 7, 8} + pieceCids := shared_testutil.GenerateCids(2) + pieceCid1 := pieceCids[0] + pieceCid2 := pieceCids[1] testCIDs := shared_testutil.GenerateCids(3) blockLocations := make([]piecestore.BlockLocation, 0, 3) for i := 0; i < 3; i++ { diff --git a/piecestore/types.go b/piecestore/types.go index 0858c4ed..4a42c71a 100644 --- a/piecestore/types.go +++ b/piecestore/types.go @@ -24,7 +24,7 @@ type BlockLocation struct { // is inside of type PieceBlockLocation struct { BlockLocation - PieceCID []byte + PieceCID cid.Cid } // CIDInfo is information about where a given CID will live inside a piece @@ -40,7 +40,7 @@ var CIDInfoUndefined = CIDInfo{} // on its PieceCID -- so that, given a pieceCID during retrieval, the miner // can determine how to unseal it if needed type PieceInfo struct { - PieceCID []byte + PieceCID cid.Cid Deals []DealInfo } @@ -49,8 +49,8 @@ var PieceInfoUndefined = PieceInfo{} // PieceStore is a saved database of piece info that can be modified and queried type PieceStore interface { - AddDealForPiece(pieceCID []byte, dealInfo DealInfo) error - AddPieceBlockLocations(pieceCID []byte, blockLocations map[cid.Cid]BlockLocation) error - GetPieceInfo(pieceCID []byte) (PieceInfo, error) + AddDealForPiece(pieceCID cid.Cid, dealInfo DealInfo) error + AddPieceBlockLocations(pieceCID cid.Cid, blockLocations map[cid.Cid]BlockLocation) error + GetPieceInfo(pieceCID cid.Cid) (PieceInfo, error) GetCIDInfo(payloadCID cid.Cid) (CIDInfo, error) } diff --git a/piecestore/types_cbor_gen.go b/piecestore/types_cbor_gen.go index 1fbb5f43..2517499c 100644 --- a/piecestore/types_cbor_gen.go +++ b/piecestore/types_cbor_gen.go @@ -21,16 +21,10 @@ func (t *PieceInfo) MarshalCBOR(w io.Writer) error { return err } - // t.PieceCID ([]uint8) (slice) - if len(t.PieceCID) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.PieceCID was too long") - } + // t.PieceCID (cid.Cid) (struct) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceCID)))); err != nil { - return err - } - if _, err := w.Write(t.PieceCID); err != nil { - return err + if err := cbg.WriteCid(w, t.PieceCID); err != nil { + return xerrors.Errorf("failed to write cid field t.PieceCID: %w", err) } // t.Deals ([]piecestore.DealInfo) (slice) @@ -64,22 +58,17 @@ func (t *PieceInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.PieceCID ([]uint8) (slice) + // t.PieceCID (cid.Cid) (struct) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) + } + + t.PieceCID = c - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.PieceCID: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.PieceCID = make([]byte, extra) - if _, err := io.ReadFull(br, t.PieceCID); err != nil { - return err } // t.Deals ([]piecestore.DealInfo) (slice) @@ -273,17 +262,12 @@ func (t *PieceBlockLocation) MarshalCBOR(w io.Writer) error { return err } - // t.PieceCID ([]uint8) (slice) - if len(t.PieceCID) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.PieceCID was too long") - } + // t.PieceCID (cid.Cid) (struct) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceCID)))); err != nil { - return err - } - if _, err := w.Write(t.PieceCID); err != nil { - return err + if err := cbg.WriteCid(w, t.PieceCID); err != nil { + return xerrors.Errorf("failed to write cid field t.PieceCID: %w", err) } + return nil } @@ -311,22 +295,17 @@ func (t *PieceBlockLocation) UnmarshalCBOR(r io.Reader) error { } } - // t.PieceCID ([]uint8) (slice) + // t.PieceCID (cid.Cid) (struct) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) + } + + t.PieceCID = c - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.PieceCID: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.PieceCID = make([]byte, extra) - if _, err := io.ReadFull(br, t.PieceCID); err != nil { - return err } return nil } diff --git a/retrievalmarket/impl/blockunsealing/blockunsealing_test.go b/retrievalmarket/impl/blockunsealing/blockunsealing_test.go index 9ed2452d..b0bdf028 100644 --- a/retrievalmarket/impl/blockunsealing/blockunsealing_test.go +++ b/retrievalmarket/impl/blockunsealing/blockunsealing_test.go @@ -54,7 +54,7 @@ func TestNewLoaderWithUnsealing(t *testing.T) { Offset: rand.Uint64(), Length: rand.Uint64(), } - pieceCID := []byte("applesauce") + pieceCID := tut.GenerateCids(1)[0] piece := piecestore.PieceInfo{ PieceCID: pieceCID, Deals: []piecestore.DealInfo{ @@ -68,7 +68,7 @@ func TestNewLoaderWithUnsealing(t *testing.T) { Offset: rand.Uint64(), Length: rand.Uint64(), } - pieceCID2 := []byte("cheesewhiz") + pieceCID2 := tut.GenerateCids(1)[0] piece2 := piecestore.PieceInfo{ PieceCID: pieceCID2, Deals: []piecestore.DealInfo{ diff --git a/retrievalmarket/impl/client.go b/retrievalmarket/impl/client.go index e3f8d6f6..12d24402 100644 --- a/retrievalmarket/impl/client.go +++ b/retrievalmarket/impl/client.go @@ -18,7 +18,7 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/blockio" "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/clientstates" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + "github.com/filecoin-project/specs-actors/actors/abi" ) var log = logging.Logger("retrieval") @@ -84,7 +84,7 @@ func (c *client) Query(ctx context.Context, p retrievalmarket.RetrievalPeer, pay } // Retrieve begins the process of requesting the data referred to by payloadCID, after a deal is accepted -func (c *client) Retrieve(ctx context.Context, payloadCID cid.Cid, params retrievalmarket.Params, totalFunds tokenamount.TokenAmount, miner peer.ID, clientWallet address.Address, minerWallet address.Address) retrievalmarket.DealID { +func (c *client) Retrieve(ctx context.Context, payloadCID cid.Cid, params retrievalmarket.Params, totalFunds abi.TokenAmount, miner peer.ID, clientWallet address.Address, minerWallet address.Address) retrievalmarket.DealID { /* The implementation of this function is just wrapper for the old code which retrieves UnixFS pieces -- it will be replaced when we do the V0 implementation of the module */ c.nextDealLk.Lock() @@ -104,8 +104,8 @@ func (c *client) Retrieve(ctx context.Context, payloadCID cid.Cid, params retrie TotalReceived: 0, CurrentInterval: params.PaymentInterval, BytesPaidFor: 0, - PaymentRequested: tokenamount.FromInt(0), - FundsSpent: tokenamount.FromInt(0), + PaymentRequested: abi.NewTokenAmount(0), + FundsSpent: abi.NewTokenAmount(0), Status: retrievalmarket.DealStatusNew, Sender: miner, } @@ -199,7 +199,7 @@ func (c *client) SubscribeToEvents(subscriber retrievalmarket.ClientSubscriber) } // V1 -func (c *client) AddMoreFunds(id retrievalmarket.DealID, amount tokenamount.TokenAmount) error { +func (c *client) AddMoreFunds(id retrievalmarket.DealID, amount abi.TokenAmount) error { panic("not implemented") } diff --git a/retrievalmarket/impl/client_test.go b/retrievalmarket/impl/client_test.go index 7fe3b9a0..ce2035af 100644 --- a/retrievalmarket/impl/client_test.go +++ b/retrievalmarket/impl/client_test.go @@ -19,8 +19,8 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/testnodes" "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" tut "github.com/filecoin-project/go-fil-markets/shared_testutil" + "github.com/filecoin-project/specs-actors/actors/abi" ) func TestClient_Query(t *testing.T) { @@ -43,7 +43,7 @@ func TestClient_Query(t *testing.T) { Status: retrievalmarket.QueryResponseAvailable, Size: 1234, PaymentAddress: address.TestAddress, - MinPricePerByte: tokenamount.FromInt(5678), + MinPricePerByte: abi.NewTokenAmount(5678), MaxPaymentInterval: 4321, MaxPaymentIntervalIncrease: 0, } diff --git a/retrievalmarket/impl/clientstates/client_states.go b/retrievalmarket/impl/clientstates/client_states.go index 4b047874..0718a904 100644 --- a/retrievalmarket/impl/clientstates/client_states.go +++ b/retrievalmarket/impl/clientstates/client_states.go @@ -8,7 +8,8 @@ import ( rm "github.com/filecoin-project/go-fil-markets/retrievalmarket" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" ) // ClientDealEnvironment is a bridge to the environment a client deal is executing in @@ -81,9 +82,9 @@ func ProposeDeal(ctx context.Context, environment ClientDealEnvironment, deal rm func ProcessPaymentRequested(ctx context.Context, environment ClientDealEnvironment, deal rm.ClientDealState) func(*rm.ClientDealState) { // check that fundsSpent + paymentRequested <= totalFunds, or fail - if tokenamount.Add(deal.FundsSpent, deal.PaymentRequested).GreaterThan(deal.TotalFunds) { + if big.Add(deal.FundsSpent, deal.PaymentRequested).GreaterThan(deal.TotalFunds) { expectedTotal := deal.TotalFunds.String() - actualTotal := tokenamount.Add(deal.FundsSpent, deal.PaymentRequested).String() + actualTotal := big.Add(deal.FundsSpent, deal.PaymentRequested).String() errMsg := fmt.Sprintf("not enough funds left: expected amt = %s, actual amt = %s", expectedTotal, actualTotal) return errorFunc(xerrors.New(errMsg)) } @@ -94,13 +95,13 @@ func ProcessPaymentRequested(ctx context.Context, environment ClientDealEnvironm } // check that paymentRequest <= (totalReceived - bytesPaidFor) * pricePerByte, or fail - if deal.PaymentRequested.GreaterThan(tokenamount.Mul(tokenamount.FromInt(deal.TotalReceived-deal.BytesPaidFor), deal.PricePerByte)) { + if deal.PaymentRequested.GreaterThan(big.Mul(abi.NewTokenAmount(int64(deal.TotalReceived-deal.BytesPaidFor)), deal.PricePerByte)) { return errorFunc(xerrors.New("too much money requested for bytes sent")) } // create payment voucher with node (or fail) for (fundsSpent + paymentRequested) // use correct payCh + lane // (node will do subtraction back to paymentRequested... slightly odd behavior but... well anyway) - voucher, err := environment.Node().CreatePaymentVoucher(ctx, deal.PayCh, tokenamount.Add(deal.FundsSpent, deal.PaymentRequested), deal.Lane) + voucher, err := environment.Node().CreatePaymentVoucher(ctx, deal.PayCh, big.Add(deal.FundsSpent, deal.PaymentRequested), deal.Lane) if err != nil { return errorFunc(xerrors.Errorf("creating payment voucher: %w", err)) } @@ -129,13 +130,13 @@ func ProcessPaymentRequested(ctx context.Context, environment ClientDealEnvironm } else { deal.Status = rm.DealStatusOngoing } - deal.FundsSpent = tokenamount.Add(deal.FundsSpent, deal.PaymentRequested) - bytesPaidFor := tokenamount.Div(deal.PaymentRequested, deal.PricePerByte).Uint64() + deal.FundsSpent = big.Add(deal.FundsSpent, deal.PaymentRequested) + bytesPaidFor := big.Div(deal.PaymentRequested, deal.PricePerByte).Uint64() if bytesPaidFor >= deal.CurrentInterval { deal.CurrentInterval += deal.DealProposal.PaymentIntervalIncrease } deal.BytesPaidFor += bytesPaidFor - deal.PaymentRequested = tokenamount.FromInt(0) + deal.PaymentRequested = abi.NewTokenAmount(0) } } diff --git a/retrievalmarket/impl/clientstates/client_states_test.go b/retrievalmarket/impl/clientstates/client_states_test.go index c48fc9cc..3875769f 100644 --- a/retrievalmarket/impl/clientstates/client_states_test.go +++ b/retrievalmarket/impl/clientstates/client_states_test.go @@ -15,9 +15,10 @@ import ( clientstates "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/clientstates" "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/testnodes" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" testnet "github.com/filecoin-project/go-fil-markets/shared_testutil" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin/paych" ) type consumeBlockResponse struct { @@ -54,7 +55,7 @@ func TestSetupPaymentChannel(t *testing.T) { ctx := context.Background() ds := testnet.NewTestRetrievalDealStream(testnet.TestDealStreamParams{}) expectedPayCh := address.TestAddress2 - expectedLane := uint64(10) + expectedLane := int64(10) environment := func(params testnodes.TestRetrievalClientNodeParams) clientstates.ClientDealEnvironment { node := testnodes.NewTestRetrievalClientNode(params) @@ -191,7 +192,7 @@ func TestProcessPaymentRequested(t *testing.T) { return &fakeEnvironment{node, ds, 0, nil} } - testVoucher := &types.SignedVoucher{} + testVoucher := &paych.SignedVoucher{} t.Run("it works", func(t *testing.T) { dealState := makeDealState(retrievalmarket.DealStatusFundsNeeded) @@ -201,8 +202,8 @@ func TestProcessPaymentRequested(t *testing.T) { f := clientstates.ProcessPaymentRequested(ctx, fe, *dealState) f(dealState) require.Empty(t, dealState.Message) - require.Equal(t, dealState.PaymentRequested, tokenamount.FromInt(0)) - require.Equal(t, dealState.FundsSpent, tokenamount.Add(defaultFundsSpent, defaultPaymentRequested)) + require.Equal(t, dealState.PaymentRequested, abi.NewTokenAmount(0)) + require.Equal(t, dealState.FundsSpent, big.Add(defaultFundsSpent, defaultPaymentRequested)) require.Equal(t, dealState.BytesPaidFor, defaultTotalReceived) require.Equal(t, dealState.CurrentInterval, defaultCurrentInterval+defaultIntervalIncrease) require.Equal(t, dealState.Status, retrievalmarket.DealStatusOngoing) @@ -216,8 +217,8 @@ func TestProcessPaymentRequested(t *testing.T) { f := clientstates.ProcessPaymentRequested(ctx, fe, *dealState) f(dealState) require.Empty(t, dealState.Message) - require.Equal(t, dealState.PaymentRequested, tokenamount.FromInt(0)) - require.Equal(t, dealState.FundsSpent, tokenamount.Add(defaultFundsSpent, defaultPaymentRequested)) + require.Equal(t, dealState.PaymentRequested, abi.NewTokenAmount(0)) + require.Equal(t, dealState.FundsSpent, big.Add(defaultFundsSpent, defaultPaymentRequested)) require.Equal(t, dealState.BytesPaidFor, defaultTotalReceived) require.Equal(t, dealState.CurrentInterval, defaultCurrentInterval+defaultIntervalIncrease) require.Equal(t, dealState.Status, retrievalmarket.DealStatusCompleted) @@ -250,7 +251,7 @@ func TestProcessPaymentRequested(t *testing.T) { t.Run("more bytes since last payment than interval works, can charge more", func(t *testing.T) { dealState := makeDealState(retrievalmarket.DealStatusFundsNeeded) dealState.BytesPaidFor = defaultBytesPaidFor - 500 - largerPaymentRequested := tokenamount.FromInt(750000) + largerPaymentRequested := abi.NewTokenAmount(750000) dealState.PaymentRequested = largerPaymentRequested fe := environment(testnet.TestDealStreamParams{}, testnodes.TestRetrievalClientNodeParams{ @@ -259,8 +260,8 @@ func TestProcessPaymentRequested(t *testing.T) { f := clientstates.ProcessPaymentRequested(ctx, fe, *dealState) f(dealState) require.Empty(t, dealState.Message) - require.Equal(t, dealState.PaymentRequested, tokenamount.FromInt(0)) - require.Equal(t, dealState.FundsSpent, tokenamount.Add(defaultFundsSpent, largerPaymentRequested)) + require.Equal(t, dealState.PaymentRequested, abi.NewTokenAmount(0)) + require.Equal(t, dealState.FundsSpent, big.Add(defaultFundsSpent, largerPaymentRequested)) require.Equal(t, dealState.BytesPaidFor, defaultTotalReceived) require.Equal(t, dealState.CurrentInterval, defaultCurrentInterval+defaultIntervalIncrease) require.Equal(t, dealState.Status, retrievalmarket.DealStatusOngoing) @@ -268,7 +269,7 @@ func TestProcessPaymentRequested(t *testing.T) { t.Run("too much payment requested", func(t *testing.T) { dealState := makeDealState(retrievalmarket.DealStatusFundsNeeded) - dealState.PaymentRequested = tokenamount.FromInt(750000) + dealState.PaymentRequested = abi.NewTokenAmount(750000) fe := environment(testnet.TestDealStreamParams{}, testnodes.TestRetrievalClientNodeParams{ Voucher: testVoucher, }) @@ -280,7 +281,7 @@ func TestProcessPaymentRequested(t *testing.T) { t.Run("too little payment requested works but records correctly", func(t *testing.T) { dealState := makeDealState(retrievalmarket.DealStatusFundsNeeded) - smallerPaymentRequested := tokenamount.FromInt(250000) + smallerPaymentRequested := abi.NewTokenAmount(250000) dealState.PaymentRequested = smallerPaymentRequested fe := environment(testnet.TestDealStreamParams{}, testnodes.TestRetrievalClientNodeParams{ Voucher: testVoucher, @@ -288,8 +289,8 @@ func TestProcessPaymentRequested(t *testing.T) { f := clientstates.ProcessPaymentRequested(ctx, fe, *dealState) f(dealState) require.Empty(t, dealState.Message) - require.Equal(t, dealState.PaymentRequested, tokenamount.FromInt(0)) - require.Equal(t, dealState.FundsSpent, tokenamount.Add(defaultFundsSpent, smallerPaymentRequested)) + require.Equal(t, dealState.PaymentRequested, abi.NewTokenAmount(0)) + require.Equal(t, dealState.FundsSpent, big.Add(defaultFundsSpent, smallerPaymentRequested)) // only records change for those bytes paid for require.Equal(t, dealState.BytesPaidFor, defaultBytesPaidFor+500) // no interval increase @@ -374,7 +375,7 @@ func TestProcessNextResponse(t *testing.T) { response := retrievalmarket.DealResponse{ Status: retrievalmarket.DealStatusFundsNeededLastPayment, ID: dealState.ID, - PaymentOwed: tokenamount.FromInt(1000), + PaymentOwed: abi.NewTokenAmount(1000), Blocks: blocks, } fe := environment(testnet.TestDealStreamParams{ @@ -411,7 +412,7 @@ func TestProcessNextResponse(t *testing.T) { response := retrievalmarket.DealResponse{ Status: retrievalmarket.DealStatusFundsNeeded, ID: dealState.ID, - PaymentOwed: tokenamount.FromInt(1000), + PaymentOwed: abi.NewTokenAmount(1000), Blocks: blocks, } fe := environment(testnet.TestDealStreamParams{ @@ -474,14 +475,14 @@ func TestProcessNextResponse(t *testing.T) { }) } -var defaultTotalFunds = tokenamount.FromInt(4000000) +var defaultTotalFunds = abi.NewTokenAmount(4000000) var defaultCurrentInterval = uint64(1000) var defaultIntervalIncrease = uint64(500) -var defaultPricePerByte = tokenamount.FromInt(500) +var defaultPricePerByte = abi.NewTokenAmount(500) var defaultTotalReceived = uint64(6000) var defaultBytesPaidFor = uint64(5000) -var defaultFundsSpent = tokenamount.FromInt(2500000) -var defaultPaymentRequested = tokenamount.FromInt(500000) +var defaultFundsSpent = abi.NewTokenAmount(2500000) +var defaultPaymentRequested = abi.NewTokenAmount(500000) func makeDealState(status retrievalmarket.DealStatus) *retrievalmarket.ClientDealState { return &retrievalmarket.ClientDealState{ @@ -489,7 +490,7 @@ func makeDealState(status retrievalmarket.DealStatus) *retrievalmarket.ClientDea MinerWallet: address.TestAddress, ClientWallet: address.TestAddress2, PayCh: address.TestAddress2, - Lane: uint64(10), + Lane: int64(10), Status: status, BytesPaidFor: defaultBytesPaidFor, TotalReceived: defaultTotalReceived, diff --git a/retrievalmarket/impl/integration_test.go b/retrievalmarket/impl/integration_test.go index cf3e9e2c..6abe8b60 100644 --- a/retrievalmarket/impl/integration_test.go +++ b/retrievalmarket/impl/integration_test.go @@ -3,7 +3,6 @@ package retrievalimpl_test import ( "bytes" "context" - "math/rand" "testing" "time" @@ -20,9 +19,10 @@ import ( retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/testnodes" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" tut "github.com/filecoin-project/go-fil-markets/shared_testutil" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin/paych" ) func TestClientCanMakeQueryToProvider(t *testing.T) { @@ -82,7 +82,7 @@ func requireSetupTestClientAndProvider(bgCtx context.Context, t *testing.T, payC providerNode := testnodes.NewTestRetrievalProviderNode() pieceStore := tut.NewTestPieceStore() expectedCIDs := tut.GenerateCids(3) - expectedPieces := [][]byte{[]byte("applesuace"), []byte("jam"), []byte("apricot")} + expectedPieceCIDs := tut.GenerateCids(3) missingCID := tut.GenerateCids(1)[0] expectedQR := tut.MakeTestQueryResponse() @@ -91,12 +91,12 @@ func requireSetupTestClientAndProvider(bgCtx context.Context, t *testing.T, payC pieceStore.ExpectCID(c, piecestore.CIDInfo{ PieceBlockLocations: []piecestore.PieceBlockLocation{ { - PieceCID: expectedPieces[i], + PieceCID: expectedPieceCIDs[i], }, }, }) } - for i, piece := range expectedPieces { + for i, piece := range expectedPieceCIDs { pieceStore.ExpectPiece(piece, piecestore.PieceInfo{ Deals: []piecestore.DealInfo{ { @@ -127,28 +127,28 @@ func TestClientCanMakeDealWithProvider(t *testing.T) { name string filename string filesize uint64 - voucherAmts []tokenamount.TokenAmount + voucherAmts []abi.TokenAmount unsealing bool }{ {name: "1 block file retrieval succeeds", filename: "lorem_under_1_block.txt", filesize: 410, - voucherAmts: []tokenamount.TokenAmount{tokenamount.FromInt(410000)}, + voucherAmts: []abi.TokenAmount{abi.NewTokenAmount(410000)}, unsealing: false}, {name: "1 block file retrieval succeeds with unsealing", filename: "lorem_under_1_block.txt", filesize: 410, - voucherAmts: []tokenamount.TokenAmount{tokenamount.FromInt(410000)}, + voucherAmts: []abi.TokenAmount{abi.NewTokenAmount(410000)}, unsealing: true}, {name: "multi-block file retrieval succeeds", filename: "lorem.txt", filesize: 19000, - voucherAmts: []tokenamount.TokenAmount{tokenamount.FromInt(10136000), tokenamount.FromInt(9784000)}, + voucherAmts: []abi.TokenAmount{abi.NewTokenAmount(10136000), abi.NewTokenAmount(9784000)}, unsealing: false}, {name: "multi-block file retrieval succeeds with unsealing", filename: "lorem.txt", filesize: 19000, - voucherAmts: []tokenamount.TokenAmount{tokenamount.FromInt(10136000), tokenamount.FromInt(9784000)}, + voucherAmts: []abi.TokenAmount{abi.NewTokenAmount(10136000), abi.NewTokenAmount(9784000)}, unsealing: true}, } @@ -170,7 +170,7 @@ func TestClientCanMakeDealWithProvider(t *testing.T) { require.NoError(t, err) paymentInterval := uint64(10000) paymentIntervalIncrease := uint64(1000) - pricePerByte := tokenamount.FromInt(1000) + pricePerByte := abi.NewTokenAmount(1000) expectedQR := retrievalmarket.QueryResponse{ Size: 1024, @@ -224,7 +224,7 @@ func TestClientCanMakeDealWithProvider(t *testing.T) { expectedVoucher := tut.MakeTestSignedVoucher() // just make sure there is enough to cover the transfer - expectedTotal := tokenamount.Mul(pricePerByte, tokenamount.FromInt(testCase.filesize*2)) + expectedTotal := big.Mul(pricePerByte, abi.NewTokenAmount(int64(testCase.filesize*2))) // voucherAmts are pulled from the actual answer so the expected keys in the test node match up. // later we compare the voucher values. The last voucherAmt is a remainder @@ -310,7 +310,7 @@ CurrentInterval: %d require.Equal(t, clientPaymentChannel, *newLaneAddr) // verify that the voucher was saved/seen by the client with correct values require.NotNil(t, createdVoucher) - assert.True(t, createdVoucher.Equals(expectedVoucher)) + tut.TestVoucherEquality(t, createdVoucher, expectedVoucher) ctx, cancel = context.WithTimeout(bgCtx, 5*time.Second) defer cancel() @@ -334,14 +334,14 @@ CurrentInterval: %d func setupClient( clientPaymentChannel address.Address, - expectedVoucher *types.SignedVoucher, + expectedVoucher *paych.SignedVoucher, nw1 rmnet.RetrievalMarketNetwork, testData *tut.Libp2pTestData) (*pmtChan, *address.Address, - *types.SignedVoucher, + *paych.SignedVoucher, retrievalmarket.RetrievalClient) { var createdChan pmtChan - paymentChannelRecorder := func(client, miner address.Address, amt tokenamount.TokenAmount) { + paymentChannelRecorder := func(client, miner address.Address, amt abi.TokenAmount) { createdChan = pmtChan{client, miner, amt} } @@ -350,8 +350,8 @@ func setupClient( newLaneAddr = paymentChannel } - var createdVoucher types.SignedVoucher - paymentVoucherRecorder := func(v *types.SignedVoucher) { + var createdVoucher paych.SignedVoucher + paymentVoucherRecorder := func(v *paych.SignedVoucher) { createdVoucher = *v } clientNode := testnodes.NewTestRetrievalClientNode(testnodes.TestRetrievalClientNodeParams{ @@ -369,9 +369,7 @@ func setupClient( func setupProvider(t *testing.T, testData *tut.Libp2pTestData, payloadCID cid.Cid, pieceInfo piecestore.PieceInfo, expectedQR retrievalmarket.QueryResponse, providerPaymentAddr address.Address, providerNode retrievalmarket.RetrievalProviderNode) retrievalmarket.RetrievalProvider { nw2 := rmnet.NewFromLibp2pHost(testData.Host2) pieceStore := tut.NewTestPieceStore() - expectedPiece := make([]byte, 32) - _, err := rand.Read(expectedPiece) - require.NoError(t, err) + expectedPiece := tut.GenerateCids(1)[0] cidInfo := piecestore.CIDInfo{ PieceBlockLocations: []piecestore.PieceBlockLocation{ { @@ -390,5 +388,5 @@ func setupProvider(t *testing.T, testData *tut.Libp2pTestData, payloadCID cid.Ci type pmtChan struct { client, miner address.Address - amt tokenamount.TokenAmount + amt abi.TokenAmount } diff --git a/retrievalmarket/impl/provider.go b/retrievalmarket/impl/provider.go index 15e4de63..5db1ee38 100644 --- a/retrievalmarket/impl/provider.go +++ b/retrievalmarket/impl/provider.go @@ -18,8 +18,7 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/blockunsealing" "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/providerstates" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/params" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + "github.com/filecoin-project/specs-actors/actors/abi" ) type provider struct { @@ -30,7 +29,7 @@ type provider struct { paymentIntervalIncrease uint64 paymentAddress address.Address pieceStore piecestore.PieceStore - pricePerByte tokenamount.TokenAmount + pricePerByte abi.TokenAmount subscribers []retrievalmarket.ProviderSubscriber subscribersLk sync.RWMutex } @@ -39,15 +38,15 @@ var _ retrievalmarket.RetrievalProvider = &provider{} // DefaultPricePerByte is the charge per byte retrieved if the miner does // not specifically set it -var DefaultPricePerByte = tokenamount.FromInt(2) +var DefaultPricePerByte = abi.NewTokenAmount(2) -// DefaultPaymentInterval is the baseline interval, set to the unixfs chunk size +// DefaultPaymentInterval is the baseline interval, set to 1Mb // if the miner does not explicitly set it otherwise -var DefaultPaymentInterval = uint64(params.UnixfsChunkSize) +var DefaultPaymentInterval = uint64(1 << 20) -// DefaultPaymentIntervalIncrease is the amount interval increases on each payment, set to the unixfs chunk size -// if the miner does not explicitly set it otherwise -var DefaultPaymentIntervalIncrease = uint64(params.UnixfsChunkSize) +// DefaultPaymentIntervalIncrease is the amount interval increases on each payment, +// set to to 1Mb if the miner does not explicitly set it otherwise +var DefaultPaymentIntervalIncrease = uint64(1 << 20) // NewProvider returns a new retrieval provider func NewProvider(paymentAddress address.Address, node retrievalmarket.RetrievalProviderNode, network rmnet.RetrievalMarketNetwork, pieceStore piecestore.PieceStore, bs blockstore.Blockstore) retrievalmarket.RetrievalProvider { @@ -75,7 +74,7 @@ func (p *provider) Start() error { // V0 // SetPricePerByte sets the price per byte a miner charges for retrievals -func (p *provider) SetPricePerByte(price tokenamount.TokenAmount) { +func (p *provider) SetPricePerByte(price abi.TokenAmount) { p.pricePerByte = price } @@ -122,7 +121,7 @@ func (p *provider) SubscribeToEvents(subscriber retrievalmarket.ProviderSubscrib } // V1 -func (p *provider) SetPricePerUnseal(price tokenamount.TokenAmount) { +func (p *provider) SetPricePerUnseal(price abi.TokenAmount) { panic("not implemented") } @@ -178,7 +177,7 @@ func (p *provider) HandleDealStream(stream rmnet.RetrievalDealStream) { dealState := retrievalmarket.ProviderDealState{ Status: retrievalmarket.DealStatusNew, TotalSent: 0, - FundsReceived: tokenamount.FromInt(0), + FundsReceived: abi.NewTokenAmount(0), } p.notifySubscribers(retrievalmarket.ProviderEventOpen, dealState) @@ -221,7 +220,7 @@ type providerDealEnvironment struct { pieceStore piecestore.PieceStore node retrievalmarket.RetrievalProviderNode br blockio.BlockReader - minPricePerByte tokenamount.TokenAmount + minPricePerByte abi.TokenAmount maxPaymentInterval uint64 maxPaymentIntervalIncrease uint64 stream rmnet.RetrievalDealStream @@ -235,7 +234,7 @@ func (pde *providerDealEnvironment) DealStream() rmnet.RetrievalDealStream { return pde.stream } -func (pde *providerDealEnvironment) CheckDealParams(pricePerByte tokenamount.TokenAmount, paymentInterval uint64, paymentIntervalIncrease uint64) error { +func (pde *providerDealEnvironment) CheckDealParams(pricePerByte abi.TokenAmount, paymentInterval uint64, paymentIntervalIncrease uint64) error { if pricePerByte.LessThan(pde.minPricePerByte) { return errors.New("Price per byte too low") } diff --git a/retrievalmarket/impl/provider_test.go b/retrievalmarket/impl/provider_test.go index eb505177..3071f208 100644 --- a/retrievalmarket/impl/provider_test.go +++ b/retrievalmarket/impl/provider_test.go @@ -15,8 +15,8 @@ import ( retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/testnodes" "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" tut "github.com/filecoin-project/go-fil-markets/shared_testutil" + "github.com/filecoin-project/specs-actors/actors/abi" ) func TestHandleQueryStream(t *testing.T) { @@ -24,7 +24,7 @@ func TestHandleQueryStream(t *testing.T) { pcid := tut.GenerateCids(1)[0] expectedPeer := peer.ID("somepeer") expectedSize := uint64(1234) - expectedPieceCID := []byte("applesauce") + expectedPieceCID := tut.GenerateCids(1)[0] expectedCIDInfo := piecestore.CIDInfo{ PieceBlockLocations: []piecestore.PieceBlockLocation{ { @@ -40,7 +40,7 @@ func TestHandleQueryStream(t *testing.T) { }, } expectedAddress := address.TestAddress2 - expectedPricePerByte := tokenamount.FromInt(4321) + expectedPricePerByte := abi.NewTokenAmount(4321) expectedPaymentInterval := uint64(4567) expectedPaymentIntervalIncrease := uint64(100) diff --git a/retrievalmarket/impl/providerstates/provider_states.go b/retrievalmarket/impl/providerstates/provider_states.go index 3e503393..6f1717ff 100644 --- a/retrievalmarket/impl/providerstates/provider_states.go +++ b/retrievalmarket/impl/providerstates/provider_states.go @@ -8,7 +8,8 @@ import ( rm "github.com/filecoin-project/go-fil-markets/retrievalmarket" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" ) // ProviderDealEnvironment is a bridge to the environment a provider deal is executing in @@ -17,7 +18,7 @@ type ProviderDealEnvironment interface { GetPieceSize(c cid.Cid) (uint64, error) DealStream() rmnet.RetrievalDealStream NextBlock(context.Context) (rm.Block, bool, error) - CheckDealParams(pricePerByte tokenamount.TokenAmount, paymentInterval uint64, paymentIntervalIncrease uint64) error + CheckDealParams(pricePerByte abi.TokenAmount, paymentInterval uint64, paymentIntervalIncrease uint64) error } func errorFunc(err error) func(*rm.ProviderDealState) { @@ -89,7 +90,7 @@ func ReceiveDeal(ctx context.Context, environment ProviderDealEnvironment, deal // SendBlocks sends blocks to the client until funds are needed func SendBlocks(ctx context.Context, environment ProviderDealEnvironment, deal rm.ProviderDealState) func(*rm.ProviderDealState) { totalSent := deal.TotalSent - totalPaidFor := tokenamount.Div(deal.FundsReceived, deal.PricePerByte).Uint64() + totalPaidFor := big.Div(deal.FundsReceived, deal.PricePerByte).Uint64() returnStatus := rm.DealStatusFundsNeeded var blocks []rm.Block @@ -107,7 +108,7 @@ func SendBlocks(ctx context.Context, environment ProviderDealEnvironment, deal r } } // send back response of blocks plus payment owed - paymentOwed := tokenamount.Mul(tokenamount.FromInt(totalSent-totalPaidFor), deal.PricePerByte) + paymentOwed := big.Mul(abi.NewTokenAmount(int64(totalSent-totalPaidFor)), deal.PricePerByte) err := environment.DealStream().WriteDealResponse(rm.DealResponse{ ID: deal.ID, Status: returnStatus, @@ -135,7 +136,7 @@ func ProcessPayment(ctx context.Context, environment ProviderDealEnvironment, de // attempt to redeem voucher // (totalSent * pricePerbyte) - fundsReceived - paymentOwed := tokenamount.Sub(tokenamount.Mul(tokenamount.FromInt(deal.TotalSent), deal.PricePerByte), deal.FundsReceived) + paymentOwed := big.Sub(big.Mul(abi.NewTokenAmount(int64(deal.TotalSent)), deal.PricePerByte), deal.FundsReceived) received, err := environment.Node().SavePaymentVoucher(ctx, payment.PaymentChannel, payment.PaymentVoucher, nil, paymentOwed) if err != nil { return responseFailure(environment.DealStream(), rm.DealStatusFailed, err.Error(), deal.ID) @@ -146,13 +147,13 @@ func ProcessPayment(ctx context.Context, environment ProviderDealEnvironment, de err := environment.DealStream().WriteDealResponse(rm.DealResponse{ ID: deal.ID, Status: deal.Status, - PaymentOwed: tokenamount.Sub(paymentOwed, received), + PaymentOwed: big.Sub(paymentOwed, received), }) if err != nil { return errorFunc(xerrors.Errorf("writing deal response", err)) } return func(deal *rm.ProviderDealState) { - deal.FundsReceived = tokenamount.Add(deal.FundsReceived, received) + deal.FundsReceived = big.Add(deal.FundsReceived, received) } } @@ -163,7 +164,7 @@ func ProcessPayment(ctx context.Context, environment ProviderDealEnvironment, de } else { deal.Status = rm.DealStatusOngoing } - deal.FundsReceived = tokenamount.Add(deal.FundsReceived, received) + deal.FundsReceived = big.Add(deal.FundsReceived, received) deal.CurrentInterval += deal.PaymentIntervalIncrease } } diff --git a/retrievalmarket/impl/providerstates/provider_states_test.go b/retrievalmarket/impl/providerstates/provider_states_test.go index e0315c28..4360d5e1 100644 --- a/retrievalmarket/impl/providerstates/provider_states_test.go +++ b/retrievalmarket/impl/providerstates/provider_states_test.go @@ -16,8 +16,9 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/providerstates" "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/testnodes" rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" testnet "github.com/filecoin-project/go-fil-markets/shared_testutil" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" ) func TestReceiveDeal(t *testing.T) { @@ -32,7 +33,7 @@ func TestReceiveDeal(t *testing.T) { return &retrievalmarket.ProviderDealState{ Status: retrievalmarket.DealStatusNew, TotalSent: 0, - FundsReceived: tokenamount.FromInt(0), + FundsReceived: abi.NewTokenAmount(0), } } @@ -250,7 +251,7 @@ func TestProcessPayment(t *testing.T) { node.VerifyExpectations(t) f(dealState) require.Equal(t, dealState.Status, retrievalmarket.DealStatusOngoing) - require.Equal(t, dealState.FundsReceived, tokenamount.Add(defaultFundsReceived, defaultPaymentPerInterval)) + require.Equal(t, dealState.FundsReceived, big.Add(defaultFundsReceived, defaultPaymentPerInterval)) require.Equal(t, dealState.CurrentInterval, defaultCurrentInterval+defaultIntervalIncrease) require.Empty(t, dealState.Message) }) @@ -272,14 +273,14 @@ func TestProcessPayment(t *testing.T) { node.VerifyExpectations(t) f(dealState) require.Equal(t, dealState.Status, retrievalmarket.DealStatusCompleted) - require.Equal(t, dealState.FundsReceived, tokenamount.Add(defaultFundsReceived, defaultPaymentPerInterval)) + require.Equal(t, dealState.FundsReceived, big.Add(defaultFundsReceived, defaultPaymentPerInterval)) require.Equal(t, dealState.CurrentInterval, defaultCurrentInterval+defaultIntervalIncrease) require.Empty(t, dealState.Message) }) t.Run("not enough funds sent", func(t *testing.T) { node := testnodes.NewTestRetrievalProviderNode() - smallerPayment := tokenamount.FromInt(400000) + smallerPayment := abi.NewTokenAmount(400000) err := node.ExpectVoucher(payCh, voucher, nil, defaultPaymentPerInterval, smallerPayment, nil) require.NoError(t, err) dealState := makeDealState(retrievalmarket.DealStatusFundsNeeded) @@ -294,14 +295,14 @@ func TestProcessPayment(t *testing.T) { ResponseWriter: testnet.ExpectDealResponseWriter(t, rm.DealResponse{ ID: dealState.ID, Status: retrievalmarket.DealStatusFundsNeeded, - PaymentOwed: tokenamount.Sub(defaultPaymentPerInterval, smallerPayment), + PaymentOwed: big.Sub(defaultPaymentPerInterval, smallerPayment), }), }) f := providerstates.ProcessPayment(ctx, fe, *dealState) node.VerifyExpectations(t) f(dealState) require.Equal(t, dealState.Status, retrievalmarket.DealStatusFundsNeeded) - require.Equal(t, dealState.FundsReceived, tokenamount.Add(defaultFundsReceived, smallerPayment)) + require.Equal(t, dealState.FundsReceived, big.Add(defaultFundsReceived, smallerPayment)) require.Equal(t, dealState.CurrentInterval, defaultCurrentInterval) require.Empty(t, dealState.Message) }) @@ -309,7 +310,7 @@ func TestProcessPayment(t *testing.T) { t.Run("failure processing payment", func(t *testing.T) { node := testnodes.NewTestRetrievalProviderNode() message := "your money's no good here" - err := node.ExpectVoucher(payCh, voucher, nil, defaultPaymentPerInterval, tokenamount.FromInt(0), errors.New(message)) + err := node.ExpectVoucher(payCh, voucher, nil, defaultPaymentPerInterval, abi.NewTokenAmount(0), errors.New(message)) require.NoError(t, err) dealState := makeDealState(retrievalmarket.DealStatusFundsNeeded) dealState.TotalSent = defaultTotalSent + defaultCurrentInterval @@ -398,7 +399,7 @@ func (te *testProviderDealEnvironment) ExpectMissingPiece(c cid.Cid) { te.expectedMissingCIDs[c] = struct{}{} } -func (te *testProviderDealEnvironment) ExpectParams(pricePerByte tokenamount.TokenAmount, +func (te *testProviderDealEnvironment) ExpectParams(pricePerByte abi.TokenAmount, paymentInterval uint64, paymentIntervalIncrease uint64, response error) { @@ -433,7 +434,7 @@ func (te *testProviderDealEnvironment) GetPieceSize(c cid.Cid) (uint64, error) { return 0, errors.New("GetPieceSize failed") } -func (te *testProviderDealEnvironment) CheckDealParams(pricePerByte tokenamount.TokenAmount, paymentInterval uint64, paymentIntervalIncrease uint64) error { +func (te *testProviderDealEnvironment) CheckDealParams(pricePerByte abi.TokenAmount, paymentInterval uint64, paymentIntervalIncrease uint64) error { key := dealParamsKey{pricePerByte.String(), paymentInterval, paymentIntervalIncrease} err, ok := te.expectedParams[key] if !ok { @@ -454,10 +455,10 @@ func (te *testProviderDealEnvironment) NextBlock(_ context.Context) (rm.Block, b var defaultCurrentInterval = uint64(1000) var defaultIntervalIncrease = uint64(500) -var defaultPricePerByte = tokenamount.FromInt(500) -var defaultPaymentPerInterval = tokenamount.Mul(defaultPricePerByte, tokenamount.FromInt(defaultCurrentInterval)) +var defaultPricePerByte = abi.NewTokenAmount(500) +var defaultPaymentPerInterval = big.Mul(defaultPricePerByte, abi.NewTokenAmount(int64(defaultCurrentInterval))) var defaultTotalSent = uint64(5000) -var defaultFundsReceived = tokenamount.FromInt(2500000) +var defaultFundsReceived = abi.NewTokenAmount(2500000) func makeDealState(status retrievalmarket.DealStatus) *retrievalmarket.ProviderDealState { return &retrievalmarket.ProviderDealState{ diff --git a/retrievalmarket/impl/testnodes/test_retrieval_client_node.go b/retrievalmarket/impl/testnodes/test_retrieval_client_node.go index 84285042..db076f67 100644 --- a/retrievalmarket/impl/testnodes/test_retrieval_client_node.go +++ b/retrievalmarket/impl/testnodes/test_retrieval_client_node.go @@ -5,8 +5,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/paych" ) // TestRetrievalClientNode is a node adapter for a retrieval client whose responses @@ -14,27 +14,27 @@ import ( type TestRetrievalClientNode struct { payCh address.Address payChErr error - lane uint64 + lane int64 laneError error - voucher *types.SignedVoucher + voucher *paych.SignedVoucher voucherError error allocateLaneRecorder func(address.Address) - createPaymentVoucherRecorder func(voucher *types.SignedVoucher) - getCreatePaymentChannelRecorder func(address.Address, address.Address, tokenamount.TokenAmount) + createPaymentVoucherRecorder func(voucher *paych.SignedVoucher) + getCreatePaymentChannelRecorder func(address.Address, address.Address, abi.TokenAmount) } // TestRetrievalClientNodeParams are parameters for initializing a TestRetrievalClientNode type TestRetrievalClientNodeParams struct { PayCh address.Address PayChErr error - Lane uint64 + Lane int64 LaneError error - Voucher *types.SignedVoucher + Voucher *paych.SignedVoucher VoucherError error AllocateLaneRecorder func(address.Address) - PaymentVoucherRecorder func(voucher *types.SignedVoucher) - PaymentChannelRecorder func(address.Address, address.Address, tokenamount.TokenAmount) + PaymentVoucherRecorder func(voucher *paych.SignedVoucher) + PaymentChannelRecorder func(address.Address, address.Address, abi.TokenAmount) } var _ retrievalmarket.RetrievalClientNode = &TestRetrievalClientNode{} @@ -55,7 +55,7 @@ func NewTestRetrievalClientNode(params TestRetrievalClientNodeParams) *TestRetri } // GetOrCreatePaymentChannel returns a mocked payment channel -func (trcn *TestRetrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable tokenamount.TokenAmount) (address.Address, error) { +func (trcn *TestRetrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable abi.TokenAmount) (address.Address, error) { if trcn.getCreatePaymentChannelRecorder != nil { trcn.getCreatePaymentChannelRecorder(clientAddress, minerAddress, clientFundsAvailable) } @@ -63,7 +63,7 @@ func (trcn *TestRetrievalClientNode) GetOrCreatePaymentChannel(ctx context.Conte } // AllocateLane creates a mock lane on a payment channel -func (trcn *TestRetrievalClientNode) AllocateLane(paymentChannel address.Address) (uint64, error) { +func (trcn *TestRetrievalClientNode) AllocateLane(paymentChannel address.Address) (int64, error) { if trcn.allocateLaneRecorder != nil { trcn.allocateLaneRecorder(paymentChannel) } @@ -71,7 +71,7 @@ func (trcn *TestRetrievalClientNode) AllocateLane(paymentChannel address.Address } // CreatePaymentVoucher creates a mock payment voucher based on a channel and lane -func (trcn *TestRetrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount tokenamount.TokenAmount, lane uint64) (*types.SignedVoucher, error) { +func (trcn *TestRetrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount abi.TokenAmount, lane int64) (*paych.SignedVoucher, error) { if trcn.createPaymentVoucherRecorder != nil { trcn.createPaymentVoucherRecorder(trcn.voucher) } diff --git a/retrievalmarket/impl/testnodes/test_retrieval_provider_node.go b/retrievalmarket/impl/testnodes/test_retrieval_provider_node.go index 1ac24088..5990cac4 100644 --- a/retrievalmarket/impl/testnodes/test_retrieval_provider_node.go +++ b/retrievalmarket/impl/testnodes/test_retrieval_provider_node.go @@ -3,6 +3,7 @@ package testnodes import ( "bytes" "context" + "encoding/base64" "errors" "io" "io/ioutil" @@ -12,8 +13,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/paych" ) type expectedVoucherKey struct { @@ -30,7 +31,7 @@ type sectorKey struct { } type voucherResult struct { - amount tokenamount.TokenAmount + amount abi.TokenAmount err error } @@ -97,30 +98,31 @@ func (trpn *TestRetrievalProviderNode) VerifyExpectations(t *testing.T) { func (trpn *TestRetrievalProviderNode) SavePaymentVoucher( ctx context.Context, paymentChannel address.Address, - voucher *types.SignedVoucher, + voucher *paych.SignedVoucher, proof []byte, - expectedAmount tokenamount.TokenAmount) (tokenamount.TokenAmount, error) { + expectedAmount abi.TokenAmount) (abi.TokenAmount, error) { key, err := trpn.toExpectedVoucherKey(paymentChannel, voucher, proof, expectedAmount) if err != nil { - return tokenamount.Empty, err + return abi.TokenAmount{}, err } result, ok := trpn.expectedVouchers[key] if ok { trpn.receivedVouchers[key] = struct{}{} return result.amount, result.err } - return tokenamount.Empty, errors.New("SavePaymentVoucher failed") + return abi.TokenAmount{}, errors.New("SavePaymentVoucher failed") } // --- Non-interface Functions // to ExpectedVoucherKey creates a lookup key for expected vouchers. -func (trpn *TestRetrievalProviderNode) toExpectedVoucherKey(paymentChannel address.Address, voucher *types.SignedVoucher, proof []byte, expectedAmount tokenamount.TokenAmount) (expectedVoucherKey, error) { +func (trpn *TestRetrievalProviderNode) toExpectedVoucherKey(paymentChannel address.Address, voucher *paych.SignedVoucher, proof []byte, expectedAmount abi.TokenAmount) (expectedVoucherKey, error) { pcString := paymentChannel.String() - voucherString, err := voucher.EncodedString() - if err != nil { + buf := new(bytes.Buffer) + if err := voucher.MarshalCBOR(buf); err != nil { return expectedVoucherKey{}, err } + voucherString := base64.RawURLEncoding.EncodeToString(buf.Bytes()) proofString := string(proof) expectedAmountString := expectedAmount.String() return expectedVoucherKey{pcString, voucherString, proofString, expectedAmountString}, nil @@ -135,10 +137,10 @@ func (trpn *TestRetrievalProviderNode) toExpectedVoucherKey(paymentChannel addre // expectedErr: an error message to expect func (trpn *TestRetrievalProviderNode) ExpectVoucher( paymentChannel address.Address, - voucher *types.SignedVoucher, + voucher *paych.SignedVoucher, proof []byte, - expectedAmount tokenamount.TokenAmount, - actualAmount tokenamount.TokenAmount, // the actual amount it should have (same unless you want to trigger an error) + expectedAmount abi.TokenAmount, + actualAmount abi.TokenAmount, // the actual amount it should have (same unless you want to trigger an error) expectedErr error) error { key, err := trpn.toExpectedVoucherKey(paymentChannel, voucher, proof, expectedAmount) if err != nil { diff --git a/retrievalmarket/network/libp2p_impl_test.go b/retrievalmarket/network/libp2p_impl_test.go index 900f64f8..a6af94a0 100644 --- a/retrievalmarket/network/libp2p_impl_test.go +++ b/retrievalmarket/network/libp2p_impl_test.go @@ -14,8 +14,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" "github.com/filecoin-project/go-fil-markets/shared_testutil" + "github.com/filecoin-project/specs-actors/actors/abi" ) type testReceiver struct { @@ -330,7 +330,7 @@ func assertDealResponseReceived(parentCtx context.Context, t *testing.T, fromNet dr := retrievalmarket.DealResponse{ Status: retrievalmarket.DealStatusCompleted, ID: retrievalmarket.DealID(rand.Uint64()), - PaymentOwed: tokenamount.TokenAmount{Int: big.NewInt(rand.Int63())}, + PaymentOwed: abi.TokenAmount{Int: big.NewInt(rand.Int63())}, Message: "some message", Blocks: []retrievalmarket.Block{fakeBlk}, } diff --git a/retrievalmarket/types.go b/retrievalmarket/types.go index ec1341a4..2a1ec5ff 100644 --- a/retrievalmarket/types.go +++ b/retrievalmarket/types.go @@ -4,14 +4,14 @@ import ( "context" "errors" "io" - "math/big" "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin/paych" ) //go:generate cbor-gen-for Query QueryResponse DealProposal DealResponse Params QueryParams DealPayment Block ClientDealState @@ -32,19 +32,19 @@ type Unsubscribe func() type ClientDealState struct { ProposalCid cid.Cid DealProposal - TotalFunds tokenamount.TokenAmount + TotalFunds abi.TokenAmount ClientWallet address.Address MinerWallet address.Address PayCh address.Address - Lane uint64 + Lane int64 Status DealStatus Sender peer.ID TotalReceived uint64 Message string BytesPaidFor uint64 CurrentInterval uint64 - PaymentRequested tokenamount.TokenAmount - FundsSpent tokenamount.TokenAmount + PaymentRequested abi.TokenAmount + FundsSpent abi.TokenAmount } // ClientEvent is an event that occurs in a deal lifecycle on the client @@ -91,7 +91,7 @@ type RetrievalClient interface { ctx context.Context, payloadCID cid.Cid, params Params, - totalFunds tokenamount.TokenAmount, + totalFunds abi.TokenAmount, miner peer.ID, clientWallet address.Address, minerWallet address.Address, @@ -101,7 +101,7 @@ type RetrievalClient interface { SubscribeToEvents(subscriber ClientSubscriber) Unsubscribe // V1 - AddMoreFunds(id DealID, amount tokenamount.TokenAmount) error + AddMoreFunds(id DealID, amount abi.TokenAmount) error CancelDeal(id DealID) error RetrievalStatus(id DealID) ListDeals() map[DealID]ClientDealState @@ -112,17 +112,17 @@ type RetrievalClientNode interface { // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist // between a client and a miner and insures the client has the given amount of funds available in the channel - GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable tokenamount.TokenAmount) (address.Address, error) + GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable abi.TokenAmount) (address.Address, error) // Allocate late creates a lane within a payment channel so that calls to // CreatePaymentVoucher will automatically make vouchers only for the difference // in total - AllocateLane(paymentChannel address.Address) (uint64, error) + AllocateLane(paymentChannel address.Address) (int64, error) // CreatePaymentVoucher creates a new payment voucher in the given lane for a // given payment channel so that all the payment vouchers in the lane add up // to the given amount (so the payment voucher will be for the difference) - CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount tokenamount.TokenAmount, lane uint64) (*types.SignedVoucher, error) + CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount abi.TokenAmount, lane int64) (*paych.SignedVoucher, error) } // ProviderDealState is the current state of a deal from the point of view @@ -132,7 +132,7 @@ type ProviderDealState struct { Status DealStatus Receiver peer.ID TotalSent uint64 - FundsReceived tokenamount.TokenAmount + FundsReceived abi.TokenAmount Message string CurrentInterval uint64 } @@ -177,7 +177,7 @@ type RetrievalProvider interface { // V0 // SetPricePerByte sets the price per byte a miner charges for retrievals - SetPricePerByte(price tokenamount.TokenAmount) + SetPricePerByte(price abi.TokenAmount) // SetPaymentInterval sets the maximum number of bytes a a provider will send before // requesting further payment, and the rate at which that value increases @@ -187,14 +187,14 @@ type RetrievalProvider interface { SubscribeToEvents(subscriber ProviderSubscriber) Unsubscribe // V1 - SetPricePerUnseal(price tokenamount.TokenAmount) + SetPricePerUnseal(price abi.TokenAmount) ListDeals() map[ProviderDealID]ProviderDealState } // RetrievalProviderNode are the node depedencies for a RetrevalProvider type RetrievalProviderNode interface { - UnsealSector(ctx context.Context, sectorId uint64, offset uint64, length uint64) (io.ReadCloser, error) - SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *types.SignedVoucher, proof []byte, expectedAmount tokenamount.TokenAmount) (tokenamount.TokenAmount, error) + UnsealSector(ctx context.Context, sectorID uint64, offset uint64, length uint64) (io.ReadCloser, error) + SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *paych.SignedVoucher, proof []byte, expectedAmount abi.TokenAmount) (abi.TokenAmount, error) } // PeerResolver is an interface for looking up providers that may have a piece @@ -250,7 +250,7 @@ const ( type QueryParams struct { //PayloadCID cid.Cid // optional, query if miner has this cid in this piece. some miners may not be able to respond. //Selector ipld.Node // optional, query if miner has this cid in this piece. some miners may not be able to respond. - //MaxPricePerByte tokenamount.TokenAmount // optional, tell miner uninterested if more expensive than this + //MaxPricePerByte abi.TokenAmount // optional, tell miner uninterested if more expensive than this //MinPaymentInterval uint64 // optional, tell miner uninterested unless payment interval is greater than this //MinPaymentIntervalIncrease uint64 // optional, tell miner uninterested unless payment interval increase is greater than this } @@ -280,7 +280,7 @@ type QueryResponse struct { //ExpectedPayloadSize uint64 // V1 - optional, if PayloadCID + selector are specified and miner knows, can offer an expected size PaymentAddress address.Address // address to send funds to -- may be different than miner addr - MinPricePerByte tokenamount.TokenAmount + MinPricePerByte abi.TokenAmount MaxPaymentInterval uint64 MaxPaymentIntervalIncrease uint64 Message string @@ -290,13 +290,13 @@ type QueryResponse struct { var QueryResponseUndefined = QueryResponse{} // PieceRetrievalPrice is the total price to retrieve the piece (size * MinPricePerByte) -func (qr QueryResponse) PieceRetrievalPrice() tokenamount.TokenAmount { - return tokenamount.Mul(qr.MinPricePerByte, tokenamount.FromInt(qr.Size)) +func (qr QueryResponse) PieceRetrievalPrice() abi.TokenAmount { + return big.Mul(qr.MinPricePerByte, abi.NewTokenAmount(int64(qr.Size))) } // PayloadRetrievalPrice is the expected price to retrieve just the given payload // & selector (V1) -//func (qr QueryResponse) PayloadRetrievalPrice() tokenamount.TokenAmount { +//func (qr QueryResponse) PayloadRetrievalPrice() abi.TokenAmount { // return types.BigMul(qr.MinPricePerByte, types.NewInt(qr.ExpectedPayloadSize)) //} @@ -369,15 +369,15 @@ func IsTerminalStatus(status DealStatus) bool { // Params are the parameters requested for a retrieval deal proposal type Params struct { //Selector ipld.Node // V1 - PricePerByte tokenamount.TokenAmount + PricePerByte abi.TokenAmount PaymentInterval uint64 // when to request payment PaymentIntervalIncrease uint64 // } // NewParamsV0 generates parameters for a retrieval deal, which is always a whole piece deal -func NewParamsV0(pricePerByte *big.Int, paymentInterval uint64, paymentIntervalIncrease uint64) Params { +func NewParamsV0(pricePerByte abi.TokenAmount, paymentInterval uint64, paymentIntervalIncrease uint64) Params { return Params{ - PricePerByte: tokenamount.TokenAmount{Int: pricePerByte}, + PricePerByte: pricePerByte, PaymentInterval: paymentInterval, PaymentIntervalIncrease: paymentIntervalIncrease, } @@ -411,7 +411,7 @@ type DealResponse struct { ID DealID // payment required to proceed - PaymentOwed tokenamount.TokenAmount + PaymentOwed abi.TokenAmount Message string Blocks []Block // V0 only @@ -424,7 +424,7 @@ var DealResponseUndefined = DealResponse{} type DealPayment struct { ID DealID PaymentChannel address.Address - PaymentVoucher *types.SignedVoucher + PaymentVoucher *paych.SignedVoucher } // DealPaymentUndefined is an undefined deal payment diff --git a/retrievalmarket/types_cbor_gen.go b/retrievalmarket/types_cbor_gen.go index bef8a0e6..4977f001 100644 --- a/retrievalmarket/types_cbor_gen.go +++ b/retrievalmarket/types_cbor_gen.go @@ -6,7 +6,7 @@ import ( "fmt" "io" - "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" @@ -86,7 +86,7 @@ func (t *QueryResponse) MarshalCBOR(w io.Writer) error { return err } - // t.MinPricePerByte (tokenamount.TokenAmount) (struct) + // t.MinPricePerByte (big.Int) (struct) if err := t.MinPricePerByte.MarshalCBOR(w); err != nil { return err } @@ -159,7 +159,7 @@ func (t *QueryResponse) UnmarshalCBOR(r io.Reader) error { } } - // t.MinPricePerByte (tokenamount.TokenAmount) (struct) + // t.MinPricePerByte (big.Int) (struct) { @@ -296,7 +296,7 @@ func (t *DealResponse) MarshalCBOR(w io.Writer) error { return err } - // t.PaymentOwed (tokenamount.TokenAmount) (struct) + // t.PaymentOwed (big.Int) (struct) if err := t.PaymentOwed.MarshalCBOR(w); err != nil { return err } @@ -364,7 +364,7 @@ func (t *DealResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.ID = DealID(extra) - // t.PaymentOwed (tokenamount.TokenAmount) (struct) + // t.PaymentOwed (big.Int) (struct) { @@ -422,7 +422,7 @@ func (t *Params) MarshalCBOR(w io.Writer) error { return err } - // t.PricePerByte (tokenamount.TokenAmount) (struct) + // t.PricePerByte (big.Int) (struct) if err := t.PricePerByte.MarshalCBOR(w); err != nil { return err } @@ -454,7 +454,7 @@ func (t *Params) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.PricePerByte (tokenamount.TokenAmount) (struct) + // t.PricePerByte (big.Int) (struct) { @@ -534,7 +534,7 @@ func (t *DealPayment) MarshalCBOR(w io.Writer) error { return err } - // t.PaymentVoucher (types.SignedVoucher) (struct) + // t.PaymentVoucher (paych.SignedVoucher) (struct) if err := t.PaymentVoucher.MarshalCBOR(w); err != nil { return err } @@ -575,7 +575,7 @@ func (t *DealPayment) UnmarshalCBOR(r io.Reader) error { } } - // t.PaymentVoucher (types.SignedVoucher) (struct) + // t.PaymentVoucher (paych.SignedVoucher) (struct) { @@ -589,7 +589,7 @@ func (t *DealPayment) UnmarshalCBOR(r io.Reader) error { return err } } else { - t.PaymentVoucher = new(types.SignedVoucher) + t.PaymentVoucher = new(paych.SignedVoucher) if err := t.PaymentVoucher.UnmarshalCBOR(br); err != nil { return err } @@ -706,7 +706,7 @@ func (t *ClientDealState) MarshalCBOR(w io.Writer) error { return err } - // t.TotalFunds (tokenamount.TokenAmount) (struct) + // t.TotalFunds (big.Int) (struct) if err := t.TotalFunds.MarshalCBOR(w); err != nil { return err } @@ -726,9 +726,15 @@ func (t *ClientDealState) MarshalCBOR(w io.Writer) error { return err } - // t.Lane (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { - return err + // t.Lane (int64) (int64) + if t.Lane >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.Lane)-1)); err != nil { + return err + } } // t.Status (retrievalmarket.DealStatus) (uint64) @@ -775,12 +781,12 @@ func (t *ClientDealState) MarshalCBOR(w io.Writer) error { return err } - // t.PaymentRequested (tokenamount.TokenAmount) (struct) + // t.PaymentRequested (big.Int) (struct) if err := t.PaymentRequested.MarshalCBOR(w); err != nil { return err } - // t.FundsSpent (tokenamount.TokenAmount) (struct) + // t.FundsSpent (big.Int) (struct) if err := t.FundsSpent.MarshalCBOR(w); err != nil { return err } @@ -823,7 +829,7 @@ func (t *ClientDealState) UnmarshalCBOR(r io.Reader) error { } } - // t.TotalFunds (tokenamount.TokenAmount) (struct) + // t.TotalFunds (big.Int) (struct) { @@ -859,16 +865,31 @@ func (t *ClientDealState) UnmarshalCBOR(r io.Reader) error { } } - // t.Lane (uint64) (uint64) + // t.Lane (int64) (int64) + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") + t.Lane = int64(extraI) } - t.Lane = uint64(extra) // t.Status (retrievalmarket.DealStatus) (uint64) maj, extra, err = cbg.CborReadHeader(br) @@ -929,7 +950,7 @@ func (t *ClientDealState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.CurrentInterval = uint64(extra) - // t.PaymentRequested (tokenamount.TokenAmount) (struct) + // t.PaymentRequested (big.Int) (struct) { @@ -938,7 +959,7 @@ func (t *ClientDealState) UnmarshalCBOR(r io.Reader) error { } } - // t.FundsSpent (tokenamount.TokenAmount) (struct) + // t.FundsSpent (big.Int) (struct) { diff --git a/shared/params/params_shared.go b/shared/params/params_shared.go deleted file mode 100644 index 31f2f255..00000000 --- a/shared/params/params_shared.go +++ /dev/null @@ -1,46 +0,0 @@ -package params - -import ( - "math/big" -) - -// Core network constants - -// ///// -// Storage - -const UnixfsChunkSize uint64 = 1 << 20 -const UnixfsLinksPerLevel = 1024 - -// ///// -// Devnet settings - -const TotalFilecoin = 2_000_000_000 -const MiningRewardTotal = 1_400_000_000 - -const InitialRewardStr = "153856861913558700202" - -var InitialReward *big.Int - -const FilecoinPrecision = 1_000_000_000_000_000_000 - -// six years -// Epochs -const HalvingPeriodEpochs = 6 * 365 * 24 * 60 * 2 - -// TODO: Move other important consts here - -func init() { - InitialReward = new(big.Int) - - var ok bool - InitialReward, ok = InitialReward. - SetString(InitialRewardStr, 10) - if !ok { - panic("could not parse InitialRewardStr") - } -} - -// assuming 4000 messages per round, this lets us not lose any messages across a -// 10 block reorg. -const BlsSignatureCacheSize = 40000 diff --git a/shared/tokenamount/tokenamount.go b/shared/tokenamount/tokenamount.go deleted file mode 100644 index a19aa1fb..00000000 --- a/shared/tokenamount/tokenamount.go +++ /dev/null @@ -1,296 +0,0 @@ -package tokenamount - -import ( - "encoding/json" - "fmt" - "io" - "math/big" - "strings" - - "github.com/filecoin-project/go-fil-markets/shared/params" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/polydawn/refmt/obj/atlas" - - cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" -) - -// BigIntMaxSerializedLen is the maximum number of bytes a big int can use when -// serialized to CBOR -const BigIntMaxSerializedLen = 128 // is this big enough? or too big? - -// TotalFilecoinAmount is all filecoin in the system, as a token amount -var TotalFilecoinAmount = FromFil(params.TotalFilecoin) - -func init() { - cbor.RegisterCborType(atlas.BuildEntry(TokenAmount{}).Transform(). - TransformMarshal(atlas.MakeMarshalTransformFunc( - func(i TokenAmount) ([]byte, error) { - return i.cborBytes(), nil - })). - TransformUnmarshal(atlas.MakeUnmarshalTransformFunc( - func(x []byte) (TokenAmount, error) { - return fromCborBytes(x) - })). - Complete()) -} - -// Empty is an empty token -var Empty = TokenAmount{} - -// TokenAmount is an amount of filecoin, represented as a big int -type TokenAmount struct { - *big.Int -} - -// FromInt creates a token amount from an integer -func FromInt(i uint64) TokenAmount { - return TokenAmount{big.NewInt(0).SetUint64(i)} -} - -// FromFil creates a token amount from a whole amount of filecoin -func FromFil(i uint64) TokenAmount { - return Mul(FromInt(i), FromInt(params.FilecoinPrecision)) -} - -// FromBytes creates a token amount from a byte string -func FromBytes(b []byte) TokenAmount { - i := big.NewInt(0).SetBytes(b) - return TokenAmount{i} -} - -// FromString creates a token amount from a string representation of a big int -func FromString(s string) (TokenAmount, error) { - v, ok := big.NewInt(0).SetString(s, 10) - if !ok { - return TokenAmount{}, fmt.Errorf("failed to parse string as a big int") - } - - return TokenAmount{v}, nil -} - -// Mul multiples two token amounts -func Mul(a, b TokenAmount) TokenAmount { - zero := big.NewInt(0) - return TokenAmount{zero.Mul(a.Int, b.Int)} -} - -// Div divides two token amounts -func Div(a, b TokenAmount) TokenAmount { - return TokenAmount{big.NewInt(0).Div(a.Int, b.Int)} -} - -// Mod computes the remainder of two token amounts -func Mod(a, b TokenAmount) TokenAmount { - return TokenAmount{big.NewInt(0).Mod(a.Int, b.Int)} -} - -// Add adds two token amounts together -func Add(a, b TokenAmount) TokenAmount { - return TokenAmount{big.NewInt(0).Add(a.Int, b.Int)} -} - -// Sub subtracts the second token amount from the first -func Sub(a, b TokenAmount) TokenAmount { - return TokenAmount{big.NewInt(0).Sub(a.Int, b.Int)} -} - -// Cmp compares two token amounts (for sorting) -func Cmp(a, b TokenAmount) int { - return a.Int.Cmp(b.Int) -} - -// Nil is true if there is no underlying token amount -func (ta TokenAmount) Nil() bool { - return ta.Int == nil -} - -// LessThan returns true if ta < o -func (ta TokenAmount) LessThan(o TokenAmount) bool { - return Cmp(ta, o) < 0 -} - -// GreaterThan returns true if ta > o -func (ta TokenAmount) GreaterThan(o TokenAmount) bool { - return Cmp(ta, o) > 0 -} - -// Equals returns true if ta == o -func (ta TokenAmount) Equals(o TokenAmount) bool { - return Cmp(ta, o) == 0 -} - -// MarshalJSON converts a token amount to a json string -func (ta *TokenAmount) MarshalJSON() ([]byte, error) { - if ta.Int == nil { - zero := FromInt(0) - return json.Marshal(zero) - } - - return json.Marshal(ta.String()) -} - -// UnmarshalJSON decodes a token amount from json -func (ta *TokenAmount) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - - res, err := ParseTokenAmount(s) - if err != nil { return err } - ta.Set(res.Int) - return nil -} - -// Scan sets a token amount value from any type -func (ta *TokenAmount) Scan(value interface{}) error { - switch value := value.(type) { - case string: - i, ok := big.NewInt(0).SetString(value, 10) - if !ok { - if value == "" { - return nil - } - return xerrors.Errorf("failed to parse bigint string: '%s'", value) - } - - ta.Int = i - - return nil - case int64: - ta.Int = big.NewInt(value) - return nil - default: - return xerrors.Errorf("non-string types unsupported: %T", value) - } -} - -func (ta *TokenAmount) cborBytes() []byte { - if ta.Int == nil { - return []byte{} - } - - switch { - case ta.Sign() > 0: - return append([]byte{0}, ta.Bytes()...) - case ta.Sign() < 0: - return append([]byte{1}, ta.Bytes()...) - default: // ta.Sign() == 0: - return []byte{} - } -} - -func fromCborBytes(buf []byte) (TokenAmount, error) { - if len(buf) == 0 { - return FromInt(0), nil - } - - var negative bool - switch buf[0] { - case 0: - negative = false - case 1: - negative = true - default: - return Empty, fmt.Errorf("big int prefix should be either 0 or 1, got %d", buf[0]) - } - - i := big.NewInt(0).SetBytes(buf[1:]) - if negative { - i.Neg(i) - } - - return TokenAmount{i}, nil -} - -// MarshalCBOR encodes a TokenAmount to a CBOR byte array -func (ta *TokenAmount) MarshalCBOR(w io.Writer) error { - if ta.Int == nil { - zero := FromInt(0) - return zero.MarshalCBOR(w) - } - - enc := ta.cborBytes() - - header := cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(enc))) - if _, err := w.Write(header); err != nil { - return err - } - - if _, err := w.Write(enc); err != nil { - return err - } - - return nil -} - -// UnmarshalCBOR decodes a TokenAmount from a CBOR byte array -func (ta *TokenAmount) UnmarshalCBOR(br io.Reader) error { - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - - if maj != cbg.MajByteString { - return fmt.Errorf("cbor input for fil big int was not a byte string (%x)", maj) - } - - if extra == 0 { - ta.Int = big.NewInt(0) - return nil - } - - if extra > BigIntMaxSerializedLen { - return fmt.Errorf("big integer byte array too long") - } - - buf := make([]byte, extra) - if _, err := io.ReadFull(br, buf); err != nil { - return err - } - - i, err := fromCborBytes(buf) - if err != nil { - return err - } - - *ta = i - - return nil -} - -// String outputs the token amount as a readable string -func (ta TokenAmount) String() string { - r := new(big.Rat).SetFrac(ta.Int, big.NewInt(params.FilecoinPrecision)) - if r.Sign() == 0 { - return "0" - } - return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") -} - -// Format converts a token amount to a string and then formats according to the -// given format -func (ta TokenAmount) Format(s fmt.State, ch rune) { - switch ch { - case 's', 'v': - fmt.Fprint(s, ta.String()) - default: - ta.Int.Format(s, ch) - } -} - -// ParseTokenAmount parses a token amount from a formatted string -func ParseTokenAmount(s string) (TokenAmount, error) { - r, ok := new(big.Rat).SetString(s) - if !ok { - return TokenAmount{}, fmt.Errorf("failed to parse %q as a decimal number", s) - } - - r = r.Mul(r, big.NewRat(params.FilecoinPrecision, 1)) - if !r.IsInt() { - return TokenAmount{}, fmt.Errorf("invalid FIL value: %q", s) - } - - return TokenAmount{r.Num()}, nil -} diff --git a/shared/tokenamount/tokenamount_test.go b/shared/tokenamount/tokenamount_test.go deleted file mode 100644 index 4fcd0a78..00000000 --- a/shared/tokenamount/tokenamount_test.go +++ /dev/null @@ -1,197 +0,0 @@ -package tokenamount_test - -import ( - "bytes" - "fmt" - "math/big" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - . "github.com/filecoin-project/go-fil-markets/shared/tokenamount" -) - -func TestBigIntSerializationRoundTrip(t *testing.T) { - testValues := []string{ - "0", "1", "10", "-10", "9999", "12345678901234567891234567890123456789012345678901234567890", - } - - for _, v := range testValues { - bi, err := FromString(v) - if err != nil { - t.Fatal(err) - } - - buf := new(bytes.Buffer) - if err := bi.MarshalCBOR(buf); err != nil { - t.Fatal(err) - } - - var out TokenAmount - if err := out.UnmarshalCBOR(buf); err != nil { - t.Fatal(err) - } - - if Cmp(out, bi) != 0 { - t.Fatal("failed to round trip BigInt through cbor") - } - - } - - // nil check - ta := TokenAmount{} - var buf bytes.Buffer - err := ta.MarshalCBOR(&buf) - require.NoError(t, err) - - assert.Equal(t, "@", buf.String()) -} - -func TestFilRoundTrip(t *testing.T) { - testValues := []string{ - "0", "1", "1.001", "100.10001", "101100", "5000.01", "5000", - } - - for _, v := range testValues { - fval, err := ParseTokenAmount(v) - if err != nil { - t.Fatal(err) - } - - if fval.String() != v { - t.Fatal("mismatch in values!", v, fval.String()) - } - } -} - -func TestFromInt(t *testing.T) { - a := uint64(999) - ta := FromInt(a) - b := big.NewInt(999) - tb := TokenAmount{Int: b} - assert.True(t, ta.Equals(tb)) - assert.Equal(t, "0.000000000000000999", ta.String()) -} - -func TestTokenAmount_MarshalUnmarshalJSON(t *testing.T) { - ta := FromInt(54321) - tb := FromInt(0) - - res, err := ta.MarshalJSON() - require.NoError(t, err) - assert.Equal(t, "\"0.000000000000054321\"", string(res[:])) - - require.NoError(t, tb.UnmarshalJSON(res)) - assert.Equal(t, ta, tb) - - assert.EqualError(t, tb.UnmarshalJSON([]byte("123garbage"[:])), "invalid character 'g' after top-level value") - - tnil := TokenAmount{} - s, err := tnil.MarshalJSON() - require.NoError(t, err) - assert.Equal(t, "\"0\"", string(s)) -} - -func TestOperations(t *testing.T) { - testCases := []struct { - name string - f func(TokenAmount, TokenAmount) TokenAmount - expected TokenAmount - }{ - {name: "Sum", f: Add, expected: FromInt(7000)}, - {name: "Sub", f: Sub, expected: FromInt(3000)}, - {name: "Mul", f: Mul, expected: FromInt(10000000)}, - {name: "Div", f: Div, expected: FromInt(2)}, - {name: "Mod", f: Mod, expected: FromInt(1000)}, - } - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - ta := TokenAmount{Int: big.NewInt(5000)} - tb := TokenAmount{Int: big.NewInt(2000)} - assert.Equal(t, testCase.expected, testCase.f(ta, tb)) - }) - } - - ta := FromInt(5000) - tb := FromInt(2000) - tc := FromInt(2000) - assert.Equal(t, Cmp(ta, tb), 1) - assert.Equal(t, Cmp(tb, ta), -1) - assert.Equal(t, Cmp(tb, tc), 0) - assert.True(t, ta.GreaterThan(tb)) - assert.False(t, ta.LessThan(tb)) - assert.True(t, tb.Equals(tc)) - - ta = TokenAmount{} - assert.True(t, ta.Nil()) -} - -func TestTokenAmount_Scan(t *testing.T) { - ta := FromFil(0) - - err := ta.Scan(54321) - assert.EqualError(t, err, "non-string types unsupported: int") - - err = ta.Scan(int64(54321)) - require.NoError(t, err) - assert.Equal(t, FromInt(54321), ta) - - err = ta.Scan("54321") - require.NoError(t, err) - assert.Equal(t, FromInt(54321), ta) - - err = ta.Scan("garbage") - assert.EqualError(t, err, "failed to parse bigint string: 'garbage'") -} - -func TestParseTokenAmount(t *testing.T) { - res, err := ParseTokenAmount("123.45") - require.NoError(t, err) - assert.Equal(t, "123.45", res.String()) - - res, err = ParseTokenAmount("12345") - require.NoError(t, err) - assert.Equal(t, FromFil(12345), res) - - _, err = ParseTokenAmount("123badnum") - assert.EqualError(t, err, "failed to parse \"123badnum\" as a decimal number") - - _, err = ParseTokenAmount("0.0000000000000000000000003") - assert.EqualError(t, err, "invalid FIL value: \"0.0000000000000000000000003\"") -} - -func TestTokenAmount_Format(t *testing.T) { - ta := FromInt(33333000000) - - s := fmt.Sprintf("%s", ta) // nolint: gosimple - assert.Equal(t, "0.000000033333", s) - - s1 := fmt.Sprintf("%v", ta) // nolint: gosimple - assert.Equal(t, "0.000000033333", s1) - - s2 := fmt.Sprintf("%-15d", ta) // nolint: gosimple - assert.Equal(t, "33333000000 ", s2) -} - -func TestFromBytes(t *testing.T) { - res := FromBytes([]byte("garbage"[:])) - // garbage in, garbage out - expected := TokenAmount{Int: big.NewInt(29099066505914213)} - assert.Equal(t, expected, res) - - expected2 := TokenAmount{Int: big.NewInt(12345)} - expectedRes := expected2.Bytes() - res = FromBytes(expectedRes) - assert.Equal(t, expected2, res) -} - -func TestFromString(t *testing.T) { - _, err := FromString("garbage") - assert.EqualError(t, err, "failed to parse string as a big int") - - res, err := FromString("12345") - require.NoError(t, err) - expected := TokenAmount{Int: big.NewInt(12345)} - assert.Equal(t, expected, res) -} diff --git a/shared/types/ask.go b/shared/types/ask.go deleted file mode 100644 index 03fc037f..00000000 --- a/shared/types/ask.go +++ /dev/null @@ -1,30 +0,0 @@ -package types - -import ( - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - cbor "github.com/ipfs/go-ipld-cbor" -) - -func init() { - cbor.RegisterCborType(SignedStorageAsk{}) - cbor.RegisterCborType(StorageAsk{}) -} - -//go:generate cbor-gen-for SignedStorageAsk StorageAsk - -type SignedStorageAsk struct { - Ask *StorageAsk - Signature *Signature -} - -type StorageAsk struct { - // Price per GiB / Epoch - Price tokenamount.TokenAmount - - MinPieceSize uint64 - Miner address.Address - Timestamp uint64 - Expiry uint64 - SeqNo uint64 -} diff --git a/shared/types/ask_cbor_gen.go b/shared/types/ask_cbor_gen.go deleted file mode 100644 index 368b4c6a..00000000 --- a/shared/types/ask_cbor_gen.go +++ /dev/null @@ -1,211 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package types - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf - -func (t *SignedStorageAsk) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Ask (types.StorageAsk) (struct) - if err := t.Ask.MarshalCBOR(w); err != nil { - return err - } - - // t.Signature (types.Signature) (struct) - if err := t.Signature.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *SignedStorageAsk) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Ask (types.StorageAsk) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Ask = new(StorageAsk) - if err := t.Ask.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - // t.Signature (types.Signature) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Signature = new(Signature) - if err := t.Signature.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *StorageAsk) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{134}); err != nil { - return err - } - - // t.Price (tokenamount.TokenAmount) (struct) - if err := t.Price.MarshalCBOR(w); err != nil { - return err - } - - // t.MinPieceSize (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinPieceSize))); err != nil { - return err - } - - // t.Miner (address.Address) (struct) - if err := t.Miner.MarshalCBOR(w); err != nil { - return err - } - - // t.Timestamp (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil { - return err - } - - // t.Expiry (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Expiry))); err != nil { - return err - } - - // t.SeqNo (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SeqNo))); err != nil { - return err - } - return nil -} - -func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 6 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Price (tokenamount.TokenAmount) (struct) - - { - - if err := t.Price.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.MinPieceSize (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.MinPieceSize = uint64(extra) - // t.Miner (address.Address) (struct) - - { - - if err := t.Miner.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Timestamp (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Timestamp = uint64(extra) - // t.Expiry (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Expiry = uint64(extra) - // t.SeqNo (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SeqNo = uint64(extra) - return nil -} diff --git a/shared/types/signature.go b/shared/types/signature.go deleted file mode 100644 index 6ac2609b..00000000 --- a/shared/types/signature.go +++ /dev/null @@ -1,123 +0,0 @@ -package types - -import ( - "bytes" - "encoding/binary" - "fmt" - logging "github.com/ipfs/go-log/v2" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" -) - -var log = logging.Logger("types") - -const SignatureMaxLength = 200 - -const ( - KTSecp256k1 = "secp256k1" - KTBLS = "bls" -) - -const ( - IKTUnknown = -1 - - IKTSecp256k1 = iota - IKTBLS -) - -type Signature struct { - Type string - Data []byte -} - -func SignatureFromBytes(x []byte) (Signature, error) { - val, nr := binary.Uvarint(x) - if nr != 1 { - return Signature{}, fmt.Errorf("signatures with type field longer than one byte are invalid") - } - var ts string - switch val { - case IKTSecp256k1: - ts = KTSecp256k1 - case IKTBLS: - ts = KTBLS - default: - return Signature{}, fmt.Errorf("unsupported signature type: %d", val) - } - - return Signature{ - Type: ts, - Data: x[1:], - }, nil -} - -func (s *Signature) TypeCode() int { - switch s.Type { - case KTSecp256k1: - return IKTSecp256k1 - case KTBLS: - return IKTBLS - default: - return IKTUnknown - } -} - -func (s *Signature) MarshalCBOR(w io.Writer) error { - if s == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - header := cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(s.Data)+1)) - - if _, err := w.Write(header); err != nil { - return err - } - - if _, err := w.Write([]byte{byte(s.TypeCode())}); err != nil { - return err - } - - if _, err := w.Write(s.Data); err != nil { - return err - } - - return nil -} - -func (s *Signature) UnmarshalCBOR(br io.Reader) error { - maj, l, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - - if maj != cbg.MajByteString { - return fmt.Errorf("cbor input for signature was not a byte string") - } - - if l > SignatureMaxLength { - return fmt.Errorf("cbor byte array for signature was too long") - } - - buf := make([]byte, l) - if _, err := io.ReadFull(br, buf); err != nil { - return err - } - - switch buf[0] { - default: - return fmt.Errorf("invalid signature type in cbor input: %d", buf[0]) - case IKTSecp256k1: - s.Type = KTSecp256k1 - case IKTBLS: - s.Type = KTBLS - } - s.Data = buf[1:] - - return nil -} - -func (s *Signature) Equals(o *Signature) bool { - return s.Type == o.Type && bytes.Equal(s.Data, o.Data) -} diff --git a/shared/types/signature_cgo.go b/shared/types/signature_cgo.go deleted file mode 100644 index d86e41d7..00000000 --- a/shared/types/signature_cgo.go +++ /dev/null @@ -1,55 +0,0 @@ -//+build cgo - -package types - -import ( - "fmt" - - bls "github.com/filecoin-project/filecoin-ffi" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-crypto" - "github.com/minio/blake2b-simd" -) - -func (s *Signature) Verify(addr address.Address, msg []byte) error { - if addr.Protocol() == address.ID { - return fmt.Errorf("must resolve ID addresses before using them to verify a signature") - } - b2sum := blake2b.Sum256(msg) - - switch s.Type { - case KTSecp256k1: - pubk, err := crypto.EcRecover(b2sum[:], s.Data) - if err != nil { - return err - } - - maybeaddr, err := address.NewSecp256k1Address(pubk) - if err != nil { - return err - } - - if addr != maybeaddr { - return fmt.Errorf("signature did not match") - } - - return nil - case KTBLS: - digests := []bls.Digest{bls.Hash(bls.Message(msg))} - - var pubk bls.PublicKey - copy(pubk[:], addr.Payload()) - pubkeys := []bls.PublicKey{pubk} - - var sig bls.Signature - copy(sig[:], s.Data) - - if !bls.Verify(&sig, digests, pubkeys) { - return fmt.Errorf("bls signature failed to verify") - } - - return nil - default: - return fmt.Errorf("cannot verify signature of unsupported type: %s", s.Type) - } -} diff --git a/shared/types/signature_test.go b/shared/types/signature_test.go deleted file mode 100644 index b45024b2..00000000 --- a/shared/types/signature_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package types - -import ( - "bytes" - "testing" -) - -func TestSignatureSerializeRoundTrip(t *testing.T) { - s := &Signature{ - Data: []byte("foo bar cat dog"), - Type: KTBLS, - } - - buf := new(bytes.Buffer) - if err := s.MarshalCBOR(buf); err != nil { - t.Fatal(err) - } - - var outs Signature - if err := outs.UnmarshalCBOR(buf); err != nil { - t.Fatal(err) - } - - if !outs.Equals(s) { - t.Fatal("serialization round trip failed") - } -} diff --git a/shared/types/voucher.go b/shared/types/voucher.go deleted file mode 100644 index 6406d1ab..00000000 --- a/shared/types/voucher.go +++ /dev/null @@ -1,92 +0,0 @@ -package types - -import ( - "bytes" - "encoding/base64" - - "github.com/filecoin-project/go-cbor-util" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - cbor "github.com/ipfs/go-ipld-cbor" -) - -//go:generate cbor-gen-for SignedVoucher ModVerifyParams Merge - -type SignedVoucher struct { - TimeLock uint64 - SecretPreimage []byte - Extra *ModVerifyParams - Lane uint64 - Nonce uint64 - Amount tokenamount.TokenAmount - MinCloseHeight uint64 - - Merges []Merge - - Signature *Signature -} - -func (sv *SignedVoucher) SigningBytes() ([]byte, error) { - osv := *sv - osv.Signature = nil - - buf := new(bytes.Buffer) - //if err := osv.MarshalCBOR(buf); err != nil { - // return nil, err - //} - - return buf.Bytes(), nil -} - -func (sv *SignedVoucher) EncodedString() (string, error) { - buf := new(bytes.Buffer) - if err := sv.MarshalCBOR(buf); err != nil { - return "", err - } - - return base64.RawURLEncoding.EncodeToString(buf.Bytes()), nil -} - -func (sv *SignedVoucher) Equals(other *SignedVoucher) bool { - // TODO: make this less bad - - selfB, err := cborutil.Dump(sv) - if err != nil { - log.Errorf("SignedVoucher.Equals: dump self: %s", err) - return false - } - - otherB, err := cborutil.Dump(other) - if err != nil { - log.Errorf("SignedVoucher.Equals: dump other: %s", err) - return false - } - - return bytes.Equal(selfB, otherB) -} - -func DecodeSignedVoucher(s string) (*SignedVoucher, error) { - data, err := base64.RawURLEncoding.DecodeString(s) - if err != nil { - return nil, err - } - - var sv SignedVoucher - if err := cbor.DecodeInto(data, &sv); err != nil { - return nil, err - } - - return &sv, nil -} - -type Merge struct { - Lane uint64 - Nonce uint64 -} - -type ModVerifyParams struct { - Actor address.Address - Method uint64 - Data []byte -} diff --git a/shared/types/voucher_cbor_gen.go b/shared/types/voucher_cbor_gen.go deleted file mode 100644 index 797ff4d8..00000000 --- a/shared/types/voucher_cbor_gen.go +++ /dev/null @@ -1,384 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package types - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf - -func (t *SignedVoucher) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{137}); err != nil { - return err - } - - // t.TimeLock (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TimeLock))); err != nil { - return err - } - - // t.SecretPreimage ([]uint8) (slice) - if len(t.SecretPreimage) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.SecretPreimage was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.SecretPreimage)))); err != nil { - return err - } - if _, err := w.Write(t.SecretPreimage); err != nil { - return err - } - - // t.Extra (types.ModVerifyParams) (struct) - if err := t.Extra.MarshalCBOR(w); err != nil { - return err - } - - // t.Lane (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { - return err - } - - // t.Nonce (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { - return err - } - - // t.Amount (tokenamount.TokenAmount) (struct) - if err := t.Amount.MarshalCBOR(w); err != nil { - return err - } - - // t.MinCloseHeight (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinCloseHeight))); err != nil { - return err - } - - // t.Merges ([]types.Merge) (slice) - if len(t.Merges) > cbg.MaxLength { - return xerrors.Errorf("Slice value in field t.Merges was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Merges)))); err != nil { - return err - } - for _, v := range t.Merges { - if err := v.MarshalCBOR(w); err != nil { - return err - } - } - - // t.Signature (types.Signature) (struct) - if err := t.Signature.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 9 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.TimeLock (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.TimeLock = uint64(extra) - // t.SecretPreimage ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.SecretPreimage: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.SecretPreimage = make([]byte, extra) - if _, err := io.ReadFull(br, t.SecretPreimage); err != nil { - return err - } - // t.Extra (types.ModVerifyParams) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Extra = new(ModVerifyParams) - if err := t.Extra.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - // t.Lane (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Lane = uint64(extra) - // t.Nonce (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Nonce = uint64(extra) - // t.Amount (tokenamount.TokenAmount) (struct) - - { - - if err := t.Amount.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.MinCloseHeight (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.MinCloseHeight = uint64(extra) - // t.Merges ([]types.Merge) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.MaxLength { - return fmt.Errorf("t.Merges: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - if extra > 0 { - t.Merges = make([]Merge, extra) - } - for i := 0; i < int(extra); i++ { - - var v Merge - if err := v.UnmarshalCBOR(br); err != nil { - return err - } - - t.Merges[i] = v - } - - // t.Signature (types.Signature) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Signature = new(Signature) - if err := t.Signature.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - -func (t *ModVerifyParams) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.Actor (address.Address) (struct) - if err := t.Actor.MarshalCBOR(w); err != nil { - return err - } - - // t.Method (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { - return err - } - - // t.Data ([]uint8) (slice) - if len(t.Data) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Data was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Data)))); err != nil { - return err - } - if _, err := w.Write(t.Data); err != nil { - return err - } - return nil -} - -func (t *ModVerifyParams) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Actor (address.Address) (struct) - - { - - if err := t.Actor.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Method (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Method = uint64(extra) - // t.Data ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Data: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Data = make([]byte, extra) - if _, err := io.ReadFull(br, t.Data); err != nil { - return err - } - return nil -} - -func (t *Merge) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Lane (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { - return err - } - - // t.Nonce (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { - return err - } - return nil -} - -func (t *Merge) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Lane (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Lane = uint64(extra) - // t.Nonce (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Nonce = uint64(extra) - return nil -} diff --git a/shared_testutil/test_piecestore.go b/shared_testutil/test_piecestore.go index 5ca54849..ecb0e7c6 100644 --- a/shared_testutil/test_piecestore.go +++ b/shared_testutil/test_piecestore.go @@ -12,9 +12,9 @@ import ( // TestPieceStore is piecestore who's query results are mocked type TestPieceStore struct { - piecesStubbed map[string]piecestore.PieceInfo - piecesExpected map[string]struct{} - piecesReceived map[string]struct{} + piecesStubbed map[cid.Cid]piecestore.PieceInfo + piecesExpected map[cid.Cid]struct{} + piecesReceived map[cid.Cid]struct{} cidInfosStubbed map[cid.Cid]piecestore.CIDInfo cidInfosExpected map[cid.Cid]struct{} cidInfosReceived map[cid.Cid]struct{} @@ -25,9 +25,9 @@ var _ piecestore.PieceStore = &TestPieceStore{} // NewTestPieceStore creates a TestPieceStore func NewTestPieceStore() *TestPieceStore { return &TestPieceStore{ - piecesStubbed: make(map[string]piecestore.PieceInfo), - piecesExpected: make(map[string]struct{}), - piecesReceived: make(map[string]struct{}), + piecesStubbed: make(map[cid.Cid]piecestore.PieceInfo), + piecesExpected: make(map[cid.Cid]struct{}), + piecesReceived: make(map[cid.Cid]struct{}), cidInfosStubbed: make(map[cid.Cid]piecestore.CIDInfo), cidInfosExpected: make(map[cid.Cid]struct{}), cidInfosReceived: make(map[cid.Cid]struct{}), @@ -36,19 +36,19 @@ func NewTestPieceStore() *TestPieceStore { // StubPiece creates a return value for the given piece cid without expecting it // to be called -func (tps *TestPieceStore) StubPiece(pieceCid []byte, pieceInfo piecestore.PieceInfo) { - tps.piecesStubbed[string(pieceCid)] = pieceInfo +func (tps *TestPieceStore) StubPiece(pieceCid cid.Cid, pieceInfo piecestore.PieceInfo) { + tps.piecesStubbed[pieceCid] = pieceInfo } // ExpectPiece records a piece being expected to be queried and return the given piece info -func (tps *TestPieceStore) ExpectPiece(pieceCid []byte, pieceInfo piecestore.PieceInfo) { - tps.piecesExpected[string(pieceCid)] = struct{}{} +func (tps *TestPieceStore) ExpectPiece(pieceCid cid.Cid, pieceInfo piecestore.PieceInfo) { + tps.piecesExpected[pieceCid] = struct{}{} tps.StubPiece(pieceCid, pieceInfo) } // ExpectMissingPiece records a piece being expected to be queried and should fail -func (tps *TestPieceStore) ExpectMissingPiece(pieceCid []byte) { - tps.piecesExpected[string(pieceCid)] = struct{}{} +func (tps *TestPieceStore) ExpectMissingPiece(pieceCid cid.Cid) { + tps.piecesExpected[pieceCid] = struct{}{} } // StubCID creates a return value for the given CID without expecting it @@ -74,22 +74,22 @@ func (tps *TestPieceStore) VerifyExpectations(t *testing.T) { require.Equal(t, tps.cidInfosExpected, tps.cidInfosReceived) } -func (tps *TestPieceStore) AddDealForPiece(pieceCID []byte, dealInfo piecestore.DealInfo) error { +func (tps *TestPieceStore) AddDealForPiece(pieceCID cid.Cid, dealInfo piecestore.DealInfo) error { panic("not implemented") } -func (tps *TestPieceStore) AddPieceBlockLocations(pieceCID []byte, blockLocations map[cid.Cid]piecestore.BlockLocation) error { +func (tps *TestPieceStore) AddPieceBlockLocations(pieceCID cid.Cid, blockLocations map[cid.Cid]piecestore.BlockLocation) error { panic("not implemented") } -func (tps *TestPieceStore) GetPieceInfo(pieceCID []byte) (piecestore.PieceInfo, error) { - tps.piecesReceived[string(pieceCID)] = struct{}{} +func (tps *TestPieceStore) GetPieceInfo(pieceCID cid.Cid) (piecestore.PieceInfo, error) { + tps.piecesReceived[pieceCID] = struct{}{} - pio, ok := tps.piecesStubbed[string(pieceCID)] + pio, ok := tps.piecesStubbed[pieceCID] if ok { return pio, nil } - _, ok = tps.piecesExpected[string(pieceCID)] + _, ok = tps.piecesExpected[pieceCID] if ok { return piecestore.PieceInfoUndefined, retrievalmarket.ErrNotFound } diff --git a/shared_testutil/test_types.go b/shared_testutil/test_types.go index 7e989513..0cb07e05 100644 --- a/shared_testutil/test_types.go +++ b/shared_testutil/test_types.go @@ -5,60 +5,62 @@ import ( "math/rand" "testing" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/go-address" "github.com/libp2p/go-libp2p-core/test" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-fil-markets/retrievalmarket" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/paych" + "github.com/filecoin-project/specs-actors/actors/crypto" ) // MakeTestSignedVoucher generates a random SignedVoucher that has all non-zero fields -func MakeTestSignedVoucher() *types.SignedVoucher { - return &types.SignedVoucher{ - TimeLock: rand.Uint64(), +func MakeTestSignedVoucher() *paych.SignedVoucher { + return &paych.SignedVoucher{ + TimeLock: abi.ChainEpoch(rand.Int63()), SecretPreimage: []byte("secret-preimage"), Extra: MakeTestModVerifyParams(), - Lane: rand.Uint64(), - Nonce: rand.Uint64(), + Lane: rand.Int63(), + Nonce: rand.Int63(), Amount: MakeTestTokenAmount(), - MinCloseHeight: rand.Uint64(), - Merges: []types.Merge{MakeTestMerge()}, + Merges: []paych.Merge{MakeTestMerge()}, Signature: MakeTestSignature(), } } // MakeTestModVerifyParams generates a random ModVerifyParams that has all non-zero fields -func MakeTestModVerifyParams() *types.ModVerifyParams { - return &types.ModVerifyParams{ +func MakeTestModVerifyParams() *paych.ModVerifyParams { + return &paych.ModVerifyParams{ Actor: address.TestAddress, - Method: rand.Uint64(), + Method: abi.MethodNum(rand.Int63()), Data: []byte("ModVerifyParams data"), } } // MakeTestMerge generates a random Merge that has all non-zero fields -func MakeTestMerge() types.Merge { - return types.Merge{ - Lane: rand.Uint64(), - Nonce: rand.Uint64(), +func MakeTestMerge() paych.Merge { + return paych.Merge{ + Lane: rand.Int63(), + Nonce: rand.Int63(), } } -// MakeTestSignagure generates a valid yet random Signature with all non-zero fields -func MakeTestSignature() *types.Signature { - return &types.Signature{ - Type: types.KTSecp256k1, +// MakeTestSignature generates a valid yet random Signature with all non-zero fields +func MakeTestSignature() *crypto.Signature { + return &crypto.Signature{ + Type: crypto.SigTypeSecp256k1, Data: []byte("signature data"), } } // MakeTestTokenAmount generates a valid yet random TokenAmount with a non-zero value. -func MakeTestTokenAmount() tokenamount.TokenAmount { - return tokenamount.TokenAmount{Int: big.NewInt(rand.Int63())} +func MakeTestTokenAmount() abi.TokenAmount { + return abi.TokenAmount{Int: big.NewInt(rand.Int63())} } // MakeTestQueryResponse generates a valid, random QueryResponse with no non-zero fields @@ -112,40 +114,47 @@ func MakeTestDealPayment() retrievalmarket.DealPayment { } } -// MakeTestStorageDealProposal generates a valid storage deal proposal -func MakeTestStorageDealProposal() *storagemarket.StorageDealProposal { - return &storagemarket.StorageDealProposal{ - PieceRef: RandomBytes(32), - PieceSize: rand.Uint64(), +// MakeTestUnsignedDealProposal generates a deal proposal with no signature +func MakeTestUnsignedDealProposal() market.DealProposal { + return market.DealProposal{ + PieceCID: GenerateCids(1)[0], + PieceSize: abi.PaddedPieceSize(rand.Int63()), Client: address.TestAddress, Provider: address.TestAddress2, - ProposalExpiration: rand.Uint64(), - Duration: rand.Uint64(), + StartEpoch: abi.ChainEpoch(rand.Int63()), + EndEpoch: abi.ChainEpoch(rand.Int63()), StoragePricePerEpoch: MakeTestTokenAmount(), - StorageCollateral: MakeTestTokenAmount(), + ProviderCollateral: MakeTestTokenAmount(), + ClientCollateral: MakeTestTokenAmount(), + } +} - ProposerSignature: MakeTestSignature(), +// MakeTestClientDealProposal generates a valid storage deal proposal +func MakeTestClientDealProposal() *market.ClientDealProposal { + return &market.ClientDealProposal{ + Proposal: MakeTestUnsignedDealProposal(), + ClientSignature: *MakeTestSignature(), } } // MakeTestStorageAsk generates a storage ask -func MakeTestStorageAsk() *types.StorageAsk { - return &types.StorageAsk{ +func MakeTestStorageAsk() *storagemarket.StorageAsk { + return &storagemarket.StorageAsk{ Price: MakeTestTokenAmount(), - MinPieceSize: rand.Uint64(), + MinPieceSize: abi.PaddedPieceSize(rand.Uint64()), Miner: address.TestAddress2, - Timestamp: rand.Uint64(), - Expiry: rand.Uint64(), + Timestamp: abi.ChainEpoch(rand.Int63()), + Expiry: abi.ChainEpoch(rand.Int63()), SeqNo: rand.Uint64(), } } // MakeTestSignedStorageAsk generates a signed storage ask -func MakeTestSignedStorageAsk() *types.SignedStorageAsk { - return &types.SignedStorageAsk{ +func MakeTestSignedStorageAsk() *storagemarket.SignedStorageAsk { + return &storagemarket.SignedStorageAsk{ Ask: MakeTestStorageAsk(), Signature: MakeTestSignature(), } @@ -155,7 +164,7 @@ func MakeTestSignedStorageAsk() *types.SignedStorageAsk { // network to a provider func MakeTestStorageNetworkProposal() smnet.Proposal { return smnet.Proposal{ - DealProposal: MakeTestStorageDealProposal(), + DealProposal: MakeTestClientDealProposal(), Piece: &storagemarket.DataRef{Root: GenerateCids(1)[0]}, } } diff --git a/shared_testutil/testutil.go b/shared_testutil/testutil.go index fbd9bcce..bb0fd2a4 100644 --- a/shared_testutil/testutil.go +++ b/shared_testutil/testutil.go @@ -2,12 +2,16 @@ package shared_testutil import ( "bytes" + "testing" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/specs-actors/actors/builtin/paych" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" blocksutil "github.com/ipfs/go-ipfs-blocksutil" "github.com/jbenet/go-random" "github.com/libp2p/go-libp2p-core/peer" + "github.com/stretchr/testify/require" ) var blockGenerator = blocksutil.NewBlockGenerator() @@ -81,3 +85,12 @@ func IndexOf(blks []blocks.Block, c cid.Cid) int { func ContainsBlock(blks []blocks.Block, block blocks.Block) bool { return IndexOf(blks, block.Cid()) != -1 } + +// TestVoucherEquality verifies that two vouchers are equal to one another +func TestVoucherEquality(t *testing.T, a, b *paych.SignedVoucher) { + aB, err := cborutil.Dump(a) + require.NoError(t, err) + bB, err := cborutil.Dump(b) + require.NoError(t, err) + require.True(t, bytes.Equal(aB, bB)) +} diff --git a/storagemarket/impl/client.go b/storagemarket/impl/client.go index 9286ff23..e7db6990 100644 --- a/storagemarket/impl/client.go +++ b/storagemarket/impl/client.go @@ -14,15 +14,17 @@ import ( "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" + commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/pieceio/cario" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-statestore" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin/market" ) //go:generate cbor-gen-for ClientDeal ClientDealProposal @@ -179,9 +181,9 @@ func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { type ClientDealProposal struct { Data *storagemarket.DataRef - PricePerEpoch tokenamount.TokenAmount - ProposalExpiration uint64 - Duration uint64 + PricePerEpoch abi.TokenAmount + StartEpoch abi.ChainEpoch + EndEpoch abi.ChainEpoch ProviderAddress address.Address Client address.Address @@ -190,7 +192,7 @@ type ClientDealProposal struct { } func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - amount := tokenamount.Mul(p.PricePerEpoch, tokenamount.FromInt(p.Duration)) + amount := big.Mul(p.PricePerEpoch, abi.NewTokenAmount(int64(p.EndEpoch)-int64(p.StartEpoch))) if err := c.node.EnsureFunds(ctx, p.Client, amount); err != nil { return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) } @@ -200,22 +202,23 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro return cid.Undef, xerrors.Errorf("computing commP failed: %w", err) } - dealProposal := &storagemarket.StorageDealProposal{ - PieceRef: commP, - PieceSize: uint64(pieceSize), + dealProposal := market.DealProposal{ + PieceCID: commcid.PieceCommitmentV1ToCID(commP), + PieceSize: abi.PaddedPieceSize(pieceSize), Client: p.Client, Provider: p.ProviderAddress, - ProposalExpiration: p.ProposalExpiration, - Duration: p.Duration, + StartEpoch: p.StartEpoch, + EndEpoch: p.EndEpoch, StoragePricePerEpoch: p.PricePerEpoch, - StorageCollateral: tokenamount.FromInt(uint64(pieceSize)), // TODO: real calc + ProviderCollateral: abi.NewTokenAmount(int64(pieceSize)), // TODO: real calc } - if err := c.node.SignProposal(ctx, p.Client, dealProposal); err != nil { + clientDealProposal, err := c.node.SignProposal(ctx, p.Client, dealProposal) + if err != nil { return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) } - proposalNd, err := cborutil.AsIpld(dealProposal) + proposalNd, err := cborutil.AsIpld(clientDealProposal) if err != nil { return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err) } @@ -225,19 +228,19 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err) } - proposal := network.Proposal{DealProposal: dealProposal, Piece: p.Data} + proposal := network.Proposal{DealProposal: clientDealProposal, Piece: p.Data} if err := s.WriteDealProposal(proposal); err != nil { return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err) } deal := &ClientDeal{ ClientDeal: storagemarket.ClientDeal{ - ProposalCid: proposalNd.Cid(), - Proposal: *dealProposal, - State: storagemarket.StorageDealUnknown, - Miner: p.MinerID, - MinerWorker: p.MinerWorker, - DataRef: p.Data, + ProposalCid: proposalNd.Cid(), + ClientDealProposal: *clientDealProposal, + State: storagemarket.StorageDealUnknown, + Miner: p.MinerID, + MinerWorker: p.MinerWorker, + DataRef: p.Data, }, s: s, @@ -251,7 +254,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro }) } -func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*types.SignedStorageAsk, error) { +func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*storagemarket.SignedStorageAsk, error) { s, err := c.net.NewAskStream(p) if err != nil { return nil, xerrors.Errorf("failed to open stream to miner: %w", err) diff --git a/storagemarket/impl/client_cbor_gen.go b/storagemarket/impl/client_cbor_gen.go index cfe21713..e11f1411 100644 --- a/storagemarket/impl/client_cbor_gen.go +++ b/storagemarket/impl/client_cbor_gen.go @@ -7,6 +7,7 @@ import ( "io" "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" @@ -71,19 +72,31 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.PricePerEpoch (tokenamount.TokenAmount) (struct) + // t.PricePerEpoch (big.Int) (struct) if err := t.PricePerEpoch.MarshalCBOR(w); err != nil { return err } - // t.ProposalExpiration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { - return err + // t.StartEpoch (abi.ChainEpoch) (int64) + if t.StartEpoch >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.StartEpoch))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.StartEpoch)-1)); err != nil { + return err + } } - // t.Duration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { - return err + // t.EndEpoch (abi.ChainEpoch) (int64) + if t.EndEpoch >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.EndEpoch))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.EndEpoch)-1)); err != nil { + return err + } } // t.ProviderAddress (address.Address) (struct) @@ -151,7 +164,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.PricePerEpoch (tokenamount.TokenAmount) (struct) + // t.PricePerEpoch (big.Int) (struct) { @@ -160,26 +173,56 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.ProposalExpiration (uint64) (uint64) + // t.StartEpoch (abi.ChainEpoch) (int64) + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + t.StartEpoch = abi.ChainEpoch(extraI) } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.ProposalExpiration = uint64(extra) - // t.Duration (uint64) (uint64) + // t.EndEpoch (abi.ChainEpoch) (int64) + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") + t.EndEpoch = abi.ChainEpoch(extraI) } - t.Duration = uint64(extra) // t.ProviderAddress (address.Address) (struct) { diff --git a/storagemarket/impl/client_states.go b/storagemarket/impl/client_states.go index 46f8d7e8..df72d443 100644 --- a/storagemarket/impl/client_states.go +++ b/storagemarket/impl/client_states.go @@ -5,7 +5,6 @@ import ( "golang.org/x/xerrors" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" ) @@ -92,6 +91,6 @@ func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal return nil, err } -func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { +func (c *Client) checkAskSignature(ask *storagemarket.SignedStorageAsk) error { return c.node.ValidateAskSignature(ask) } diff --git a/storagemarket/impl/client_storagemarket.go b/storagemarket/impl/client_storagemarket.go index 44da3e3f..e197b743 100644 --- a/storagemarket/impl/client_storagemarket.go +++ b/storagemarket/impl/client_storagemarket.go @@ -5,13 +5,12 @@ package storageimpl import ( "context" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" ) @@ -50,13 +49,13 @@ func (c *Client) ListInProgressDeals(ctx context.Context) ([]storagemarket.Clien out := make([]storagemarket.ClientDeal, len(deals)) for k, v := range deals { out[k] = storagemarket.ClientDeal{ - ProposalCid: v.ProposalCid, - Proposal: v.Proposal, - State: v.State, - Miner: v.Miner, - MinerWorker: v.MinerWorker, - DealID: v.DealID, - PublishMessage: v.PublishMessage, + ProposalCid: v.ProposalCid, + ClientDealProposal: v.ClientDealProposal, + State: v.State, + Miner: v.Miner, + MinerWorker: v.MinerWorker, + DealID: v.DealID, + PublishMessage: v.PublishMessage, } } @@ -78,7 +77,7 @@ func (c *Client) GetInProgressDeal(ctx context.Context, cid cid.Cid) (storagemar return storagemarket.ClientDeal{}, xerrors.Errorf("couldn't find client deal") } -func (c *Client) GetAsk(ctx context.Context, info storagemarket.StorageProviderInfo) (*types.SignedStorageAsk, error) { +func (c *Client) GetAsk(ctx context.Context, info storagemarket.StorageProviderInfo) (*storagemarket.SignedStorageAsk, error) { return c.QueryAsk(ctx, info.PeerID, info.Address) } @@ -87,21 +86,21 @@ func (c *Client) ProposeStorageDeal( addr address.Address, info *storagemarket.StorageProviderInfo, data *storagemarket.DataRef, - proposalExpiration storagemarket.Epoch, - duration storagemarket.Epoch, - price tokenamount.TokenAmount, - collateral tokenamount.TokenAmount, + startEpoch abi.ChainEpoch, + endEpoch abi.ChainEpoch, + price abi.TokenAmount, + collateral abi.TokenAmount, ) (*storagemarket.ProposeStorageDealResult, error) { proposal := ClientDealProposal{ - Data: data, - PricePerEpoch: price, - ProposalExpiration: uint64(proposalExpiration), - Duration: uint64(duration), - Client: addr, - ProviderAddress: info.Address, - MinerWorker: info.Worker, - MinerID: info.PeerID, + Data: data, + PricePerEpoch: price, + StartEpoch: startEpoch, + EndEpoch: endEpoch, + Client: addr, + ProviderAddress: info.Address, + MinerWorker: info.Worker, + MinerID: info.PeerID, } proposalCid, err := c.Start(ctx, proposal) @@ -120,7 +119,7 @@ func (c *Client) GetPaymentEscrow(ctx context.Context, addr address.Address) (st return balance, err } -func (c *Client) AddPaymentEscrow(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { +func (c *Client) AddPaymentEscrow(ctx context.Context, addr address.Address, amount abi.TokenAmount) error { return c.node.AddFunds(ctx, addr, amount) } diff --git a/storagemarket/impl/client_utils.go b/storagemarket/impl/client_utils.go index e7b44cd6..61a29745 100644 --- a/storagemarket/impl/client_utils.go +++ b/storagemarket/impl/client_utils.go @@ -13,6 +13,8 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + cborutil "github.com/filecoin-project/go-cbor-util" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-statestore" @@ -49,6 +51,18 @@ func (c *Client) commP(ctx context.Context, root cid.Cid) ([]byte, uint64, error return commp[:], paddedSize, nil } +func (c *Client) verifyResponse(resp network.SignedResponse, minerAddr address.Address) error { + b, err := cborutil.Dump(&resp.Response) + if err != nil { + return err + } + verified := c.node.VerifySignature(*resp.Signature, minerAddr, b) + if !verified { + return xerrors.New("could not verify signature") + } + return nil +} + func (c *Client) readStorageDealResp(deal ClientDeal) (*network.Response, error) { s, ok := c.conns[deal.ProposalCid] if !ok { @@ -62,8 +76,8 @@ func (c *Client) readStorageDealResp(deal ClientDeal) (*network.Response, error) return nil, err } - if err := resp.Verify(deal.MinerWorker, c.node.VerifySignature); err != nil { - return nil, xerrors.Errorf("verifying response signature failed", err) + if err := c.verifyResponse(resp, deal.MinerWorker); err != nil { + return nil, xerrors.Errorf("verifying response signature failed: %w", err) } if resp.Response.Proposal != deal.ProposalCid { @@ -139,7 +153,7 @@ func (c *ClientRequestValidator) ValidatePull( return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer) } if !deal.DataRef.Root.Equals(baseCid) { - return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) + return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", deal.Proposal.PieceCID.String(), baseCid.String(), ErrWrongPiece) } for _, state := range DataTransferStates { if deal.State == state { diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index b1ef54ab..dab68202 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -1,7 +1,6 @@ package storageimpl import ( - "bytes" "context" "errors" "io" @@ -16,15 +15,15 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" datatransfer "github.com/filecoin-project/go-data-transfer" + commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/pieceio/cario" "github.com/filecoin-project/go-fil-markets/piecestore" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-statestore" + "github.com/filecoin-project/specs-actors/actors/abi" ) var ProviderDsPrefix = "/deals/provider" @@ -39,10 +38,10 @@ type MinerDeal struct { type Provider struct { net network.StorageMarketNetwork - pricePerByteBlock tokenamount.TokenAmount // how much we want for storing one byte for one block - minPieceSize uint64 + pricePerByteBlock abi.TokenAmount // how much we want for storing one byte for one block + minPieceSize abi.PaddedPieceSize - ask *types.SignedStorageAsk + ask *storagemarket.SignedStorageAsk askLk sync.Mutex spn storagemarket.StorageProviderNode @@ -91,8 +90,8 @@ func NewProvider(net network.StorageMarketNetwork, ds datastore.Batching, bs blo dataTransfer: dataTransfer, spn: spn, - pricePerByteBlock: tokenamount.FromInt(3), // TODO: allow setting - minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up)) + pricePerByteBlock: abi.NewTokenAmount(3), // TODO: allow setting + minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up)) conns: map[cid.Cid]network.StorageDealStream{}, @@ -114,7 +113,7 @@ func NewProvider(net network.StorageMarketNetwork, ds datastore.Batching, bs blo if h.ask == nil { // TODO: we should be fine with this state, and just say it means 'not actively accepting deals' // for now... lets just set a price - if err := h.SetPrice(tokenamount.FromInt(500_000_000), 1000000); err != nil { + if err := h.SetPrice(abi.NewTokenAmount(500_000_000), 1000000); err != nil { return nil, xerrors.Errorf("failed setting a default price: %w", err) } } @@ -258,10 +257,10 @@ func (p *Provider) newDeal(s network.StorageDealStream, proposal network.Proposa return MinerDeal{ MinerDeal: storagemarket.MinerDeal{ - Client: s.RemotePeer(), - Proposal: *proposal.DealProposal, - ProposalCid: proposalNd.Cid(), - State: storagemarket.StorageDealUnknown, + Client: s.RemotePeer(), + ClientDealProposal: *proposal.DealProposal, + ProposalCid: proposalNd.Cid(), + State: storagemarket.StorageDealUnknown, Ref: proposal.Piece, }, @@ -313,18 +312,22 @@ func (p *Provider) ImportDataForDeal(ctx context.Context, propCid cid.Cid, data } _ = n // TODO: verify n? + pieceSize := uint64(tempfi.Size()) + _, err = tempfi.Seek(0, io.SeekStart) if err != nil { return xerrors.Errorf("failed to seek through temp imported file: %w", err) } - commP, err := p.pio.ReadPiece(tempfi) + commP, _, err := pieceio.GeneratePieceCommitment(tempfi, pieceSize) if err != nil { return xerrors.Errorf("failed to generate commP") } - if !bytes.Equal(commP.Bytes(), d.Proposal.PieceRef) { - return xerrors.Errorf("given data does not match expected commP (got: %x, expected %x)", commP.Bytes(), d.Proposal.PieceRef) + pieceCid := commcid.PieceCommitmentV1ToCID(commP) + // Verify CommP matches + if !pieceCid.Equals(d.Proposal.PieceCID) { + return xerrors.Errorf("given data does not match expected commP (got: %x, expected %x)", pieceCid, d.Proposal.PieceCID) } select { diff --git a/storagemarket/impl/provider_asks.go b/storagemarket/impl/provider_asks.go index bfc24bdd..77c63547 100644 --- a/storagemarket/impl/provider_asks.go +++ b/storagemarket/impl/provider_asks.go @@ -3,19 +3,18 @@ package storageimpl import ( "bytes" "context" - "time" "github.com/ipfs/go-datastore" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" + "github.com/filecoin-project/specs-actors/actors/abi" ) -func (p *Provider) SetPrice(price tokenamount.TokenAmount, ttlsecs int64) error { +func (p *Provider) SetPrice(price abi.TokenAmount, duration abi.ChainEpoch) error { p.askLk.Lock() defer p.askLk.Unlock() @@ -24,11 +23,14 @@ func (p *Provider) SetPrice(price tokenamount.TokenAmount, ttlsecs int64) error seqno = p.ask.Ask.SeqNo + 1 } - now := time.Now().Unix() - ask := &types.StorageAsk{ + stateKey, err := p.spn.MostRecentStateId(context.TODO()) + if err != nil { + return err + } + ask := &storagemarket.StorageAsk{ Price: price, - Timestamp: uint64(now), - Expiry: uint64(now + ttlsecs), + Timestamp: stateKey.Height(), + Expiry: stateKey.Height() + duration, Miner: p.actor, SeqNo: seqno, MinPieceSize: p.minPieceSize, @@ -42,7 +44,7 @@ func (p *Provider) SetPrice(price tokenamount.TokenAmount, ttlsecs int64) error return p.saveAsk(ssa) } -func (p *Provider) GetAsk(m address.Address) *types.SignedStorageAsk { +func (p *Provider) GetAsk(m address.Address) *storagemarket.SignedStorageAsk { p.askLk.Lock() defer p.askLk.Unlock() if m != p.actor { @@ -98,7 +100,7 @@ func (p *Provider) loadAsk() error { return xerrors.Errorf("failed to load most recent ask from disk: %w", err) } - var ssa types.SignedStorageAsk + var ssa storagemarket.SignedStorageAsk if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil { return err } @@ -107,7 +109,7 @@ func (p *Provider) loadAsk() error { return nil } -func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) { +func (p *Provider) signAsk(a *storagemarket.StorageAsk) (*storagemarket.SignedStorageAsk, error) { b, err := cborutil.Dump(a) if err != nil { return nil, err @@ -123,13 +125,13 @@ func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) return nil, err } - return &types.SignedStorageAsk{ + return &storagemarket.SignedStorageAsk{ Ask: a, Signature: sig, }, nil } -func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { +func (p *Provider) saveAsk(a *storagemarket.SignedStorageAsk) error { b, err := cborutil.Dump(a) if err != nil { return err diff --git a/storagemarket/impl/provider_states.go b/storagemarket/impl/provider_states.go index 212021ec..20af700c 100644 --- a/storagemarket/impl/provider_states.go +++ b/storagemarket/impl/provider_states.go @@ -1,9 +1,9 @@ package storageimpl import ( - "bytes" "context" + commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-padreader" "github.com/ipfs/go-cid" ipldfree "github.com/ipld/go-ipld-prime/impl/free" @@ -12,9 +12,10 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-fil-markets/piecestore" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" ) type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) @@ -45,13 +46,13 @@ func (p *Provider) validating(ctx context.Context, deal MinerDeal) (func(*MinerD if err != nil { return nil, err } - if head.Height() >= deal.Proposal.ProposalExpiration { + if head.Height() >= deal.Proposal.StartEpoch { return nil, xerrors.Errorf("deal proposal already expired") } // TODO: check StorageCollateral - minPrice := tokenamount.Div(tokenamount.Mul(p.ask.Ask.Price, tokenamount.FromInt(deal.Proposal.PieceSize)), tokenamount.FromInt(1<<30)) + minPrice := big.Div(big.Mul(p.ask.Ask.Price, abi.NewTokenAmount(int64(deal.Proposal.PieceSize))), abi.NewTokenAmount(1<<30)) if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) { return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice) } @@ -68,7 +69,7 @@ func (p *Provider) validating(ctx context.Context, deal MinerDeal) (func(*MinerD // This doesn't guarantee that the client won't withdraw / lock those funds // but it's a decent first filter - if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStoragePrice()) { + if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStorageFee()) { return nil, xerrors.New("clientMarketBalance.Available too small") } @@ -120,8 +121,9 @@ func (p *Provider) verifydata(ctx context.Context, deal MinerDeal) (func(*MinerD return nil, err } + pieceCid := commcid.PieceCommitmentV1ToCID(commp) // Verify CommP matches - if !bytes.Equal(commp, deal.Proposal.PieceRef) { + if !pieceCid.Equals(deal.Proposal.PieceCID) { return nil, xerrors.Errorf("proposal CommP doesn't match calculated CommP") } @@ -138,16 +140,16 @@ func (p *Provider) publishing(ctx context.Context, deal MinerDeal) (func(*MinerD } // TODO: check StorageCollateral (may be too large (or too small)) - if err := p.spn.EnsureFunds(ctx, waddr, deal.Proposal.StorageCollateral); err != nil { + if err := p.spn.EnsureFunds(ctx, waddr, deal.Proposal.ProviderCollateral); err != nil { return nil, err } smDeal := storagemarket.MinerDeal{ - Client: deal.Client, - Proposal: deal.Proposal, - ProposalCid: deal.ProposalCid, - State: deal.State, - Ref: deal.Ref, + Client: deal.Client, + ClientDealProposal: deal.ClientDealProposal, + ProposalCid: deal.ProposalCid, + State: deal.State, + Ref: deal.Ref, } dealId, mcid, err := p.spn.PublishDeals(ctx, smDeal) @@ -184,12 +186,12 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) err = p.spn.OnDealComplete( ctx, storagemarket.MinerDeal{ - Client: deal.Client, - Proposal: deal.Proposal, - ProposalCid: deal.ProposalCid, - State: deal.State, - Ref: deal.Ref, - DealID: deal.DealID, + Client: deal.Client, + ClientDealProposal: deal.ClientDealProposal, + ProposalCid: deal.ProposalCid, + State: deal.State, + Ref: deal.Ref, + DealID: deal.DealID, }, paddedSize, paddedReader, @@ -229,13 +231,13 @@ func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDea return nil, err } // TODO: Record actual block locations for all CIDs in piece by improving car writing - err = p.pieceStore.AddPieceBlockLocations(deal.Proposal.PieceRef, map[cid.Cid]piecestore.BlockLocation{ + err = p.pieceStore.AddPieceBlockLocations(deal.Proposal.PieceCID, map[cid.Cid]piecestore.BlockLocation{ deal.Ref.Root: {}, }) if err != nil { return nil, err } - return nil, p.pieceStore.AddDealForPiece(deal.Proposal.PieceRef, piecestore.DealInfo{ + return nil, p.pieceStore.AddDealForPiece(deal.Proposal.PieceCID, piecestore.DealInfo{ DealID: deal.DealID, SectorID: sectorID, Offset: offset, diff --git a/storagemarket/impl/provider_storagemarket.go b/storagemarket/impl/provider_storagemarket.go index cee75cc1..435be13e 100644 --- a/storagemarket/impl/provider_storagemarket.go +++ b/storagemarket/impl/provider_storagemarket.go @@ -6,20 +6,19 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/specs-actors/actors/abi" ) -func (p *Provider) AddAsk(price tokenamount.TokenAmount, ttlsecs int64) error { - return p.SetPrice(price, ttlsecs) +func (p *Provider) AddAsk(price abi.TokenAmount, duration abi.ChainEpoch) error { + return p.SetPrice(price, duration) } -func (p *Provider) ListAsks(addr address.Address) []*types.SignedStorageAsk { +func (p *Provider) ListAsks(addr address.Address) []*storagemarket.SignedStorageAsk { ask := p.GetAsk(addr) if ask != nil { - return []*types.SignedStorageAsk{ask} + return []*storagemarket.SignedStorageAsk{ask} } return nil @@ -29,7 +28,7 @@ func (p *Provider) ListDeals(ctx context.Context) ([]storagemarket.StorageDeal, return p.spn.ListProviderDeals(ctx, p.actor) } -func (p *Provider) AddStorageCollateral(ctx context.Context, amount tokenamount.TokenAmount) error { +func (p *Provider) AddStorageCollateral(ctx context.Context, amount abi.TokenAmount) error { return p.spn.AddFunds(ctx, p.actor, amount) } @@ -49,12 +48,12 @@ func (p *Provider) ListIncompleteDeals() ([]storagemarket.MinerDeal, error) { for _, deal := range deals { out = append(out, storagemarket.MinerDeal{ - Client: deal.Client, - Proposal: deal.Proposal, - ProposalCid: deal.ProposalCid, - State: deal.State, - Ref: deal.Ref, - DealID: deal.DealID, + Client: deal.Client, + ClientDealProposal: deal.ClientDealProposal, + ProposalCid: deal.ProposalCid, + State: deal.State, + Ref: deal.Ref, + DealID: deal.DealID, }) } diff --git a/storagemarket/impl/provider_utils.go b/storagemarket/impl/provider_utils.go index fb002f64..45c21a5c 100644 --- a/storagemarket/impl/provider_utils.go +++ b/storagemarket/impl/provider_utils.go @@ -1,6 +1,7 @@ package storageimpl import ( + "bytes" "context" "runtime" @@ -9,6 +10,7 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" + "github.com/filecoin-project/specs-actors/actors/builtin/market" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-statestore" @@ -47,6 +49,17 @@ func (p *Provider) failDeal(ctx context.Context, id cid.Cid, cerr error) { } } +func (p *Provider) verifyProposal(sdp *market.ClientDealProposal) error { + var buf bytes.Buffer + if err := sdp.Proposal.MarshalCBOR(&buf); err != nil { + return err + } + verified := p.spn.VerifySignature(sdp.ClientSignature, sdp.Proposal.Client, buf.Bytes()) + if !verified { + return xerrors.New("could not verify signature") + } + return nil +} func (p *Provider) readProposal(s network.StorageDealStream) (proposal network.Proposal, err error) { proposal, err = s.ReadDealProposal() if err != nil { @@ -54,16 +67,12 @@ func (p *Provider) readProposal(s network.StorageDealStream) (proposal network.P return proposal, err } - if proposal.DealProposal.ProposerSignature == nil { - return proposal, xerrors.Errorf("incoming deal proposal has no signature") - } - - if err := proposal.DealProposal.Verify(p.spn.VerifySignature); err != nil { + if err := p.verifyProposal(proposal.DealProposal); err != nil { return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) } - if proposal.DealProposal.Provider != p.actor { - log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider) + if proposal.DealProposal.Proposal.Provider != p.actor { + log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Proposal.Provider) return proposal, err } @@ -160,7 +169,7 @@ func (m *ProviderRequestValidator) ValidatePush( } if !deal.Ref.Root.Equals(baseCid) { - return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) + return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", deal.Proposal.PieceCID.String(), baseCid.String(), ErrWrongPiece) } for _, state := range DataTransferStates { if deal.State == state { diff --git a/storagemarket/impl/request_validation_test.go b/storagemarket/impl/request_validation_test.go index cf62aeae..aba85bda 100644 --- a/storagemarket/impl/request_validation_test.go +++ b/storagemarket/impl/request_validation_test.go @@ -5,7 +5,8 @@ import ( "math/rand" "testing" - "github.com/ipfs/go-cid" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" dss "github.com/ipfs/go-datastore/sync" @@ -15,10 +16,10 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-statestore" + "github.com/filecoin-project/specs-actors/actors/crypto" ) var blockGenerator = blocksutil.NewBlockGenerator() @@ -38,22 +39,24 @@ func (wrongDTType) Type() string { return "WrongDTTYPE" } -func uniqueStorageDealProposal() (storagemarket.StorageDealProposal, error) { +func uniqueStorageDealProposal() (market.ClientDealProposal, error) { clientAddr, err := address.NewIDAddress(uint64(rand.Int())) if err != nil { - return storagemarket.StorageDealProposal{}, err + return market.ClientDealProposal{}, err } providerAddr, err := address.NewIDAddress(uint64(rand.Int())) if err != nil { - return storagemarket.StorageDealProposal{}, err + return market.ClientDealProposal{}, err } - return storagemarket.StorageDealProposal{ - PieceRef: blockGenerator.Next().Cid().Bytes(), - Client: clientAddr, - Provider: providerAddr, - ProposerSignature: &types.Signature{ + return market.ClientDealProposal{ + Proposal: market.DealProposal{ + PieceCID: blockGenerator.Next().Cid(), + Client: clientAddr, + Provider: providerAddr, + }, + ClientSignature: crypto.Signature{ Data: []byte("foo bar cat dog"), - Type: types.KTBLS, + Type: crypto.SigTypeBLS, }, }, nil } @@ -74,8 +77,8 @@ func newClientDeal(minerID peer.ID, state storagemarket.StorageDealStatus) (deal return deals.ClientDeal{ ClientDeal: storagemarket.ClientDeal{ - Proposal: newProposal, - ProposalCid: proposalNd.Cid(), + ClientDealProposal: newProposal, + ProposalCid: proposalNd.Cid(), DataRef: &storagemarket.DataRef{ Root: blockGenerator.Next().Cid(), }, @@ -99,11 +102,11 @@ func newMinerDeal(clientID peer.ID, state storagemarket.StorageDealStatus) (deal return deals.MinerDeal{ MinerDeal: storagemarket.MinerDeal{ - Proposal: newProposal, - ProposalCid: proposalNd.Cid(), - Client: clientID, - State: state, - Ref: &storagemarket.DataRef{Root: ref}, + ClientDealProposal: newProposal, + ProposalCid: proposalNd.Cid(), + Client: clientID, + State: state, + Ref: &storagemarket.DataRef{Root: ref}, }, }, nil } @@ -129,11 +132,7 @@ func TestClientRequestValidation(t *testing.T) { if err != nil { t.Fatal("error serializing proposal") } - pieceRef, err := cid.Cast(proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid()}, pieceRef, nil), deals.ErrNoDeal) { + if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid()}, proposal.Proposal.PieceCID, nil), deals.ErrNoDeal) { t.Fatal("Pull should fail if there is no deal stored") } }) @@ -213,11 +212,7 @@ func TestProviderRequestValidation(t *testing.T) { if err != nil { t.Fatal("error serializing proposal") } - pieceRef, err := cid.Cast(proposal.PieceRef) - if err != nil { - t.Fatal("unable to construct piece cid") - } - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid()}, pieceRef, nil), deals.ErrNoDeal) { + if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid()}, proposal.Proposal.PieceCID, nil), deals.ErrNoDeal) { t.Fatal("Push should fail if there is no deal stored") } }) diff --git a/storagemarket/integration_test.go b/storagemarket/integration_test.go index 7b60d016..d059b090 100644 --- a/storagemarket/integration_test.go +++ b/storagemarket/integration_test.go @@ -1,6 +1,7 @@ package storagemarket_test import ( + "bytes" "context" "io" "io/ioutil" @@ -21,19 +22,22 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-fil-markets/filestore" + "github.com/filecoin-project/go-fil-markets/pieceio/cario" "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/shared_testutil" "github.com/filecoin-project/go-fil-markets/storagemarket" storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl" "github.com/filecoin-project/go-fil-markets/storagemarket/network" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/specs-actors/actors/crypto" ) func TestMakeDeal(t *testing.T) { ctx := context.Background() - epoch := uint64(100) + epoch := abi.ChainEpoch(100) nodeCommon := fakeCommon{newStorageMarketState()} ds1 := datastore.NewMapDatastore() td := shared_testutil.NewLibp2pTestData(ctx, t) @@ -84,7 +88,7 @@ func TestMakeDeal(t *testing.T) { assert.NoError(t, err) // set ask price where we'll accept any price - err = provider.AddAsk(tokenamount.FromInt(0), 50_000) + err = provider.AddAsk(big.NewInt(0), 50_000) assert.NoError(t, err) err = provider.Start(ctx) @@ -108,7 +112,7 @@ func TestMakeDeal(t *testing.T) { TransferType: storagemarket.TTGraphsync, Root: payloadCid, } - result, err := client.ProposeStorageDeal(ctx, providerAddr, &providerInfo, dataRef, storagemarket.Epoch(epoch+100), 20000, tokenamount.FromInt(1), tokenamount.FromInt(0)) + result, err := client.ProposeStorageDeal(ctx, providerAddr, &providerInfo, dataRef, abi.ChainEpoch(epoch+100), abi.ChainEpoch(epoch+20100), big.NewInt(1), big.NewInt(0)) assert.NoError(t, err) proposalCid = result.ProposalCid @@ -128,6 +132,123 @@ func TestMakeDeal(t *testing.T) { assert.Equal(t, pd.State, storagemarket.StorageDealActive) } +func TestMakeDealOffline(t *testing.T) { + ctx := context.Background() + epoch := abi.ChainEpoch(100) + nodeCommon := fakeCommon{newStorageMarketState()} + ds1 := datastore.NewMapDatastore() + td := shared_testutil.NewLibp2pTestData(ctx, t) + rootLink := td.LoadUnixFSFile(t, "payload.txt", false) + payloadCid := rootLink.(cidlink.Link).Cid + + clientNode := fakeClientNode{ + fakeCommon: nodeCommon, + ClientAddr: address.TestAddress, + } + + providerAddr := address.TestAddress2 + ds2 := datastore.NewMapDatastore() + tempPath, err := ioutil.TempDir("", "storagemarket_test") + assert.NoError(t, err) + ps := piecestore.NewPieceStore(ds2) + providerNode := fakeProviderNode{ + fakeCommon: nodeCommon, + MinerAddr: providerAddr, + } + fs, err := filestore.NewLocalFileStore(filestore.OsPath(tempPath)) + assert.NoError(t, err) + + // create provider and client + dt1 := graphsync.NewGraphSyncDataTransfer(td.Host1, td.GraphSync1) + require.NoError(t, dt1.RegisterVoucherType(reflect.TypeOf(&storageimpl.StorageDataTransferVoucher{}), &fakeDTValidator{})) + + client := storageimpl.NewClient( + network.NewFromLibp2pHost(td.Host1), + td.Bs1, + dt1, + discovery.NewLocal(ds1), + statestore.New(ds1), + &clientNode, + ) + + dt2 := graphsync.NewGraphSyncDataTransfer(td.Host2, td.GraphSync2) + provider, err := storageimpl.NewProvider( + network.NewFromLibp2pHost(td.Host2), + ds2, + td.Bs2, + fs, + ps, + dt2, + &providerNode, + providerAddr, + ) + assert.NoError(t, err) + + // set ask price where we'll accept any price + err = provider.AddAsk(big.NewInt(0), 50_000) + assert.NoError(t, err) + + err = provider.Start(ctx) + assert.NoError(t, err) + + // Closely follows the MinerInfo struct in the spec + providerInfo := storagemarket.StorageProviderInfo{ + Address: providerAddr, + Owner: providerAddr, + Worker: providerAddr, + SectorSize: 32, + PeerID: td.Host2.ID(), + } + + var proposalCid cid.Cid + + // make a deal + go func() { + client.Run(ctx) + dataRef := &storagemarket.DataRef{ + TransferType: storagemarket.TTManual, + Root: payloadCid, + } + result, err := client.ProposeStorageDeal(ctx, providerAddr, &providerInfo, dataRef, abi.ChainEpoch(epoch+100), abi.ChainEpoch(epoch+20100), big.NewInt(1), big.NewInt(0)) + assert.NoError(t, err) + + proposalCid = result.ProposalCid + }() + + time.Sleep(time.Millisecond * 100) + + cd, err := client.GetInProgressDeal(ctx, proposalCid) + assert.NoError(t, err) + assert.Equal(t, cd.State, storagemarket.StorageDealUnknown) + + providerDeals, err := provider.ListIncompleteDeals() + assert.NoError(t, err) + + pd := providerDeals[0] + assert.True(t, pd.ProposalCid.Equals(proposalCid)) + assert.Equal(t, pd.State, storagemarket.StorageDealTransferring) + + carBuf := new(bytes.Buffer) + + err = cario.NewCarIO().WriteCar(ctx, td.Bs1, payloadCid, td.AllSelector, carBuf) + require.NoError(t, err) + err = provider.ImportDataForDeal(ctx, pd.ProposalCid, carBuf) + require.NoError(t, err) + + time.Sleep(time.Millisecond * 100) + + cd, err = client.GetInProgressDeal(ctx, proposalCid) + assert.NoError(t, err) + assert.Equal(t, cd.State, storagemarket.StorageDealActive) + + providerDeals, err = provider.ListIncompleteDeals() + assert.NoError(t, err) + + pd = providerDeals[0] + assert.True(t, pd.ProposalCid.Equals(proposalCid)) + assert.Equal(t, pd.State, storagemarket.StorageDealActive) +} + type fakeDTValidator struct{} func (v *fakeDTValidator) ValidatePush(sender peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, selector ipld.Node) error { @@ -141,16 +262,16 @@ func (v *fakeDTValidator) ValidatePull(receiver peer.ID, voucher datatransfer.Vo var _ datatransfer.RequestValidator = (*fakeDTValidator)(nil) // Below fake node implementations -type testStateKey struct{ Epoch uint64 } +type testStateKey struct{ Epoch abi.ChainEpoch } -func (k *testStateKey) Height() uint64 { +func (k *testStateKey) Height() abi.ChainEpoch { return k.Epoch } type storageMarketState struct { - Epoch uint64 + Epoch abi.ChainEpoch DealId uint64 - Balances map[address.Address]tokenamount.TokenAmount + Balances map[address.Address]abi.TokenAmount StorageDeals map[address.Address][]storagemarket.StorageDeal Providers []*storagemarket.StorageProviderInfo } @@ -159,15 +280,15 @@ func newStorageMarketState() *storageMarketState { return &storageMarketState{ Epoch: 0, DealId: 0, - Balances: map[address.Address]tokenamount.TokenAmount{}, + Balances: map[address.Address]abi.TokenAmount{}, StorageDeals: map[address.Address][]storagemarket.StorageDeal{}, Providers: nil, } } -func (sma *storageMarketState) AddFunds(addr address.Address, amount tokenamount.TokenAmount) { +func (sma *storageMarketState) AddFunds(addr address.Address, amount abi.TokenAmount) { if existing, ok := sma.Balances[addr]; ok { - sma.Balances[addr] = tokenamount.Add(existing, amount) + sma.Balances[addr] = big.Add(existing, amount) } else { sma.Balances[addr] = amount } @@ -175,9 +296,9 @@ func (sma *storageMarketState) AddFunds(addr address.Address, amount tokenamount func (sma *storageMarketState) Balance(addr address.Address) storagemarket.Balance { if existing, ok := sma.Balances[addr]; ok { - return storagemarket.Balance{tokenamount.FromInt(0), existing} + return storagemarket.Balance{big.NewInt(0), existing} } - return storagemarket.Balance{tokenamount.FromInt(0), tokenamount.FromInt(0)} + return storagemarket.Balance{big.NewInt(0), big.NewInt(0)} } func (sma *storageMarketState) Deals(addr address.Address) []storagemarket.StorageDeal { @@ -210,15 +331,15 @@ func (n *fakeCommon) MostRecentStateId(ctx context.Context) (storagemarket.State return n.SMState.StateKey(), nil } -func (n *fakeCommon) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { +func (n *fakeCommon) AddFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) error { n.SMState.AddFunds(addr, amount) return nil } -func (n *fakeCommon) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error { +func (n *fakeCommon) EnsureFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) error { balance := n.SMState.Balance(addr) if balance.Available.LessThan(amount) { - n.SMState.AddFunds(addr, tokenamount.Sub(amount, balance.Available)) + n.SMState.AddFunds(addr, big.Sub(amount, balance.Available)) } return nil } @@ -227,8 +348,8 @@ func (n *fakeCommon) GetBalance(ctx context.Context, addr address.Address) (stor return n.SMState.Balance(addr), nil } -func (n *fakeCommon) VerifySignature(addr address.Address, data []byte) error { - return nil +func (n *fakeCommon) VerifySignature(signature crypto.Signature, addr address.Address, data []byte) bool { + return true } type fakeClientNode struct { @@ -249,9 +370,11 @@ func (n *fakeClientNode) ValidatePublishedDeal(ctx context.Context, deal storage return 0, nil } -func (n *fakeClientNode) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error { - proposal.ProposerSignature = shared_testutil.MakeTestSignature() - return nil +func (n *fakeClientNode) SignProposal(ctx context.Context, signer address.Address, proposal market.DealProposal) (*market.ClientDealProposal, error) { + return &market.ClientDealProposal{ + Proposal: proposal, + ClientSignature: *shared_testutil.MakeTestSignature(), + }, nil } func (n *fakeClientNode) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { @@ -263,7 +386,7 @@ func (n *fakeClientNode) OnDealSectorCommitted(ctx context.Context, provider add return nil } -func (n *fakeClientNode) ValidateAskSignature(ask *types.SignedStorageAsk) error { +func (n *fakeClientNode) ValidateAskSignature(ask *storagemarket.SignedStorageAsk) error { return n.ValidationError } @@ -280,17 +403,10 @@ type fakeProviderNode struct { } func (n *fakeProviderNode) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) { - p := deal.Proposal sd := storagemarket.StorageDeal{ - PieceRef: p.PieceRef, - PieceSize: p.PieceSize, - Client: p.Client, - Provider: p.Provider, - ProposalExpiration: p.ProposalExpiration, - Duration: p.Duration, - StoragePricePerEpoch: p.StoragePricePerEpoch, - StorageCollateral: p.StorageCollateral, + deal.Proposal, + market.DealState{}, } n.SMState.AddDeal(sd) @@ -310,7 +426,7 @@ func (n *fakeProviderNode) GetMinerWorker(ctx context.Context, miner address.Add return n.MinerAddr, nil } -func (n *fakeProviderNode) SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) { +func (n *fakeProviderNode) SignBytes(ctx context.Context, signer address.Address, b []byte) (*crypto.Signature, error) { return shared_testutil.MakeTestSignature(), nil } diff --git a/storagemarket/network/types.go b/storagemarket/network/types.go index b511acdd..7f4bb33c 100644 --- a/storagemarket/network/types.go +++ b/storagemarket/network/types.go @@ -1,12 +1,12 @@ package network import ( + "github.com/filecoin-project/specs-actors/actors/builtin/market" "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" - cborutil "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/specs-actors/actors/crypto" ) //go:generate cbor-gen-for AskRequest AskResponse Proposal Response SignedResponse @@ -14,7 +14,7 @@ import ( // Proposal is the data sent over the network from client to provider when proposing // a deal type Proposal struct { - DealProposal *storagemarket.StorageDealProposal + DealProposal *market.ClientDealProposal Piece *storagemarket.DataRef } @@ -37,21 +37,11 @@ type Response struct { type SignedResponse struct { Response Response - Signature *types.Signature + Signature *crypto.Signature } var SignedResponseUndefined = SignedResponse{} -// Verify verifies that a proposal was signed by the given provider -func (r *SignedResponse) Verify(addr address.Address, verifier storagemarket.SignatureVerifier) error { - b, err := cborutil.Dump(&r.Response) - if err != nil { - return err - } - - return verifier(addr, b) -} - // AskRequest is a request for current ask parameters for a given miner type AskRequest struct { Miner address.Address @@ -62,7 +52,7 @@ var AskRequestUndefined = AskRequest{} // AskResponse is the response sent over the network in response // to an ask request type AskResponse struct { - Ask *types.SignedStorageAsk + Ask *storagemarket.SignedStorageAsk } var AskResponseUndefined = AskResponse{} diff --git a/storagemarket/network/types_cbor_gen.go b/storagemarket/network/types_cbor_gen.go index 07c3f388..6859bbdf 100644 --- a/storagemarket/network/types_cbor_gen.go +++ b/storagemarket/network/types_cbor_gen.go @@ -6,8 +6,9 @@ import ( "fmt" "io" - "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/specs-actors/actors/crypto" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -66,7 +67,7 @@ func (t *AskResponse) MarshalCBOR(w io.Writer) error { return err } - // t.Ask (types.SignedStorageAsk) (struct) + // t.Ask (storagemarket.SignedStorageAsk) (struct) if err := t.Ask.MarshalCBOR(w); err != nil { return err } @@ -88,7 +89,7 @@ func (t *AskResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Ask (types.SignedStorageAsk) (struct) + // t.Ask (storagemarket.SignedStorageAsk) (struct) { @@ -102,7 +103,7 @@ func (t *AskResponse) UnmarshalCBOR(r io.Reader) error { return err } } else { - t.Ask = new(types.SignedStorageAsk) + t.Ask = new(storagemarket.SignedStorageAsk) if err := t.Ask.UnmarshalCBOR(br); err != nil { return err } @@ -121,7 +122,7 @@ func (t *Proposal) MarshalCBOR(w io.Writer) error { return err } - // t.DealProposal (storagemarket.StorageDealProposal) (struct) + // t.DealProposal (market.ClientDealProposal) (struct) if err := t.DealProposal.MarshalCBOR(w); err != nil { return err } @@ -148,7 +149,7 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.DealProposal (storagemarket.StorageDealProposal) (struct) + // t.DealProposal (market.ClientDealProposal) (struct) { @@ -162,7 +163,7 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error { return err } } else { - t.DealProposal = new(storagemarket.StorageDealProposal) + t.DealProposal = new(market.ClientDealProposal) if err := t.DealProposal.UnmarshalCBOR(br); err != nil { return err } @@ -328,7 +329,7 @@ func (t *SignedResponse) MarshalCBOR(w io.Writer) error { return err } - // t.Signature (types.Signature) (struct) + // t.Signature (crypto.Signature) (struct) if err := t.Signature.MarshalCBOR(w); err != nil { return err } @@ -359,7 +360,7 @@ func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { } } - // t.Signature (types.Signature) (struct) + // t.Signature (crypto.Signature) (struct) { @@ -373,7 +374,7 @@ func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { return err } } else { - t.Signature = new(types.Signature) + t.Signature = new(crypto.Signature) if err := t.Signature.UnmarshalCBOR(br); err != nil { return err } diff --git a/storagemarket/types.go b/storagemarket/types.go index 7a432ef9..8af43754 100644 --- a/storagemarket/types.go +++ b/storagemarket/types.go @@ -1,29 +1,28 @@ package storagemarket import ( - "bytes" "context" "io" "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/peer" - xerrors "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-fil-markets/filestore" - "github.com/filecoin-project/go-fil-markets/shared/tokenamount" - "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/specs-actors/actors/crypto" ) -//go:generate cbor-gen-for ClientDeal MinerDeal StorageDeal Balance StorageDealProposal DataRef +//go:generate cbor-gen-for ClientDeal MinerDeal Balance SignedStorageAsk StorageAsk StorageDeal DataRef const DealProtocolID = "/fil/storage/mk/1.0.1" const AskProtocolID = "/fil/storage/ask/1.0.1" type Balance struct { - Locked tokenamount.TokenAmount - Available tokenamount.TokenAmount + Locked abi.TokenAmount + Available abi.TokenAmount } type StorageDealStatus = uint64 @@ -80,91 +79,35 @@ var DealStates = []string{ type DealID uint64 -type StorageDealProposal struct { - PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea - PieceSize uint64 - - Client address.Address - Provider address.Address - - ProposalExpiration uint64 - Duration uint64 // TODO: spec - - StoragePricePerEpoch tokenamount.TokenAmount - StorageCollateral tokenamount.TokenAmount - - ProposerSignature *types.Signature -} - -func (sdp *StorageDealProposal) TotalStoragePrice() tokenamount.TokenAmount { - return tokenamount.Mul(sdp.StoragePricePerEpoch, tokenamount.FromInt(sdp.Duration)) +func init() { + cbor.RegisterCborType(SignedStorageAsk{}) + cbor.RegisterCborType(StorageAsk{}) } -type SignFunc = func(context.Context, []byte) (*types.Signature, error) - -func (sdp *StorageDealProposal) Sign(ctx context.Context, sign SignFunc) error { - if sdp.ProposerSignature != nil { - return xerrors.New("signature already present in StorageDealProposal") - } - var buf bytes.Buffer - if err := sdp.MarshalCBOR(&buf); err != nil { - return err - } - sig, err := sign(ctx, buf.Bytes()) - if err != nil { - return err - } - sdp.ProposerSignature = sig - return nil -} - -func (sdp *StorageDealProposal) Cid() (cid.Cid, error) { - nd, err := cborutil.AsIpld(sdp) - if err != nil { - return cid.Undef, err - } - - return nd.Cid(), nil -} - -type SignatureVerifier func(address.Address, []byte) error - -func (sdp *StorageDealProposal) Verify(verifier SignatureVerifier) error { - unsigned := *sdp - unsigned.ProposerSignature = nil - var buf bytes.Buffer - if err := unsigned.MarshalCBOR(&buf); err != nil { - return err - } - - return verifier(sdp.Client, buf.Bytes()) +type SignedStorageAsk struct { + Ask *StorageAsk + Signature *crypto.Signature } -type StorageDeal struct { - PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea - PieceSize uint64 - - Client address.Address - Provider address.Address +type StorageAsk struct { + // Price per GiB / Epoch + Price abi.TokenAmount - ProposalExpiration uint64 - Duration uint64 // TODO: spec - - StoragePricePerEpoch tokenamount.TokenAmount - StorageCollateral tokenamount.TokenAmount - ActivationEpoch uint64 // 0 = inactive + MinPieceSize abi.PaddedPieceSize + Miner address.Address + Timestamp abi.ChainEpoch + Expiry abi.ChainEpoch + SeqNo uint64 } type StateKey interface { - Height() uint64 + Height() abi.ChainEpoch } -type Epoch uint64 - // Duplicated from deals package for now type MinerDeal struct { + market.ClientDealProposal ProposalCid cid.Cid - Proposal StorageDealProposal Miner peer.ID Client peer.ID State StorageDealStatus @@ -176,8 +119,8 @@ type MinerDeal struct { } type ClientDeal struct { + market.ClientDealProposal ProposalCid cid.Cid - Proposal StorageDealProposal State StorageDealStatus Miner peer.ID MinerWorker address.Address @@ -187,16 +130,22 @@ type ClientDeal struct { PublishMessage *cid.Cid } -// The interface provided for storage providers +// StorageDeal is a local combination of a proposal and a current deal state +type StorageDeal struct { + market.DealProposal + market.DealState +} + +// StorageProvider is the interface provided for storage providers type StorageProvider interface { Start(ctx context.Context) error Stop() error - AddAsk(price tokenamount.TokenAmount, ttlsecs int64) error + AddAsk(price abi.TokenAmount, duration abi.ChainEpoch) error // ListAsks lists current asks - ListAsks(addr address.Address) []*types.SignedStorageAsk + ListAsks(addr address.Address) []*SignedStorageAsk // ListDeals lists on-chain deals associated with this provider ListDeals(ctx context.Context) ([]StorageDeal, error) @@ -205,7 +154,7 @@ type StorageProvider interface { ListIncompleteDeals() ([]MinerDeal, error) // AddStorageCollateral adds storage collateral - AddStorageCollateral(ctx context.Context, amount tokenamount.TokenAmount) error + AddStorageCollateral(ctx context.Context, amount abi.TokenAmount) error // GetStorageCollateral returns the current collateral balance GetStorageCollateral(ctx context.Context) (Balance, error) @@ -217,11 +166,14 @@ type StorageProvider interface { type StorageProviderNode interface { MostRecentStateId(ctx context.Context) (StateKey, error) + // Verify a signature against an address + data + VerifySignature(signature crypto.Signature, signer address.Address, plaintext []byte) bool + // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. - AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error + AddFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) error // Ensures that a storage market participant has a certain amount of available funds - EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error + EnsureFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) error // GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients. GetBalance(ctx context.Context, addr address.Address) (Balance, error) @@ -239,13 +191,11 @@ type StorageProviderNode interface { GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) // Signs bytes - SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) + SignBytes(ctx context.Context, signer address.Address, b []byte) (*crypto.Signature, error) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealID uint64, cb DealSectorCommittedCallback) error LocatePieceForDealWithinSector(ctx context.Context, dealID uint64) (sectorID uint64, offset uint64, length uint64, err error) - - VerifySignature(addr address.Address, data []byte) error } type DealSectorCommittedCallback func(err error) @@ -254,10 +204,13 @@ type DealSectorCommittedCallback func(err error) type StorageClientNode interface { MostRecentStateId(ctx context.Context) (StateKey, error) + // Verify a signature against an address + data + VerifySignature(signature crypto.Signature, signer address.Address, plaintext []byte) bool + // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. - AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error + AddFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) error - EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error + EnsureFunds(ctx context.Context, addr address.Address, amount abi.TokenAmount) error // GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients. GetBalance(ctx context.Context, addr address.Address) (Balance, error) @@ -280,15 +233,13 @@ type StorageClientNode interface { ValidatePublishedDeal(ctx context.Context, deal ClientDeal) (uint64, error) // SignProposal signs a proposal - SignProposal(ctx context.Context, signer address.Address, proposal *StorageDealProposal) error + SignProposal(ctx context.Context, signer address.Address, proposal market.DealProposal) (*market.ClientDealProposal, error) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb DealSectorCommittedCallback) error - ValidateAskSignature(ask *types.SignedStorageAsk) error - - VerifySignature(addr address.Address, data []byte) error + ValidateAskSignature(ask *SignedStorageAsk) error } type StorageClientProofs interface { @@ -338,17 +289,17 @@ type StorageClient interface { GetInProgressDeal(ctx context.Context, cid cid.Cid) (ClientDeal, error) // GetAsk returns the current ask for a storage provider - GetAsk(ctx context.Context, info StorageProviderInfo) (*types.SignedStorageAsk, error) + GetAsk(ctx context.Context, info StorageProviderInfo) (*SignedStorageAsk, error) //// FindStorageOffers lists providers and queries them to find offers that satisfy some criteria based on price, duration, etc. //FindStorageOffers(criteria AskCriteria, limit uint) []*StorageOffer // ProposeStorageDeal initiates deal negotiation with a Storage Provider - ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, data *DataRef, proposalExpiration Epoch, duration Epoch, price tokenamount.TokenAmount, collateral tokenamount.TokenAmount) (*ProposeStorageDealResult, error) + ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, data *DataRef, startEpoch abi.ChainEpoch, endEpoch abi.ChainEpoch, price abi.TokenAmount, collateral abi.TokenAmount) (*ProposeStorageDealResult, error) // GetPaymentEscrow returns the current funds available for deal payment GetPaymentEscrow(ctx context.Context, addr address.Address) (Balance, error) // AddStorageCollateral adds storage collateral - AddPaymentEscrow(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error + AddPaymentEscrow(ctx context.Context, addr address.Address, amount abi.TokenAmount) error } diff --git a/storagemarket/types_cbor_gen.go b/storagemarket/types_cbor_gen.go index 4d3da936..f9c85be5 100644 --- a/storagemarket/types_cbor_gen.go +++ b/storagemarket/types_cbor_gen.go @@ -7,7 +7,8 @@ import ( "io" "github.com/filecoin-project/go-fil-markets/filestore" - "github.com/filecoin-project/go-fil-markets/shared/types" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" @@ -24,17 +25,17 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { return err } + // t.ClientDealProposal (market.ClientDealProposal) (struct) + if err := t.ClientDealProposal.MarshalCBOR(w); err != nil { + return err + } + // t.ProposalCid (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ProposalCid); err != nil { return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) } - // t.Proposal (storagemarket.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - // t.State (uint64) (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { return err @@ -97,26 +98,26 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.ProposalCid (cid.Cid) (struct) + // t.ClientDealProposal (market.ClientDealProposal) (struct) { - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) + if err := t.ClientDealProposal.UnmarshalCBOR(br); err != nil { + return err } - t.ProposalCid = c - } - // t.Proposal (storagemarket.StorageDealProposal) (struct) + // t.ProposalCid (cid.Cid) (struct) { - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) } + t.ProposalCid = c + } // t.State (uint64) (uint64) @@ -214,17 +215,17 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { return err } + // t.ClientDealProposal (market.ClientDealProposal) (struct) + if err := t.ClientDealProposal.MarshalCBOR(w); err != nil { + return err + } + // t.ProposalCid (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ProposalCid); err != nil { return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) } - // t.Proposal (storagemarket.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - // t.Miner (peer.ID) (string) if len(t.Miner) > cbg.MaxLength { return xerrors.Errorf("Value in field t.Miner was too long") @@ -293,26 +294,26 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.ProposalCid (cid.Cid) (struct) + // t.ClientDealProposal (market.ClientDealProposal) (struct) { - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) + if err := t.ClientDealProposal.UnmarshalCBOR(br); err != nil { + return err } - t.ProposalCid = c - } - // t.Proposal (storagemarket.StorageDealProposal) (struct) + // t.ProposalCid (cid.Cid) (struct) { - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) } + t.ProposalCid = c + } // t.Miner (peer.ID) (string) @@ -388,70 +389,28 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *StorageDeal) MarshalCBOR(w io.Writer) error { +func (t *Balance) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{137}); err != nil { - return err - } - - // t.PieceRef ([]uint8) (slice) - if len(t.PieceRef) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.PieceRef was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceRef)))); err != nil { - return err - } - if _, err := w.Write(t.PieceRef); err != nil { - return err - } - - // t.PieceSize (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSize))); err != nil { - return err - } - - // t.Client (address.Address) (struct) - if err := t.Client.MarshalCBOR(w); err != nil { - return err - } - - // t.Provider (address.Address) (struct) - if err := t.Provider.MarshalCBOR(w); err != nil { - return err - } - - // t.ProposalExpiration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { - return err - } - - // t.Duration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { - return err - } - - // t.StoragePricePerEpoch (tokenamount.TokenAmount) (struct) - if err := t.StoragePricePerEpoch.MarshalCBOR(w); err != nil { + if _, err := w.Write([]byte{130}); err != nil { return err } - // t.StorageCollateral (tokenamount.TokenAmount) (struct) - if err := t.StorageCollateral.MarshalCBOR(w); err != nil { + // t.Locked (big.Int) (struct) + if err := t.Locked.MarshalCBOR(w); err != nil { return err } - // t.ActivationEpoch (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ActivationEpoch))); err != nil { + // t.Available (big.Int) (struct) + if err := t.Available.MarshalCBOR(w); err != nil { return err } return nil } -func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { +func (t *Balance) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -462,107 +421,32 @@ func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 9 { + if extra != 2 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.PieceRef ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.PieceRef: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.PieceRef = make([]byte, extra) - if _, err := io.ReadFull(br, t.PieceRef); err != nil { - return err - } - // t.PieceSize (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.PieceSize = uint64(extra) - // t.Client (address.Address) (struct) + // t.Locked (big.Int) (struct) { - if err := t.Client.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.Provider (address.Address) (struct) - - { - - if err := t.Provider.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.ProposalExpiration (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.ProposalExpiration = uint64(extra) - // t.Duration (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Duration = uint64(extra) - // t.StoragePricePerEpoch (tokenamount.TokenAmount) (struct) - - { - - if err := t.StoragePricePerEpoch.UnmarshalCBOR(br); err != nil { + if err := t.Locked.UnmarshalCBOR(br); err != nil { return err } } - // t.StorageCollateral (tokenamount.TokenAmount) (struct) + // t.Available (big.Int) (struct) { - if err := t.StorageCollateral.UnmarshalCBOR(br); err != nil { + if err := t.Available.UnmarshalCBOR(br); err != nil { return err } } - // t.ActivationEpoch (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.ActivationEpoch = uint64(extra) return nil } -func (t *Balance) MarshalCBOR(w io.Writer) error { +func (t *SignedStorageAsk) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -571,19 +455,19 @@ func (t *Balance) MarshalCBOR(w io.Writer) error { return err } - // t.Locked (tokenamount.TokenAmount) (struct) - if err := t.Locked.MarshalCBOR(w); err != nil { + // t.Ask (storagemarket.StorageAsk) (struct) + if err := t.Ask.MarshalCBOR(w); err != nil { return err } - // t.Available (tokenamount.TokenAmount) (struct) - if err := t.Available.MarshalCBOR(w); err != nil { + // t.Signature (crypto.Signature) (struct) + if err := t.Signature.MarshalCBOR(w); err != nil { return err } return nil } -func (t *Balance) UnmarshalCBOR(r io.Reader) error { +func (t *SignedStorageAsk) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -598,91 +482,105 @@ func (t *Balance) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Locked (tokenamount.TokenAmount) (struct) + // t.Ask (storagemarket.StorageAsk) (struct) { - if err := t.Locked.UnmarshalCBOR(br); err != nil { + pb, err := br.PeekByte() + if err != nil { return err } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Ask = new(StorageAsk) + if err := t.Ask.UnmarshalCBOR(br); err != nil { + return err + } + } } - // t.Available (tokenamount.TokenAmount) (struct) + // t.Signature (crypto.Signature) (struct) { - if err := t.Available.UnmarshalCBOR(br); err != nil { + pb, err := br.PeekByte() + if err != nil { return err } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Signature = new(crypto.Signature) + if err := t.Signature.UnmarshalCBOR(br); err != nil { + return err + } + } } return nil } -func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { +func (t *StorageAsk) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{137}); err != nil { + if _, err := w.Write([]byte{134}); err != nil { return err } - // t.PieceRef ([]uint8) (slice) - if len(t.PieceRef) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.PieceRef was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceRef)))); err != nil { - return err - } - if _, err := w.Write(t.PieceRef); err != nil { - return err - } - - // t.PieceSize (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSize))); err != nil { - return err - } - - // t.Client (address.Address) (struct) - if err := t.Client.MarshalCBOR(w); err != nil { - return err - } - - // t.Provider (address.Address) (struct) - if err := t.Provider.MarshalCBOR(w); err != nil { + // t.Price (big.Int) (struct) + if err := t.Price.MarshalCBOR(w); err != nil { return err } - // t.ProposalExpiration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { + // t.MinPieceSize (abi.PaddedPieceSize) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinPieceSize))); err != nil { return err } - // t.Duration (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { + // t.Miner (address.Address) (struct) + if err := t.Miner.MarshalCBOR(w); err != nil { return err } - // t.StoragePricePerEpoch (tokenamount.TokenAmount) (struct) - if err := t.StoragePricePerEpoch.MarshalCBOR(w); err != nil { - return err + // t.Timestamp (abi.ChainEpoch) (int64) + if t.Timestamp >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.Timestamp)-1)); err != nil { + return err + } } - // t.StorageCollateral (tokenamount.TokenAmount) (struct) - if err := t.StorageCollateral.MarshalCBOR(w); err != nil { - return err + // t.Expiry (abi.ChainEpoch) (int64) + if t.Expiry >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Expiry))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.Expiry)-1)); err != nil { + return err + } } - // t.ProposerSignature (types.Signature) (struct) - if err := t.ProposerSignature.MarshalCBOR(w); err != nil { + // t.SeqNo (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SeqNo))); err != nil { return err } return nil } -func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { +func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -693,28 +591,20 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 9 { + if extra != 6 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.PieceRef ([]uint8) (slice) + // t.Price (big.Int) (struct) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } + { + + if err := t.Price.UnmarshalCBOR(br); err != nil { + return err + } - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.PieceRef: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.PieceRef = make([]byte, extra) - if _, err := io.ReadFull(br, t.PieceRef); err != nil { - return err } - // t.PieceSize (uint64) (uint64) + // t.MinPieceSize (abi.PaddedPieceSize) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -723,26 +613,67 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.PieceSize = uint64(extra) - // t.Client (address.Address) (struct) + t.MinPieceSize = abi.PaddedPieceSize(extra) + // t.Miner (address.Address) (struct) { - if err := t.Client.UnmarshalCBOR(br); err != nil { + if err := t.Miner.UnmarshalCBOR(br); err != nil { return err } } - // t.Provider (address.Address) (struct) - + // t.Timestamp (abi.ChainEpoch) (int64) { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } - if err := t.Provider.UnmarshalCBOR(br); err != nil { + t.Timestamp = abi.ChainEpoch(extraI) + } + // t.Expiry (abi.ChainEpoch) (int64) + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { return err } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + t.Expiry = abi.ChainEpoch(extraI) } - // t.ProposalExpiration (uint64) (uint64) + // t.SeqNo (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -751,54 +682,62 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ProposalExpiration = uint64(extra) - // t.Duration (uint64) (uint64) + t.SeqNo = uint64(extra) + return nil +} - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { +func (t *StorageDeal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) return err } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") + if _, err := w.Write([]byte{130}); err != nil { + return err } - t.Duration = uint64(extra) - // t.StoragePricePerEpoch (tokenamount.TokenAmount) (struct) - { + // t.DealProposal (market.DealProposal) (struct) + if err := t.DealProposal.MarshalCBOR(w); err != nil { + return err + } - if err := t.StoragePricePerEpoch.UnmarshalCBOR(br); err != nil { - return err - } + // t.DealState (market.DealState) (struct) + if err := t.DealState.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") } - // t.StorageCollateral (tokenamount.TokenAmount) (struct) + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.DealProposal (market.DealProposal) (struct) { - if err := t.StorageCollateral.UnmarshalCBOR(br); err != nil { + if err := t.DealProposal.UnmarshalCBOR(br); err != nil { return err } } - // t.ProposerSignature (types.Signature) (struct) + // t.DealState (market.DealState) (struct) { - pb, err := br.PeekByte() - if err != nil { + if err := t.DealState.UnmarshalCBOR(br); err != nil { return err } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.ProposerSignature = new(types.Signature) - if err := t.ProposerSignature.UnmarshalCBOR(br); err != nil { - return err - } - } } return nil From 2202549b86eee4cfd4c19defc9eeb2d46c3d14e2 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Wed, 12 Feb 2020 14:29:58 -0800 Subject: [PATCH 5/7] Update go-sectorbuilder to latest that uses specs-actors types (#121) * fix(deps): update filecoin-ffi * feat(deps): update ffi+sectorbuilder use spec-actors lower level types --- extern/filecoin-ffi | 2 +- go.mod | 6 +++--- go.sum | 14 ++++++-------- pieceio/pieceio.go | 7 ++++--- pieceio/types.go | 5 +++-- storagemarket/impl/client.go | 2 +- storagemarket/impl/client_utils.go | 4 +++- storagemarket/integration_test.go | 2 +- storagemarket/types.go | 2 +- 9 files changed, 23 insertions(+), 21 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 2383ce07..cb6019e0 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 2383ce072e9514e60dc055b45d32ffa61e810064 +Subproject commit cb6019e085428c845dc29c95b9e70569019e619f diff --git a/go.mod b/go.mod index f2c66db1..2095312b 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 - github.com/filecoin-project/go-padreader v0.0.0-20200130212543-892867c4edf9 - github.com/filecoin-project/go-sectorbuilder v0.0.1 + github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 + github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200210220012-eb75ec747d6b github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099 @@ -26,7 +26,7 @@ require ( github.com/ipfs/go-ipfs-files v0.0.4 github.com/ipfs/go-ipld-cbor v0.0.4 github.com/ipfs/go-ipld-format v0.0.2 - github.com/ipfs/go-log/v2 v2.0.1 + github.com/ipfs/go-log/v2 v2.0.2 github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 diff --git a/go.sum b/go.sum index 1814305c..300a8336 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= -github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2jaVU8TvWTx7iZPhlYvUE8vkoOnj778XuKavz8W36g= -github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5 h1:/MmWluswvDIbuPvBct4q6HeQgVm62O2DzWYTB38kt4A= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e h1:IOoff6yAZSJ5zHCPY2jzGNwQYQU6ygsRVe/cSnJrY+o= @@ -74,12 +72,12 @@ github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= -github.com/filecoin-project/go-padreader v0.0.0-20200130212543-892867c4edf9 h1:CQsjS+oWG96rk5YbeKpPw84fhbgc5H6/BGvrlPgd63A= -github.com/filecoin-project/go-padreader v0.0.0-20200130212543-892867c4edf9/go.mod h1:r0gyD7zvnqyRKSY8stil5G/LF0kXFgNzW/yR4vjga+Y= +github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= +github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= -github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCstYuDKiI6RCXiYz4GaRs= -github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200210220012-eb75ec747d6b h1:ds4TQay8wuV+2ucC6ENAeSYQDdl9CWYXnX0gvxzGKHg= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200210220012-eb75ec747d6b/go.mod h1:qsuPYsbKTHH2phNk81aUF9VJIilUxFrnxxnryJh4FOM= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf h1:fbxBG12yrxilPFV1EG2lYqpUyAlRZWkvtqjk2svSeXY= @@ -220,8 +218,8 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0 h1:BW3LQIiZzpNyolt84yvKNCd3FU+AK4VDw1hnHR+1aiI= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= -github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= -github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= +github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw= +github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.4 h1:ZSHQSe9BENfixUjT+MaLeHEeZGxrZQfgo3KT3SLosF8= github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= diff --git a/pieceio/pieceio.go b/pieceio/pieceio.go index a491b9c9..11188141 100644 --- a/pieceio/pieceio.go +++ b/pieceio/pieceio.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipld/go-ipld-prime" @@ -50,7 +51,7 @@ func NewPieceIOWithStore(carIO CarIO, store filestore.FileStore, bs blockstore.B return &pieceIOWithStore{pieceIO{carIO, bs}, store} } -func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, uint64, error) { +func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, abi.UnpaddedPieceSize, error) { preparedCar, err := pio.carIO.PrepareCar(context.Background(), pio.bs, payloadCid, selector) if err != nil { return nil, 0, err @@ -86,7 +87,7 @@ func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.No return commitment, paddedSize, nil } -func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, uint64, error) { +func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, abi.UnpaddedPieceSize, error) { f, err := pio.store.CreateTemp() if err != nil { return nil, "", 0, err @@ -115,7 +116,7 @@ func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(payloadCid cid.Cid, s return commitment, f.Path(), paddedSize, nil } -func GeneratePieceCommitment(rd io.Reader, pieceSize uint64) ([]byte, uint64, error) { +func GeneratePieceCommitment(rd io.Reader, pieceSize uint64) ([]byte, abi.UnpaddedPieceSize, error) { paddedReader, paddedSize := padreader.New(rd, pieceSize) commitment, err := sectorbuilder.GeneratePieceCommitment(paddedReader, paddedSize) if err != nil { diff --git a/pieceio/types.go b/pieceio/types.go index a9326eeb..c965d48e 100644 --- a/pieceio/types.go +++ b/pieceio/types.go @@ -4,6 +4,7 @@ import ( "io" "github.com/filecoin-project/go-fil-markets/filestore" + "github.com/filecoin-project/specs-actors/actors/abi" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -20,11 +21,11 @@ type ReadStore interface { // PieceIO converts between payloads and pieces type PieceIO interface { - GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, uint64, error) + GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, abi.UnpaddedPieceSize, error) ReadPiece(r io.Reader) (cid.Cid, error) } type PieceIOWithStore interface { PieceIO - GeneratePieceCommitmentToFile(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, uint64, error) + GeneratePieceCommitmentToFile(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, abi.UnpaddedPieceSize, error) } diff --git a/storagemarket/impl/client.go b/storagemarket/impl/client.go index e7db6990..05ba01c1 100644 --- a/storagemarket/impl/client.go +++ b/storagemarket/impl/client.go @@ -204,7 +204,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro dealProposal := market.DealProposal{ PieceCID: commcid.PieceCommitmentV1ToCID(commP), - PieceSize: abi.PaddedPieceSize(pieceSize), + PieceSize: pieceSize.Padded(), Client: p.Client, Provider: p.ProviderAddress, StartEpoch: p.StartEpoch, diff --git a/storagemarket/impl/client_utils.go b/storagemarket/impl/client_utils.go index 61a29745..c2ed9363 100644 --- a/storagemarket/impl/client_utils.go +++ b/storagemarket/impl/client_utils.go @@ -4,6 +4,8 @@ import ( "context" "runtime" + "github.com/filecoin-project/specs-actors/actors/abi" + ipldfree "github.com/ipld/go-ipld-prime/impl/free" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" @@ -36,7 +38,7 @@ func (c *Client) failDeal(id cid.Cid, cerr error) { log.Errorf("deal %s failed: %+v", id, cerr) } -func (c *Client) commP(ctx context.Context, root cid.Cid) ([]byte, uint64, error) { +func (c *Client) commP(ctx context.Context, root cid.Cid) ([]byte, abi.UnpaddedPieceSize, error) { ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) // entire DAG selector diff --git a/storagemarket/integration_test.go b/storagemarket/integration_test.go index d059b090..4fdf0f4d 100644 --- a/storagemarket/integration_test.go +++ b/storagemarket/integration_test.go @@ -418,7 +418,7 @@ func (n *fakeProviderNode) ListProviderDeals(ctx context.Context, addr address.A return n.SMState.Deals(addr), nil } -func (n *fakeProviderNode) OnDealComplete(ctx context.Context, deal storagemarket.MinerDeal, pieceSize uint64, pieceReader io.Reader) error { +func (n *fakeProviderNode) OnDealComplete(ctx context.Context, deal storagemarket.MinerDeal, pieceSize abi.UnpaddedPieceSize, pieceReader io.Reader) error { return nil } diff --git a/storagemarket/types.go b/storagemarket/types.go index 8af43754..63c50dd0 100644 --- a/storagemarket/types.go +++ b/storagemarket/types.go @@ -185,7 +185,7 @@ type StorageProviderNode interface { ListProviderDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error) // Called when a deal is complete and on chain, and data has been transferred and is ready to be added to a sector - OnDealComplete(ctx context.Context, deal MinerDeal, pieceSize uint64, pieceReader io.Reader) error + OnDealComplete(ctx context.Context, deal MinerDeal, pieceSize abi.UnpaddedPieceSize, pieceReader io.Reader) error // returns the worker address associated with a miner GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) From bfd030852a59ea880fd176fd6e72a002baeb13b4 Mon Sep 17 00:00:00 2001 From: Shannon Wells Date: Thu, 20 Feb 2020 10:19:18 -0800 Subject: [PATCH 6/7] use latest specs-actors with uint64 lane and nonce from paych.Actor (#122) --- go.mod | 2 +- go.sum | 2 + .../impl/clientstates/client_states_test.go | 4 +- .../testnodes/test_retrieval_client_node.go | 8 ++-- retrievalmarket/types.go | 6 +-- retrievalmarket/types_cbor_gen.go | 43 +++++-------------- shared_testutil/test_types.go | 8 ++-- 7 files changed, 27 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index 2095312b..96893bea 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200210220012-eb75ec747d6b github.com/filecoin-project/go-statestore v0.1.0 - github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf + github.com/filecoin-project/specs-actors v0.0.0-20200220011005-b2a2fbf40362 github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c diff --git a/go.sum b/go.sum index 300a8336..9f3b176d 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf h1:fbxBG12yrxilPFV1EG2lYqpUyAlRZWkvtqjk2svSeXY= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= +github.com/filecoin-project/specs-actors v0.0.0-20200220011005-b2a2fbf40362 h1:6FKMQbj6UIsg7t2qm1YlSbHjh/ehnCjaiOWzFwJefO0= +github.com/filecoin-project/specs-actors v0.0.0-20200220011005-b2a2fbf40362/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= diff --git a/retrievalmarket/impl/clientstates/client_states_test.go b/retrievalmarket/impl/clientstates/client_states_test.go index 3875769f..b8bbc19a 100644 --- a/retrievalmarket/impl/clientstates/client_states_test.go +++ b/retrievalmarket/impl/clientstates/client_states_test.go @@ -55,7 +55,7 @@ func TestSetupPaymentChannel(t *testing.T) { ctx := context.Background() ds := testnet.NewTestRetrievalDealStream(testnet.TestDealStreamParams{}) expectedPayCh := address.TestAddress2 - expectedLane := int64(10) + expectedLane := uint64(10) environment := func(params testnodes.TestRetrievalClientNodeParams) clientstates.ClientDealEnvironment { node := testnodes.NewTestRetrievalClientNode(params) @@ -490,7 +490,7 @@ func makeDealState(status retrievalmarket.DealStatus) *retrievalmarket.ClientDea MinerWallet: address.TestAddress, ClientWallet: address.TestAddress2, PayCh: address.TestAddress2, - Lane: int64(10), + Lane: uint64(10), Status: status, BytesPaidFor: defaultBytesPaidFor, TotalReceived: defaultTotalReceived, diff --git a/retrievalmarket/impl/testnodes/test_retrieval_client_node.go b/retrievalmarket/impl/testnodes/test_retrieval_client_node.go index db076f67..a2c4aee8 100644 --- a/retrievalmarket/impl/testnodes/test_retrieval_client_node.go +++ b/retrievalmarket/impl/testnodes/test_retrieval_client_node.go @@ -14,7 +14,7 @@ import ( type TestRetrievalClientNode struct { payCh address.Address payChErr error - lane int64 + lane uint64 laneError error voucher *paych.SignedVoucher voucherError error @@ -28,7 +28,7 @@ type TestRetrievalClientNode struct { type TestRetrievalClientNodeParams struct { PayCh address.Address PayChErr error - Lane int64 + Lane uint64 LaneError error Voucher *paych.SignedVoucher VoucherError error @@ -63,7 +63,7 @@ func (trcn *TestRetrievalClientNode) GetOrCreatePaymentChannel(ctx context.Conte } // AllocateLane creates a mock lane on a payment channel -func (trcn *TestRetrievalClientNode) AllocateLane(paymentChannel address.Address) (int64, error) { +func (trcn *TestRetrievalClientNode) AllocateLane(paymentChannel address.Address) (uint64, error) { if trcn.allocateLaneRecorder != nil { trcn.allocateLaneRecorder(paymentChannel) } @@ -71,7 +71,7 @@ func (trcn *TestRetrievalClientNode) AllocateLane(paymentChannel address.Address } // CreatePaymentVoucher creates a mock payment voucher based on a channel and lane -func (trcn *TestRetrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount abi.TokenAmount, lane int64) (*paych.SignedVoucher, error) { +func (trcn *TestRetrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount abi.TokenAmount, lane uint64) (*paych.SignedVoucher, error) { if trcn.createPaymentVoucherRecorder != nil { trcn.createPaymentVoucherRecorder(trcn.voucher) } diff --git a/retrievalmarket/types.go b/retrievalmarket/types.go index 2a1ec5ff..187b42e1 100644 --- a/retrievalmarket/types.go +++ b/retrievalmarket/types.go @@ -36,7 +36,7 @@ type ClientDealState struct { ClientWallet address.Address MinerWallet address.Address PayCh address.Address - Lane int64 + Lane uint64 Status DealStatus Sender peer.ID TotalReceived uint64 @@ -117,12 +117,12 @@ type RetrievalClientNode interface { // Allocate late creates a lane within a payment channel so that calls to // CreatePaymentVoucher will automatically make vouchers only for the difference // in total - AllocateLane(paymentChannel address.Address) (int64, error) + AllocateLane(paymentChannel address.Address) (uint64, error) // CreatePaymentVoucher creates a new payment voucher in the given lane for a // given payment channel so that all the payment vouchers in the lane add up // to the given amount (so the payment voucher will be for the difference) - CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount abi.TokenAmount, lane int64) (*paych.SignedVoucher, error) + CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount abi.TokenAmount, lane uint64) (*paych.SignedVoucher, error) } // ProviderDealState is the current state of a deal from the point of view diff --git a/retrievalmarket/types_cbor_gen.go b/retrievalmarket/types_cbor_gen.go index 4977f001..78144a1f 100644 --- a/retrievalmarket/types_cbor_gen.go +++ b/retrievalmarket/types_cbor_gen.go @@ -726,15 +726,9 @@ func (t *ClientDealState) MarshalCBOR(w io.Writer) error { return err } - // t.Lane (int64) (int64) - if t.Lane >= 0 { - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { - return err - } - } else { - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.Lane)-1)); err != nil { - return err - } + // t.Lane (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { + return err } // t.Status (retrievalmarket.DealStatus) (uint64) @@ -865,31 +859,16 @@ func (t *ClientDealState) UnmarshalCBOR(r io.Reader) error { } } - // t.Lane (int64) (int64) - { - maj, extra, err := cbg.CborReadHeader(br) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } + // t.Lane (uint64) (uint64) - t.Lane = int64(extraI) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") } + t.Lane = uint64(extra) // t.Status (retrievalmarket.DealStatus) (uint64) maj, extra, err = cbg.CborReadHeader(br) diff --git a/shared_testutil/test_types.go b/shared_testutil/test_types.go index 0cb07e05..474b42f5 100644 --- a/shared_testutil/test_types.go +++ b/shared_testutil/test_types.go @@ -25,8 +25,8 @@ func MakeTestSignedVoucher() *paych.SignedVoucher { TimeLock: abi.ChainEpoch(rand.Int63()), SecretPreimage: []byte("secret-preimage"), Extra: MakeTestModVerifyParams(), - Lane: rand.Int63(), - Nonce: rand.Int63(), + Lane: rand.Uint64(), + Nonce: rand.Uint64(), Amount: MakeTestTokenAmount(), Merges: []paych.Merge{MakeTestMerge()}, Signature: MakeTestSignature(), @@ -45,8 +45,8 @@ func MakeTestModVerifyParams() *paych.ModVerifyParams { // MakeTestMerge generates a random Merge that has all non-zero fields func MakeTestMerge() paych.Merge { return paych.Merge{ - Lane: rand.Int63(), - Nonce: rand.Int63(), + Lane: rand.Uint64(), + Nonce: rand.Uint64(), } } From 1f82dd52a21041d357aa5471afe6f665e4376741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 24 Feb 2020 19:04:10 +0100 Subject: [PATCH 7/7] storage client: Call EnsureFunds more correctly (#123) * More correct client EnsureFunds call * market client: Set ClientCollateral field --- storagemarket/impl/client.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/storagemarket/impl/client.go b/storagemarket/impl/client.go index 05ba01c1..7552e239 100644 --- a/storagemarket/impl/client.go +++ b/storagemarket/impl/client.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket/network" cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" logging "github.com/ipfs/go-log/v2" @@ -23,7 +24,6 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin/market" ) @@ -192,11 +192,6 @@ type ClientDealProposal struct { } func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - amount := big.Mul(p.PricePerEpoch, abi.NewTokenAmount(int64(p.EndEpoch)-int64(p.StartEpoch))) - if err := c.node.EnsureFunds(ctx, p.Client, amount); err != nil { - return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) - } - commP, pieceSize, err := c.commP(ctx, p.Data.Root) if err != nil { return cid.Undef, xerrors.Errorf("computing commP failed: %w", err) @@ -211,6 +206,11 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro EndEpoch: p.EndEpoch, StoragePricePerEpoch: p.PricePerEpoch, ProviderCollateral: abi.NewTokenAmount(int64(pieceSize)), // TODO: real calc + ClientCollateral: big.Zero(), + } + + if err := c.node.EnsureFunds(ctx, p.Client, dealProposal.ClientBalanceRequirement()); err != nil { + return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) } clientDealProposal, err := c.node.SignProposal(ctx, p.Client, dealProposal)