diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index cb6019e0..4da0bccc 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit cb6019e085428c845dc29c95b9e70569019e619f +Subproject commit 4da0bccccefb48df4bce63869d058f9bbf02b686 diff --git a/go.mod b/go.mod index 9d16833a..ad93c258 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,14 @@ module github.com/filecoin-project/go-fil-markets go 1.13 require ( - github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5 + github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce - github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 - github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200210220012-eb75ec747d6b + github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56 github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 github.com/filecoin-project/go-statestore v0.1.0 - github.com/filecoin-project/specs-actors v0.0.0-20200220011005-b2a2fbf40362 + github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775 github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c diff --git a/go.sum b/go.sum index ac8fdb2f..0e0a27b9 100644 --- a/go.sum +++ b/go.sum @@ -5,14 +5,10 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= -github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= -github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -42,8 +38,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSY github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= -github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY= -github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -57,11 +51,13 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= +github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5 h1:/MmWluswvDIbuPvBct4q6HeQgVm62O2DzWYTB38kt4A= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= +github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be h1:TooKBwR/g8jG0hZ3lqe9S5sy2vTUcLOZLlz3M5wGn2E= +github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e h1:IOoff6yAZSJ5zHCPY2jzGNwQYQU6ygsRVe/cSnJrY+o= github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -74,18 +70,18 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE= -github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= -github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200210220012-eb75ec747d6b h1:ds4TQay8wuV+2ucC6ENAeSYQDdl9CWYXnX0gvxzGKHg= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200210220012-eb75ec747d6b/go.mod h1:qsuPYsbKTHH2phNk81aUF9VJIilUxFrnxxnryJh4FOM= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h1:eYxi6vI5CyeXD15X1bB3bledDXbqKxqf0wQzTLgwYwA= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56 h1:CqldkHf9HtAsewneyOZdi19Btc6kEvktVVeZsP9N7NQ= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf h1:fbxBG12yrxilPFV1EG2lYqpUyAlRZWkvtqjk2svSeXY= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= -github.com/filecoin-project/specs-actors v0.0.0-20200220011005-b2a2fbf40362 h1:6FKMQbj6UIsg7t2qm1YlSbHjh/ehnCjaiOWzFwJefO0= -github.com/filecoin-project/specs-actors v0.0.0-20200220011005-b2a2fbf40362/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= +github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775 h1:j6EdTc5hIHe31ydt9SjgJNDi/ck50y5wa1OJ+4YZ6zk= +github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= @@ -413,6 +409,9 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -481,7 +480,6 @@ github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHx github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= @@ -559,8 +557,6 @@ github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -654,6 +650,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pieceio/pieceio.go b/pieceio/pieceio.go index 11188141..20636c6f 100644 --- a/pieceio/pieceio.go +++ b/pieceio/pieceio.go @@ -51,15 +51,15 @@ func NewPieceIOWithStore(carIO CarIO, store filestore.FileStore, bs blockstore.B return &pieceIOWithStore{pieceIO{carIO, bs}, store} } -func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, abi.UnpaddedPieceSize, error) { +func (pio *pieceIO) GeneratePieceCommitment(rt abi.RegisteredProof, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, abi.UnpaddedPieceSize, error) { preparedCar, err := pio.carIO.PrepareCar(context.Background(), pio.bs, payloadCid, selector) if err != nil { - return nil, 0, err + return cid.Undef, 0, err } pieceSize := uint64(preparedCar.Size()) r, w, err := os.Pipe() if err != nil { - return nil, 0, err + return cid.Undef, 0, err } var stop sync.WaitGroup stop.Add(1) @@ -72,25 +72,25 @@ func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.No werr = err } }() - commitment, paddedSize, err := GeneratePieceCommitment(r, pieceSize) + commitment, paddedSize, err := GeneratePieceCommitment(rt, r, pieceSize) closeErr := r.Close() if err != nil { - return nil, 0, err + return cid.Undef, 0, err } if closeErr != nil { - return nil, 0, closeErr + return cid.Undef, 0, closeErr } stop.Wait() if werr != nil { - return nil, 0, werr + return cid.Undef, 0, werr } return commitment, paddedSize, nil } -func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, abi.UnpaddedPieceSize, error) { +func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(rt abi.RegisteredProof, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, abi.UnpaddedPieceSize, error) { f, err := pio.store.CreateTemp() if err != nil { - return nil, "", 0, err + return cid.Undef, "", 0, err } cleanup := func() { f.Close() @@ -99,30 +99,30 @@ func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(payloadCid cid.Cid, s err = pio.carIO.WriteCar(context.Background(), pio.bs, payloadCid, selector, f) if err != nil { cleanup() - return nil, "", 0, err + return cid.Undef, "", 0, err } pieceSize := uint64(f.Size()) _, err = f.Seek(0, io.SeekStart) if err != nil { cleanup() - return nil, "", 0, err + return cid.Undef, "", 0, err } - commitment, paddedSize, err := GeneratePieceCommitment(f, pieceSize) + commitment, paddedSize, err := GeneratePieceCommitment(rt, f, pieceSize) if err != nil { cleanup() - return nil, "", 0, err + return cid.Undef, "", 0, err } _ = f.Close() return commitment, f.Path(), paddedSize, nil } -func GeneratePieceCommitment(rd io.Reader, pieceSize uint64) ([]byte, abi.UnpaddedPieceSize, error) { +func GeneratePieceCommitment(rt abi.RegisteredProof, rd io.Reader, pieceSize uint64) (cid.Cid, abi.UnpaddedPieceSize, error) { paddedReader, paddedSize := padreader.New(rd, pieceSize) - commitment, err := sectorbuilder.GeneratePieceCommitment(paddedReader, paddedSize) + commitment, err := sectorbuilder.GeneratePieceCIDFromFile(rt, paddedReader, paddedSize) if err != nil { - return nil, 0, err + return cid.Undef, 0, err } - return commitment[:], paddedSize, nil + return commitment, paddedSize, nil } func (pio *pieceIO) ReadPiece(r io.Reader) (cid.Cid, error) { diff --git a/pieceio/pieceio_test.go b/pieceio/pieceio_test.go index c5a92658..f3a8f487 100644 --- a/pieceio/pieceio_test.go +++ b/pieceio/pieceio_test.go @@ -5,12 +5,19 @@ import ( "context" "fmt" + "io" + "testing" + "github.com/filecoin-project/go-fil-markets/filestore" fsmocks "github.com/filecoin-project/go-fil-markets/filestore/mocks" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/pieceio/cario" pmocks "github.com/filecoin-project/go-fil-markets/pieceio/mocks" + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/ipfs/go-cid" dag "github.com/ipfs/go-merkledag" dstest "github.com/ipfs/go-merkledag/test" ipldfree "github.com/ipld/go-ipld-prime/impl/free" @@ -18,8 +25,6 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "io" - "testing" ) func Test_ThereAndBackAgain(t *testing.T) { @@ -65,7 +70,7 @@ func Test_ThereAndBackAgain(t *testing.T) { ssb.ExploreIndex(1, ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())))) }).Node() - bytes, tmpPath, _, err := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node) + pcid, tmpPath, _, err := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node) require.NoError(t, err) tmpFile, err := store.Open(tmpPath) require.NoError(t, err) @@ -75,9 +80,7 @@ func Test_ThereAndBackAgain(t *testing.T) { deferErr = store.Delete(tmpFile.Path()) require.NoError(t, deferErr) }() - for _, b := range bytes { - require.NotEqual(t, 0, b) - } + require.NotEqual(t, pcid, cid.Undef) bufSize := int64(16) // small buffer to illustrate the logic buf := make([]byte, bufSize) var readErr error @@ -155,7 +158,7 @@ func Test_StoreRestoreMemoryBuffer(t *testing.T) { ssb.ExploreIndex(1, ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())))) }).Node() - commitment, tmpPath, paddedSize, err := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node) + commitment, tmpPath, paddedSize, err := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node) require.NoError(t, err) tmpFile, err := store.Open(tmpPath) require.NoError(t, err) @@ -169,16 +172,14 @@ func Test_StoreRestoreMemoryBuffer(t *testing.T) { _, err = tmpFile.Seek(0, io.SeekStart) require.NoError(t, err) - for _, b := range commitment { - require.NotEqual(t, 0, b) - } + require.NotEqual(t, commitment, cid.Undef) buf := make([]byte, paddedSize) _, err = tmpFile.Read(buf) require.NoError(t, err) buffer := bytes.NewBuffer(buf) - secondCommitment, err := sectorbuilder.GeneratePieceCommitment(buffer, paddedSize) + secondCommitment, err := sectorbuilder.GeneratePieceCIDFromFile(abi.RegisteredProof_StackedDRG2KiBPoSt, buffer, paddedSize) require.NoError(t, err) - require.Equal(t, commitment, secondCommitment[:]) + require.Equal(t, commitment, secondCommitment) } func Test_PieceCommitmentEquivalenceMemoryFile(t *testing.T) { @@ -222,13 +223,13 @@ func Test_PieceCommitmentEquivalenceMemoryFile(t *testing.T) { ssb.ExploreIndex(1, ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())))) }).Node() - fcommitment, tmpPath, fpaddedSize, ferr := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node) + fcommitment, tmpPath, fpaddedSize, ferr := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node) defer func() { deferErr := store.Delete(tmpPath) require.NoError(t, deferErr) }() - mcommitment, mpaddedSize, merr := pio.GeneratePieceCommitment(nd3.Cid(), node) + mcommitment, mpaddedSize, merr := pio.GeneratePieceCommitment(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node) require.Equal(t, fcommitment, mcommitment) require.Equal(t, fpaddedSize, mpaddedSize) require.Equal(t, ferr, merr) @@ -273,7 +274,7 @@ func Test_Failures(t *testing.T) { fsmock := fsmocks.FileStore{} fsmock.On("CreateTemp").Return(nil, fmt.Errorf("Failed")) pio := pieceio.NewPieceIOWithStore(nil, &fsmock, sourceBs) - _, _, _, err := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node) + _, _, _, err := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node) require.Error(t, err) }) t.Run("write CAR fails", func(t *testing.T) { @@ -285,7 +286,7 @@ func Test_Failures(t *testing.T) { any := mock.Anything ciomock.On("WriteCar", any, any, any, any, any).Return(fmt.Errorf("failed to write car")) pio := pieceio.NewPieceIOWithStore(&ciomock, store, sourceBs) - _, _, _, err = pio.GeneratePieceCommitmentToFile(nd3.Cid(), node) + _, _, _, err = pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node) require.Error(t, err) }) t.Run("prepare CAR fails", func(t *testing.T) { @@ -294,7 +295,7 @@ func Test_Failures(t *testing.T) { any := mock.Anything ciomock.On("PrepareCar", any, any, any, any).Return(nil, fmt.Errorf("failed to prepare car")) pio := pieceio.NewPieceIO(&ciomock, sourceBs) - _, _, err := pio.GeneratePieceCommitment(nd3.Cid(), node) + _, _, err := pio.GeneratePieceCommitment(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node) require.Error(t, err) }) t.Run("PreparedCard dump operation fails", func(t *testing.T) { @@ -305,7 +306,7 @@ func Test_Failures(t *testing.T) { preparedCarMock.On("Size").Return(uint64(1000)) preparedCarMock.On("Dump", any).Return(fmt.Errorf("failed to write car")) pio := pieceio.NewPieceIO(&ciomock, sourceBs) - _, _, err := pio.GeneratePieceCommitment(nd3.Cid(), node) + _, _, err := pio.GeneratePieceCommitment(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node) require.Error(t, err) }) t.Run("seek fails", func(t *testing.T) { @@ -332,7 +333,7 @@ func Test_Failures(t *testing.T) { mockfile.On("Seek", mock.Anything, mock.Anything).Return(int64(0), fmt.Errorf("seek failed")) pio := pieceio.NewPieceIOWithStore(cio, &fsmock, sourceBs) - _, _, _, err := pio.GeneratePieceCommitmentToFile(nd3.Cid(), node) + _, _, _, err := pio.GeneratePieceCommitmentToFile(abi.RegisteredProof_StackedDRG2KiBPoSt, nd3.Cid(), node) require.Error(t, err) }) } diff --git a/pieceio/types.go b/pieceio/types.go index c965d48e..4c345295 100644 --- a/pieceio/types.go +++ b/pieceio/types.go @@ -21,11 +21,11 @@ type ReadStore interface { // PieceIO converts between payloads and pieces type PieceIO interface { - GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, abi.UnpaddedPieceSize, error) + GeneratePieceCommitment(rt abi.RegisteredProof, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, abi.UnpaddedPieceSize, error) ReadPiece(r io.Reader) (cid.Cid, error) } type PieceIOWithStore interface { PieceIO - GeneratePieceCommitmentToFile(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, abi.UnpaddedPieceSize, error) + GeneratePieceCommitmentToFile(rt abi.RegisteredProof, payloadCid cid.Cid, selector ipld.Node) (cid.Cid, filestore.Path, abi.UnpaddedPieceSize, error) } diff --git a/shared_testutil/test_types.go b/shared_testutil/test_types.go index 474b42f5..401c5176 100644 --- a/shared_testutil/test_types.go +++ b/shared_testutil/test_types.go @@ -22,7 +22,8 @@ import ( // MakeTestSignedVoucher generates a random SignedVoucher that has all non-zero fields func MakeTestSignedVoucher() *paych.SignedVoucher { return &paych.SignedVoucher{ - TimeLock: abi.ChainEpoch(rand.Int63()), + TimeLockMin: abi.ChainEpoch(rand.Int63()), + TimeLockMax: 0, SecretPreimage: []byte("secret-preimage"), Extra: MakeTestModVerifyParams(), Lane: rand.Uint64(), diff --git a/storagemarket/impl/client.go b/storagemarket/impl/client.go index 7552e239..e1b3f35f 100644 --- a/storagemarket/impl/client.go +++ b/storagemarket/impl/client.go @@ -15,7 +15,6 @@ import ( "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" - commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/pieceio/cario" @@ -186,19 +185,20 @@ type ClientDealProposal struct { EndEpoch abi.ChainEpoch ProviderAddress address.Address + ProofType abi.RegisteredProof Client address.Address MinerWorker address.Address MinerID peer.ID } func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - commP, pieceSize, err := c.commP(ctx, p.Data.Root) + commP, pieceSize, err := c.commP(ctx, p.ProofType, p.Data.Root) if err != nil { return cid.Undef, xerrors.Errorf("computing commP failed: %w", err) } dealProposal := market.DealProposal{ - PieceCID: commcid.PieceCommitmentV1ToCID(commP), + PieceCID: commP, PieceSize: pieceSize.Padded(), Client: p.Client, Provider: p.ProviderAddress, diff --git a/storagemarket/impl/client_storagemarket.go b/storagemarket/impl/client_storagemarket.go index e197b743..10b6a5c2 100644 --- a/storagemarket/impl/client_storagemarket.go +++ b/storagemarket/impl/client_storagemarket.go @@ -90,6 +90,7 @@ func (c *Client) ProposeStorageDeal( endEpoch abi.ChainEpoch, price abi.TokenAmount, collateral abi.TokenAmount, + rt abi.RegisteredProof, ) (*storagemarket.ProposeStorageDealResult, error) { proposal := ClientDealProposal{ @@ -101,6 +102,7 @@ func (c *Client) ProposeStorageDeal( ProviderAddress: info.Address, MinerWorker: info.Worker, MinerID: info.PeerID, + ProofType: rt, } proposalCid, err := c.Start(ctx, proposal) diff --git a/storagemarket/impl/client_utils.go b/storagemarket/impl/client_utils.go index c2ed9363..fa958cb0 100644 --- a/storagemarket/impl/client_utils.go +++ b/storagemarket/impl/client_utils.go @@ -38,19 +38,19 @@ func (c *Client) failDeal(id cid.Cid, cerr error) { log.Errorf("deal %s failed: %+v", id, cerr) } -func (c *Client) commP(ctx context.Context, root cid.Cid) ([]byte, abi.UnpaddedPieceSize, error) { +func (c *Client) commP(ctx context.Context, rt abi.RegisteredProof, root cid.Cid) (cid.Cid, abi.UnpaddedPieceSize, error) { ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) // entire DAG selector allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() - commp, paddedSize, err := c.pio.GeneratePieceCommitment(root, allSelector) + commp, paddedSize, err := c.pio.GeneratePieceCommitment(rt, root, allSelector) if err != nil { - return nil, 0, xerrors.Errorf("generating CommP: %w", err) + return cid.Undef, 0, xerrors.Errorf("generating CommP: %w", err) } - return commp[:], paddedSize, nil + return commp, paddedSize, nil } func (c *Client) verifyResponse(resp network.SignedResponse, minerAddr address.Address) error { diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index dab68202..442b9d76 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -15,7 +15,6 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" datatransfer "github.com/filecoin-project/go-data-transfer" - commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/pieceio/cario" @@ -40,6 +39,7 @@ type Provider struct { pricePerByteBlock abi.TokenAmount // how much we want for storing one byte for one block minPieceSize abi.PaddedPieceSize + proofType abi.RegisteredProof ask *storagemarket.SignedStorageAsk askLk sync.Mutex @@ -78,7 +78,7 @@ var ( ErrDataTransferFailed = errors.New("deal data transfer failed") ) -func NewProvider(net network.StorageMarketNetwork, ds datastore.Batching, bs blockstore.Blockstore, fs filestore.FileStore, pieceStore piecestore.PieceStore, dataTransfer datatransfer.Manager, spn storagemarket.StorageProviderNode, minerAddress address.Address) (storagemarket.StorageProvider, error) { +func NewProvider(net network.StorageMarketNetwork, ds datastore.Batching, bs blockstore.Blockstore, fs filestore.FileStore, pieceStore piecestore.PieceStore, dataTransfer datatransfer.Manager, spn storagemarket.StorageProviderNode, minerAddress address.Address, rt abi.RegisteredProof) (storagemarket.StorageProvider, error) { carIO := cario.NewCarIO() pio := pieceio.NewPieceIOWithStore(carIO, fs, bs) @@ -92,6 +92,7 @@ func NewProvider(net network.StorageMarketNetwork, ds datastore.Batching, bs blo pricePerByteBlock: abi.NewTokenAmount(3), // TODO: allow setting minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up)) + proofType: rt, conns: map[cid.Cid]network.StorageDealStream{}, @@ -319,12 +320,11 @@ func (p *Provider) ImportDataForDeal(ctx context.Context, propCid cid.Cid, data return xerrors.Errorf("failed to seek through temp imported file: %w", err) } - commP, _, err := pieceio.GeneratePieceCommitment(tempfi, pieceSize) + pieceCid, _, err := pieceio.GeneratePieceCommitment(p.proofType, tempfi, pieceSize) if err != nil { return xerrors.Errorf("failed to generate commP") } - pieceCid := commcid.PieceCommitmentV1ToCID(commP) // Verify CommP matches if !pieceCid.Equals(d.Proposal.PieceCID) { return xerrors.Errorf("given data does not match expected commP (got: %x, expected %x)", pieceCid, d.Proposal.PieceCID) diff --git a/storagemarket/impl/provider_states.go b/storagemarket/impl/provider_states.go index 20af700c..6127166f 100644 --- a/storagemarket/impl/provider_states.go +++ b/storagemarket/impl/provider_states.go @@ -3,7 +3,6 @@ package storageimpl import ( "context" - commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-padreader" "github.com/ipfs/go-cid" ipldfree "github.com/ipld/go-ipld-prime/impl/free" @@ -116,14 +115,13 @@ func (p *Provider) verifydata(ctx context.Context, deal MinerDeal) (func(*MinerD allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() - commp, path, _, err := p.pio.GeneratePieceCommitmentToFile(deal.Ref.Root, allSelector) + pieceCid, path, _, err := p.pio.GeneratePieceCommitmentToFile(p.proofType, deal.Ref.Root, allSelector) if err != nil { return nil, err } - pieceCid := commcid.PieceCommitmentV1ToCID(commp) // Verify CommP matches - if !pieceCid.Equals(deal.Proposal.PieceCID) { + if pieceCid != deal.Proposal.PieceCID { return nil, xerrors.Errorf("proposal CommP doesn't match calculated CommP") } diff --git a/storagemarket/integration_test.go b/storagemarket/integration_test.go index 4fdf0f4d..278fe9a1 100644 --- a/storagemarket/integration_test.go +++ b/storagemarket/integration_test.go @@ -84,6 +84,7 @@ func TestMakeDeal(t *testing.T) { dt2, &providerNode, providerAddr, + abi.RegisteredProof_StackedDRG2KiBPoSt, ) assert.NoError(t, err) @@ -112,7 +113,7 @@ func TestMakeDeal(t *testing.T) { TransferType: storagemarket.TTGraphsync, Root: payloadCid, } - result, err := client.ProposeStorageDeal(ctx, providerAddr, &providerInfo, dataRef, abi.ChainEpoch(epoch+100), abi.ChainEpoch(epoch+20100), big.NewInt(1), big.NewInt(0)) + result, err := client.ProposeStorageDeal(ctx, providerAddr, &providerInfo, dataRef, abi.ChainEpoch(epoch+100), abi.ChainEpoch(epoch+20100), big.NewInt(1), big.NewInt(0), abi.RegisteredProof_StackedDRG2KiBPoSt) assert.NoError(t, err) proposalCid = result.ProposalCid @@ -181,6 +182,7 @@ func TestMakeDealOffline(t *testing.T) { dt2, &providerNode, providerAddr, + abi.RegisteredProof_StackedDRG2KiBPoSt, ) assert.NoError(t, err) @@ -209,7 +211,7 @@ func TestMakeDealOffline(t *testing.T) { TransferType: storagemarket.TTManual, Root: payloadCid, } - result, err := client.ProposeStorageDeal(ctx, providerAddr, &providerInfo, dataRef, abi.ChainEpoch(epoch+100), abi.ChainEpoch(epoch+20100), big.NewInt(1), big.NewInt(0)) + result, err := client.ProposeStorageDeal(ctx, providerAddr, &providerInfo, dataRef, abi.ChainEpoch(epoch+100), abi.ChainEpoch(epoch+20100), big.NewInt(1), big.NewInt(0), abi.RegisteredProof_StackedDRG2KiBPoSt) assert.NoError(t, err) proposalCid = result.ProposalCid diff --git a/storagemarket/types.go b/storagemarket/types.go index 63c50dd0..c0c049a0 100644 --- a/storagemarket/types.go +++ b/storagemarket/types.go @@ -295,7 +295,7 @@ type StorageClient interface { //FindStorageOffers(criteria AskCriteria, limit uint) []*StorageOffer // ProposeStorageDeal initiates deal negotiation with a Storage Provider - ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, data *DataRef, startEpoch abi.ChainEpoch, endEpoch abi.ChainEpoch, price abi.TokenAmount, collateral abi.TokenAmount) (*ProposeStorageDealResult, error) + ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, data *DataRef, startEpoch abi.ChainEpoch, endEpoch abi.ChainEpoch, price abi.TokenAmount, collateral abi.TokenAmount, rt abi.RegisteredProof) (*ProposeStorageDealResult, error) // GetPaymentEscrow returns the current funds available for deal payment GetPaymentEscrow(ctx context.Context, addr address.Address) (Balance, error)