Merge pull request #116 from filecoin-project/release/02-05-2020

Release: Car Features, Retrieval fixes, etc

go.mod

@@ -8,12 +8,13 @@ require (
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce
+	github.com/filecoin-project/go-padreader v0.0.0-20200130212543-892867c4edf9
 	github.com/filecoin-project/go-sectorbuilder v0.0.1
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099
 	github.com/ipfs/go-block-format v0.0.2
 	github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c
-	github.com/ipfs/go-car v0.0.3-0.20200124090545-1a340009d896
+	github.com/ipfs/go-car v0.0.3-0.20200131220434-3f68f6ebd093
 	github.com/ipfs/go-cid v0.0.4
 	github.com/ipfs/go-datastore v0.1.1
 	github.com/ipfs/go-graphsync v0.0.4

go.sum

@@ -66,6 +66,8 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY=
 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww=
+github.com/filecoin-project/go-padreader v0.0.0-20200130212543-892867c4edf9 h1:CQsjS+oWG96rk5YbeKpPw84fhbgc5H6/BGvrlPgd63A=
+github.com/filecoin-project/go-padreader v0.0.0-20200130212543-892867c4edf9/go.mod h1:r0gyD7zvnqyRKSY8stil5G/LF0kXFgNzW/yR4vjga+Y=
 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
 github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCstYuDKiI6RCXiYz4GaRs=
@@ -142,8 +144,8 @@ github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJ
 github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0=
 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
-github.com/ipfs/go-car v0.0.3-0.20200124090545-1a340009d896 h1:l8gnU1VBhftugMKzfh+n7nuDhOw3X1iqfrA33GVBMMY=
-github.com/ipfs/go-car v0.0.3-0.20200124090545-1a340009d896/go.mod h1:rmd887mJxQRDfndfDEY3Liyx8gQVyfFFRSHdsnDSAlk=
+github.com/ipfs/go-car v0.0.3-0.20200131220434-3f68f6ebd093 h1:mYq7vJKGUzxIkkYfqXfO0uEO8gOmV9F38Tcpvi/p8P8=
+github.com/ipfs/go-car v0.0.3-0.20200131220434-3f68f6ebd093/go.mod h1:rEkw0S1sHd5kHL3rUSGEhwNanYqTwwNhjtpp0rwjrr4=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=

pieceio/cario/cario.go

@@ -8,24 +8,26 @@ import (
 	"github.com/ipfs/go-car"
 	"github.com/ipfs/go-cid"
 	"github.com/ipld/go-ipld-prime"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
 
 	"github.com/filecoin-project/go-fil-markets/pieceio"
 )
 
 type carIO struct {
 }
 
+// NewCarIO returns a new CarIO utility module that wraps go-car
 func NewCarIO() pieceio.CarIO {
 	return &carIO{}
 }
 
-func (c carIO) WriteCar(ctx context.Context, bs pieceio.ReadStore, payloadCid cid.Cid, node ipld.Node, w io.Writer) error {
-	selector, err := selector.ParseSelector(node)
-	if err != nil {
-		return err
-	}
-	return car.WriteSelectiveCar(ctx, bs, []car.CarDag{{Root: payloadCid, Selector: selector}}, w)
+func (c carIO) WriteCar(ctx context.Context, bs pieceio.ReadStore, payloadCid cid.Cid, selector ipld.Node, w io.Writer) error {
+	sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: payloadCid, Selector: selector}})
+	return sc.Write(w)
+}
+
+func (c carIO) PrepareCar(ctx context.Context, bs pieceio.ReadStore, payloadCid cid.Cid, selector ipld.Node) (pieceio.PreparedCar, error) {
+	sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: payloadCid, Selector: selector}})
+	return sc.Prepare()
 }
 
 func (c carIO) LoadCar(bs pieceio.WriteStore, r io.Reader) (cid.Cid, error) {

pieceio/pieceio.go

@@ -3,34 +3,90 @@ package pieceio
 import (
 	"context"
 	"io"
+	"os"
+	"sync"
 
+	"github.com/filecoin-project/go-padreader"
+	"github.com/filecoin-project/go-sectorbuilder"
 	"github.com/ipfs/go-cid"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	"github.com/ipld/go-ipld-prime"
 
 	"github.com/filecoin-project/go-fil-markets/filestore"
-	"github.com/filecoin-project/go-fil-markets/pieceio/padreader"
-	"github.com/filecoin-project/go-sectorbuilder"
 )
 
+type PreparedCar interface {
+	Size() uint64
+	Dump(w io.Writer) error
+}
+
 type CarIO interface {
 	// WriteCar writes a given payload to a CAR file and into the passed IO stream
-	WriteCar(ctx context.Context, bs ReadStore, payloadCid cid.Cid, selector ipld.Node, w io.Writer) error
+	WriteCar(ctx context.Context, bs ReadStore, payloadCid cid.Cid, node ipld.Node, w io.Writer) error
+
+	// PrepareCar prepares a car so that it's total size can be calculated without writing it to a file.
+	// It can then be written with PreparedCar.Dump
+	PrepareCar(ctx context.Context, bs ReadStore, payloadCid cid.Cid, node ipld.Node) (PreparedCar, error)
+
 	// LoadCar loads blocks into the a store from a given CAR file
 	LoadCar(bs WriteStore, r io.Reader) (cid.Cid, error)
 }
 
 type pieceIO struct {
 	carIO CarIO
-	store filestore.FileStore
 	bs    blockstore.Blockstore
 }
 
-func NewPieceIO(carIO CarIO, store filestore.FileStore, bs blockstore.Blockstore) PieceIO {
-	return &pieceIO{carIO, store, bs}
+func NewPieceIO(carIO CarIO, bs blockstore.Blockstore) PieceIO {
+	return &pieceIO{carIO, bs}
+}
+
+type pieceIOWithStore struct {
+	pieceIO
+	store filestore.FileStore
+}
+
+func NewPieceIOWithStore(carIO CarIO, store filestore.FileStore, bs blockstore.Blockstore) PieceIOWithStore {
+	return &pieceIOWithStore{pieceIO{carIO, bs}, store}
+}
+
+func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, uint64, error) {
+	preparedCar, err := pio.carIO.PrepareCar(context.Background(), pio.bs, payloadCid, selector)
+	if err != nil {
+		return nil, 0, err
+	}
+	pieceSize := uint64(preparedCar.Size())
+	r, w, err := os.Pipe()
+	if err != nil {
+		return nil, 0, err
+	}
+	var stop sync.WaitGroup
+	stop.Add(1)
+	var werr error
+	go func() {
+		defer stop.Done()
+		werr = preparedCar.Dump(w)
+		err := w.Close()
+		if werr == nil && err != nil {
+			werr = err
+		}
+	}()
+	commitment, paddedSize, err := GeneratePieceCommitment(r, pieceSize)
+	closeErr := r.Close()
+	if err != nil {
+		return nil, 0, err
+	}
+	if closeErr != nil {
+		return nil, 0, closeErr
+	}
+	stop.Wait()
+	if werr != nil {
+		return nil, 0, werr
+	}
+	return commitment, paddedSize, nil
 }
 
-func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, uint64, error) {
+func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(payloadCid cid.Cid, selector ipld.Node) ([]byte, filestore.Path, uint64, error) {
 	f, err := pio.store.CreateTemp()
 	if err != nil {
 		return nil, "", 0, err
@@ -60,7 +116,7 @@ func (pio *pieceIO) GeneratePieceCommitment(payloadCid cid.Cid, selector ipld.No
 }
 
 func GeneratePieceCommitment(rd io.Reader, pieceSize uint64) ([]byte, uint64, error) {
-	paddedReader, paddedSize := padreader.NewPaddedReader(rd, pieceSize)
+	paddedReader, paddedSize := padreader.New(rd, pieceSize)
 	commitment, err := sectorbuilder.GeneratePieceCommitment(paddedReader, paddedSize)
 	if err != nil {
 		return nil, 0, err