Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove temp files #449

Merged
merged 6 commits into from
Nov 19, 2020
Merged

Remove temp files #449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
84b3044
feat(pieceio): remove temp files
hannahhoward Nov 17, 2020
638384a
fix: GeneratePieceCommitment return values
dirkmc Nov 18, 2020
5a9364c
fix: only delete deal store once deal activated
dirkmc Nov 18, 2020
932e70e
fix: lint
dirkmc Nov 18, 2020
ab5a86b
style(imports): fix imports
hannahhoward Nov 19, 2020
47cd348
fix(deps): update go-car ref
hannahhoward Nov 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified docs/storageclient.mmd.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
6 changes: 3 additions & 3 deletions docs/storageclient.mmd.svg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ require (
github.com/ipfs/go-log/v2 v2.0.5
github.com/ipfs/go-merkledag v0.3.2
github.com/ipfs/go-unixfs v0.2.4
github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d
github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
github.com/jpillora/backoff v1.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,8 +360,8 @@ github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo=
github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4 h1:6phjU3kXvCEWOZpu+Ob0w6DzgPFZmDLgLPxJhD8RxEY=
github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D1vEwl+YAXsg/JfA98XGpXDwnkIL4Aimqw=
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U=
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ=
github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e h1:ZISbJlM0urTANR9KRfRaqlBmyOj5uUtxs2r4Up9IXsA=
github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f h1:XpOuNQ5GbXxUcSukbQcW9jkE7REpaFGJU2/T00fo9kA=
Expand Down
4 changes: 2 additions & 2 deletions pieceio/cario/cario.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ func (c carIO) WriteCar(ctx context.Context, bs pieceio.ReadStore, payloadCid ci
return sc.Write(w, userOnNewCarBlocks...)
}

func (c carIO) PrepareCar(ctx context.Context, bs pieceio.ReadStore, payloadCid cid.Cid, selector ipld.Node) (pieceio.PreparedCar, error) {
func (c carIO) PrepareCar(ctx context.Context, bs pieceio.ReadStore, payloadCid cid.Cid, selector ipld.Node, userOnNewCarBlocks ...car.OnNewCarBlockFunc) (pieceio.PreparedCar, error) {
sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: payloadCid, Selector: selector}})
return sc.Prepare()
return sc.Prepare(userOnNewCarBlocks...)
}

func (c carIO) LoadCar(bs pieceio.WriteStore, r io.Reader) (cid.Cid, error) {
Expand Down
21 changes: 14 additions & 7 deletions pieceio/mocks/CarIO.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

74 changes: 18 additions & 56 deletions pieceio/pieceio.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ import (
"github.com/filecoin-project/go-multistore"
"github.com/filecoin-project/go-padreader"
"github.com/filecoin-project/go-state-types/abi"

"github.com/filecoin-project/go-fil-markets/filestore"
)

type PreparedCar interface {
Expand All @@ -32,7 +30,7 @@ type CarIO interface {

// PrepareCar prepares a car so that its total size can be calculated without writing it to a file.
// It can then be written with PreparedCar.Dump
PrepareCar(ctx context.Context, bs ReadStore, payloadCid cid.Cid, node ipld.Node) (PreparedCar, error)
PrepareCar(ctx context.Context, bs ReadStore, payloadCid cid.Cid, node ipld.Node, userOnNewCarBlocks ...car.OnNewCarBlockFunc) (PreparedCar, error)

// LoadCar loads blocks into the a store from a given CAR file
LoadCar(bs WriteStore, r io.Reader) (cid.Cid, error)
Expand All @@ -52,40 +50,37 @@ func NewPieceIO(carIO CarIO, bs blockstore.Blockstore, multiStore MultiStore) Pi
return &pieceIO{carIO, bs, multiStore}
}

type pieceIOWithStore struct {
pieceIO
store filestore.FileStore
}

func NewPieceIOWithStore(carIO CarIO, store filestore.FileStore, bs blockstore.Blockstore, multiStore MultiStore) PieceIOWithStore {
return &pieceIOWithStore{pieceIO{carIO, bs, multiStore}, store}
}

func (pio *pieceIO) GeneratePieceCommitment(rt abi.RegisteredSealProof, payloadCid cid.Cid, selector ipld.Node, storeID *multistore.StoreID) (cid.Cid, abi.UnpaddedPieceSize, error) {
func (pio *pieceIO) GeneratePieceReader(payloadCid cid.Cid, selector ipld.Node, storeID *multistore.StoreID, userOnNewCarBlocks ...car.OnNewCarBlockFunc) (io.ReadCloser, uint64, error, <-chan error) {
bstore, err := pio.bstore(storeID)
if err != nil {
return cid.Undef, 0, err
return nil, 0, err, nil
}
preparedCar, err := pio.carIO.PrepareCar(context.Background(), bstore, payloadCid, selector)
preparedCar, err := pio.carIO.PrepareCar(context.Background(), bstore, payloadCid, selector, userOnNewCarBlocks...)
if err != nil {
return cid.Undef, 0, err
return nil, 0, err, nil
}
pieceSize := uint64(preparedCar.Size())
r, w, err := os.Pipe()
if err != nil {
return cid.Undef, 0, err
return nil, 0, err, nil
}
var stop sync.WaitGroup
stop.Add(1)
var werr error
writeErr := make(chan error, 1)
go func() {
defer stop.Done()
werr = preparedCar.Dump(w)
werr := preparedCar.Dump(w)
err := w.Close()
if werr == nil && err != nil {
werr = err
}
writeErr <- werr
}()
return r, pieceSize, nil, writeErr
}

func (pio *pieceIO) GeneratePieceCommitment(rt abi.RegisteredSealProof, payloadCid cid.Cid, selector ipld.Node, storeID *multistore.StoreID, userOnNewCarBlocks ...car.OnNewCarBlockFunc) (cid.Cid, abi.UnpaddedPieceSize, error) {
r, pieceSize, err, writeErrChan := pio.GeneratePieceReader(payloadCid, selector, storeID, userOnNewCarBlocks...)
if err != nil {
return cid.Undef, 0, err
}
commitment, paddedSize, err := GeneratePieceCommitment(rt, r, pieceSize)
closeErr := r.Close()
if err != nil {
Expand All @@ -94,46 +89,13 @@ func (pio *pieceIO) GeneratePieceCommitment(rt abi.RegisteredSealProof, payloadC
if closeErr != nil {
return cid.Undef, 0, closeErr
}
stop.Wait()
werr := <-writeErrChan
if werr != nil {
return cid.Undef, 0, werr
}
return commitment, paddedSize, nil
}

func (pio *pieceIOWithStore) GeneratePieceCommitmentToFile(rt abi.RegisteredSealProof, payloadCid cid.Cid, selector ipld.Node, storeID *multistore.StoreID, userOnNewCarBlocks ...car.OnNewCarBlockFunc) (cid.Cid, filestore.Path, abi.UnpaddedPieceSize, error) {
bstore, err := pio.bstore(storeID)
if err != nil {
return cid.Undef, "", 0, err
}
f, err := pio.store.CreateTemp()
if err != nil {
return cid.Undef, "", 0, err
}
cleanup := func() {
f.Close()
_ = pio.store.Delete(f.Path())
}
err = pio.carIO.WriteCar(context.Background(), bstore, payloadCid, selector, f, userOnNewCarBlocks...)
if err != nil {
cleanup()
return cid.Undef, "", 0, err
}
pieceSize := uint64(f.Size())
_, err = f.Seek(0, io.SeekStart)
if err != nil {
cleanup()
return cid.Undef, "", 0, err
}
commitment, paddedSize, err := GeneratePieceCommitment(rt, f, pieceSize)
if err != nil {
cleanup()
return cid.Undef, "", 0, err
}
_ = f.Close()
return commitment, f.Path(), paddedSize, nil
}

func GeneratePieceCommitment(rt abi.RegisteredSealProof, rd io.Reader, pieceSize uint64) (cid.Cid, abi.UnpaddedPieceSize, error) {
paddedReader, paddedSize := padreader.New(rd, pieceSize)
commitment, err := GeneratePieceCIDFromFile(rt, paddedReader, paddedSize)
Expand Down
Loading