From 24e317603af74d59d1ff334e440a6c5b440ffd8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 18 Dec 2018 14:23:55 +0100 Subject: [PATCH] coreapi/unixfs: Use path instead of raw hash in AddEvent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/add.go | 32 ++++++++++++++++++++++++----- core/commands/tar.go | 7 +++---- core/coreapi/interface/path.go | 1 + core/coreapi/interface/unixfs.go | 7 +++---- core/coreapi/unixfs_test.go | 35 ++++++++++++++++++++++---------- core/coreunix/add.go | 24 +++++----------------- core/coreunix/add_test.go | 6 +++--- 7 files changed, 66 insertions(+), 46 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index fd5da1b3c29..c3399835d59 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -1,6 +1,7 @@ package commands import ( + "errors" "fmt" "io" "os" @@ -19,6 +20,13 @@ import ( // ErrDepthLimitExceeded indicates that the max depth has been exceeded. var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded") +type AddEvent struct { + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` +} + const ( quietOptionName = "quiet" quieterOptionName = "quieter" @@ -210,9 +218,23 @@ You can now check what blocks have been created by: _, err = api.Unixfs().Add(req.Context, req.Files, opts...) }() - err = res.Emit(events) - if err != nil { - return err + for event := range events { + output, ok := event.(*coreiface.AddEvent) + if !ok { + return errors.New("unknown event type") + } + + h := "" + if output.Path != nil { + h = output.Path.Cid().String() + } + + res.Emit(&AddEvent{ + Name: output.Name, + Hash: h, + Bytes: output.Bytes, + Size: output.Size, + }) } return <-errCh @@ -269,7 +291,7 @@ You can now check what blocks have been created by: break LOOP } - output := out.(*coreiface.AddEvent) + output := out.(*AddEvent) if len(output.Hash) > 0 { lastHash = output.Hash if quieter { @@ -357,5 +379,5 @@ You can now check what blocks have been created by: } }, }, - Type: coreiface.AddEvent{}, + Type: AddEvent{}, } diff --git a/core/commands/tar.go b/core/commands/tar.go index 9604cc3de7f..2004369b516 100644 --- a/core/commands/tar.go +++ b/core/commands/tar.go @@ -6,7 +6,6 @@ import ( "github.com/ipfs/go-ipfs/core" "github.com/ipfs/go-ipfs/core/commands/cmdenv" - coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" tar "github.com/ipfs/go-ipfs/tar" "gx/ipfs/QmZErC2Ay6WuGi96CPg316PwitdwgLo6RxZRqVjJjRj2MR/go-path" @@ -57,14 +56,14 @@ represent it. c := node.Cid() - return cmds.EmitOnce(res, &coreiface.AddEvent{ + return cmds.EmitOnce(res, &AddEvent{ Name: it.Name(), Hash: c.String(), }) }, - Type: coreiface.AddEvent{}, + Type: AddEvent{}, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *coreiface.AddEvent) error { + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddEvent) error { fmt.Fprintln(w, out.Hash) return nil }), diff --git a/core/coreapi/interface/path.go b/core/coreapi/interface/path.go index 57ef4c21bd3..01dda97d5a8 100644 --- a/core/coreapi/interface/path.go +++ b/core/coreapi/interface/path.go @@ -46,6 +46,7 @@ type ResolvedPath interface { // cidRoot := {"A": {"/": cidA }} // // And resolve paths: + // // * "/ipfs/${cidRoot}" // * Calling Cid() will return `cidRoot` // * Calling Root() will return `cidRoot` diff --git a/core/coreapi/interface/unixfs.go b/core/coreapi/interface/unixfs.go index 589083c6b83..b42b454cc2a 100644 --- a/core/coreapi/interface/unixfs.go +++ b/core/coreapi/interface/unixfs.go @@ -9,12 +9,11 @@ import ( ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" ) -// TODO: ideas on making this more coreapi-ish without breaking the http API? type AddEvent struct { Name string - Hash string `json:",omitempty"` - Bytes int64 `json:",omitempty"` - Size string `json:",omitempty"` + Path ResolvedPath `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` } // UnixfsAPI is the basic interface to immutable files in IPFS diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 967ffd46db5..91cb8ea87af 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "fmt" + "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" "io" "io/ioutil" "math" @@ -178,6 +179,14 @@ func TestAdd(t *testing.T) { t.Error(err) } + p := func(h string) coreiface.ResolvedPath { + c, err := cid.Parse(h) + if err != nil { + t.Fatal(err) + } + return coreiface.IpfsPath(c) + } + cases := []struct { name string data func() files.Node @@ -406,7 +415,7 @@ func TestAdd(t *testing.T) { data: strFile(helloStr), path: "/ipfs/zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", events: []coreiface.AddEvent{ - {Name: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Hash: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Size: strconv.Itoa(len(helloStr))}, + {Name: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Path: p("zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd"), Size: strconv.Itoa(len(helloStr))}, }, opts: []options.UnixfsAddOption{options.Unixfs.RawLeaves(true)}, }, @@ -415,8 +424,8 @@ func TestAdd(t *testing.T) { data: twoLevelDir(), path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", events: []coreiface.AddEvent{ - {Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"}, - {Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"}, + {Name: "t/abc", Path: p("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"}, + {Name: "t", Path: p("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"}, }, wrap: "t", opts: []options.UnixfsAddOption{options.Unixfs.Silent(true)}, @@ -426,11 +435,11 @@ func TestAdd(t *testing.T) { data: twoLevelDir(), path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", events: []coreiface.AddEvent{ - {Name: "t/abc/def", Hash: "QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk", Size: "13"}, - {Name: "t/bar", Hash: "QmS21GuXiRMvJKHos4ZkEmQDmRBqRaF5tQS2CQCu2ne9sY", Size: "14"}, - {Name: "t/foo", Hash: "QmfAjGiVpTN56TXi6SBQtstit5BEw3sijKj1Qkxn6EXKzJ", Size: "14"}, - {Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"}, - {Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"}, + {Name: "t/abc/def", Path: p("QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk"), Size: "13"}, + {Name: "t/bar", Path: p("QmS21GuXiRMvJKHos4ZkEmQDmRBqRaF5tQS2CQCu2ne9sY"), Size: "14"}, + {Name: "t/foo", Path: p("QmfAjGiVpTN56TXi6SBQtstit5BEw3sijKj1Qkxn6EXKzJ"), Size: "14"}, + {Name: "t/abc", Path: p("QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt"), Size: "62"}, + {Name: "t", Path: p("QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr"), Size: "229"}, }, wrap: "t", }, @@ -445,7 +454,7 @@ func TestAdd(t *testing.T) { {Name: "", Bytes: 524288}, {Name: "", Bytes: 786432}, {Name: "", Bytes: 1000000}, - {Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Hash: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Size: "1000256"}, + {Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Path: p("QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD"), Size: "1000256"}, }, wrap: "", opts: []options.UnixfsAddOption{options.Unixfs.Progress(true)}, @@ -497,8 +506,12 @@ func TestAdd(t *testing.T) { t.Errorf("Event.Name didn't match, %s != %s", expected[0].Name, event.Name) } - if expected[0].Hash != event.Hash { - t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Hash, event.Hash) + if expected[0].Path != nil && event.Path != nil { + if expected[0].Path.Cid().String() != event.Path.Cid().String() { + t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) + } + } else if event.Path != expected[0].Path { + t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) } if expected[0].Bytes != event.Bytes { t.Errorf("Event.Bytes didn't match, %d != %d", expected[0].Bytes, event.Bytes) diff --git a/core/coreunix/add.go b/core/coreunix/add.go index b4636a5ba71..d3ded586aee 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -41,12 +41,6 @@ type Link struct { Size uint64 } -type Object struct { - Hash string - Links []Link - Size string -} - // NewAdder Returns a new Adder used for a file add operation. func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAGService) (*Adder, error) { bufferedDS := ipld.NewBufferedDAG(ctx, ds) @@ -580,7 +574,7 @@ func outputDagnode(out chan<- interface{}, name string, dn ipld.Node) error { } out <- &coreiface.AddEvent{ - Hash: o.Hash, + Path: o.Path, Name: name, Size: o.Size, } @@ -589,24 +583,16 @@ func outputDagnode(out chan<- interface{}, name string, dn ipld.Node) error { } // from core/commands/object.go -func getOutput(dagnode ipld.Node) (*Object, error) { +func getOutput(dagnode ipld.Node) (*coreiface.AddEvent, error) { c := dagnode.Cid() s, err := dagnode.Size() if err != nil { return nil, err } - output := &Object{ - Hash: c.String(), - Size: strconv.FormatUint(s, 10), - Links: make([]Link, len(dagnode.Links())), - } - - for i, link := range dagnode.Links() { - output.Links[i] = Link{ - Name: link.Name, - Size: link.Size, - } + output := &coreiface.AddEvent{ + Path: coreiface.IpfsPath(c), + Size: strconv.FormatUint(s, 10), } return output, nil diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index eed9e537833..270e8094afe 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -100,7 +100,7 @@ func TestAddGCLive(t *testing.T) { addedHashes := make(map[string]struct{}) select { case o := <-out: - addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{} + addedHashes[o.(*coreiface.AddEvent).Path.Cid().String()] = struct{}{} case <-addDone: t.Fatal("add shouldnt complete yet") } @@ -128,7 +128,7 @@ func TestAddGCLive(t *testing.T) { // receive next object from adder o := <-out - addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{} + addedHashes[o.(*coreiface.AddEvent).Path.Cid().String()] = struct{}{} <-gcstarted @@ -144,7 +144,7 @@ func TestAddGCLive(t *testing.T) { var last cid.Cid for a := range out { // wait for it to finish - c, err := cid.Decode(a.(*coreiface.AddEvent).Hash) + c, err := cid.Decode(a.(*coreiface.AddEvent).Path.Cid().String()) if err != nil { t.Fatal(err) }