Skip to content

Commit

Permalink
Merge pull request #5501 from ipfs/feat/coreapi/unixfs
Browse files Browse the repository at this point in the history 
Refactor UnixFS CoreAPI
  • Loading branch information
@Stebalien
Stebalien authored Oct 5, 2018
2 parents ab9e211 + 676403b commit 5ba0976
Show file tree
Hide file tree
Showing 12 changed files with 1,177 additions and 236 deletions.
163 changes: 35 additions & 128 deletions core/commands/add.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,15 @@ import (
"os"
"strings"

core "github.com/ipfs/go-ipfs/core"
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
"github.com/ipfs/go-ipfs/core/coreunix"
filestore "github.com/ipfs/go-ipfs/filestore"
blockservice "gx/ipfs/QmNozJswSuwiZspexEHcQo5GMqpzM5exUGjNW6s4AAipUX/go-blockservice"
dag "gx/ipfs/QmTGpm48qm4fUZ9E5hMXy4ZngJUYCMKu15rTMVR3BSEnPm/go-merkledag"
dagtest "gx/ipfs/QmTGpm48qm4fUZ9E5hMXy4ZngJUYCMKu15rTMVR3BSEnPm/go-merkledag/test"
ft "gx/ipfs/QmavvHwEZTkNShKWK1jRejv2Y8oF6ZYxdGxytL3Mwvices/go-unixfs"

offline "gx/ipfs/QmPXcrGQQEEPswwg6YiE2WLk8qkmvncZ7zphMKKP8bXqY3/go-ipfs-exchange-offline"
coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
options "github.com/ipfs/go-ipfs/core/coreapi/interface/options"

mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
pb "gx/ipfs/QmPtj12fdwuAqj9sBSTNUxBNu8kCGNp8b3o8yUzMm5GHpq/pb"
cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil"
mfs "gx/ipfs/QmQUjAGdPuNA9tpzrx5osWnPMhht7B5YzJNddjB45DUq2U/go-mfs"
cmdkit "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files"
cmds "gx/ipfs/QmXTmUCBtDUrzDYVzASogLiNph7EBuYqEgPL7QoHNMzUnz/go-ipfs-cmds"
bstore "gx/ipfs/QmfUhZX9KpvJiuiziUzP2cjhRAyqHJURsPgRKn1cdDZMKa/go-ipfs-blockstore"
)

// ErrDepthLimitExceeded indicates that the max depth has been exceeded.
Expand Down Expand Up @@ -148,22 +139,10 @@ You can now check what blocks have been created by:
return nil
},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
n, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := n.Repo.Config()
api, err := cmdenv.GetApi(env)
if err != nil {
return err
}
// check if repo will exceed storage limit if added
// TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated)
// TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon
//if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil {
// res.SetError(err, cmdkit.ErrNormal)
// return
//}

progress, _ := req.Options[progressOptionName].(bool)
trickle, _ := req.Options[trickleOptionName].(bool)
Expand All @@ -181,131 +160,59 @@ You can now check what blocks have been created by:
inline, _ := req.Options[inlineOptionName].(bool)
inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
pathName, _ := req.Options[stdinPathName].(string)

// The arguments are subject to the following constraints.
//
// nocopy -> filestoreEnabled
// nocopy -> rawblocks
// (hash != sha2-256) -> cidv1

// NOTE: 'rawblocks -> cidv1' is missing. Legacy reasons.

// nocopy -> filestoreEnabled
if nocopy && !cfg.Experimental.FilestoreEnabled {
return cmdkit.Errorf(cmdkit.ErrClient, filestore.ErrFilestoreNotEnabled.Error())
}

// nocopy -> rawblocks
if nocopy && !rawblks {
// fixed?
if rbset {
return fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well")
}
// No, satisfy mandatory constraint.
rawblks = true
}

// (hash != "sha2-256") -> CIDv1
if hashFunStr != "sha2-256" && cidVer == 0 {
if cidVerSet {
return cmdkit.Errorf(cmdkit.ErrClient, "CIDv0 only supports sha2-256")
}
cidVer = 1
}

// cidV1 -> raw blocks (by default)
if cidVer > 0 && !rbset {
rawblks = true
}

prefix, err := dag.PrefixForCidVersion(cidVer)
if err != nil {
return err
}
local, _ := req.Options["local"].(bool)

hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
if !ok {
return fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr))
}

prefix.MhType = hashFunCode
prefix.MhLength = -1

if hash {
nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
//TODO: need this to be true or all files
// hashed will be stored in memory!
NilRepo: true,
})
if err != nil {
return err
}
n = nilnode
}
events := make(chan interface{}, adderOutChanSize)

addblockstore := n.Blockstore
if !(fscache || nocopy) {
addblockstore = bstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker)
}
opts := []options.UnixfsAddOption{
options.Unixfs.Hash(hashFunCode),

exch := n.Exchange
local, _ := req.Options["local"].(bool)
if local {
exch = offline.Exchange(addblockstore)
}
options.Unixfs.Inline(inline),
options.Unixfs.InlineLimit(inlineLimit),

bserv := blockservice.New(addblockstore, exch) // hash security 001
dserv := dag.NewDAGService(bserv)
options.Unixfs.Chunker(chunker),

outChan := make(chan interface{}, adderOutChanSize)
options.Unixfs.Pin(dopin),
options.Unixfs.HashOnly(hash),
options.Unixfs.Local(local),
options.Unixfs.FsCache(fscache),
options.Unixfs.Nocopy(nocopy),

fileAdder, err := coreunix.NewAdder(req.Context, n.Pinning, n.Blockstore, dserv)
if err != nil {
return err
options.Unixfs.Wrap(wrap),
options.Unixfs.Hidden(hidden),
options.Unixfs.StdinName(pathName),

options.Unixfs.Progress(progress),
options.Unixfs.Silent(silent),
options.Unixfs.Events(events),
}

fileAdder.Out = outChan
fileAdder.Chunker = chunker
fileAdder.Progress = progress
fileAdder.Hidden = hidden
fileAdder.Trickle = trickle
fileAdder.Wrap = wrap
fileAdder.Pin = dopin && !hash
fileAdder.Silent = silent
fileAdder.RawLeaves = rawblks
fileAdder.NoCopy = nocopy
fileAdder.Name = pathName
fileAdder.CidBuilder = prefix

if inline {
fileAdder.CidBuilder = cidutil.InlineBuilder{
Builder: fileAdder.CidBuilder,
Limit: inlineLimit,
}
if cidVerSet {
opts = append(opts, options.Unixfs.CidVersion(cidVer))
}

if hash {
md := dagtest.Mock()
emptyDirNode := ft.EmptyDirNode()
// Use the same prefix for the "empty" MFS root as for the file adder.
emptyDirNode.SetCidBuilder(fileAdder.CidBuilder)
mr, err := mfs.NewRoot(req.Context, md, emptyDirNode, nil)
if err != nil {
return err
}
if rbset {
opts = append(opts, options.Unixfs.RawLeaves(rawblks))
}

fileAdder.SetMfsRoot(mr)
if trickle {
opts = append(opts, options.Unixfs.Layout(options.TrickleLayout))
}

errCh := make(chan error)
go func() {
var err error
defer func() { errCh <- err }()
defer close(outChan)
err = fileAdder.AddAllAndPin(req.Files)
defer close(events)
_, err = api.Unixfs().Add(req.Context, req.Files, opts...)
}()

err = res.Emit(outChan)
err = res.Emit(events)
if err != nil {
return err
}
Expand Down Expand Up @@ -371,7 +278,7 @@ You can now check what blocks have been created by:

break LOOP
}
output := out.(*coreunix.AddedObject)
output := out.(*coreiface.AddEvent)
if len(output.Hash) > 0 {
lastHash = output.Hash
if quieter {
Expand Down Expand Up @@ -451,5 +358,5 @@ You can now check what blocks have been created by:
}
},
},
Type: coreunix.AddedObject{},
Type: coreiface.AddEvent{},
}
8 changes: 4 additions & 4 deletions core/commands/tar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
cmds "github.com/ipfs/go-ipfs/commands"
core "github.com/ipfs/go-ipfs/core"
e "github.com/ipfs/go-ipfs/core/commands/e"
"github.com/ipfs/go-ipfs/core/coreunix"
coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
tar "github.com/ipfs/go-ipfs/tar"
dag "gx/ipfs/QmTGpm48qm4fUZ9E5hMXy4ZngJUYCMKu15rTMVR3BSEnPm/go-merkledag"
path "gx/ipfs/QmV4QxScV9Y7LbaWhHazFfRd8uyeUd4pAH8a7fFFbi5odJ/go-path"
Expand Down Expand Up @@ -60,20 +60,20 @@ represent it.
c := node.Cid()

fi.FileName()
res.SetOutput(&coreunix.AddedObject{
res.SetOutput(&coreiface.AddEvent{
Name: fi.FileName(),
Hash: c.String(),
})
},
Type: coreunix.AddedObject{},
Type: coreiface.AddEvent{},
Marshalers: cmds.MarshalerMap{
cmds.Text: func(res cmds.Response) (io.Reader, error) {
v, err := unwrapOutput(res.Output())
if err != nil {
return nil, err
}

o, ok := v.(*coreunix.AddedObject)
o, ok := v.(*coreiface.AddEvent)
if !ok {
return nil, e.TypeErr(o, v)
}
Expand Down
Loading

0 comments on commit 5ba0976

Please sign in to comment.