From 235a7674929ec787c141abec96b11534023a159c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Sep 2014 16:55:49 +0000 Subject: [PATCH 001/105] implement namesys resolvers (thanks to bren2010 for dns and proquint) --- cmd/ipfs/pin.go | 44 +++++++++++++++++++++++++ core/core.go | 2 +- namesys/dns.go | 33 +++++++++++++++++++ namesys/entry.pb.go | 48 +++++++++++++++++++++++++++ namesys/entry.proto | 6 ++++ namesys/nsresolver.go | 5 +++ namesys/proquint.go | 22 +++++++++++++ namesys/resolver.go | 21 ++++++++++++ namesys/routing.go | 77 +++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 cmd/ipfs/pin.go create mode 100644 namesys/dns.go create mode 100644 namesys/entry.pb.go create mode 100644 namesys/entry.proto create mode 100644 namesys/nsresolver.go create mode 100644 namesys/proquint.go create mode 100644 namesys/resolver.go create mode 100644 namesys/routing.go diff --git a/cmd/ipfs/pin.go b/cmd/ipfs/pin.go new file mode 100644 index 00000000000..88f8e850563 --- /dev/null +++ b/cmd/ipfs/pin.go @@ -0,0 +1,44 @@ +package main + +import ( + "os" + + "github.com/gonuts/flag" + "github.com/jbenet/commander" + "github.com/jbenet/go-ipfs/daemon" + u "github.com/jbenet/go-ipfs/util" +) + +var cmdIpfsPin = &commander.Command{ + UsageLine: "pin", + Short: "pin an ipfs object to local storage.", + Long: `ipfs pin - pin ipfs object to local storage. + + Retrieves the object named by and stores it locally + on disk. +`, + Run: pinCmd, + Flag: *flag.NewFlagSet("ipfs-pin", flag.ExitOnError), +} + +func pinCmd(c *commander.Command, inp []string) error { + if len(inp) < 1 { + u.POut(c.Long) + return nil + } + + com := daemon.NewCommand() + com.Command = "pin" + com.Args = inp + + err := daemon.SendCommand(com, "localhost:12345") + if err != nil { + n, err := localNode(false) + if err != nil { + return err + } + + daemon.ExecuteCommand(com, n, os.Stdout) + } + return nil +} diff --git a/core/core.go b/core/core.go index f641e8b2d93..459a32aabfc 100644 --- a/core/core.go +++ b/core/core.go @@ -60,7 +60,7 @@ type IpfsNode struct { Resolver *path.Resolver // the name system, resolves paths to hashes - // Namesys *namesys.Namesys + Namesys namesys.NSResolver } // NewIpfsNode constructs a new IpfsNode based on the given config. diff --git a/namesys/dns.go b/namesys/dns.go new file mode 100644 index 00000000000..b12bc0d38a0 --- /dev/null +++ b/namesys/dns.go @@ -0,0 +1,33 @@ +package namesys + +import ( + "net" + "strings" + + u "github.com/jbenet/go-ipfs/util" +) + +type DNSResolver struct { + // TODO: maybe some sort of caching? + // cache would need a timeout +} + +func (r *DNSResolver) Resolve(name string) (string, error) { + txt, err := net.LookupTXT(name) + if err != nil { + return "", err + } + + for _, t := range txt { + pair := strings.Split(t, "=") + if len(pair) < 2 { + // Log error? + u.DErr("Incorrectly formatted text record.") + continue + } + if pair[0] == name { + return pair[1], nil + } + } + return "", u.ErrNotFound +} diff --git a/namesys/entry.pb.go b/namesys/entry.pb.go new file mode 100644 index 00000000000..c05efeb2f28 --- /dev/null +++ b/namesys/entry.pb.go @@ -0,0 +1,48 @@ +// Code generated by protoc-gen-go. +// source: entry.proto +// DO NOT EDIT! + +/* +Package namesys is a generated protocol buffer package. + +It is generated from these files: + entry.proto + +It has these top-level messages: + InpsEntry +*/ +package namesys + +import proto "code.google.com/p/goprotobuf/proto" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = math.Inf + +type InpsEntry struct { + Value []byte `protobuf:"bytes,1,req,name=value" json:"value,omitempty"` + Signature []byte `protobuf:"bytes,2,req,name=signature" json:"signature,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *InpsEntry) Reset() { *m = InpsEntry{} } +func (m *InpsEntry) String() string { return proto.CompactTextString(m) } +func (*InpsEntry) ProtoMessage() {} + +func (m *InpsEntry) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func (m *InpsEntry) GetSignature() []byte { + if m != nil { + return m.Signature + } + return nil +} + +func init() { +} diff --git a/namesys/entry.proto b/namesys/entry.proto new file mode 100644 index 00000000000..b7b09f0417d --- /dev/null +++ b/namesys/entry.proto @@ -0,0 +1,6 @@ +package namesys; + +message InpsEntry { + required bytes value = 1; + required bytes signature = 2; +} diff --git a/namesys/nsresolver.go b/namesys/nsresolver.go new file mode 100644 index 00000000000..ef5bc65cbd7 --- /dev/null +++ b/namesys/nsresolver.go @@ -0,0 +1,5 @@ +package namesys + +type NSResolver interface { + Resolve(string) (string, error) +} diff --git a/namesys/proquint.go b/namesys/proquint.go new file mode 100644 index 00000000000..8583a539080 --- /dev/null +++ b/namesys/proquint.go @@ -0,0 +1,22 @@ +package namesys + +import ( + "errors" + + proquint "github.com/bren2010/proquint" +) + +var _ = proquint.Encode + +type ProquintResolver struct{} + +func (r *ProquintResolver) Resolve(name string) (string, error) { + ok, err := proquint.IsProquint(name) + if err != nil { + return "", err + } + if !ok { + return "", errors.New("not a valid proquint string") + } + return string(proquint.Decode(name)), nil +} diff --git a/namesys/resolver.go b/namesys/resolver.go new file mode 100644 index 00000000000..3498cbd463e --- /dev/null +++ b/namesys/resolver.go @@ -0,0 +1,21 @@ +package namesys + +import "strings" + +type MasterResolver struct { + dns *DNSResolver + routing *RoutingResolver + pro *ProquintResolver +} + +func (mr *MasterResolver) Resolve(name string) (string, error) { + if strings.Contains(name, ".") { + return mr.dns.Resolve(name) + } + + if strings.Contains(name, "-") { + return mr.pro.Resolve(name) + } + + return mr.routing.Resolve(name) +} diff --git a/namesys/routing.go b/namesys/routing.go new file mode 100644 index 00000000000..f37b6485fd0 --- /dev/null +++ b/namesys/routing.go @@ -0,0 +1,77 @@ +package namesys + +import ( + "fmt" + "time" + + "code.google.com/p/goprotobuf/proto" + + ci "github.com/jbenet/go-ipfs/crypto" + mdag "github.com/jbenet/go-ipfs/merkledag" + "github.com/jbenet/go-ipfs/routing" + u "github.com/jbenet/go-ipfs/util" + mh "github.com/jbenet/go-multihash" +) + +// RoutingName is the de-serialized name structure that is stored (serialized) +// in the routing system. Basically, a hash + a digital signature. (serialization can be +// protobuf, or a simple binary format) +type RoutingName struct { + Hash u.Key + Signature []byte +} + +// RoutingResolver implements NSResolver for the main IPFS SFS-like naming +type RoutingResolver struct { + routing routing.IpfsRouting + dag mdag.DAGService +} + +func (r *RoutingResolver) Resolve(name string) (string, error) { + hash, err := mh.FromB58String(name) + if err != nil { + return "", err + } + // name should be a multihash. if it isn't, error out here. + + // use the routing system to get the name. + // /ipns/ + h, err := u.Hash([]byte("ipns:" + name)) + if err != nil { + return "", err + } + + inpsKey := u.Key(h) + val, err := r.routing.GetValue(inpsKey, time.Second*10) + if err != nil { + return "", err + } + + entry := new(InpsEntry) + err = proto.Unmarshal(val, entry) + if err != nil { + return "", err + } + + // name should be a public key retrievable from ipfs + // /ipfs/ + key := u.Key(hash) + node, err := r.dag.Get(key) + if err != nil { + return "", err + } + + // get PublicKey from node.Data + pk, err := ci.UnmarshalPublicKey(node.Data) + if err != nil { + return "", err + } + + // check sig with pk + if ok, err := pk.Verify(entry.GetValue(), entry.GetSignature()); err != nil && ok { + return "", fmt.Errorf("Invalid value. Not signed by PrivateKey corresponding to %v", pk) + } + + // ok sig checks out. this is a valid name. + return string(entry.GetValue()), nil +} From b5fd9492d70527dddc49482a2ff536d22403b6e2 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Sep 2014 20:56:29 +0000 Subject: [PATCH 002/105] fixes to make interface more usable --- core/commands/add.go | 10 ++++++++++ core/commands/commands.go | 4 ++++ core/core.go | 1 + fuse/readonly/readonly_unix.go | 31 ++++++++++++++++++++++++------- merkledag/merkledag.go | 8 ++++++-- routing/dht/routing.go | 1 + 6 files changed, 46 insertions(+), 9 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 48a6ccae9f8..c98ee145186 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -103,6 +103,16 @@ func addFile(n *core.IpfsNode, fpath string, depth int) (*dag.Node, error) { return nil, err } + k, err := root.Key() + if err != nil { + return nil, err + } + + log.Info("Adding file: %s = %s\n", fpath, k.Pretty()) + for _, l := range root.Links { + log.Info("SubBlock: %s\n", l.Hash.B58String()) + } + return root, addNode(n, root, fpath) } diff --git a/core/commands/commands.go b/core/commands/commands.go index 5bef54a7b18..b356d9bbb91 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -4,6 +4,10 @@ import ( "io" "github.com/jbenet/go-ipfs/core" + + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) +var log = logging.MustGetLogger("commands") + type CmdFunc func(*core.IpfsNode, []string, map[string]interface{}, io.Writer) error diff --git a/core/core.go b/core/core.go index 459a32aabfc..005b6e62135 100644 --- a/core/core.go +++ b/core/core.go @@ -16,6 +16,7 @@ import ( exchange "github.com/jbenet/go-ipfs/exchange" bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" merkledag "github.com/jbenet/go-ipfs/merkledag" + namesys "github.com/jbenet/go-ipfs/namesys" inet "github.com/jbenet/go-ipfs/net" mux "github.com/jbenet/go-ipfs/net/mux" netservice "github.com/jbenet/go-ipfs/net/service" diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 897c90d78a9..03dba781069 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -14,6 +14,8 @@ import ( "syscall" "time" + "code.google.com/p/goprotobuf/proto" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" core "github.com/jbenet/go-ipfs/core" @@ -72,20 +74,35 @@ func (*Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { // Node is the core object representing a filesystem tree node. type Node struct { - Ipfs *core.IpfsNode - Nd *mdag.Node - fd *mdag.DagReader + Ipfs *core.IpfsNode + Nd *mdag.Node + fd *mdag.DagReader + cached *mdag.PBData +} + +func (s *Node) loadData() error { + s.cached = new(mdag.PBData) + return proto.Unmarshal(s.Nd.Data, s.cached) } // Attr returns the attributes of a given node. func (s *Node) Attr() fuse.Attr { u.DOut("Node attr.\n") - if len(s.Nd.Links) > 0 { + if s.cached == nil { + s.loadData() + } + switch s.cached.GetType() { + case mdag.PBData_Directory: + u.DOut("this is a directory.\n") return fuse.Attr{Mode: os.ModeDir | 0555} + case mdag.PBData_File, mdag.PBData_Raw: + u.DOut("this is a file.\n") + size, _ := s.Nd.Size() + return fuse.Attr{Mode: 0444, Size: uint64(size)} + default: + u.PErr("Invalid data type.") + return fuse.Attr{} } - - size, _ := s.Nd.Size() - return fuse.Attr{Mode: 0444, Size: uint64(size)} } // Lookup performs a lookup under this node. diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 1ec5f3c5e75..1cc26278345 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -3,7 +3,8 @@ package merkledag import ( "fmt" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" blocks "github.com/jbenet/go-ipfs/blocks" @@ -11,6 +12,8 @@ import ( u "github.com/jbenet/go-ipfs/util" ) +var log = logging.MustGetLogger("commands") + // NodeMap maps u.Keys to Nodes. // We cannot use []byte/Multihash for keys :( // so have to convert Multihash bytes to string (u.Key) @@ -105,7 +108,7 @@ type DAGService struct { // Add adds a node to the DAGService, storing the block in the BlockService func (n *DAGService) Add(nd *Node) (u.Key, error) { k, _ := nd.Key() - u.DOut("DagService Add [%s]\n", k.Pretty()) + log.Debug("DagService Add [%s]\n", k.Pretty()) if n == nil { return "", fmt.Errorf("DAGService is nil") } @@ -126,6 +129,7 @@ func (n *DAGService) Add(nd *Node) (u.Key, error) { func (n *DAGService) AddRecursive(nd *Node) error { _, err := n.Add(nd) if err != nil { + log.Info("AddRecursive Error: %s\n", err) return err } diff --git a/routing/dht/routing.go b/routing/dht/routing.go index 66ae0984879..16f74380b12 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -101,6 +101,7 @@ func (dht *IpfsDHT) Provide(ctx context.Context, key u.Key) error { dht.providers.AddProvider(key, dht.self) peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), PoolSize) if len(peers) == 0 { + // Early out for no targets return nil } From da1890e5efbadb9ffbc2e1a309909bd87db69b92 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 10 Sep 2014 23:07:01 +0000 Subject: [PATCH 003/105] add run command --- cmd/ipfs/ipfs.go | 1 + cmd/ipfs/run.go | 34 ++++++++++++++++++++++++++++++++++ fuse/readonly/readonly_unix.go | 6 +++++- 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 cmd/ipfs/run.go diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 88dbacce5bb..d6fbac0bf42 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -51,6 +51,7 @@ Use "ipfs help " for more information about a command. cmdIpfsMount, cmdIpfsInit, cmdIpfsServe, + cmdIpfsRun, }, Flag: *flag.NewFlagSet("ipfs", flag.ExitOnError), } diff --git a/cmd/ipfs/run.go b/cmd/ipfs/run.go new file mode 100644 index 00000000000..94158ecd435 --- /dev/null +++ b/cmd/ipfs/run.go @@ -0,0 +1,34 @@ +package main + +import ( + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" + "github.com/jbenet/go-ipfs/daemon" + u "github.com/jbenet/go-ipfs/util" +) + +var cmdIpfsRun = &commander.Command{ + UsageLine: "run", + Short: "run local ifps node.", + Long: `run a local ipfs node with no other interface. +`, + Run: runCmd, + Flag: *flag.NewFlagSet("ipfs-run", flag.ExitOnError), +} + +func runCmd(c *commander.Command, inp []string) error { + u.Debug = true + + n, err := localNode(true) + if err != nil { + return err + } + + dl, err := daemon.NewDaemonListener(n, "localhost:12345") + if err != nil { + return err + } + dl.Listen() + dl.Close() + return nil +} diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 03dba781069..aa62429a4a0 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -98,7 +98,11 @@ func (s *Node) Attr() fuse.Attr { case mdag.PBData_File, mdag.PBData_Raw: u.DOut("this is a file.\n") size, _ := s.Nd.Size() - return fuse.Attr{Mode: 0444, Size: uint64(size)} + return fuse.Attr{ + Mode: 0444, + Size: uint64(size), + Blocks: uint64(len(s.Nd.Links)), + } default: u.PErr("Invalid data type.") return fuse.Attr{} From c41a9717fe56a3f8e92c9694a594271818cc430d Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 11 Sep 2014 02:20:04 +0000 Subject: [PATCH 004/105] make disconnects and reconnects work a little better --- cmd/ipfs/ipfs.go | 2 +- crypto/spipe/handshake.go | 4 ++++ fuse/readonly/readonly_unix.go | 1 + path/path.go | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index d6fbac0bf42..1627131d02d 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" "os" - "runtime/pprof" + "runtime" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" diff --git a/crypto/spipe/handshake.go b/crypto/spipe/handshake.go index f617c75b3aa..67af992fdd0 100644 --- a/crypto/spipe/handshake.go +++ b/crypto/spipe/handshake.go @@ -229,6 +229,8 @@ func (s *SecurePipe) handleSecureIn(hashType string, tIV, tCKey, tMKey []byte) { for { data, ok := <-s.insecure.In if !ok { + u.DOut("Closing incoming proxy.\n") + close(secureIn) return } @@ -265,6 +267,8 @@ func (s *SecurePipe) handleSecureOut(hashType string, mIV, mCKey, mMKey []byte) for { data, ok := <-s.Out if !ok { + u.DOut("Closing outgoing proxy.\n") + close(secureOut) return } diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index aa62429a4a0..0ce5fab4cf8 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -168,6 +168,7 @@ func Mount(ipfs *core.IpfsNode, fpath string) error { } time.Sleep(time.Millisecond * 10) } + ipfs.Swarm.Close() }() c, err := fuse.Mount(fpath) diff --git a/path/path.go b/path/path.go index a06fb98cb29..eb5ce6660fd 100644 --- a/path/path.go +++ b/path/path.go @@ -38,6 +38,7 @@ func (s *Resolver) ResolvePath(fpath string) (*merkledag.Node, error) { // first element in the path is a b58 hash (for now) h, err := mh.FromB58String(parts[0]) if err != nil { + u.DOut("given path element is not a base58 string.\n") return nil, err } From 8ddfb57c59ba452e87f278c55ec2cbcdd0467c2c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 11 Sep 2014 19:12:14 +0000 Subject: [PATCH 005/105] some bugfixes and added logging --- core/commands/add.go | 1 + core/core.go | 2 ++ routing/dht/routing.go | 9 ++++----- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index c98ee145186..f6c5492ad16 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -94,6 +94,7 @@ func addDir(n *core.IpfsNode, fpath string, depth int) (*dag.Node, error) { } } + fmt.Printf("Adding dir: %s = %s\n", fpath) return tree, addNode(n, tree, fpath) } diff --git a/core/core.go b/core/core.go index 005b6e62135..f29f87abcdd 100644 --- a/core/core.go +++ b/core/core.go @@ -238,3 +238,5 @@ func (n *IpfsNode) PinDagNode(nd *merkledag.Node) error { u.DOut("Pinning node. Currently No-Op\n") return nil } + +func (n *IpfsNode) SetIpnsEntry() {} diff --git a/routing/dht/routing.go b/routing/dht/routing.go index 16f74380b12..3d7a4bf0a76 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -152,15 +152,14 @@ func (dht *IpfsDHT) FindProvidersAsync2(ctx context.Context, key u.Key, count in peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), AlphaValue) for _, pp := range peers { - ppp := pp - go func() { - pmes, err := dht.findProvidersSingle(ctx, ppp, key, 0) + go func(p *peer.Peer) { + pmes, err := dht.findProvidersSingle(p, key, 0, timeout) if err != nil { u.PErr("%v\n", err) return } - dht.addPeerListAsync(key, pmes.GetProviderPeers(), ps, count, peerOut) - }() + dht.addPeerListAsync(key, pmes.GetPeers(), ps, count, peerOut) + }(pp) } }() From 4c0f18cbbde4982ee5f47b8c885e510650bd689c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 12 Sep 2014 02:41:46 +0000 Subject: [PATCH 006/105] implement ipns publisher code --- core/core.go | 2 -- namesys/entry.pb.go | 14 +++++----- namesys/entry.proto | 2 +- namesys/publisher.go | 64 ++++++++++++++++++++++++++++++++++++++++++++ namesys/resolver.go | 15 ++++++++++- namesys/routing.go | 19 +++++++------ 6 files changed, 95 insertions(+), 21 deletions(-) create mode 100644 namesys/publisher.go diff --git a/core/core.go b/core/core.go index f29f87abcdd..005b6e62135 100644 --- a/core/core.go +++ b/core/core.go @@ -238,5 +238,3 @@ func (n *IpfsNode) PinDagNode(nd *merkledag.Node) error { u.DOut("Pinning node. Currently No-Op\n") return nil } - -func (n *IpfsNode) SetIpnsEntry() {} diff --git a/namesys/entry.pb.go b/namesys/entry.pb.go index c05efeb2f28..e4420e9f978 100644 --- a/namesys/entry.pb.go +++ b/namesys/entry.pb.go @@ -9,7 +9,7 @@ It is generated from these files: entry.proto It has these top-level messages: - InpsEntry + IpnsEntry */ package namesys @@ -20,24 +20,24 @@ import math "math" var _ = proto.Marshal var _ = math.Inf -type InpsEntry struct { +type IpnsEntry struct { Value []byte `protobuf:"bytes,1,req,name=value" json:"value,omitempty"` Signature []byte `protobuf:"bytes,2,req,name=signature" json:"signature,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *InpsEntry) Reset() { *m = InpsEntry{} } -func (m *InpsEntry) String() string { return proto.CompactTextString(m) } -func (*InpsEntry) ProtoMessage() {} +func (m *IpnsEntry) Reset() { *m = IpnsEntry{} } +func (m *IpnsEntry) String() string { return proto.CompactTextString(m) } +func (*IpnsEntry) ProtoMessage() {} -func (m *InpsEntry) GetValue() []byte { +func (m *IpnsEntry) GetValue() []byte { if m != nil { return m.Value } return nil } -func (m *InpsEntry) GetSignature() []byte { +func (m *IpnsEntry) GetSignature() []byte { if m != nil { return m.Signature } diff --git a/namesys/entry.proto b/namesys/entry.proto index b7b09f0417d..fee830d7ebb 100644 --- a/namesys/entry.proto +++ b/namesys/entry.proto @@ -1,6 +1,6 @@ package namesys; -message InpsEntry { +message IpnsEntry { required bytes value = 1; required bytes signature = 2; } diff --git a/namesys/publisher.go b/namesys/publisher.go new file mode 100644 index 00000000000..4588ccb6a89 --- /dev/null +++ b/namesys/publisher.go @@ -0,0 +1,64 @@ +package namesys + +import ( + "code.google.com/p/goprotobuf/proto" + + ci "github.com/jbenet/go-ipfs/crypto" + mdag "github.com/jbenet/go-ipfs/merkledag" + routing "github.com/jbenet/go-ipfs/routing" + u "github.com/jbenet/go-ipfs/util" +) + +type IpnsPublisher struct { + dag *mdag.DAGService + routing routing.IpfsRouting +} + +// Publish accepts a keypair and a value, +func (p *IpnsPublisher) Publish(k ci.PrivKey, value u.Key) error { + data, err := CreateEntryData(k, value) + if err != nil { + return err + } + pubkey := k.GetPublic() + pkbytes, err := pubkey.Bytes() + if err != nil { + return nil + } + + nameb, err := u.Hash(pkbytes) + if err != nil { + return nil + } + namekey := u.Key(nameb).Pretty() + + ipnskey, err := u.Hash([]byte("ipns:" + namekey)) + if err != nil { + return err + } + + // Store associated public key + err = p.routing.PutValue(u.Key(nameb), pkbytes) + if err != nil { + return err + } + + // Store ipns entry at h("ipns:"+b58(h(pubkey))) + err = p.routing.PutValue(u.Key(ipnskey), data) + if err != nil { + return err + } + + return nil +} + +func CreateEntryData(pk ci.PrivKey, val u.Key) ([]byte, error) { + entry := new(IpnsEntry) + sig, err := pk.Sign([]byte(val)) + if err != nil { + return nil, err + } + entry.Signature = sig + entry.Value = []byte(val) + return proto.Marshal(entry) +} diff --git a/namesys/resolver.go b/namesys/resolver.go index 3498cbd463e..e440946b985 100644 --- a/namesys/resolver.go +++ b/namesys/resolver.go @@ -1,6 +1,11 @@ package namesys -import "strings" +import ( + "strings" + + mdag "github.com/jbenet/go-ipfs/merkledag" + "github.com/jbenet/go-ipfs/routing" +) type MasterResolver struct { dns *DNSResolver @@ -8,6 +13,14 @@ type MasterResolver struct { pro *ProquintResolver } +func NewMasterResolver(r routing.IpfsRouting, dag *mdag.DAGService) *MasterResolver { + mr := new(MasterResolver) + mr.dns = new(DNSResolver) + mr.pro = new(ProquintResolver) + mr.routing = NewRoutingResolver(r, dag) + return mr +} + func (mr *MasterResolver) Resolve(name string) (string, error) { if strings.Contains(name, ".") { return mr.dns.Resolve(name) diff --git a/namesys/routing.go b/namesys/routing.go index f37b6485fd0..ff0c2df397d 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -13,18 +13,17 @@ import ( mh "github.com/jbenet/go-multihash" ) -// RoutingName is the de-serialized name structure that is stored (serialized) -// in the routing system. Basically, a hash + a digital signature. (serialization can be -// protobuf, or a simple binary format) -type RoutingName struct { - Hash u.Key - Signature []byte -} - // RoutingResolver implements NSResolver for the main IPFS SFS-like naming type RoutingResolver struct { routing routing.IpfsRouting - dag mdag.DAGService + dag *mdag.DAGService +} + +func NewRoutingResolver(route routing.IpfsRouting, dagservice *mdag.DAGService) *RoutingResolver { + return &RoutingResolver{ + routing: route, + dag: dagservice, + } } func (r *RoutingResolver) Resolve(name string) (string, error) { @@ -47,7 +46,7 @@ func (r *RoutingResolver) Resolve(name string) (string, error) { return "", err } - entry := new(InpsEntry) + entry := new(IpnsEntry) err = proto.Unmarshal(val, entry) if err != nil { return "", err From 2aa2b8c6c7f591c629c855ab352e12e59a93f54c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 12 Sep 2014 17:34:07 +0000 Subject: [PATCH 007/105] add routing resolver test --- core/core.go | 4 ++- namesys/nsresolver.go | 2 +- namesys/resolve_test.go | 66 +++++++++++++++++++++++++++++++++++++++++ namesys/routing.go | 7 +++-- routing/dht/routing.go | 9 ++++-- 5 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 namesys/resolve_test.go diff --git a/core/core.go b/core/core.go index 005b6e62135..82cb3411683 100644 --- a/core/core.go +++ b/core/core.go @@ -61,7 +61,7 @@ type IpfsNode struct { Resolver *path.Resolver // the name system, resolves paths to hashes - Namesys namesys.NSResolver + Namesys namesys.Resolver } // NewIpfsNode constructs a new IpfsNode based on the given config. @@ -143,6 +143,7 @@ func NewIpfsNode(cfg *config.Config, online bool) (*IpfsNode, error) { } dag := &merkledag.DAGService{Blocks: bs} + resolve := namesys.NewMasterResolver(route, dag) success = true return &IpfsNode{ @@ -155,6 +156,7 @@ func NewIpfsNode(cfg *config.Config, online bool) (*IpfsNode, error) { Exchange: exchangeSession, Identity: local, Routing: route, + Namesys: resolve, }, nil } diff --git a/namesys/nsresolver.go b/namesys/nsresolver.go index ef5bc65cbd7..5dd9813306e 100644 --- a/namesys/nsresolver.go +++ b/namesys/nsresolver.go @@ -1,5 +1,5 @@ package namesys -type NSResolver interface { +type Resolver interface { Resolve(string) (string, error) } diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go new file mode 100644 index 00000000000..0620bd446e6 --- /dev/null +++ b/namesys/resolve_test.go @@ -0,0 +1,66 @@ +package namesys + +import ( + "testing" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + bs "github.com/jbenet/go-ipfs/blockservice" + ci "github.com/jbenet/go-ipfs/crypto" + mdag "github.com/jbenet/go-ipfs/merkledag" + "github.com/jbenet/go-ipfs/peer" + "github.com/jbenet/go-ipfs/routing/dht" + "github.com/jbenet/go-ipfs/swarm" + u "github.com/jbenet/go-ipfs/util" +) + +func TestRoutingResolve(t *testing.T) { + local := &peer.Peer{ + ID: []byte("testID"), + } + net := swarm.NewSwarm(local) + lds := ds.NewMapDatastore() + d := dht.NewDHT(local, net, lds) + + bserv, err := bs.NewBlockService(lds, nil) + if err != nil { + t.Fatal(err) + } + + dag := &mdag.DAGService{Blocks: bserv} + + resolve := NewMasterResolver(d, dag) + + pub := IpnsPublisher{ + dag: dag, + routing: d, + } + + privk, pubk, err := ci.GenerateKeyPair(ci.RSA, 512) + if err != nil { + t.Fatal(err) + } + + err = pub.Publish(privk, u.Key("Hello")) + if err != nil { + t.Fatal(err) + } + + pubkb, err := pubk.Bytes() + if err != nil { + t.Fatal(err) + } + + pkhash, err := u.Hash(pubkb) + if err != nil { + t.Fatal(err) + } + + res, err := resolve.Resolve(u.Key(pkhash).Pretty()) + if err != nil { + t.Fatal(err) + } + + if res != "Hello" { + t.Fatal("Got back incorrect value.") + } +} diff --git a/namesys/routing.go b/namesys/routing.go index ff0c2df397d..e667f5b68f1 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -29,6 +29,7 @@ func NewRoutingResolver(route routing.IpfsRouting, dagservice *mdag.DAGService) func (r *RoutingResolver) Resolve(name string) (string, error) { hash, err := mh.FromB58String(name) if err != nil { + u.DOut("RoutingResolve: bad input hash: [%s]\n", name) return "", err } // name should be a multihash. if it isn't, error out here. @@ -43,6 +44,7 @@ func (r *RoutingResolver) Resolve(name string) (string, error) { inpsKey := u.Key(h) val, err := r.routing.GetValue(inpsKey, time.Second*10) if err != nil { + u.DOut("RoutingResolve get failed.\n") return "", err } @@ -55,13 +57,14 @@ func (r *RoutingResolver) Resolve(name string) (string, error) { // name should be a public key retrievable from ipfs // /ipfs/ key := u.Key(hash) - node, err := r.dag.Get(key) + pkval, err := r.routing.GetValue(key, time.Second*10) if err != nil { + u.DOut("RoutingResolve PubKey Get failed.\n") return "", err } // get PublicKey from node.Data - pk, err := ci.UnmarshalPublicKey(node.Data) + pk, err := ci.UnmarshalPublicKey(pkval) if err != nil { return "", err } diff --git a/routing/dht/routing.go b/routing/dht/routing.go index 3d7a4bf0a76..b2c40380327 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -17,10 +17,13 @@ import ( // PutValue adds value corresponding to given Key. // This is the top level "Store" operation of the DHT -func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error { - peers := []*peer.Peer{} +func (dht *IpfsDHT) PutValue(key u.Key, value []byte) error { + err := dht.putLocal(key, value) + if err != nil { + return err + } - // get the peers we need to announce to + var peers []*peer.Peer for _, route := range dht.routingTables { npeers := route.NearestPeers(kb.ConvertKey(key), KValue) peers = append(peers, npeers...) From d1f011244f7991a9332a1f238be08a0e308b11e4 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 15 Sep 2014 20:32:37 +0000 Subject: [PATCH 008/105] address comments from the PR #45 --- crypto/key.go | 3 +++ crypto/rsa.go | 17 +++++++++++++++++ crypto/spipe/handshake.go | 4 ++-- namesys/dns.go | 4 ++++ namesys/nsresolver.go | 1 + namesys/proquint.go | 5 +++++ namesys/publisher.go | 4 ++-- namesys/resolver.go | 34 +++++++++++++++++++++------------- namesys/routing.go | 13 +++++++++---- 9 files changed, 64 insertions(+), 21 deletions(-) diff --git a/crypto/key.go b/crypto/key.go index f0a35c698ab..95cbfb44005 100644 --- a/crypto/key.go +++ b/crypto/key.go @@ -27,6 +27,9 @@ type Key interface { // Bytes returns a serialized, storeable representation of this key Bytes() ([]byte, error) + // Hash returns the hash of this key + Hash() ([]byte, error) + // Equals checks whether two PubKeys are the same Equals(Key) bool } diff --git a/crypto/rsa.go b/crypto/rsa.go index e582b59c297..124c3e0706f 100644 --- a/crypto/rsa.go +++ b/crypto/rsa.go @@ -9,6 +9,7 @@ import ( "errors" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + u "github.com/jbenet/go-ipfs/util" ) type RsaPrivateKey struct { @@ -46,6 +47,14 @@ func (pk *RsaPublicKey) Equals(k Key) bool { return KeyEqual(pk, k) } +func (pk *RsaPublicKey) Hash() ([]byte, error) { + pkb, err := pk.Bytes() + if err != nil { + return nil, err + } + return u.Hash(pkb) +} + func (sk *RsaPrivateKey) GenSecret() []byte { buf := make([]byte, 16) rand.Read(buf) @@ -75,6 +84,14 @@ func (sk *RsaPrivateKey) Equals(k Key) bool { return KeyEqual(sk, k) } +func (sk *RsaPrivateKey) Hash() ([]byte, error) { + skb, err := sk.Bytes() + if err != nil { + return nil, err + } + return u.Hash(skb) +} + func UnmarshalRsaPrivateKey(b []byte) (*RsaPrivateKey, error) { sk, err := x509.ParsePKCS1PrivateKey(b) if err != nil { diff --git a/crypto/spipe/handshake.go b/crypto/spipe/handshake.go index 67af992fdd0..dc9e2f7613a 100644 --- a/crypto/spipe/handshake.go +++ b/crypto/spipe/handshake.go @@ -230,7 +230,7 @@ func (s *SecurePipe) handleSecureIn(hashType string, tIV, tCKey, tMKey []byte) { data, ok := <-s.insecure.In if !ok { u.DOut("Closing incoming proxy.\n") - close(secureIn) + close(s.In) return } @@ -268,7 +268,7 @@ func (s *SecurePipe) handleSecureOut(hashType string, mIV, mCKey, mMKey []byte) data, ok := <-s.Out if !ok { u.DOut("Closing outgoing proxy.\n") - close(secureOut) + close(s.Out) return } diff --git a/namesys/dns.go b/namesys/dns.go index b12bc0d38a0..51e99ed78b7 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -12,6 +12,10 @@ type DNSResolver struct { // cache would need a timeout } +func (r *DNSResolver) Matches(name string) bool { + return strings.Contains(name, ".") +} + func (r *DNSResolver) Resolve(name string) (string, error) { txt, err := net.LookupTXT(name) if err != nil { diff --git a/namesys/nsresolver.go b/namesys/nsresolver.go index 5dd9813306e..c8d40dcb3db 100644 --- a/namesys/nsresolver.go +++ b/namesys/nsresolver.go @@ -2,4 +2,5 @@ package namesys type Resolver interface { Resolve(string) (string, error) + Matches(string) bool } diff --git a/namesys/proquint.go b/namesys/proquint.go index 8583a539080..8c4db27994c 100644 --- a/namesys/proquint.go +++ b/namesys/proquint.go @@ -10,6 +10,11 @@ var _ = proquint.Encode type ProquintResolver struct{} +func (r *ProquintResolver) Matches(name string) bool { + ok, err := proquint.IsProquint(name) + return err == nil && ok +} + func (r *ProquintResolver) Resolve(name string) (string, error) { ok, err := proquint.IsProquint(name) if err != nil { diff --git a/namesys/publisher.go b/namesys/publisher.go index 4588ccb6a89..d64d7954b86 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -32,7 +32,7 @@ func (p *IpnsPublisher) Publish(k ci.PrivKey, value u.Key) error { } namekey := u.Key(nameb).Pretty() - ipnskey, err := u.Hash([]byte("ipns:" + namekey)) + ipnskey, err := u.Hash([]byte("/ipns/" + namekey)) if err != nil { return err } @@ -43,7 +43,7 @@ func (p *IpnsPublisher) Publish(k ci.PrivKey, value u.Key) error { return err } - // Store ipns entry at h("ipns:"+b58(h(pubkey))) + // Store ipns entry at h("/ipns/"+b58(h(pubkey))) err = p.routing.PutValue(u.Key(ipnskey), data) if err != nil { return err diff --git a/namesys/resolver.go b/namesys/resolver.go index e440946b985..2fedf210c09 100644 --- a/namesys/resolver.go +++ b/namesys/resolver.go @@ -1,34 +1,42 @@ package namesys import ( - "strings" + "errors" mdag "github.com/jbenet/go-ipfs/merkledag" "github.com/jbenet/go-ipfs/routing" ) +var ErrCouldntResolve = errors.New("could not resolve name.") + type MasterResolver struct { - dns *DNSResolver - routing *RoutingResolver - pro *ProquintResolver + res []Resolver } func NewMasterResolver(r routing.IpfsRouting, dag *mdag.DAGService) *MasterResolver { mr := new(MasterResolver) - mr.dns = new(DNSResolver) - mr.pro = new(ProquintResolver) - mr.routing = NewRoutingResolver(r, dag) + mr.res = []Resolver{ + new(DNSResolver), + new(ProquintResolver), + NewRoutingResolver(r, dag), + } return mr } func (mr *MasterResolver) Resolve(name string) (string, error) { - if strings.Contains(name, ".") { - return mr.dns.Resolve(name) + for _, r := range mr.res { + if r.Matches(name) { + return r.Resolve(name) + } } + return "", ErrCouldntResolve +} - if strings.Contains(name, "-") { - return mr.pro.Resolve(name) +func (mr *MasterResolver) Matches(name string) bool { + for _, r := range mr.res { + if r.Matches(name) { + return true + } } - - return mr.routing.Resolve(name) + return false } diff --git a/namesys/routing.go b/namesys/routing.go index e667f5b68f1..5f85e942fb5 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -26,6 +26,11 @@ func NewRoutingResolver(route routing.IpfsRouting, dagservice *mdag.DAGService) } } +func (r *RoutingResolver) Matches(name string) bool { + _, err := mh.FromB58String(name) + return err == nil +} + func (r *RoutingResolver) Resolve(name string) (string, error) { hash, err := mh.FromB58String(name) if err != nil { @@ -36,13 +41,13 @@ func (r *RoutingResolver) Resolve(name string) (string, error) { // use the routing system to get the name. // /ipns/ - h, err := u.Hash([]byte("ipns:" + name)) + h, err := u.Hash([]byte("/ipns/" + name)) if err != nil { return "", err } - inpsKey := u.Key(h) - val, err := r.routing.GetValue(inpsKey, time.Second*10) + ipnsKey := u.Key(h) + val, err := r.routing.GetValue(ipnsKey, time.Second*10) if err != nil { u.DOut("RoutingResolve get failed.\n") return "", err @@ -70,7 +75,7 @@ func (r *RoutingResolver) Resolve(name string) (string, error) { } // check sig with pk - if ok, err := pk.Verify(entry.GetValue(), entry.GetSignature()); err != nil && ok { + if ok, err := pk.Verify(entry.GetValue(), entry.GetSignature()); err != nil || !ok { return "", fmt.Errorf("Invalid value. Not signed by PrivateKey corresponding to %v", pk) } From 1a3886597d2ce4c0af30a14d7d4d7a5d4753b495 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 16 Sep 2014 03:44:53 +0000 Subject: [PATCH 009/105] Address concerns from PR --- namesys/proquint.go | 7 +------ namesys/resolver.go | 10 +++++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/namesys/proquint.go b/namesys/proquint.go index 8c4db27994c..9b64f3fde51 100644 --- a/namesys/proquint.go +++ b/namesys/proquint.go @@ -6,8 +6,6 @@ import ( proquint "github.com/bren2010/proquint" ) -var _ = proquint.Encode - type ProquintResolver struct{} func (r *ProquintResolver) Matches(name string) bool { @@ -16,10 +14,7 @@ func (r *ProquintResolver) Matches(name string) bool { } func (r *ProquintResolver) Resolve(name string) (string, error) { - ok, err := proquint.IsProquint(name) - if err != nil { - return "", err - } + ok := r.Matches(name) if !ok { return "", errors.New("not a valid proquint string") } diff --git a/namesys/resolver.go b/namesys/resolver.go index 2fedf210c09..7765a4ba074 100644 --- a/namesys/resolver.go +++ b/namesys/resolver.go @@ -9,12 +9,12 @@ import ( var ErrCouldntResolve = errors.New("could not resolve name.") -type MasterResolver struct { +type masterResolver struct { res []Resolver } -func NewMasterResolver(r routing.IpfsRouting, dag *mdag.DAGService) *MasterResolver { - mr := new(MasterResolver) +func NewMasterResolver(r routing.IpfsRouting, dag *mdag.DAGService) Resolver { + mr := new(masterResolver) mr.res = []Resolver{ new(DNSResolver), new(ProquintResolver), @@ -23,7 +23,7 @@ func NewMasterResolver(r routing.IpfsRouting, dag *mdag.DAGService) *MasterResol return mr } -func (mr *MasterResolver) Resolve(name string) (string, error) { +func (mr *masterResolver) Resolve(name string) (string, error) { for _, r := range mr.res { if r.Matches(name) { return r.Resolve(name) @@ -32,7 +32,7 @@ func (mr *MasterResolver) Resolve(name string) (string, error) { return "", ErrCouldntResolve } -func (mr *MasterResolver) Matches(name string) bool { +func (mr *masterResolver) Matches(name string) bool { for _, r := range mr.res { if r.Matches(name) { return true From a65e8600a3e40c0238db17eac0fe0eb71bb1b7cd Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 22 Sep 2014 15:14:19 -0700 Subject: [PATCH 010/105] catch ipns branch up to master and make all things compile --- cmd/ipfs/ipfs.go | 1 - cmd/ipfs/pin.go | 21 +++++++++++++-------- cmd/ipfs/run.go | 21 +++++++++++++++++++-- crypto/spipe/handshake.go | 5 +---- crypto/spipe/pipe.go | 1 - fuse/readonly/readonly_unix.go | 2 +- namesys/dns.go | 18 ++++++++++++------ namesys/publisher.go | 6 ++++-- namesys/resolve_test.go | 2 +- namesys/routing.go | 16 ++++++++++------ routing/dht/dht.go | 1 + routing/dht/routing.go | 16 ++++++---------- 12 files changed, 68 insertions(+), 42 deletions(-) diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 1627131d02d..ed8c26a5823 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "os" - "runtime" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" diff --git a/cmd/ipfs/pin.go b/cmd/ipfs/pin.go index 88f8e850563..bea074c5103 100644 --- a/cmd/ipfs/pin.go +++ b/cmd/ipfs/pin.go @@ -3,8 +3,9 @@ package main import ( "os" - "github.com/gonuts/flag" - "github.com/jbenet/commander" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" + "github.com/jbenet/go-ipfs/core/commands" "github.com/jbenet/go-ipfs/daemon" u "github.com/jbenet/go-ipfs/util" ) @@ -27,18 +28,22 @@ func pinCmd(c *commander.Command, inp []string) error { return nil } - com := daemon.NewCommand() - com.Command = "pin" - com.Args = inp + cmd := daemon.NewCommand() + cmd.Command = "pin" + cmd.Args = inp - err := daemon.SendCommand(com, "localhost:12345") + err := daemon.SendCommand(cmd, "localhost:12345") if err != nil { - n, err := localNode(false) + conf, err := getConfigDir(c.Parent) + if err != nil { + return err + } + n, err := localNode(conf, false) if err != nil { return err } - daemon.ExecuteCommand(com, n, os.Stdout) + return commands.Pin(n, cmd.Args, cmd.Opts, os.Stdout) } return nil } diff --git a/cmd/ipfs/run.go b/cmd/ipfs/run.go index 94158ecd435..803647c77c8 100644 --- a/cmd/ipfs/run.go +++ b/cmd/ipfs/run.go @@ -1,8 +1,11 @@ package main import ( + "errors" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" + ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" "github.com/jbenet/go-ipfs/daemon" u "github.com/jbenet/go-ipfs/util" ) @@ -19,12 +22,26 @@ var cmdIpfsRun = &commander.Command{ func runCmd(c *commander.Command, inp []string) error { u.Debug = true - n, err := localNode(true) + conf, err := getConfigDir(c.Parent) + if err != nil { + return err + } + n, err := localNode(conf, true) + if err != nil { + return err + } + + // launch the RPC endpoint. + if n.Config.RPCAddress == "" { + return errors.New("no config.RPCAddress endpoint supplied") + } + + maddr, err := ma.NewMultiaddr(n.Config.RPCAddress) if err != nil { return err } - dl, err := daemon.NewDaemonListener(n, "localhost:12345") + dl, err := daemon.NewDaemonListener(n, maddr) if err != nil { return err } diff --git a/crypto/spipe/handshake.go b/crypto/spipe/handshake.go index dc9e2f7613a..f42f13aaa63 100644 --- a/crypto/spipe/handshake.go +++ b/crypto/spipe/handshake.go @@ -229,8 +229,7 @@ func (s *SecurePipe) handleSecureIn(hashType string, tIV, tCKey, tMKey []byte) { for { data, ok := <-s.insecure.In if !ok { - u.DOut("Closing incoming proxy.\n") - close(s.In) + close(s.Duplex.In) return } @@ -267,8 +266,6 @@ func (s *SecurePipe) handleSecureOut(hashType string, mIV, mCKey, mMKey []byte) for { data, ok := <-s.Out if !ok { - u.DOut("Closing outgoing proxy.\n") - close(s.Out) return } diff --git a/crypto/spipe/pipe.go b/crypto/spipe/pipe.go index 8d0db0d5dbe..7f9ccc30f62 100644 --- a/crypto/spipe/pipe.go +++ b/crypto/spipe/pipe.go @@ -82,6 +82,5 @@ func (s *SecurePipe) Close() error { s.cancel() s.cancel = nil - close(s.In) return nil } diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 0ce5fab4cf8..54eac2bbcbd 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -168,7 +168,7 @@ func Mount(ipfs *core.IpfsNode, fpath string) error { } time.Sleep(time.Millisecond * 10) } - ipfs.Swarm.Close() + ipfs.Network.Close() }() c, err := fuse.Mount(fpath) diff --git a/namesys/dns.go b/namesys/dns.go index 51e99ed78b7..55d00f94514 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -4,6 +4,8 @@ import ( "net" "strings" + b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" + mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" u "github.com/jbenet/go-ipfs/util" ) @@ -16,6 +18,8 @@ func (r *DNSResolver) Matches(name string) bool { return strings.Contains(name, ".") } +// TXT records for a given domain name should contain a b58 +// encoded multihash. func (r *DNSResolver) Resolve(name string) (string, error) { txt, err := net.LookupTXT(name) if err != nil { @@ -23,15 +27,17 @@ func (r *DNSResolver) Resolve(name string) (string, error) { } for _, t := range txt { - pair := strings.Split(t, "=") - if len(pair) < 2 { - // Log error? - u.DErr("Incorrectly formatted text record.") + chk := b58.Decode(t) + if len(chk) == 0 { continue } - if pair[0] == name { - return pair[1], nil + + _, err := mh.Cast(chk) + if err != nil { + continue } + return t, nil } + return "", u.ErrNotFound } diff --git a/namesys/publisher.go b/namesys/publisher.go index d64d7954b86..83f418a5714 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -1,6 +1,7 @@ package namesys import ( + "code.google.com/p/go.net/context" "code.google.com/p/goprotobuf/proto" ci "github.com/jbenet/go-ipfs/crypto" @@ -16,6 +17,7 @@ type IpnsPublisher struct { // Publish accepts a keypair and a value, func (p *IpnsPublisher) Publish(k ci.PrivKey, value u.Key) error { + ctx := context.TODO() data, err := CreateEntryData(k, value) if err != nil { return err @@ -38,13 +40,13 @@ func (p *IpnsPublisher) Publish(k ci.PrivKey, value u.Key) error { } // Store associated public key - err = p.routing.PutValue(u.Key(nameb), pkbytes) + err = p.routing.PutValue(ctx, u.Key(nameb), pkbytes) if err != nil { return err } // Store ipns entry at h("/ipns/"+b58(h(pubkey))) - err = p.routing.PutValue(u.Key(ipnskey), data) + err = p.routing.PutValue(ctx, u.Key(ipnskey), data) if err != nil { return err } diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index 0620bd446e6..32a12cf76aa 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -7,9 +7,9 @@ import ( bs "github.com/jbenet/go-ipfs/blockservice" ci "github.com/jbenet/go-ipfs/crypto" mdag "github.com/jbenet/go-ipfs/merkledag" + "github.com/jbenet/go-ipfs/net/swarm" "github.com/jbenet/go-ipfs/peer" "github.com/jbenet/go-ipfs/routing/dht" - "github.com/jbenet/go-ipfs/swarm" u "github.com/jbenet/go-ipfs/util" ) diff --git a/namesys/routing.go b/namesys/routing.go index 5f85e942fb5..2b1d06c8064 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -2,8 +2,8 @@ package namesys import ( "fmt" - "time" + "code.google.com/p/go.net/context" "code.google.com/p/goprotobuf/proto" ci "github.com/jbenet/go-ipfs/crypto" @@ -11,8 +11,11 @@ import ( "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" mh "github.com/jbenet/go-multihash" + "github.com/op/go-logging" ) +var log = logging.MustGetLogger("namesys") + // RoutingResolver implements NSResolver for the main IPFS SFS-like naming type RoutingResolver struct { routing routing.IpfsRouting @@ -32,9 +35,10 @@ func (r *RoutingResolver) Matches(name string) bool { } func (r *RoutingResolver) Resolve(name string) (string, error) { + ctx := context.TODO() hash, err := mh.FromB58String(name) if err != nil { - u.DOut("RoutingResolve: bad input hash: [%s]\n", name) + log.Warning("RoutingResolve: bad input hash: [%s]\n", name) return "", err } // name should be a multihash. if it isn't, error out here. @@ -47,9 +51,9 @@ func (r *RoutingResolver) Resolve(name string) (string, error) { } ipnsKey := u.Key(h) - val, err := r.routing.GetValue(ipnsKey, time.Second*10) + val, err := r.routing.GetValue(ctx, ipnsKey) if err != nil { - u.DOut("RoutingResolve get failed.\n") + log.Warning("RoutingResolve get failed.") return "", err } @@ -62,9 +66,9 @@ func (r *RoutingResolver) Resolve(name string) (string, error) { // name should be a public key retrievable from ipfs // /ipfs/ key := u.Key(hash) - pkval, err := r.routing.GetValue(key, time.Second*10) + pkval, err := r.routing.GetValue(ctx, key) if err != nil { - u.DOut("RoutingResolve PubKey Get failed.\n") + log.Warning("RoutingResolve PubKey Get failed.") return "", err } diff --git a/routing/dht/dht.go b/routing/dht/dht.go index 8ebecd5bd88..11712338b55 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -345,6 +345,7 @@ func (dht *IpfsDHT) putLocal(key u.Key, value []byte) error { // Update signals to all routingTables to Update their last-seen status // on the given peer. func (dht *IpfsDHT) Update(p *peer.Peer) { + u.DOut("updating peer: [%s] latency = %f\n", p.ID.Pretty(), p.GetLatency().Seconds()) removedCount := 0 for _, route := range dht.routingTables { removed := route.Update(p) diff --git a/routing/dht/routing.go b/routing/dht/routing.go index b2c40380327..778aaba7586 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -17,7 +17,7 @@ import ( // PutValue adds value corresponding to given Key. // This is the top level "Store" operation of the DHT -func (dht *IpfsDHT) PutValue(key u.Key, value []byte) error { +func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error { err := dht.putLocal(key, value) if err != nil { return err @@ -38,7 +38,7 @@ func (dht *IpfsDHT) PutValue(key u.Key, value []byte) error { return &dhtQueryResult{success: true}, nil }) - _, err := query.Run(ctx, peers) + _, err = query.Run(ctx, peers) u.DOut("[%s] PutValue %v %v\n", dht.self.ID.Pretty(), key, value) return err } @@ -104,8 +104,7 @@ func (dht *IpfsDHT) Provide(ctx context.Context, key u.Key) error { dht.providers.AddProvider(key, dht.self) peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), PoolSize) if len(peers) == 0 { - // Early out for no targets - return nil + return kb.ErrLookupFailure } //TODO FIX: this doesn't work! it needs to be sent to the actual nearest peers. @@ -156,12 +155,12 @@ func (dht *IpfsDHT) FindProvidersAsync2(ctx context.Context, key u.Key, count in peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), AlphaValue) for _, pp := range peers { go func(p *peer.Peer) { - pmes, err := dht.findProvidersSingle(p, key, 0, timeout) + pmes, err := dht.findProvidersSingle(ctx, p, key, 0) if err != nil { u.PErr("%v\n", err) return } - dht.addPeerListAsync(key, pmes.GetPeers(), ps, count, peerOut) + dht.addPeerListAsync(key, pmes.GetProviderPeers(), ps, count, peerOut) }(pp) } @@ -190,11 +189,8 @@ func (dht *IpfsDHT) addPeerListAsync(k u.Key, peers []*Message_Peer, ps *peerSet // FindProviders searches for peers who can provide the value for given key. func (dht *IpfsDHT) FindProviders(ctx context.Context, key u.Key) ([]*peer.Peer, error) { - ll := startNewRPC("FindProviders") - ll.EndAndPrint() - // get closest peer - u.DOut("Find providers for: '%s'\n", key) + u.DOut("Find providers for: '%s'\n", key.Pretty()) p := dht.routingTables[0].NearestPeer(kb.ConvertKey(key)) if p == nil { return nil, nil From 16aa636273355f0c9c71e82d9c265e4823cc8372 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 25 Sep 2014 11:23:52 -0700 Subject: [PATCH 011/105] allow ipns to be mounted by mount command --- cmd/ipfs/mount_unix.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmd/ipfs/mount_unix.go b/cmd/ipfs/mount_unix.go index 58523230a56..8bec0251e20 100644 --- a/cmd/ipfs/mount_unix.go +++ b/cmd/ipfs/mount_unix.go @@ -11,6 +11,7 @@ import ( ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" "github.com/jbenet/go-ipfs/daemon" + ipns "github.com/jbenet/go-ipfs/fuse/ipns" rofs "github.com/jbenet/go-ipfs/fuse/readonly" u "github.com/jbenet/go-ipfs/util" ) @@ -29,6 +30,10 @@ var cmdIpfsMount = &commander.Command{ Flag: *flag.NewFlagSet("ipfs-mount", flag.ExitOnError), } +func init() { + cmdIpfsMount.Flag.String("n", "", "specify a mountpoint for ipns") +} + func mountCmd(c *commander.Command, inp []string) error { if len(inp) < 1 || len(inp[0]) == 0 { u.POut(c.Long) @@ -66,5 +71,16 @@ func mountCmd(c *commander.Command, inp []string) error { mp := inp[0] fmt.Printf("Mounting at %s\n", mp) + + ns, ok := c.Flag.Lookup("n").Value.Get().(string) + if ok { + go func() { + err = ipns.Mount(n, ns, mp) + if err != nil { + fmt.Printf("Error mounting ipns: %s\n", err) + } + }() + } + return rofs.Mount(n, mp) } From 375a38c5f77aea82ef5677a8bae5e7d1d7e4118b Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 25 Sep 2014 15:10:57 -0700 Subject: [PATCH 012/105] add basic publish command, needs polish --- cmd/ipfs/ipfs.go | 2 ++ cmd/ipfs/run.go | 2 +- daemon/daemon.go | 2 ++ namesys/publisher.go | 16 ++++++++++++++-- routing/dht/dht.go | 3 +++ routing/dht/query.go | 7 ++++++- routing/dht/routing.go | 4 ++-- 7 files changed, 30 insertions(+), 6 deletions(-) diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index ed8c26a5823..69e5ee5df6a 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "runtime/pprof" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" @@ -51,6 +52,7 @@ Use "ipfs help " for more information about a command. cmdIpfsInit, cmdIpfsServe, cmdIpfsRun, + cmdIpfsPub, }, Flag: *flag.NewFlagSet("ipfs", flag.ExitOnError), } diff --git a/cmd/ipfs/run.go b/cmd/ipfs/run.go index 803647c77c8..a3010cb0750 100644 --- a/cmd/ipfs/run.go +++ b/cmd/ipfs/run.go @@ -41,7 +41,7 @@ func runCmd(c *commander.Command, inp []string) error { return err } - dl, err := daemon.NewDaemonListener(n, maddr) + dl, err := daemon.NewDaemonListener(n, maddr, conf) if err != nil { return err } diff --git a/daemon/daemon.go b/daemon/daemon.go index 8180131bad0..755de7df931 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -131,6 +131,8 @@ func (dl *DaemonListener) handleConnection(conn net.Conn) { err = commands.Ls(dl.node, command.Args, command.Opts, conn) case "pin": err = commands.Pin(dl.node, command.Args, command.Opts, conn) + case "publish": + err = commands.Publish(dl.node, command.Args, command.Opts, conn) default: err = fmt.Errorf("Invalid Command: '%s'", command.Command) } diff --git a/namesys/publisher.go b/namesys/publisher.go index 83f418a5714..67830eb6bea 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -1,6 +1,8 @@ package namesys import ( + "time" + "code.google.com/p/go.net/context" "code.google.com/p/goprotobuf/proto" @@ -15,8 +17,16 @@ type IpnsPublisher struct { routing routing.IpfsRouting } +func NewPublisher(dag *mdag.DAGService, route routing.IpfsRouting) *IpnsPublisher { + return &IpnsPublisher{ + dag: dag, + routing: route, + } +} + // Publish accepts a keypair and a value, func (p *IpnsPublisher) Publish(k ci.PrivKey, value u.Key) error { + log.Debug("namesys: Publish %s", value.Pretty()) ctx := context.TODO() data, err := CreateEntryData(k, value) if err != nil { @@ -40,13 +50,15 @@ func (p *IpnsPublisher) Publish(k ci.PrivKey, value u.Key) error { } // Store associated public key - err = p.routing.PutValue(ctx, u.Key(nameb), pkbytes) + timectx, _ := context.WithDeadline(ctx, time.Now().Add(time.Second*4)) + err = p.routing.PutValue(timectx, u.Key(nameb), pkbytes) if err != nil { return err } // Store ipns entry at h("/ipns/"+b58(h(pubkey))) - err = p.routing.PutValue(ctx, u.Key(ipnskey), data) + timectx, _ = context.WithDeadline(ctx, time.Now().Add(time.Second*4)) + err = p.routing.PutValue(timectx, u.Key(ipnskey), data) if err != nil { return err } diff --git a/routing/dht/dht.go b/routing/dht/dht.go index 11712338b55..f9f52b10828 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -13,6 +13,7 @@ import ( peer "github.com/jbenet/go-ipfs/peer" kb "github.com/jbenet/go-ipfs/routing/kbucket" u "github.com/jbenet/go-ipfs/util" + "github.com/op/go-logging" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" @@ -21,6 +22,8 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ) +var log = logging.MustGetLogger("dht") + // TODO. SEE https://github.com/jbenet/node-ipfs/blob/master/submodules/ipfs-dht/index.js // IpfsDHT is an implementation of Kademlia with Coral and S/Kademlia modifications. diff --git a/routing/dht/query.go b/routing/dht/query.go index 4db3f70e723..cc709d7e9b1 100644 --- a/routing/dht/query.go +++ b/routing/dht/query.go @@ -101,6 +101,11 @@ func newQueryRunner(ctx context.Context, q *dhtQuery) *dhtQueryRunner { } func (r *dhtQueryRunner) Run(peers []*peer.Peer) (*dhtQueryResult, error) { + log.Debug("Run query with %d peers.", len(peers)) + if len(peers) == 0 { + log.Warning("Running query with no peers!") + return nil, nil + } // setup concurrency rate limiting for i := 0; i < r.query.concurrency; i++ { r.rateLimit <- struct{}{} @@ -164,7 +169,7 @@ func (r *dhtQueryRunner) addPeerToQuery(next *peer.Peer, benchmark *peer.Peer) { r.peersSeen[next.Key()] = next r.Unlock() - u.DOut("adding peer to query: %v\n", next.ID.Pretty()) + log.Debug("adding peer to query: %v\n", next.ID.Pretty()) // do this after unlocking to prevent possible deadlocks. r.peersRemaining.Increment(1) diff --git a/routing/dht/routing.go b/routing/dht/routing.go index 778aaba7586..6a2a0cdcb6b 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -18,6 +18,7 @@ import ( // PutValue adds value corresponding to given Key. // This is the top level "Store" operation of the DHT func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error { + log.Debug("[%s] PutValue %v %v", dht.self.ID.Pretty(), key, value) err := dht.putLocal(key, value) if err != nil { return err @@ -30,7 +31,7 @@ func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error } query := newQuery(key, func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) { - u.DOut("[%s] PutValue qry part %v\n", dht.self.ID.Pretty(), p.ID.Pretty()) + log.Debug("[%s] PutValue qry part %v", dht.self.ID.Pretty(), p.ID.Pretty()) err := dht.putValueToNetwork(ctx, p, string(key), value) if err != nil { return nil, err @@ -39,7 +40,6 @@ func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error }) _, err = query.Run(ctx, peers) - u.DOut("[%s] PutValue %v %v\n", dht.self.ID.Pretty(), key, value) return err } From e61c59758bba88d13d0f0f9e81f6686b513313b9 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 25 Sep 2014 18:29:05 -0700 Subject: [PATCH 013/105] implement initial ipns filesystem interface as well as plumbing command for publishing --- cmd/ipfs/publish.go | 59 ++++++++ core/commands/publish.go | 39 +++++ fuse/ipns/ipns_unix.go | 316 +++++++++++++++++++++++++++++++++++++++ routing/dht/query.go | 1 + routing/dht/routing.go | 5 +- 5 files changed, 417 insertions(+), 3 deletions(-) create mode 100644 cmd/ipfs/publish.go create mode 100644 core/commands/publish.go create mode 100644 fuse/ipns/ipns_unix.go diff --git a/cmd/ipfs/publish.go b/cmd/ipfs/publish.go new file mode 100644 index 00000000000..649767614da --- /dev/null +++ b/cmd/ipfs/publish.go @@ -0,0 +1,59 @@ +package main + +import ( + "os" + + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" + "github.com/jbenet/go-ipfs/core/commands" + "github.com/jbenet/go-ipfs/daemon" + u "github.com/jbenet/go-ipfs/util" +) + +var cmdIpfsPub = &commander.Command{ + UsageLine: "publish", + Short: "Publish an object to ipns under your key.", + Long: `ipfs publish - Publish object to ipns. + +`, + Run: pubCmd, + Flag: *flag.NewFlagSet("ipfs-publish", flag.ExitOnError), +} + +func init() { + cmdIpfsPub.Flag.String("k", "", "Specify key to use for publishing.") +} + +func pubCmd(c *commander.Command, inp []string) error { + u.Debug = true + if len(inp) < 1 { + u.POut(c.Long) + return nil + } + + conf, err := getConfigDir(c.Parent) + if err != nil { + return err + } + + cmd := daemon.NewCommand() + cmd.Command = "publish" + cmd.Args = inp + cmd.Opts["k"] = c.Flag.Lookup("k").Value.Get() + err = daemon.SendCommand(cmd, conf) + if err != nil { + u.DOut("Executing command locally.\n") + // Do locally + conf, err := getConfigDir(c.Parent) + if err != nil { + return err + } + n, err := localNode(conf, true) + if err != nil { + return err + } + + return commands.Publish(n, cmd.Args, cmd.Opts, os.Stdout) + } + return nil +} diff --git a/core/commands/publish.go b/core/commands/publish.go new file mode 100644 index 00000000000..8fae4235d95 --- /dev/null +++ b/core/commands/publish.go @@ -0,0 +1,39 @@ +package commands + +import ( + "errors" + "fmt" + "io" + + "github.com/jbenet/go-ipfs/core" + u "github.com/jbenet/go-ipfs/util" + "github.com/op/go-logging" + + nsys "github.com/jbenet/go-ipfs/namesys" +) + +var log = logging.MustGetLogger("commands") + +func Publish(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error { + log.Debug("Begin Publish") + if n.Identity == nil { + return errors.New("Identity not loaded!") + } + + k := n.Identity.PrivKey + val := u.Key(args[0]) + + pub := nsys.NewPublisher(n.DAG, n.Routing) + err := pub.Publish(k, val) + if err != nil { + return err + } + + hash, err := k.GetPublic().Hash() + if err != nil { + return err + } + fmt.Fprintf(out, "Published %s to %s\n", val, u.Key(hash).Pretty()) + + return nil +} diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go new file mode 100644 index 00000000000..2713538aef9 --- /dev/null +++ b/fuse/ipns/ipns_unix.go @@ -0,0 +1,316 @@ +package ipns + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "syscall" + "time" + + "bazil.org/fuse" + "bazil.org/fuse/fs" + "code.google.com/p/goprotobuf/proto" + "github.com/jbenet/go-ipfs/core" + ci "github.com/jbenet/go-ipfs/crypto" + mdag "github.com/jbenet/go-ipfs/merkledag" + u "github.com/jbenet/go-ipfs/util" + "github.com/op/go-logging" +) + +var log = logging.MustGetLogger("ipns") + +// FileSystem is the readonly Ipfs Fuse Filesystem. +type FileSystem struct { + Ipfs *core.IpfsNode + RootNode *Root +} + +// NewFileSystem constructs new fs using given core.IpfsNode instance. +func NewIpns(ipfs *core.IpfsNode, ipfspath string) (*FileSystem, error) { + root, err := CreateRoot(ipfs, []ci.PrivKey{ipfs.Identity.PrivKey}, ipfspath) + if err != nil { + return nil, err + } + return &FileSystem{Ipfs: ipfs, RootNode: root}, nil +} + +func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, error) { + root := new(Root) + root.LocalDirs = make(map[string]*Node) + root.Ipfs = n + abspath, err := filepath.Abs(ipfsroot) + if err != nil { + return nil, err + } + root.IpfsRoot = abspath + + root.Keys = keys + + if len(keys) == 0 { + log.Warning("No keys given for ipns root creation") + } else { + k := keys[0] + pub := k.GetPublic() + hash, err := pub.Hash() + if err != nil { + log.Error("Read Root Error: %s", err) + return nil, err + } + root.LocalLink = &Link{u.Key(hash).Pretty()} + } + + return root, nil +} + +// Root constructs the Root of the filesystem, a Root object. +func (f FileSystem) Root() (fs.Node, fuse.Error) { + return f.RootNode, nil +} + +// Root is the root object of the filesystem tree. +type Root struct { + Ipfs *core.IpfsNode + Keys []ci.PrivKey + + // Used for symlinking into ipfs + IpfsRoot string + LocalDirs map[string]*Node + + LocalLink *Link +} + +// Attr returns file attributes. +func (*Root) Attr() fuse.Attr { + return fuse.Attr{Mode: os.ModeDir | 0111} // -rw+x +} + +// Lookup performs a lookup under this node. +func (s *Root) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { + log.Debug("ipns: Root Lookup: '%s'", name) + switch name { + case "mach_kernel", ".hidden", "._.": + // Just quiet some log noise on OS X. + return nil, fuse.ENOENT + } + + if name == "local" { + if s.LocalLink == nil { + return nil, fuse.ENOENT + } + return s.LocalLink, nil + } + + nd, ok := s.LocalDirs[name] + if ok { + return nd, nil + } + + log.Debug("ipns: Falling back to resolution.") + resolved, err := s.Ipfs.Namesys.Resolve(name) + if err != nil { + log.Error("ipns: namesys resolve error: %s", err) + return nil, fuse.ENOENT + } + + return &Link{s.IpfsRoot + "/" + resolved}, nil +} + +// ReadDir reads a particular directory. Disallowed for root. +func (r *Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { + u.DOut("Read Root.\n") + listing := []fuse.Dirent{ + fuse.Dirent{ + Name: "local", + Type: fuse.DT_Link, + }, + } + for _, k := range r.Keys { + pub := k.GetPublic() + hash, err := pub.Hash() + if err != nil { + log.Error("Read Root Error: %s", err) + continue + } + ent := fuse.Dirent{ + Name: u.Key(hash).Pretty(), + Type: fuse.DT_Dir, + } + listing = append(listing, ent) + } + return listing, nil +} + +// Node is the core object representing a filesystem tree node. +type Node struct { + nsRoot *Node + Ipfs *core.IpfsNode + Nd *mdag.Node + fd *mdag.DagReader + cached *mdag.PBData +} + +func (s *Node) loadData() error { + s.cached = new(mdag.PBData) + return proto.Unmarshal(s.Nd.Data, s.cached) +} + +// Attr returns the attributes of a given node. +func (s *Node) Attr() fuse.Attr { + u.DOut("Node attr.\n") + if s.cached == nil { + s.loadData() + } + switch s.cached.GetType() { + case mdag.PBData_Directory: + u.DOut("this is a directory.\n") + return fuse.Attr{Mode: os.ModeDir | 0555} + case mdag.PBData_File, mdag.PBData_Raw: + u.DOut("this is a file.\n") + size, _ := s.Nd.Size() + return fuse.Attr{ + Mode: 0444, + Size: uint64(size), + Blocks: uint64(len(s.Nd.Links)), + } + default: + u.PErr("Invalid data type.") + return fuse.Attr{} + } +} + +// Lookup performs a lookup under this node. +func (s *Node) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { + u.DOut("Lookup '%s'\n", name) + nd, err := s.Ipfs.Resolver.ResolveLinks(s.Nd, []string{name}) + if err != nil { + // todo: make this error more versatile. + return nil, fuse.ENOENT + } + + return &Node{Ipfs: s.Ipfs, Nd: nd}, nil +} + +// ReadDir reads the link structure as directory entries +func (s *Node) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { + u.DOut("Node ReadDir\n") + entries := make([]fuse.Dirent, len(s.Nd.Links)) + for i, link := range s.Nd.Links { + n := link.Name + if len(n) == 0 { + n = link.Hash.B58String() + } + entries[i] = fuse.Dirent{Name: n, Type: fuse.DT_File} + } + + if len(entries) > 0 { + return entries, nil + } + return nil, fuse.ENOENT +} + +// ReadAll reads the object data as file data +func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { + log.Debug("ipns: ReadAll Node") + r, err := mdag.NewDagReader(s.Nd, s.Ipfs.DAG) + if err != nil { + return nil, err + } + // this is a terrible function... 'ReadAll'? + // what if i have a 6TB file? GG RAM. + return ioutil.ReadAll(r) +} + +// Mount mounts an IpfsNode instance at a particular path. It +// serves until the process receives exit signals (to Unmount). +func Mount(ipfs *core.IpfsNode, fpath string, ipfspath string) error { + + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT, + syscall.SIGTERM, syscall.SIGQUIT) + + go func() { + <-sigc + for { + err := Unmount(fpath) + if err == nil { + return + } + time.Sleep(time.Millisecond * 10) + } + ipfs.Network.Close() + }() + + c, err := fuse.Mount(fpath) + if err != nil { + return err + } + defer c.Close() + + fsys, err := NewIpns(ipfs, ipfspath) + if err != nil { + return err + } + + err = fs.Serve(c, fsys) + if err != nil { + return err + } + + // check if the mount process has an error to report + <-c.Ready + if err := c.MountError; err != nil { + return err + } + return nil +} + +// Unmount attempts to unmount the provided FUSE mount point, forcibly +// if necessary. +func Unmount(point string) error { + fmt.Printf("Unmounting %s...\n", point) + + var cmd *exec.Cmd + switch runtime.GOOS { + case "darwin": + cmd = exec.Command("diskutil", "umount", "force", point) + case "linux": + cmd = exec.Command("fusermount", "-u", point) + default: + return fmt.Errorf("unmount: unimplemented") + } + + errc := make(chan error, 1) + go func() { + if err := exec.Command("umount", point).Run(); err == nil { + errc <- err + } + // retry to unmount with the fallback cmd + errc <- cmd.Run() + }() + + select { + case <-time.After(1 * time.Second): + return fmt.Errorf("umount timeout") + case err := <-errc: + return err + } +} + +type Link struct { + Target string +} + +func (l *Link) Attr() fuse.Attr { + log.Debug("Link attr.") + return fuse.Attr{ + Mode: os.ModeSymlink | 0555, + } +} + +func (l *Link) Readlink(req *fuse.ReadlinkRequest, intr fs.Intr) (string, fuse.Error) { + return l.Target, nil +} diff --git a/routing/dht/query.go b/routing/dht/query.go index cc709d7e9b1..a62646f05b2 100644 --- a/routing/dht/query.go +++ b/routing/dht/query.go @@ -106,6 +106,7 @@ func (r *dhtQueryRunner) Run(peers []*peer.Peer) (*dhtQueryResult, error) { log.Warning("Running query with no peers!") return nil, nil } + // setup concurrency rate limiting for i := 0; i < r.query.concurrency; i++ { r.rateLimit <- struct{}{} diff --git a/routing/dht/routing.go b/routing/dht/routing.go index 6a2a0cdcb6b..a62f4c01beb 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -47,14 +47,13 @@ func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error // If the search does not succeed, a multiaddr string of a closer peer is // returned along with util.ErrSearchIncomplete func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) { - ll := startNewRPC("GET") - defer ll.EndAndPrint() + log.Debug("Get Value [%s]", key.Pretty()) // If we have it local, dont bother doing an RPC! // NOTE: this might not be what we want to do... val, err := dht.getLocal(key) if err == nil { - ll.Success = true + log.Debug("Got value locally!") return val, nil } From 7f5e13716deedcc9d040739cf6b797e4697be1f6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 25 Sep 2014 22:00:05 -0700 Subject: [PATCH 014/105] writes to ipns work if the top object is the written file (no directories yet!) --- core/core.go | 5 ++ fuse/ipns/ipns_unix.go | 115 +++++++++++++++++++++++++++++++++++++++-- routing/dht/routing.go | 2 +- 3 files changed, 118 insertions(+), 4 deletions(-) diff --git a/core/core.go b/core/core.go index 82cb3411683..1761a38ac59 100644 --- a/core/core.go +++ b/core/core.go @@ -62,6 +62,9 @@ type IpfsNode struct { // the name system, resolves paths to hashes Namesys namesys.Resolver + + // the routing publisher + Publisher *namesys.IpnsPublisher } // NewIpfsNode constructs a new IpfsNode based on the given config. @@ -144,6 +147,7 @@ func NewIpfsNode(cfg *config.Config, online bool) (*IpfsNode, error) { dag := &merkledag.DAGService{Blocks: bs} resolve := namesys.NewMasterResolver(route, dag) + publisher := namesys.NewPublisher(dag, route) success = true return &IpfsNode{ @@ -157,6 +161,7 @@ func NewIpfsNode(cfg *config.Config, online bool) (*IpfsNode, error) { Identity: local, Routing: route, Namesys: resolve, + Publisher: publisher, }, nil } diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 2713538aef9..d8cff44139e 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -11,11 +11,14 @@ import ( "syscall" "time" + "bytes" + "bazil.org/fuse" "bazil.org/fuse/fs" "code.google.com/p/goprotobuf/proto" "github.com/jbenet/go-ipfs/core" ci "github.com/jbenet/go-ipfs/crypto" + imp "github.com/jbenet/go-ipfs/importer" mdag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" "github.com/op/go-logging" @@ -63,6 +66,33 @@ func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, er root.LocalLink = &Link{u.Key(hash).Pretty()} } + for _, k := range keys { + hash, err := k.GetPublic().Hash() + if err != nil { + log.Error("failed to hash public key.") + continue + } + name := u.Key(hash).Pretty() + nd := new(Node) + nd.Ipfs = n + nd.key = k + + pointsTo, err := n.Namesys.Resolve(name) + if err != nil { + log.Warning("Could not resolve value for local ipns entry") + continue + } + + node, err := n.Resolver.ResolvePath(pointsTo) + if err != nil { + log.Warning("Failed to resolve value from ipns entry in ipfs") + continue + } + + nd.Nd = node + root.LocalDirs[name] = nd + } + return root, nil } @@ -147,10 +177,17 @@ func (r *Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { // Node is the core object representing a filesystem tree node. type Node struct { nsRoot *Node + + // Private keys held by nodes at the root of a keyspace + key ci.PrivKey + Ipfs *core.IpfsNode Nd *mdag.Node fd *mdag.DagReader cached *mdag.PBData + + dataBuf *bytes.Buffer + changed bool } func (s *Node) loadData() error { @@ -172,7 +209,7 @@ func (s *Node) Attr() fuse.Attr { u.DOut("this is a file.\n") size, _ := s.Nd.Size() return fuse.Attr{ - Mode: 0444, + Mode: 0666, Size: uint64(size), Blocks: uint64(len(s.Nd.Links)), } @@ -184,14 +221,25 @@ func (s *Node) Attr() fuse.Attr { // Lookup performs a lookup under this node. func (s *Node) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { - u.DOut("Lookup '%s'\n", name) + log.Debug("ipns node Lookup '%s'", name) nd, err := s.Ipfs.Resolver.ResolveLinks(s.Nd, []string{name}) if err != nil { // todo: make this error more versatile. return nil, fuse.ENOENT } - return &Node{Ipfs: s.Ipfs, Nd: nd}, nil + child := &Node{ + Ipfs: s.Ipfs, + Nd: nd, + } + + if s.nsRoot == nil { + child.nsRoot = s + } else { + child.nsRoot = s.nsRoot + } + + return child, nil } // ReadDir reads the link structure as directory entries @@ -224,6 +272,67 @@ func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { return ioutil.ReadAll(r) } +func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error { + if n.dataBuf == nil { + n.dataBuf = new(bytes.Buffer) + } + log.Debug("ipns Node Write: flags = %s, offset = %d, size = %d", req.Flags.String(), req.Offset, len(req.Data)) + if req.Offset == 0 { + n.dataBuf.Reset() + n.dataBuf.Write(req.Data) + n.changed = true + resp.Size = len(req.Data) + } else { + log.Error("Unhandled write to offset!") + } + return nil +} + +func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { + log.Debug("Got flush request!") + + if n.changed { + //TODO: + // This operation holds everything in memory, + // should be changed to stream the block creation/storage + // but for now, since the buf is all in memory anyways... + nnode, err := imp.NewDagFromReader(n.dataBuf) + if err != nil { + log.Error("ipns Flush error: %s", err) + // return fuse.EVERYBAD + return fuse.ENODATA + } + + err = n.Ipfs.DAG.AddRecursive(nnode) + if err != nil { + log.Critical("ipns Dag Add Error: %s", err) + } + + n.Nd = nnode + n.changed = false + n.dataBuf = nil + + ndkey, err := nnode.Key() + if err != nil { + log.Error("getKey error: %s", err) + // return fuse.ETHISREALLYSUCKS + return fuse.ENODATA + } + log.Debug("Publishing changes!") + + err = n.Ipfs.Publisher.Publish(n.key, ndkey) + if err != nil { + log.Error("ipns Publish Failed: %s", err) + } + } + return nil +} + +func (n *Node) Fsync(req *fuse.FsyncRequest, intr fs.Intr) fuse.Error { + log.Debug("Got fsync request!") + return nil +} + // Mount mounts an IpfsNode instance at a particular path. It // serves until the process receives exit signals (to Unmount). func Mount(ipfs *core.IpfsNode, fpath string, ipfspath string) error { diff --git a/routing/dht/routing.go b/routing/dht/routing.go index a62f4c01beb..65e4e3b54f6 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -103,7 +103,7 @@ func (dht *IpfsDHT) Provide(ctx context.Context, key u.Key) error { dht.providers.AddProvider(key, dht.self) peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), PoolSize) if len(peers) == 0 { - return kb.ErrLookupFailure + return nil } //TODO FIX: this doesn't work! it needs to be sent to the actual nearest peers. From 006b68b558c0de83db2c5519caae4a4cfc3a46ea Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 26 Sep 2014 00:09:27 -0700 Subject: [PATCH 015/105] WIP: getting closer to being able to write in ipns dirs --- fuse/ipns/ipns_unix.go | 22 +++++++++++++++++----- importer/importer.go | 23 ++++++++++++++++++++++- merkledag/merkledag.go | 38 +++++++++++++++++++++++++++++++------- routing/dht/routing.go | 2 +- 4 files changed, 71 insertions(+), 14 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index d8cff44139e..56e2dfb8e2a 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -296,23 +296,35 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { // This operation holds everything in memory, // should be changed to stream the block creation/storage // but for now, since the buf is all in memory anyways... - nnode, err := imp.NewDagFromReader(n.dataBuf) + err := imp.NewDagInNode(n.dataBuf, n.Nd) if err != nil { log.Error("ipns Flush error: %s", err) // return fuse.EVERYBAD return fuse.ENODATA } - err = n.Ipfs.DAG.AddRecursive(nnode) + var root *Node + if n.nsRoot != nil { + root = n.nsRoot + } else { + root = n + } + + err = root.Nd.Update() + if err != nil { + log.Error("ipns dag tree update failed: %s", err) + return fuse.ENODATA + } + + err = n.Ipfs.DAG.AddRecursive(root.Nd) if err != nil { log.Critical("ipns Dag Add Error: %s", err) } - n.Nd = nnode n.changed = false n.dataBuf = nil - ndkey, err := nnode.Key() + ndkey, err := root.Nd.Key() if err != nil { log.Error("getKey error: %s", err) // return fuse.ETHISREALLYSUCKS @@ -320,7 +332,7 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { } log.Debug("Publishing changes!") - err = n.Ipfs.Publisher.Publish(n.key, ndkey) + err = n.Ipfs.Publisher.Publish(root.key, ndkey) if err != nil { log.Error("ipns Publish Failed: %s", err) } diff --git a/importer/importer.go b/importer/importer.go index 197eaef19fe..5a60167ac17 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -14,13 +14,15 @@ var BlockSizeLimit = int64(1048576) // 1 MB // ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit. var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded") +var DefaultSplitter = &SizeSplitter{1024 * 512} + // todo: incremental construction with an ipfs node. dumping constructed // objects into the datastore, to avoid buffering all in memory // NewDagFromReader constructs a Merkle DAG from the given io.Reader. // size required for block construction. func NewDagFromReader(r io.Reader) (*dag.Node, error) { - return NewDagFromReaderWithSplitter(r, &SizeSplitter{1024 * 512}) + return NewDagFromReaderWithSplitter(r, DefaultSplitter) } func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, error) { @@ -58,3 +60,22 @@ func NewDagFromFile(fpath string) (*dag.Node, error) { return NewDagFromReader(f) } + +// TODO: this needs a better name +func NewDagInNode(r io.Reader, n *dag.Node) error { + n.Links = nil + + blkChan := DefaultSplitter.Split(r) + first := <-blkChan + n.Data = first + + for blk := range blkChan { + child := &dag.Node{Data: dag.WrapData(blk)} + err := n.AddNodeLink("", child) + if err != nil { + return err + } + } + + return nil +} diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 1cc26278345..4c57fb95ca1 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -12,7 +12,7 @@ import ( u "github.com/jbenet/go-ipfs/util" ) -var log = logging.MustGetLogger("commands") +var log = logging.MustGetLogger("merkledag") // NodeMap maps u.Keys to Nodes. // We cannot use []byte/Multihash for keys :( @@ -96,6 +96,31 @@ func (n *Node) Key() (u.Key, error) { return u.Key(h), err } +// Recursively update all hash links and size values in the tree +func (n *Node) Update() error { + log.Debug("node update") + for _, l := range n.Links { + if l.Node != nil { + err := l.Node.Update() + if err != nil { + return err + } + nhash, err := l.Node.Multihash() + if err != nil { + return err + } + l.Hash = nhash + size, err := l.Node.Size() + if err != nil { + return err + } + l.Size = size + } + } + _, err := n.Encoded(true) + return err +} + // DAGService is an IPFS Merkle DAG service. // - the root is virtual (like a forest) // - stores nodes' data in a BlockService @@ -134,12 +159,11 @@ func (n *DAGService) AddRecursive(nd *Node) error { } for _, link := range nd.Links { - if link.Node == nil { - panic("Why does this node have a nil link?\n") - } - err := n.AddRecursive(link.Node) - if err != nil { - return err + if link.Node != nil { + err := n.AddRecursive(link.Node) + if err != nil { + return err + } } } diff --git a/routing/dht/routing.go b/routing/dht/routing.go index 65e4e3b54f6..e7f98ad4eb9 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -18,7 +18,7 @@ import ( // PutValue adds value corresponding to given Key. // This is the top level "Store" operation of the DHT func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error { - log.Debug("[%s] PutValue %v %v", dht.self.ID.Pretty(), key, value) + log.Debug("[%s] PutValue %v %v", dht.self.ID.Pretty(), key.Pretty(), value) err := dht.putLocal(key, value) if err != nil { return err From 6080944af992a0793df0917e55a86c8f8f5578c1 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 26 Sep 2014 12:08:00 -0700 Subject: [PATCH 016/105] writing files inside ipns works now! also implemented resolve cli command --- cmd/ipfs/ipfs.go | 1 + cmd/ipfs/resolve.go | 54 ++++++++++++++++++++++++++++++++++++++++ core/commands/publish.go | 5 ++-- core/commands/resolve.go | 18 ++++++++++++++ daemon/daemon.go | 2 ++ fuse/ipns/ipns_unix.go | 34 +++++++++++++++++-------- importer/importer.go | 2 +- namesys/nsresolver.go | 2 ++ namesys/publisher.go | 6 ++--- namesys/resolve_test.go | 8 +++--- namesys/routing.go | 1 + path/path.go | 20 +++++++++++---- routing/dht/routing.go | 2 +- util/util.go | 17 +++++++++++++ 14 files changed, 144 insertions(+), 28 deletions(-) create mode 100644 cmd/ipfs/resolve.go create mode 100644 core/commands/resolve.go diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 69e5ee5df6a..56c07945391 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -53,6 +53,7 @@ Use "ipfs help " for more information about a command. cmdIpfsServe, cmdIpfsRun, cmdIpfsPub, + cmdIpfsResolve, }, Flag: *flag.NewFlagSet("ipfs", flag.ExitOnError), } diff --git a/cmd/ipfs/resolve.go b/cmd/ipfs/resolve.go new file mode 100644 index 00000000000..18841624b16 --- /dev/null +++ b/cmd/ipfs/resolve.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" + "github.com/jbenet/go-ipfs/core/commands" + "github.com/jbenet/go-ipfs/daemon" + u "github.com/jbenet/go-ipfs/util" +) + +var cmdIpfsResolve = &commander.Command{ + UsageLine: "resolve", + Short: "resolve an ipns link to a hash", + Long: `ipfs resolve ... - Resolve hash. + +`, + Run: resolveCmd, + Flag: *flag.NewFlagSet("ipfs-resolve", flag.ExitOnError), +} + +func resolveCmd(c *commander.Command, inp []string) error { + u.Debug = true + if len(inp) < 1 { + u.POut(c.Long) + return nil + } + conf, err := getConfigDir(c.Parent) + if err != nil { + return err + } + + cmd := daemon.NewCommand() + cmd.Command = "resolve" + cmd.Args = inp + err = daemon.SendCommand(cmd, conf) + if err != nil { + now := time.Now() + // Resolve requires working DHT + n, err := localNode(conf, true) + if err != nil { + return err + } + + took := time.Now().Sub(now) + fmt.Printf("localNode creation took %s\n", took.String()) + + return commands.Resolve(n, cmd.Args, cmd.Opts, os.Stdout) + } + return nil +} diff --git a/core/commands/publish.go b/core/commands/publish.go index 8fae4235d95..145e9b06edf 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -21,10 +21,9 @@ func Publish(n *core.IpfsNode, args []string, opts map[string]interface{}, out i } k := n.Identity.PrivKey - val := u.Key(args[0]) pub := nsys.NewPublisher(n.DAG, n.Routing) - err := pub.Publish(k, val) + err := pub.Publish(k, args[0]) if err != nil { return err } @@ -33,7 +32,7 @@ func Publish(n *core.IpfsNode, args []string, opts map[string]interface{}, out i if err != nil { return err } - fmt.Fprintf(out, "Published %s to %s\n", val, u.Key(hash).Pretty()) + fmt.Fprintf(out, "Published %s to %s\n", args[0], u.Key(hash).Pretty()) return nil } diff --git a/core/commands/resolve.go b/core/commands/resolve.go new file mode 100644 index 00000000000..bd3672e185e --- /dev/null +++ b/core/commands/resolve.go @@ -0,0 +1,18 @@ +package commands + +import ( + "fmt" + "io" + + "github.com/jbenet/go-ipfs/core" +) + +func Resolve(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error { + res, err := n.Namesys.Resolve(args[0]) + if err != nil { + return err + } + + fmt.Fprintf(out, "%s -> %s\n", args[0], res) + return nil +} diff --git a/daemon/daemon.go b/daemon/daemon.go index 755de7df931..2e4681592de 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -133,6 +133,8 @@ func (dl *DaemonListener) handleConnection(conn net.Conn) { err = commands.Pin(dl.node, command.Args, command.Opts, conn) case "publish": err = commands.Publish(dl.node, command.Args, command.Opts, conn) + case "resolve": + err = commands.Resolve(dl.node, command.Args, command.Opts, conn) default: err = fmt.Errorf("Invalid Command: '%s'", command.Command) } diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 56e2dfb8e2a..08514321949 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -2,6 +2,7 @@ package ipns import ( "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -83,6 +84,11 @@ func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, er continue } + if !u.IsValidHash(pointsTo) { + log.Critical("Got back bad data from namesys resolve! [%s]", pointsTo) + return nil, nil + } + node, err := n.Resolver.ResolvePath(pointsTo) if err != nil { log.Warning("Failed to resolve value from ipns entry in ipfs") @@ -120,7 +126,7 @@ func (*Root) Attr() fuse.Attr { // Lookup performs a lookup under this node. func (s *Root) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { - log.Debug("ipns: Root Lookup: '%s'", name) + log.Debug("ipns: Root Lookup: '%s' [intr = %s]", name, intr.String()) switch name { case "mach_kernel", ".hidden", "._.": // Just quiet some log noise on OS X. @@ -139,14 +145,14 @@ func (s *Root) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { return nd, nil } - log.Debug("ipns: Falling back to resolution.") + log.Debug("ipns: Falling back to resolution for [%s].", name) resolved, err := s.Ipfs.Namesys.Resolve(name) if err != nil { log.Error("ipns: namesys resolve error: %s", err) return nil, fuse.ENOENT } - return &Link{s.IpfsRoot + "/" + resolved}, nil + return &Link{s.IpfsRoot + "/" + u.Key(resolved).Pretty()}, nil } // ReadDir reads a particular directory. Disallowed for root. @@ -221,7 +227,7 @@ func (s *Node) Attr() fuse.Attr { // Lookup performs a lookup under this node. func (s *Node) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { - log.Debug("ipns node Lookup '%s'", name) + log.Debug("ipns: node Lookup '%s'", name) nd, err := s.Ipfs.Resolver.ResolveLinks(s.Nd, []string{name}) if err != nil { // todo: make this error more versatile. @@ -276,7 +282,7 @@ func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.I if n.dataBuf == nil { n.dataBuf = new(bytes.Buffer) } - log.Debug("ipns Node Write: flags = %s, offset = %d, size = %d", req.Flags.String(), req.Offset, len(req.Data)) + log.Debug("ipns: Node Write: flags = %s, offset = %d, size = %d", req.Flags.String(), req.Offset, len(req.Data)) if req.Offset == 0 { n.dataBuf.Reset() n.dataBuf.Write(req.Data) @@ -298,11 +304,18 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { // but for now, since the buf is all in memory anyways... err := imp.NewDagInNode(n.dataBuf, n.Nd) if err != nil { - log.Error("ipns Flush error: %s", err) + log.Error("ipns: Flush error: %s", err) // return fuse.EVERYBAD return fuse.ENODATA } + read, err := mdag.NewDagReader(n.Nd, n.Ipfs.DAG) + if err != nil { + panic(err) + } + + io.Copy(os.Stdout, read) + var root *Node if n.nsRoot != nil { root = n.nsRoot @@ -312,13 +325,13 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { err = root.Nd.Update() if err != nil { - log.Error("ipns dag tree update failed: %s", err) + log.Error("ipns: dag tree update failed: %s", err) return fuse.ENODATA } err = n.Ipfs.DAG.AddRecursive(root.Nd) if err != nil { - log.Critical("ipns Dag Add Error: %s", err) + log.Critical("ipns: Dag Add Error: %s", err) } n.changed = false @@ -332,9 +345,9 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { } log.Debug("Publishing changes!") - err = n.Ipfs.Publisher.Publish(root.key, ndkey) + err = n.Ipfs.Publisher.Publish(root.key, ndkey.Pretty()) if err != nil { - log.Error("ipns Publish Failed: %s", err) + log.Error("ipns: Publish Failed: %s", err) } } return nil @@ -433,5 +446,6 @@ func (l *Link) Attr() fuse.Attr { } func (l *Link) Readlink(req *fuse.ReadlinkRequest, intr fs.Intr) (string, fuse.Error) { + log.Debug("ReadLink: %s", l.Target) return l.Target, nil } diff --git a/importer/importer.go b/importer/importer.go index 5a60167ac17..b71118317e2 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -67,7 +67,7 @@ func NewDagInNode(r io.Reader, n *dag.Node) error { blkChan := DefaultSplitter.Split(r) first := <-blkChan - n.Data = first + n.Data = dag.FilePBData(first) for blk := range blkChan { child := &dag.Node{Data: dag.WrapData(blk)} diff --git a/namesys/nsresolver.go b/namesys/nsresolver.go index c8d40dcb3db..89ef9ff5a87 100644 --- a/namesys/nsresolver.go +++ b/namesys/nsresolver.go @@ -1,6 +1,8 @@ package namesys type Resolver interface { + // Resolve returns a base58 encoded string Resolve(string) (string, error) + Matches(string) bool } diff --git a/namesys/publisher.go b/namesys/publisher.go index 67830eb6bea..73a4197e1bb 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -25,8 +25,8 @@ func NewPublisher(dag *mdag.DAGService, route routing.IpfsRouting) *IpnsPublishe } // Publish accepts a keypair and a value, -func (p *IpnsPublisher) Publish(k ci.PrivKey, value u.Key) error { - log.Debug("namesys: Publish %s", value.Pretty()) +func (p *IpnsPublisher) Publish(k ci.PrivKey, value string) error { + log.Debug("namesys: Publish %s", value) ctx := context.TODO() data, err := CreateEntryData(k, value) if err != nil { @@ -66,7 +66,7 @@ func (p *IpnsPublisher) Publish(k ci.PrivKey, value u.Key) error { return nil } -func CreateEntryData(pk ci.PrivKey, val u.Key) ([]byte, error) { +func CreateEntryData(pk ci.PrivKey, val string) ([]byte, error) { entry := new(IpnsEntry) sig, err := pk.Sign([]byte(val)) if err != nil { diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index 32a12cf76aa..7cad8bef098 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -7,9 +7,8 @@ import ( bs "github.com/jbenet/go-ipfs/blockservice" ci "github.com/jbenet/go-ipfs/crypto" mdag "github.com/jbenet/go-ipfs/merkledag" - "github.com/jbenet/go-ipfs/net/swarm" "github.com/jbenet/go-ipfs/peer" - "github.com/jbenet/go-ipfs/routing/dht" + mock "github.com/jbenet/go-ipfs/routing/mock" u "github.com/jbenet/go-ipfs/util" ) @@ -17,9 +16,8 @@ func TestRoutingResolve(t *testing.T) { local := &peer.Peer{ ID: []byte("testID"), } - net := swarm.NewSwarm(local) lds := ds.NewMapDatastore() - d := dht.NewDHT(local, net, lds) + d := mock.NewMockRouter(local, lds) bserv, err := bs.NewBlockService(lds, nil) if err != nil { @@ -40,7 +38,7 @@ func TestRoutingResolve(t *testing.T) { t.Fatal(err) } - err = pub.Publish(privk, u.Key("Hello")) + err = pub.Publish(privk, "Hello") if err != nil { t.Fatal(err) } diff --git a/namesys/routing.go b/namesys/routing.go index 2b1d06c8064..605458eef04 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -35,6 +35,7 @@ func (r *RoutingResolver) Matches(name string) bool { } func (r *RoutingResolver) Resolve(name string) (string, error) { + log.Debug("RoutingResolve: '%s'", name) ctx := context.TODO() hash, err := mh.FromB58String(name) if err != nil { diff --git a/path/path.go b/path/path.go index eb5ce6660fd..533afc7cd17 100644 --- a/path/path.go +++ b/path/path.go @@ -8,8 +8,11 @@ import ( mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" merkledag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" + "github.com/op/go-logging" ) +var log = logging.MustGetLogger("path") + // Resolver provides path resolution to IPFS // It has a pointer to a DAGService, which is uses to resolve nodes. type Resolver struct { @@ -20,7 +23,7 @@ type Resolver struct { // path component as a hash (key) of the first node, then resolves // all other components walking the links, with ResolveLinks. func (s *Resolver) ResolvePath(fpath string) (*merkledag.Node, error) { - u.DOut("Resolve: '%s'\n", fpath) + log.Debug("Resolve: '%s'", fpath) fpath = path.Clean(fpath) parts := strings.Split(fpath, "/") @@ -66,10 +69,12 @@ func (s *Resolver) ResolveLinks(ndd *merkledag.Node, names []string) ( for _, name := range names { var next u.Key + var nlink *merkledag.Link // for each of the links in nd, the current object for _, link := range nd.Links { if link.Name == name { next = u.Key(link.Hash) + nlink = link break } } @@ -80,10 +85,15 @@ func (s *Resolver) ResolveLinks(ndd *merkledag.Node, names []string) ( return nil, fmt.Errorf("no link named %q under %s", name, h2) } - // fetch object for link and assign to nd - nd, err = s.DAG.Get(next) - if err != nil { - return nd, err + if nlink.Node == nil { + // fetch object for link and assign to nd + nd, err = s.DAG.Get(next) + if err != nil { + return nd, err + } + nlink.Node = nd + } else { + nd = nlink.Node } } return diff --git a/routing/dht/routing.go b/routing/dht/routing.go index e7f98ad4eb9..c1b85cefafa 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -18,7 +18,7 @@ import ( // PutValue adds value corresponding to given Key. // This is the top level "Store" operation of the DHT func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error { - log.Debug("[%s] PutValue %v %v", dht.self.ID.Pretty(), key.Pretty(), value) + log.Debug("PutValue %s %v", key.Pretty(), value) err := dht.putLocal(key, value) if err != nil { return err diff --git a/util/util.go b/util/util.go index 41f6afede47..f2ce35a344e 100644 --- a/util/util.go +++ b/util/util.go @@ -11,8 +11,17 @@ import ( ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" + "github.com/op/go-logging" ) +var format = "%{color}%{time} %{shortfile} %{level}: %{color:reset}%{message}" + +func init() { + backend := logging.NewLogBackend(os.Stderr, "", 0) + logging.SetBackend(backend) + logging.SetFormatter(logging.MustStringFormatter(format)) +} + // Debug is a global flag for debugging. var Debug bool @@ -42,6 +51,14 @@ func Hash(data []byte) (mh.Multihash, error) { return mh.Sum(data, mh.SHA2_256, -1) } +func IsValidHash(s string) bool { + out := b58.Decode(s) + if out == nil || len(out) == 0 { + return false + } + return true +} + // TildeExpansion expands a filename, which may begin with a tilde. func TildeExpansion(filename string) (string, error) { if strings.HasPrefix(filename, "~/") { From 3b8b9e6847f405724eef226900d65b858476202d Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 26 Sep 2014 15:24:30 -0700 Subject: [PATCH 017/105] new file creation inside ipns dirs now works --- fuse/ipns/ipns_unix.go | 70 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 08514321949..0454278d004 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -183,6 +183,7 @@ func (r *Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { // Node is the core object representing a filesystem tree node. type Node struct { nsRoot *Node + name string // Private keys held by nodes at the root of a keyspace key ci.PrivKey @@ -192,6 +193,7 @@ type Node struct { fd *mdag.DagReader cached *mdag.PBData + // For writing dataBuf *bytes.Buffer changed bool } @@ -227,25 +229,30 @@ func (s *Node) Attr() fuse.Attr { // Lookup performs a lookup under this node. func (s *Node) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { - log.Debug("ipns: node Lookup '%s'", name) + log.Debug("ipns: node[%s] Lookup '%s' [intr= %s]", s.name, name, intr.String()) nd, err := s.Ipfs.Resolver.ResolveLinks(s.Nd, []string{name}) if err != nil { // todo: make this error more versatile. return nil, fuse.ENOENT } + return s.makeChild(name, nd), nil +} + +func (n *Node) makeChild(name string, node *mdag.Node) *Node { child := &Node{ - Ipfs: s.Ipfs, - Nd: nd, + Ipfs: n.Ipfs, + Nd: node, + name: name, } - if s.nsRoot == nil { - child.nsRoot = s + if n.nsRoot == nil { + child.nsRoot = n } else { - child.nsRoot = s.nsRoot + child.nsRoot = n.nsRoot } - return child, nil + return child } // ReadDir reads the link structure as directory entries @@ -358,6 +365,55 @@ func (n *Node) Fsync(req *fuse.FsyncRequest, intr fs.Intr) fuse.Error { return nil } +func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) { + log.Debug("Got mkdir request!") + dagnd := new(mdag.Node) + dagnd.Data = mdag.FolderPBData() + n.Nd.AddNodeLink(req.Name, dagnd) + n.changed = true + + child := &Node{ + Ipfs: n.Ipfs, + Nd: dagnd, + } + + if n.nsRoot == nil { + child.nsRoot = n + } else { + child.nsRoot = n.nsRoot + } + + return child, nil +} + +func (n *Node) Mknod(req *fuse.MknodRequest, intr fs.Intr) (fs.Node, fuse.Error) { + log.Debug("Got mknod request!") + return nil, nil +} +func (n *Node) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr) (fs.Handle, fuse.Error) { + log.Debug("[%s] Received open request! flags = %s", n.name, req.Flags.String()) + return n, nil +} + +func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr fs.Intr) (fs.Node, fs.Handle, fuse.Error) { + log.Debug("Got create request!") + nd := new(mdag.Node) + nd.Data = mdag.FilePBData(nil) + child := n.makeChild(req.Name, nd) + + err := n.Nd.AddNodeLink(req.Name, nd) + if err != nil { + log.Error("Error adding child to node: %s", err) + return nil, nil, fuse.ENOENT + } + return child, nil, nil +} + +func (n *Node) Remove(req *fuse.RemoveRequest, intr fs.Intr) fuse.Error { + log.Debug("Got Remove request!") + return fuse.EIO +} + // Mount mounts an IpfsNode instance at a particular path. It // serves until the process receives exit signals (to Unmount). func Mount(ipfs *core.IpfsNode, fpath string, ipfspath string) error { From ed2fbd0512eb96fe74c641eefcc721b797675207 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 27 Sep 2014 12:38:32 -0700 Subject: [PATCH 018/105] new files and directories appear to work properly --- fuse/ipns/ipns_unix.go | 130 +++++++++++++++++++++++++++-------------- merkledag/merkledag.go | 10 ++++ namesys/dns.go | 2 +- 3 files changed, 96 insertions(+), 46 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 0454278d004..7c24f716cac 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -2,7 +2,6 @@ package ipns import ( "fmt" - "io" "io/ioutil" "os" "os/exec" @@ -126,7 +125,7 @@ func (*Root) Attr() fuse.Attr { // Lookup performs a lookup under this node. func (s *Root) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { - log.Debug("ipns: Root Lookup: '%s' [intr = %s]", name, intr.String()) + log.Debug("ipns: Root Lookup: '%s'", name) switch name { case "mach_kernel", ".hidden", "._.": // Just quiet some log noise on OS X. @@ -183,7 +182,9 @@ func (r *Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { // Node is the core object representing a filesystem tree node. type Node struct { nsRoot *Node - name string + + // Name really only for logging purposes + name string // Private keys held by nodes at the root of a keyspace key ci.PrivKey @@ -205,16 +206,13 @@ func (s *Node) loadData() error { // Attr returns the attributes of a given node. func (s *Node) Attr() fuse.Attr { - u.DOut("Node attr.\n") if s.cached == nil { s.loadData() } switch s.cached.GetType() { case mdag.PBData_Directory: - u.DOut("this is a directory.\n") return fuse.Attr{Mode: os.ModeDir | 0555} case mdag.PBData_File, mdag.PBData_Raw: - u.DOut("this is a file.\n") size, _ := s.Nd.Size() return fuse.Attr{ Mode: 0666, @@ -222,14 +220,14 @@ func (s *Node) Attr() fuse.Attr { Blocks: uint64(len(s.Nd.Links)), } default: - u.PErr("Invalid data type.") + log.Error("Invalid data type.") return fuse.Attr{} } } // Lookup performs a lookup under this node. func (s *Node) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { - log.Debug("ipns: node[%s] Lookup '%s' [intr= %s]", s.name, name, intr.String()) + log.Debug("ipns: node[%s] Lookup '%s'", s.name, name) nd, err := s.Ipfs.Resolver.ResolveLinks(s.Nd, []string{name}) if err != nil { // todo: make this error more versatile. @@ -243,7 +241,7 @@ func (n *Node) makeChild(name string, node *mdag.Node) *Node { child := &Node{ Ipfs: n.Ipfs, Nd: node, - name: name, + name: n.name + "/" + name, } if n.nsRoot == nil { @@ -286,10 +284,10 @@ func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { } func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error { + log.Debug("ipns: Node Write: flags = %s, offset = %d, size = %d", req.Flags.String(), req.Offset, len(req.Data)) if n.dataBuf == nil { n.dataBuf = new(bytes.Buffer) } - log.Debug("ipns: Node Write: flags = %s, offset = %d, size = %d", req.Flags.String(), req.Offset, len(req.Data)) if req.Offset == 0 { n.dataBuf.Reset() n.dataBuf.Write(req.Data) @@ -316,46 +314,51 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { return fuse.ENODATA } - read, err := mdag.NewDagReader(n.Nd, n.Ipfs.DAG) + err = n.updateTree() if err != nil { - panic(err) + log.Error("updateTree failed: %s", err) + return fuse.ENODATA } - io.Copy(os.Stdout, read) + } + return nil +} - var root *Node - if n.nsRoot != nil { - root = n.nsRoot - } else { - root = n - } +func (n *Node) updateTree() error { + var root *Node + if n.nsRoot != nil { + root = n.nsRoot + } else { + root = n + } - err = root.Nd.Update() - if err != nil { - log.Error("ipns: dag tree update failed: %s", err) - return fuse.ENODATA - } + err := root.Nd.Update() + if err != nil { + log.Error("ipns: dag tree update failed: %s", err) + return err + } - err = n.Ipfs.DAG.AddRecursive(root.Nd) - if err != nil { - log.Critical("ipns: Dag Add Error: %s", err) - } + err = n.Ipfs.DAG.AddRecursive(root.Nd) + if err != nil { + log.Critical("ipns: Dag Add Error: %s", err) + return err + } - n.changed = false - n.dataBuf = nil + n.changed = false + n.dataBuf = nil - ndkey, err := root.Nd.Key() - if err != nil { - log.Error("getKey error: %s", err) - // return fuse.ETHISREALLYSUCKS - return fuse.ENODATA - } - log.Debug("Publishing changes!") + ndkey, err := root.Nd.Key() + if err != nil { + log.Error("getKey error: %s", err) + // return fuse.ETHISREALLYSUCKS + return err + } + log.Debug("Publishing changes!") - err = n.Ipfs.Publisher.Publish(root.key, ndkey.Pretty()) - if err != nil { - log.Error("ipns: Publish Failed: %s", err) - } + err = n.Ipfs.Publisher.Publish(root.key, ndkey.Pretty()) + if err != nil { + log.Error("ipns: Publish Failed: %s", err) + return err } return nil } @@ -383,6 +386,8 @@ func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) child.nsRoot = n.nsRoot } + n.updateTree() + return child, nil } @@ -390,13 +395,15 @@ func (n *Node) Mknod(req *fuse.MknodRequest, intr fs.Intr) (fs.Node, fuse.Error) log.Debug("Got mknod request!") return nil, nil } + func (n *Node) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr) (fs.Handle, fuse.Error) { log.Debug("[%s] Received open request! flags = %s", n.name, req.Flags.String()) + //TODO: check open flags and truncate if necessary return n, nil } func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr fs.Intr) (fs.Node, fs.Handle, fuse.Error) { - log.Debug("Got create request!") + log.Debug("Got create request: %s", req.Name) nd := new(mdag.Node) nd.Data = mdag.FilePBData(nil) child := n.makeChild(req.Name, nd) @@ -406,12 +413,45 @@ func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr f log.Error("Error adding child to node: %s", err) return nil, nil, fuse.ENOENT } - return child, nil, nil + return child, child, nil } func (n *Node) Remove(req *fuse.RemoveRequest, intr fs.Intr) fuse.Error { - log.Debug("Got Remove request!") - return fuse.EIO + log.Debug("[%s] Got Remove request: %s", n.name, req.Name) + err := n.Nd.RemoveNodeLink(req.Name) + if err != nil { + log.Error("Remove: No such file.") + return fuse.ENOENT + } + return nil +} + +func (n *Node) Rename(req *fuse.RenameRequest, newDir fs.Node, intr fs.Intr) fuse.Error { + log.Debug("Got Rename request '%s' -> '%s'", req.OldName, req.NewName) + var mdn *mdag.Node + for _, l := range n.Nd.Links { + if l.Name == req.OldName { + mdn = l.Node + } + } + if mdn == nil { + log.Critical("nil Link found on rename!") + return fuse.ENOENT + } + n.Nd.RemoveNodeLink(req.OldName) + + switch newDir := newDir.(type) { + case *Node: + err := newDir.Nd.AddNodeLink(req.NewName, mdn) + if err != nil { + log.Error("Error adding node to new dir on rename: %s", err) + return fuse.ENOENT + } + default: + log.Critical("Unknown node type for rename target dir!") + return fuse.ENOENT + } + return nil } // Mount mounts an IpfsNode instance at a particular path. It diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 4c57fb95ca1..b8e96e4651d 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -65,6 +65,16 @@ func (n *Node) AddNodeLink(name string, that *Node) error { return nil } +func (n *Node) RemoveNodeLink(name string) error { + for i, l := range n.Links { + if l.Name == name { + n.Links = append(n.Links[:i], n.Links[i+1:]...) + return nil + } + } + return u.ErrNotFound +} + // Size returns the total size of the data addressed by node, // including the total sizes of references. func (n *Node) Size() (uint64, error) { diff --git a/namesys/dns.go b/namesys/dns.go index 55d00f94514..1154a66fec1 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -15,7 +15,7 @@ type DNSResolver struct { } func (r *DNSResolver) Matches(name string) bool { - return strings.Contains(name, ".") + return strings.Contains(name, ".") && !strings.HasPrefix(name, ".") } // TXT records for a given domain name should contain a b58 From 9b0c578fc0a60b014981cd8e723a5ff3bcee0f58 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 27 Sep 2014 13:10:31 -0700 Subject: [PATCH 019/105] better cleanup of ipfs on program exit --- cmd/ipfs/mount_unix.go | 9 ++++++++- daemon/daemon_client.go | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/ipfs/mount_unix.go b/cmd/ipfs/mount_unix.go index 8bec0251e20..2f7e623f06a 100644 --- a/cmd/ipfs/mount_unix.go +++ b/cmd/ipfs/mount_unix.go @@ -72,15 +72,22 @@ func mountCmd(c *commander.Command, inp []string) error { mp := inp[0] fmt.Printf("Mounting at %s\n", mp) + var ipnsDone chan struct{} ns, ok := c.Flag.Lookup("n").Value.Get().(string) if ok { + ipnsDone = make(chan struct{}) go func() { err = ipns.Mount(n, ns, mp) if err != nil { fmt.Printf("Error mounting ipns: %s\n", err) } + ipnsDone <- struct{}{} }() } - return rofs.Mount(n, mp) + err = rofs.Mount(n, mp) + if ipnsDone != nil { + <-ipnsDone + } + return err } diff --git a/daemon/daemon_client.go b/daemon/daemon_client.go index 5ca0e3a3797..fe5b1f9efa0 100644 --- a/daemon/daemon_client.go +++ b/daemon/daemon_client.go @@ -60,7 +60,7 @@ func SendCommand(command *Command, confdir string) error { return ErrDaemonNotRunning } - log.Info("Daemon is running! %s", err) + log.Info("Daemon is running! [reason = %s]", err) server, err := getDaemonAddr(confdir) if err != nil { From 54142b2173e07c493659bd280420360cbe87284c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 28 Sep 2014 00:13:07 -0700 Subject: [PATCH 020/105] update logging in multiple packages --- blockservice/blockservice.go | 15 +++++---- fuse/ipns/ipns_unix.go | 56 ++++++++++++++++++++++++++++------ fuse/readonly/readonly_unix.go | 17 ++++++----- routing/dht/dht.go | 30 +++++++++--------- routing/dht/routing.go | 10 +++--- util/util.go | 2 +- 6 files changed, 85 insertions(+), 45 deletions(-) diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index 1fbbfcb4405..51a6e0014e9 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -7,12 +7,15 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" + "github.com/op/go-logging" blocks "github.com/jbenet/go-ipfs/blocks" exchange "github.com/jbenet/go-ipfs/exchange" u "github.com/jbenet/go-ipfs/util" ) +var log = logging.MustGetLogger("blockservice") + // BlockService is a block datastore. // It uses an internal `datastore.Datastore` instance to store values. type BlockService struct { @@ -26,7 +29,7 @@ func NewBlockService(d ds.Datastore, rem exchange.Interface) (*BlockService, err return nil, fmt.Errorf("BlockService requires valid datastore") } if rem == nil { - u.DErr("Caution: blockservice running in local (offline) mode.\n") + log.Warning("blockservice running in local (offline) mode.") } return &BlockService{Datastore: d, Remote: rem}, nil } @@ -35,7 +38,7 @@ func NewBlockService(d ds.Datastore, rem exchange.Interface) (*BlockService, err func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { k := b.Key() dsk := ds.NewKey(string(k)) - u.DOut("storing [%s] in datastore\n", k.Pretty()) + log.Debug("storing [%s] in datastore", k.Pretty()) // TODO(brian): define a block datastore with a Put method which accepts a // block parameter err := s.Datastore.Put(dsk, b.Data) @@ -52,11 +55,11 @@ func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { // GetBlock retrieves a particular block from the service, // Getting it from the datastore using the key (hash). func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) { - u.DOut("BlockService GetBlock: '%s'\n", k.Pretty()) + log.Debug("BlockService GetBlock: '%s'", k.Pretty()) dsk := ds.NewKey(string(k)) datai, err := s.Datastore.Get(dsk) if err == nil { - u.DOut("Blockservice: Got data in datastore.\n") + log.Debug("Blockservice: Got data in datastore.") bdata, ok := datai.([]byte) if !ok { return nil, fmt.Errorf("data associated with %s is not a []byte", k) @@ -66,7 +69,7 @@ func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) { Data: bdata, }, nil } else if err == ds.ErrNotFound && s.Remote != nil { - u.DOut("Blockservice: Searching bitswap.\n") + log.Debug("Blockservice: Searching bitswap.") ctx, _ := context.WithTimeout(context.TODO(), 5*time.Second) blk, err := s.Remote.Block(ctx, k) if err != nil { @@ -74,7 +77,7 @@ func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) { } return blk, nil } else { - u.DOut("Blockservice GetBlock: Not found.\n") + log.Debug("Blockservice GetBlock: Not found.") return nil, u.ErrNotFound } } diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 7c24f716cac..b15ee11e969 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -79,7 +79,9 @@ func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, er pointsTo, err := n.Namesys.Resolve(name) if err != nil { - log.Warning("Could not resolve value for local ipns entry") + log.Warning("Could not resolve value for local ipns entry, providing empty dir") + nd.Nd = &mdag.Node{Data: mdag.FolderPBData()} + root.LocalDirs[name] = nd continue } @@ -147,16 +149,16 @@ func (s *Root) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { log.Debug("ipns: Falling back to resolution for [%s].", name) resolved, err := s.Ipfs.Namesys.Resolve(name) if err != nil { - log.Error("ipns: namesys resolve error: %s", err) + log.Warning("ipns: namesys resolve error: %s", err) return nil, fuse.ENOENT } - return &Link{s.IpfsRoot + "/" + u.Key(resolved).Pretty()}, nil + return &Link{s.IpfsRoot + "/" + resolved}, nil } // ReadDir reads a particular directory. Disallowed for root. func (r *Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { - u.DOut("Read Root.\n") + log.Debug("Read Root.") listing := []fuse.Dirent{ fuse.Dirent{ Name: "local", @@ -255,7 +257,7 @@ func (n *Node) makeChild(name string, node *mdag.Node) *Node { // ReadDir reads the link structure as directory entries func (s *Node) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { - u.DOut("Node ReadDir\n") + log.Debug("Node ReadDir") entries := make([]fuse.Dirent, len(s.Nd.Links)) for i, link := range s.Nd.Links { n := link.Name @@ -370,10 +372,8 @@ func (n *Node) Fsync(req *fuse.FsyncRequest, intr fs.Intr) fuse.Error { func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) { log.Debug("Got mkdir request!") - dagnd := new(mdag.Node) - dagnd.Data = mdag.FolderPBData() + dagnd := &mdag.Node{Data: mdag.FolderPBData()} n.Nd.AddNodeLink(req.Name, dagnd) - n.changed = true child := &Node{ Ipfs: n.Ipfs, @@ -386,6 +386,7 @@ func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) child.nsRoot = n.nsRoot } + n.changed = true n.updateTree() return child, nil @@ -404,8 +405,9 @@ func (n *Node) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr fs.Intr) (fs.Node, fs.Handle, fuse.Error) { log.Debug("Got create request: %s", req.Name) - nd := new(mdag.Node) - nd.Data = mdag.FilePBData(nil) + + // New 'empty' file + nd := &mdag.Node{Data: mdag.FilePBData(nil)} child := n.makeChild(req.Name, nd) err := n.Nd.AddNodeLink(req.Name, nd) @@ -545,3 +547,37 @@ func (l *Link) Readlink(req *fuse.ReadlinkRequest, intr fs.Intr) (string, fuse.E log.Debug("ReadLink: %s", l.Target) return l.Target, nil } + +type Republisher struct { + Timeout time.Duration + Publish chan struct{} + node *Node +} + +func NewRepublisher(n *Node, tout time.Duration) *Republisher { + return &Republisher{ + Timeout: tout, + Publish: make(chan struct{}), + node: n, + } +} + +func (np *Republisher) Run() { + for _ := range np.Publish { + timer := time.After(np.Timeout) + for { + select { + case <-timer: + //Do the publish! + err := np.node.updateTree() + if err != nil { + log.Critical("updateTree error: %s", err) + } + goto done + case <-np.Publish: + timer = time.After(np.Timeout) + } + } + done: + } +} diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 54eac2bbcbd..1fa7bfb0d24 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -21,8 +21,11 @@ import ( core "github.com/jbenet/go-ipfs/core" mdag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" + "github.com/op/go-logging" ) +var log = logging.MustGetLogger("ipfs") + // FileSystem is the readonly Ipfs Fuse Filesystem. type FileSystem struct { Ipfs *core.IpfsNode @@ -50,7 +53,7 @@ func (*Root) Attr() fuse.Attr { // Lookup performs a lookup under this node. func (s *Root) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { - u.DOut("Root Lookup: '%s'\n", name) + log.Debug("Root Lookup: '%s'", name) switch name { case "mach_kernel", ".hidden", "._.": // Just quiet some log noise on OS X. @@ -68,7 +71,7 @@ func (s *Root) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { // ReadDir reads a particular directory. Disallowed for root. func (*Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { - u.DOut("Read Root.\n") + log.Debug("Read Root.") return nil, fuse.EPERM } @@ -87,16 +90,14 @@ func (s *Node) loadData() error { // Attr returns the attributes of a given node. func (s *Node) Attr() fuse.Attr { - u.DOut("Node attr.\n") + log.Debug("Node attr.") if s.cached == nil { s.loadData() } switch s.cached.GetType() { case mdag.PBData_Directory: - u.DOut("this is a directory.\n") return fuse.Attr{Mode: os.ModeDir | 0555} case mdag.PBData_File, mdag.PBData_Raw: - u.DOut("this is a file.\n") size, _ := s.Nd.Size() return fuse.Attr{ Mode: 0444, @@ -111,7 +112,7 @@ func (s *Node) Attr() fuse.Attr { // Lookup performs a lookup under this node. func (s *Node) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { - u.DOut("Lookup '%s'\n", name) + log.Debug("Lookup '%s'", name) nd, err := s.Ipfs.Resolver.ResolveLinks(s.Nd, []string{name}) if err != nil { // todo: make this error more versatile. @@ -123,7 +124,7 @@ func (s *Node) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { // ReadDir reads the link structure as directory entries func (s *Node) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { - u.DOut("Node ReadDir\n") + log.Debug("Node ReadDir") entries := make([]fuse.Dirent, len(s.Nd.Links)) for i, link := range s.Nd.Links { n := link.Name @@ -193,7 +194,7 @@ func Mount(ipfs *core.IpfsNode, fpath string) error { // Unmount attempts to unmount the provided FUSE mount point, forcibly // if necessary. func Unmount(point string) error { - fmt.Printf("Unmounting %s...\n", point) + log.Info("Unmounting %s...", point) var cmd *exec.Cmd switch runtime.GOOS { diff --git a/routing/dht/dht.go b/routing/dht/dht.go index f9f52b10828..4e844ecd6f4 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -77,7 +77,7 @@ func NewDHT(p *peer.Peer, ps peer.Peerstore, net inet.Network, sender inet.Sende // Connect to a new peer at the given address, ping and add to the routing table func (dht *IpfsDHT) Connect(ctx context.Context, npeer *peer.Peer) (*peer.Peer, error) { - u.DOut("Connect to new peer: %s\n", npeer.ID.Pretty()) + log.Debug("Connect to new peer: %s\n", npeer.ID.Pretty()) // TODO(jbenet,whyrusleeping) // @@ -132,7 +132,7 @@ func (dht *IpfsDHT) HandleMessage(ctx context.Context, mes msg.NetMessage) msg.N dht.Update(mPeer) // Print out diagnostic - u.DOut("[peer: %s] Got message type: '%s' [from = %s]\n", + log.Debug("[peer: %s] Got message type: '%s' [from = %s]\n", dht.self.ID.Pretty(), Message_MessageType_name[int32(pmes.GetType())], mPeer.ID.Pretty()) @@ -177,7 +177,7 @@ func (dht *IpfsDHT) sendRequest(ctx context.Context, p *peer.Peer, pmes *Message start := time.Now() // Print out diagnostic - u.DOut("[peer: %s] Sent message type: '%s' [to = %s]\n", + log.Debug("[peer: %s] Sent message type: '%s' [to = %s]\n", dht.self.ID.Pretty(), Message_MessageType_name[int32(pmes.GetType())], p.ID.Pretty()) @@ -224,7 +224,7 @@ func (dht *IpfsDHT) putProvider(ctx context.Context, p *peer.Peer, key string) e return err } - u.DOut("[%s] putProvider: %s for %s\n", dht.self.ID.Pretty(), p.ID.Pretty(), key) + log.Debug("[%s] putProvider: %s for %s", dht.self.ID.Pretty(), p.ID.Pretty(), key) if *rpmes.Key != *pmes.Key { return errors.New("provider not added correctly") } @@ -240,10 +240,10 @@ func (dht *IpfsDHT) getValueOrPeers(ctx context.Context, p *peer.Peer, return nil, nil, err } - u.DOut("pmes.GetValue() %v\n", pmes.GetValue()) + log.Debug("pmes.GetValue() %v", pmes.GetValue()) if value := pmes.GetValue(); value != nil { // Success! We were given the value - u.DOut("getValueOrPeers: got value\n") + log.Debug("getValueOrPeers: got value") return value, nil, nil } @@ -253,7 +253,7 @@ func (dht *IpfsDHT) getValueOrPeers(ctx context.Context, p *peer.Peer, if err != nil { return nil, nil, err } - u.DOut("getValueOrPeers: get from providers\n") + log.Debug("getValueOrPeers: get from providers") return val, nil, nil } @@ -266,7 +266,7 @@ func (dht *IpfsDHT) getValueOrPeers(ctx context.Context, p *peer.Peer, addr, err := ma.NewMultiaddr(pb.GetAddr()) if err != nil { - u.PErr("%v\n", err.Error()) + log.Error("%v", err.Error()) continue } @@ -281,11 +281,11 @@ func (dht *IpfsDHT) getValueOrPeers(ctx context.Context, p *peer.Peer, } if len(peers) > 0 { - u.DOut("getValueOrPeers: peers\n") + u.DOut("getValueOrPeers: peers") return nil, peers, nil } - u.DOut("getValueOrPeers: u.ErrNotFound\n") + log.Warning("getValueOrPeers: u.ErrNotFound") return nil, nil, u.ErrNotFound } @@ -307,13 +307,13 @@ func (dht *IpfsDHT) getFromPeerList(ctx context.Context, key u.Key, for _, pinfo := range peerlist { p, err := dht.ensureConnectedToPeer(pinfo) if err != nil { - u.DErr("getFromPeers error: %s\n", err) + log.Error("getFromPeers error: %s", err) continue } pmes, err := dht.getValueSingle(ctx, p, key, level) if err != nil { - u.DErr("getFromPeers error: %s\n", err) + log.Error("getFromPeers error: %s\n", err) continue } @@ -348,7 +348,7 @@ func (dht *IpfsDHT) putLocal(key u.Key, value []byte) error { // Update signals to all routingTables to Update their last-seen status // on the given peer. func (dht *IpfsDHT) Update(p *peer.Peer) { - u.DOut("updating peer: [%s] latency = %f\n", p.ID.Pretty(), p.GetLatency().Seconds()) + log.Debug("updating peer: [%s] latency = %f\n", p.ID.Pretty(), p.GetLatency().Seconds()) removedCount := 0 for _, route := range dht.routingTables { removed := route.Update(p) @@ -404,7 +404,7 @@ func (dht *IpfsDHT) addProviders(key u.Key, peers []*Message_Peer) []*peer.Peer continue } - u.DOut("[%s] adding provider: %s for %s", dht.self.ID.Pretty(), p, key) + log.Debug("[%s] adding provider: %s for %s", dht.self.ID.Pretty(), p, key) // Dont add outselves to the list if p.ID.Equal(dht.self.ID) { @@ -439,7 +439,7 @@ func (dht *IpfsDHT) betterPeerToQuery(pmes *Message) *peer.Peer { // == to self? nil if closer.ID.Equal(dht.self.ID) { - u.DOut("Attempted to return self! this shouldnt happen...\n") + log.Error("Attempted to return self! this shouldnt happen...") return nil } diff --git a/routing/dht/routing.go b/routing/dht/routing.go index c1b85cefafa..d1f50afb889 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -86,7 +86,7 @@ func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) { return nil, err } - u.DOut("[%s] GetValue %v %v\n", dht.self.ID.Pretty(), key, result.value) + log.Debug("GetValue %v %v", key, result.value) if result.value == nil { return nil, u.ErrNotFound } @@ -189,7 +189,7 @@ func (dht *IpfsDHT) addPeerListAsync(k u.Key, peers []*Message_Peer, ps *peerSet // FindProviders searches for peers who can provide the value for given key. func (dht *IpfsDHT) FindProviders(ctx context.Context, key u.Key) ([]*peer.Peer, error) { // get closest peer - u.DOut("Find providers for: '%s'\n", key.Pretty()) + log.Debug("Find providers for: '%s'", key.Pretty()) p := dht.routingTables[0].NearestPeer(kb.ConvertKey(key)) if p == nil { return nil, nil @@ -333,17 +333,17 @@ func (dht *IpfsDHT) findPeerMultiple(ctx context.Context, id peer.ID) (*peer.Pee // Ping a peer, log the time it took func (dht *IpfsDHT) Ping(ctx context.Context, p *peer.Peer) error { // Thoughts: maybe this should accept an ID and do a peer lookup? - u.DOut("[%s] ping %s start\n", dht.self.ID.Pretty(), p.ID.Pretty()) + log.Info("ping %s start", p.ID.Pretty()) pmes := newMessage(Message_PING, "", 0) _, err := dht.sendRequest(ctx, p, pmes) - u.DOut("[%s] ping %s end (err = %s)\n", dht.self.ID.Pretty(), p.ID.Pretty(), err) + log.Info("ping %s end (err = %s)", p.ID.Pretty(), err) return err } func (dht *IpfsDHT) getDiagnostic(ctx context.Context) ([]*diagInfo, error) { - u.DOut("Begin Diagnostic") + log.Info("Begin Diagnostic") peers := dht.routingTables[0].NearestPeers(kb.ConvertPeerID(dht.self.ID), 10) var out []*diagInfo diff --git a/util/util.go b/util/util.go index f2ce35a344e..b33960471a8 100644 --- a/util/util.go +++ b/util/util.go @@ -14,7 +14,7 @@ import ( "github.com/op/go-logging" ) -var format = "%{color}%{time} %{shortfile} %{level}: %{color:reset}%{message}" +var format = "%{color}%{time:01-02 15:04:05.9999} %{shortfile} %{level}: %{color:reset}%{message}" func init() { backend := logging.NewLogBackend(os.Stderr, "", 0) From 9652ada0d2029f8d97ca1219dbbf2e8d33e13699 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 29 Sep 2014 17:47:13 +0000 Subject: [PATCH 021/105] implement publisher for ipns to wait until moments of rapid churn die down --- fuse/ipns/ipns_unix.go | 30 +++++++++++++++++++++--------- routing/dht/dht.go | 7 +++---- routing/dht/routing.go | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index b15ee11e969..c5b31a13c70 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -76,6 +76,9 @@ func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, er nd := new(Node) nd.Ipfs = n nd.key = k + nd.repub = NewRepublisher(nd, time.Millisecond*10) + + go nd.repub.Run() pointsTo, err := n.Namesys.Resolve(name) if err != nil { @@ -185,6 +188,8 @@ func (r *Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { type Node struct { nsRoot *Node + repub *Republisher + // Name really only for logging purposes name string @@ -316,16 +321,20 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { return fuse.ENODATA } - err = n.updateTree() - if err != nil { - log.Error("updateTree failed: %s", err) - return fuse.ENODATA - } - + n.sendPublishSignal() } return nil } +func (n *Node) sendPublishSignal() { + root := n.nsRoot + if root == nil { + root = n + } + + root.repub.Publish <- struct{}{} +} + func (n *Node) updateTree() error { var root *Node if n.nsRoot != nil { @@ -387,7 +396,7 @@ func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) } n.changed = true - n.updateTree() + n.sendPublishSignal() return child, nil } @@ -425,6 +434,8 @@ func (n *Node) Remove(req *fuse.RemoveRequest, intr fs.Intr) fuse.Error { log.Error("Remove: No such file.") return fuse.ENOENT } + n.changed = true + n.sendPublishSignal() return nil } @@ -471,7 +482,7 @@ func Mount(ipfs *core.IpfsNode, fpath string, ipfspath string) error { if err == nil { return } - time.Sleep(time.Millisecond * 10) + time.Sleep(time.Millisecond * 100) } ipfs.Network.Close() }() @@ -563,12 +574,13 @@ func NewRepublisher(n *Node, tout time.Duration) *Republisher { } func (np *Republisher) Run() { - for _ := range np.Publish { + for _ = range np.Publish { timer := time.After(np.Timeout) for { select { case <-timer: //Do the publish! + log.Info("Publishing Changes!") err := np.node.updateTree() if err != nil { log.Critical("updateTree error: %s", err) diff --git a/routing/dht/dht.go b/routing/dht/dht.go index 4e844ecd6f4..eccfd7e451e 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -177,8 +177,7 @@ func (dht *IpfsDHT) sendRequest(ctx context.Context, p *peer.Peer, pmes *Message start := time.Now() // Print out diagnostic - log.Debug("[peer: %s] Sent message type: '%s' [to = %s]\n", - dht.self.ID.Pretty(), + log.Debug("Sent message type: '%s' [to = %s]", Message_MessageType_name[int32(pmes.GetType())], p.ID.Pretty()) rmes, err := dht.sender.SendRequest(ctx, mes) @@ -281,7 +280,7 @@ func (dht *IpfsDHT) getValueOrPeers(ctx context.Context, p *peer.Peer, } if len(peers) > 0 { - u.DOut("getValueOrPeers: peers") + log.Debug("getValueOrPeers: peers") return nil, peers, nil } @@ -400,7 +399,7 @@ func (dht *IpfsDHT) addProviders(key u.Key, peers []*Message_Peer) []*peer.Peer for _, prov := range peers { p, err := dht.peerFromInfo(prov) if err != nil { - u.PErr("error getting peer from info: %v\n", err) + log.Error("error getting peer from info: %v", err) continue } diff --git a/routing/dht/routing.go b/routing/dht/routing.go index d1f50afb889..4fa6c8c9466 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -18,7 +18,7 @@ import ( // PutValue adds value corresponding to given Key. // This is the top level "Store" operation of the DHT func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error { - log.Debug("PutValue %s %v", key.Pretty(), value) + log.Debug("PutValue %s", key.Pretty()) err := dht.putLocal(key, value) if err != nil { return err From f085d594aab6d1e416442821d99e5d2817ceb477 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 00:44:22 -0700 Subject: [PATCH 022/105] vendoring ipns things --- Godeps/Godeps.json | 4 + .../github.com/bren2010/proquint/README.md | 6 + .../github.com/bren2010/proquint/proquint.go | 123 ++++++++++++++++++ blockservice/blockservice.go | 2 +- core/commands/publish.go | 2 +- fuse/ipns/ipns_unix.go | 8 +- fuse/readonly/readonly_unix.go | 4 +- namesys/entry.pb.go | 2 +- namesys/proquint.go | 2 +- namesys/publisher.go | 4 +- namesys/routing.go | 8 +- path/path.go | 2 +- routing/dht/dht.go | 2 +- util/util.go | 2 +- 14 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/bren2010/proquint/README.md create mode 100644 Godeps/_workspace/src/github.com/bren2010/proquint/proquint.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 985b0bccf28..fbe7494b0aa 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -38,6 +38,10 @@ "Comment": "null-15", "Rev": "12e4b4183793ac4b061921e7980845e750679fd0" }, + { + "ImportPath": "github.com/bren2010/proquint", + "Rev": "5958552242606512f714d2e93513b380f43f9991" + }, { "ImportPath": "github.com/camlistore/lock", "Rev": "ae27720f340952636b826119b58130b9c1a847a0" diff --git a/Godeps/_workspace/src/github.com/bren2010/proquint/README.md b/Godeps/_workspace/src/github.com/bren2010/proquint/README.md new file mode 100644 index 00000000000..13e7b0b5eba --- /dev/null +++ b/Godeps/_workspace/src/github.com/bren2010/proquint/README.md @@ -0,0 +1,6 @@ +Proquint +------- + +Golang implementation of [Proquint Pronounceable Identifiers](https://github.com/deoxxa/proquint). + + diff --git a/Godeps/_workspace/src/github.com/bren2010/proquint/proquint.go b/Godeps/_workspace/src/github.com/bren2010/proquint/proquint.go new file mode 100644 index 00000000000..60e1cf9816f --- /dev/null +++ b/Godeps/_workspace/src/github.com/bren2010/proquint/proquint.go @@ -0,0 +1,123 @@ +/* +Copyright (c) 2014 Brendan McMillion + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package proquint + +import ( + "bytes" + "strings" + "regexp" +) + +var ( + conse = [...]byte{'b', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', + 'p', 'r', 's', 't', 'v', 'z'} + vowse = [...]byte{'a', 'i', 'o', 'u'} + + consd = map[byte] uint16 { + 'b' : 0, 'd' : 1, 'f' : 2, 'g' : 3, + 'h' : 4, 'j' : 5, 'k' : 6, 'l' : 7, + 'm' : 8, 'n' : 9, 'p' : 10, 'r' : 11, + 's' : 12, 't' : 13, 'v' : 14, 'z' : 15, + } + + vowsd = map[byte] uint16 { + 'a' : 0, 'i' : 1, 'o' : 2, 'u' : 3, + } +) + +/** +* Tests if a given string is a Proquint identifier +* +* @param {string} str The candidate string. +* +* @return {bool} Whether or not it qualifies. +* @return {error} Error +*/ +func IsProquint(str string) (bool, error) { + exp := "^([abdfghijklmnoprstuvz]{5}-)*[abdfghijklmnoprstuvz]{5}$" + ok, err := regexp.MatchString(exp, str) + + return ok, err +} + +/** +* Encodes an arbitrary byte slice into an identifier. +* +* @param {[]byte} buf Slice of bytes to encode. +* +* @return {string} The given byte slice as an identifier. +*/ +func Encode(buf []byte) string { + var out bytes.Buffer + + for i := 0; i < len(buf); i = i + 2 { + var n uint16 = (uint16(buf[i]) * 256) + uint16(buf[i + 1]) + + var ( + c1 = n & 0x0f + v1 = (n >> 4) & 0x03 + c2 = (n >> 6) & 0x0f + v2 = (n >> 10) & 0x03 + c3 = (n >> 12) & 0x0f + ) + + out.WriteByte(conse[c1]) + out.WriteByte(vowse[v1]) + out.WriteByte(conse[c2]) + out.WriteByte(vowse[v2]) + out.WriteByte(conse[c3]) + + if (i + 2) < len(buf) { + out.WriteByte('-') + } + } + + return out.String() +} + +/** +* Decodes an identifier into its corresponding byte slice. +* +* @param {string} str Identifier to convert. +* +* @return {[]byte} The identifier as a byte slice. +*/ +func Decode(str string) []byte { + var ( + out bytes.Buffer + bits []string = strings.Split(str, "-") + ) + + for i := 0; i < len(bits); i++ { + var x uint16 = consd[bits[i][0]] + + (vowsd[bits[i][1]] << 4) + + (consd[bits[i][2]] << 6) + + (vowsd[bits[i][3]] << 10) + + (consd[bits[i][4]] << 12) + + out.WriteByte(byte(x >> 8)) + out.WriteByte(byte(x)) + } + + return out.Bytes() +} diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index 51a6e0014e9..36c3ed607b0 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -7,7 +7,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - "github.com/op/go-logging" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" blocks "github.com/jbenet/go-ipfs/blocks" exchange "github.com/jbenet/go-ipfs/exchange" diff --git a/core/commands/publish.go b/core/commands/publish.go index 145e9b06edf..bd281b44b8e 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -7,7 +7,7 @@ import ( "github.com/jbenet/go-ipfs/core" u "github.com/jbenet/go-ipfs/util" - "github.com/op/go-logging" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" nsys "github.com/jbenet/go-ipfs/namesys" ) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index c5b31a13c70..e7247034b1c 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -13,15 +13,15 @@ import ( "bytes" - "bazil.org/fuse" - "bazil.org/fuse/fs" - "code.google.com/p/goprotobuf/proto" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" "github.com/jbenet/go-ipfs/core" ci "github.com/jbenet/go-ipfs/crypto" imp "github.com/jbenet/go-ipfs/importer" mdag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" - "github.com/op/go-logging" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) var log = logging.MustGetLogger("ipns") diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 1fa7bfb0d24..8641916e251 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -14,14 +14,14 @@ import ( "syscall" "time" - "code.google.com/p/goprotobuf/proto" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" core "github.com/jbenet/go-ipfs/core" mdag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" - "github.com/op/go-logging" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) var log = logging.MustGetLogger("ipfs") diff --git a/namesys/entry.pb.go b/namesys/entry.pb.go index e4420e9f978..d9dc5160b4f 100644 --- a/namesys/entry.pb.go +++ b/namesys/entry.pb.go @@ -13,7 +13,7 @@ It has these top-level messages: */ package namesys -import proto "code.google.com/p/goprotobuf/proto" +import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" import math "math" // Reference imports to suppress errors if they are not otherwise used. diff --git a/namesys/proquint.go b/namesys/proquint.go index 9b64f3fde51..bf34c3b6c99 100644 --- a/namesys/proquint.go +++ b/namesys/proquint.go @@ -3,7 +3,7 @@ package namesys import ( "errors" - proquint "github.com/bren2010/proquint" + proquint "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/bren2010/proquint" ) type ProquintResolver struct{} diff --git a/namesys/publisher.go b/namesys/publisher.go index 73a4197e1bb..a4292d7fed9 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -3,8 +3,8 @@ package namesys import ( "time" - "code.google.com/p/go.net/context" - "code.google.com/p/goprotobuf/proto" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ci "github.com/jbenet/go-ipfs/crypto" mdag "github.com/jbenet/go-ipfs/merkledag" diff --git a/namesys/routing.go b/namesys/routing.go index 605458eef04..4c2b0d81655 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -3,15 +3,15 @@ package namesys import ( "fmt" - "code.google.com/p/go.net/context" - "code.google.com/p/goprotobuf/proto" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ci "github.com/jbenet/go-ipfs/crypto" mdag "github.com/jbenet/go-ipfs/merkledag" "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" - mh "github.com/jbenet/go-multihash" - "github.com/op/go-logging" + mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) var log = logging.MustGetLogger("namesys") diff --git a/path/path.go b/path/path.go index 533afc7cd17..ebc9d81d665 100644 --- a/path/path.go +++ b/path/path.go @@ -8,7 +8,7 @@ import ( mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" merkledag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" - "github.com/op/go-logging" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) var log = logging.MustGetLogger("path") diff --git a/routing/dht/dht.go b/routing/dht/dht.go index eccfd7e451e..f02be471146 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -13,7 +13,7 @@ import ( peer "github.com/jbenet/go-ipfs/peer" kb "github.com/jbenet/go-ipfs/routing/kbucket" u "github.com/jbenet/go-ipfs/util" - "github.com/op/go-logging" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" diff --git a/util/util.go b/util/util.go index b33960471a8..4778146c0c4 100644 --- a/util/util.go +++ b/util/util.go @@ -11,7 +11,7 @@ import ( ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - "github.com/op/go-logging" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) var format = "%{color}%{time:01-02 15:04:05.9999} %{shortfile} %{level}: %{color:reset}%{message}" From 7cd9e4db3e7da26f14a737268445bbb90eefb4f4 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 00:45:36 -0700 Subject: [PATCH 023/105] remove merge issue (logger) --- core/commands/publish.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/commands/publish.go b/core/commands/publish.go index bd281b44b8e..90f4d221131 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -7,13 +7,10 @@ import ( "github.com/jbenet/go-ipfs/core" u "github.com/jbenet/go-ipfs/util" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" nsys "github.com/jbenet/go-ipfs/namesys" ) -var log = logging.MustGetLogger("commands") - func Publish(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error { log.Debug("Begin Publish") if n.Identity == nil { From 11a8826d4fa29eae26d420f635172a5371835b79 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 00:46:39 -0700 Subject: [PATCH 024/105] RPCAddress -> ddresses.API --- cmd/ipfs/run.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/ipfs/run.go b/cmd/ipfs/run.go index a3010cb0750..cc9fe8483c1 100644 --- a/cmd/ipfs/run.go +++ b/cmd/ipfs/run.go @@ -32,11 +32,11 @@ func runCmd(c *commander.Command, inp []string) error { } // launch the RPC endpoint. - if n.Config.RPCAddress == "" { - return errors.New("no config.RPCAddress endpoint supplied") + if n.Config.Addresses.API == "" { + return errors.New("no config.Addresses.API endpoint supplied") } - maddr, err := ma.NewMultiaddr(n.Config.RPCAddress) + maddr, err := ma.NewMultiaddr(n.Config.Addresses.API) if err != nil { return err } From 26a481a904ab51501184da6c2693e17a6b53a0e1 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 00:52:26 -0700 Subject: [PATCH 025/105] fixed key hashing interface + test --- crypto/key.go | 11 +++++++++++ crypto/key_test.go | 4 ++++ crypto/rsa.go | 13 ++----------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/crypto/key.go b/crypto/key.go index 95cbfb44005..4f702479a46 100644 --- a/crypto/key.go +++ b/crypto/key.go @@ -15,6 +15,8 @@ import ( "math/big" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + + u "github.com/jbenet/go-ipfs/util" ) var ErrBadKeyType = errors.New("invalid or unsupported key type") @@ -249,3 +251,12 @@ func KeyEqual(k1, k2 Key) bool { b2, err2 := k2.Bytes() return bytes.Equal(b1, b2) && err1 == err2 } + +// KeyHash hashes a key. +func KeyHash(k Key) ([]byte, error) { + kb, err := k.Bytes() + if err != nil { + return nil, err + } + return u.Hash(kb) +} diff --git a/crypto/key_test.go b/crypto/key_test.go index 13c94215e80..112b99bee57 100644 --- a/crypto/key_test.go +++ b/crypto/key_test.go @@ -92,3 +92,7 @@ func (pk testkey) Bytes() ([]byte, error) { func (pk testkey) Equals(k Key) bool { return KeyEqual(pk, k) } + +func (pk testkey) Hash() ([]byte, error) { + return KeyHash(pk) +} diff --git a/crypto/rsa.go b/crypto/rsa.go index 124c3e0706f..bad04bfb53d 100644 --- a/crypto/rsa.go +++ b/crypto/rsa.go @@ -9,7 +9,6 @@ import ( "errors" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" - u "github.com/jbenet/go-ipfs/util" ) type RsaPrivateKey struct { @@ -48,11 +47,7 @@ func (pk *RsaPublicKey) Equals(k Key) bool { } func (pk *RsaPublicKey) Hash() ([]byte, error) { - pkb, err := pk.Bytes() - if err != nil { - return nil, err - } - return u.Hash(pkb) + return KeyHash(pk) } func (sk *RsaPrivateKey) GenSecret() []byte { @@ -85,11 +80,7 @@ func (sk *RsaPrivateKey) Equals(k Key) bool { } func (sk *RsaPrivateKey) Hash() ([]byte, error) { - skb, err := sk.Bytes() - if err != nil { - return nil, err - } - return u.Hash(skb) + return KeyHash(sk) } func UnmarshalRsaPrivateKey(b []byte) (*RsaPrivateKey, error) { From 728f17d3c9f43aa88d88b98dec959604576fcc73 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 01:10:57 -0700 Subject: [PATCH 026/105] cmd/ipfs/pin.go now uses MakeCommand + added recursive pinning func --- cmd/ipfs/pin.go | 32 ++++---------------------------- core/commands/add.go | 2 +- core/commands/pin.go | 9 ++++++++- core/core.go | 9 +++++++-- 4 files changed, 20 insertions(+), 32 deletions(-) diff --git a/cmd/ipfs/pin.go b/cmd/ipfs/pin.go index bea074c5103..fcebe6c4d77 100644 --- a/cmd/ipfs/pin.go +++ b/cmd/ipfs/pin.go @@ -1,13 +1,9 @@ package main import ( - "os" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" "github.com/jbenet/go-ipfs/core/commands" - "github.com/jbenet/go-ipfs/daemon" - u "github.com/jbenet/go-ipfs/util" ) var cmdIpfsPin = &commander.Command{ @@ -22,28 +18,8 @@ var cmdIpfsPin = &commander.Command{ Flag: *flag.NewFlagSet("ipfs-pin", flag.ExitOnError), } -func pinCmd(c *commander.Command, inp []string) error { - if len(inp) < 1 { - u.POut(c.Long) - return nil - } - - cmd := daemon.NewCommand() - cmd.Command = "pin" - cmd.Args = inp - - err := daemon.SendCommand(cmd, "localhost:12345") - if err != nil { - conf, err := getConfigDir(c.Parent) - if err != nil { - return err - } - n, err := localNode(conf, false) - if err != nil { - return err - } - - return commands.Pin(n, cmd.Args, cmd.Opts, os.Stdout) - } - return nil +func init() { + cmdIpfsPin.Flag.Bool("r", false, "pin objects recursively") } + +var pinCmd = MakeCommand("pin", []string{"r"}, commands.Pin) diff --git a/core/commands/add.go b/core/commands/add.go index f6c5492ad16..1af9de79575 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -133,5 +133,5 @@ func addNode(n *core.IpfsNode, nd *dag.Node, fpath string) error { u.POut("added %s %s\n", k.Pretty(), fpath) // ensure we keep it. atm no-op - return n.PinDagNode(nd) + return n.PinDagNodeRecursively(nd, -1) } diff --git a/core/commands/pin.go b/core/commands/pin.go index 8594ee9489c..5dbbf6304e3 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -8,13 +8,20 @@ import ( ) func Pin(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error { + + // if recursive, set flag + depth := 1 + if r, ok := opts["r"].(bool); r && ok { + depth = -1 + } + for _, fn := range args { dagnode, err := n.Resolver.ResolvePath(fn) if err != nil { return fmt.Errorf("pin error: %v", err) } - err = n.PinDagNode(dagnode) + err = n.PinDagNodeRecursively(dagnode, depth) if err != nil { return fmt.Errorf("pin: %v", err) } diff --git a/core/core.go b/core/core.go index 1761a38ac59..4b17d6cebe9 100644 --- a/core/core.go +++ b/core/core.go @@ -240,8 +240,13 @@ func initConnections(ctx context.Context, cfg *config.Config, pstore peer.Peerst } } -// PinDagNode ensures a given node is stored persistently locally. +// PinDagNode ensures a given node is stored persistently locally func (n *IpfsNode) PinDagNode(nd *merkledag.Node) error { - u.DOut("Pinning node. Currently No-Op\n") + return n.PinDagNodeRecursively(nd, 1) +} + +// PinDagNodeRecursively ensures a given node is stored persistently locally +func (n *IpfsNode) PinDagNodeRecursively(nd *merkledag.Node, depth int) error { + u.DOut("Pinning node recursively. Currently No-Op\n") return nil } From 14a384d82679b3efcd708d9694575d93adf310cb Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 01:26:48 -0700 Subject: [PATCH 027/105] pin: add depth arg. --- cmd/ipfs/pin.go | 3 ++- core/commands/pin.go | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/cmd/ipfs/pin.go b/cmd/ipfs/pin.go index fcebe6c4d77..5919168b115 100644 --- a/cmd/ipfs/pin.go +++ b/cmd/ipfs/pin.go @@ -20,6 +20,7 @@ var cmdIpfsPin = &commander.Command{ func init() { cmdIpfsPin.Flag.Bool("r", false, "pin objects recursively") + cmdIpfsPin.Flag.Int("d", 1, "recursive depth") } -var pinCmd = MakeCommand("pin", []string{"r"}, commands.Pin) +var pinCmd = MakeCommand("pin", []string{"r", "d"}, commands.Pin) diff --git a/core/commands/pin.go b/core/commands/pin.go index 5dbbf6304e3..40b452ffae8 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -9,12 +9,20 @@ import ( func Pin(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error { - // if recursive, set flag - depth := 1 - if r, ok := opts["r"].(bool); r && ok { - depth = -1 + // set recursive flag + recursive, _ := opts["r"].(bool) // false if cast fails. + + // if recursive, set depth flag + depth := 1 // default (non recursive) + if d, ok := opts["d"].(int); recursive && ok { + depth = d + } + if depth < -1 { + return fmt.Errorf("ipfs pin: called with invalid depth: %v", depth) } + fmt.Printf("recursive, depth: %v, %v\n", recursive, depth) + for _, fn := range args { dagnode, err := n.Resolver.ResolvePath(fn) if err != nil { From 1cfb0ffd0566e2623baed3e664f805c344490e40 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 01:28:20 -0700 Subject: [PATCH 028/105] command output nit --- core/commands/publish.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/publish.go b/core/commands/publish.go index 90f4d221131..b292361cd26 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -29,7 +29,7 @@ func Publish(n *core.IpfsNode, args []string, opts map[string]interface{}, out i if err != nil { return err } - fmt.Fprintf(out, "Published %s to %s\n", args[0], u.Key(hash).Pretty()) + fmt.Fprintf(out, "published mapping %s to %s\n", u.Key(hash).Pretty(), args[0]) return nil } From b787c705e2e62357f7ef87fe02a1fa6df12ee81b Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 01:28:55 -0700 Subject: [PATCH 029/105] command output nit --- core/commands/resolve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/resolve.go b/core/commands/resolve.go index bd3672e185e..f4bd0c5462d 100644 --- a/core/commands/resolve.go +++ b/core/commands/resolve.go @@ -13,6 +13,6 @@ func Resolve(n *core.IpfsNode, args []string, opts map[string]interface{}, out i return err } - fmt.Fprintf(out, "%s -> %s\n", args[0], res) + fmt.Fprintf(out, "%s\n", res) return nil } From 41aab9f46f495e5a48ef1aeae574b42bbafcd466 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 01:30:01 -0700 Subject: [PATCH 030/105] comment + import nits --- fuse/ipns/ipns_unix.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index e7247034b1c..7bd3e3e1b50 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -16,17 +16,18 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" + "github.com/jbenet/go-ipfs/core" ci "github.com/jbenet/go-ipfs/crypto" imp "github.com/jbenet/go-ipfs/importer" mdag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) var log = logging.MustGetLogger("ipns") -// FileSystem is the readonly Ipfs Fuse Filesystem. +// FileSystem is the readwrite IPNS Fuse Filesystem. type FileSystem struct { Ipfs *core.IpfsNode RootNode *Root From 19b0a28d43c8de879c05870da665451a75da6561 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 01:31:04 -0700 Subject: [PATCH 031/105] security TODO warning --- fuse/ipns/ipns_unix.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 7bd3e3e1b50..56f964b1270 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -195,6 +195,8 @@ type Node struct { name string // Private keys held by nodes at the root of a keyspace + // WARNING(security): the PrivKey interface is currently insecure + // (holds the raw key). It will be secured later. key ci.PrivKey Ipfs *core.IpfsNode From ba510cbd6dd432519b43eb594dd2a214720a0970 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 01:36:21 -0700 Subject: [PATCH 032/105] IpnsPublicher -> Publisher interface --- core/core.go | 2 +- namesys/publisher.go | 12 ++++++++---- namesys/resolve_test.go | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core/core.go b/core/core.go index 4b17d6cebe9..dd5887ef54c 100644 --- a/core/core.go +++ b/core/core.go @@ -64,7 +64,7 @@ type IpfsNode struct { Namesys namesys.Resolver // the routing publisher - Publisher *namesys.IpnsPublisher + Publisher namesys.Publisher } // NewIpfsNode constructs a new IpfsNode based on the given config. diff --git a/namesys/publisher.go b/namesys/publisher.go index a4292d7fed9..76eb6a55b01 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -12,20 +12,24 @@ import ( u "github.com/jbenet/go-ipfs/util" ) -type IpnsPublisher struct { +type ipnsPublisher struct { dag *mdag.DAGService routing routing.IpfsRouting } -func NewPublisher(dag *mdag.DAGService, route routing.IpfsRouting) *IpnsPublisher { - return &IpnsPublisher{ +type Publisher interface { + Publish(ci.PrivKey, string) error +} + +func NewPublisher(dag *mdag.DAGService, route routing.IpfsRouting) Publisher { + return &ipnsPublisher{ dag: dag, routing: route, } } // Publish accepts a keypair and a value, -func (p *IpnsPublisher) Publish(k ci.PrivKey, value string) error { +func (p *ipnsPublisher) Publish(k ci.PrivKey, value string) error { log.Debug("namesys: Publish %s", value) ctx := context.TODO() data, err := CreateEntryData(k, value) diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index 7cad8bef098..35898b50f23 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -28,7 +28,7 @@ func TestRoutingResolve(t *testing.T) { resolve := NewMasterResolver(d, dag) - pub := IpnsPublisher{ + pub := ipnsPublisher{ dag: dag, routing: d, } From aa79f38580661570304a9b72a32f7ccd1e2c7ac8 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 01:56:27 -0700 Subject: [PATCH 033/105] cleanup logging setup. - should not be on init, because need debug flag --- cmd/ipfs/ipfs.go | 35 ++++++++++++++++++++++------------- cmd/ipfs/publish.go | 1 - cmd/ipfs/resolve.go | 1 - cmd/ipfs/run.go | 3 --- util/util.go | 24 ++++++++++++++++-------- 5 files changed, 38 insertions(+), 26 deletions(-) diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 56c07945391..32d24ad120d 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -6,9 +6,10 @@ import ( "os" "runtime/pprof" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" - "github.com/jbenet/go-ipfs/config" + flag "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" + commander "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" + + config "github.com/jbenet/go-ipfs/config" core "github.com/jbenet/go-ipfs/core" u "github.com/jbenet/go-ipfs/util" ) @@ -72,16 +73,24 @@ func ipfsCmd(c *commander.Command, args []string) error { } func main() { - u.Debug = true - ofi, err := os.Create("cpu.prof") - if err != nil { - fmt.Println(err) - return + u.Debug = false + + // setup logging + u.SetupLogging() + + // if debugging, setup profiling. + if u.Debug { + ofi, err := os.Create("cpu.prof") + if err != nil { + fmt.Println(err) + return + } + pprof.StartCPUProfile(ofi) + defer ofi.Close() + defer pprof.StopCPUProfile() } - pprof.StartCPUProfile(ofi) - defer ofi.Close() - defer pprof.StopCPUProfile() - err = CmdIpfs.Dispatch(os.Args[1:]) + + err := CmdIpfs.Dispatch(os.Args[1:]) if err != nil { if len(err.Error()) > 0 { fmt.Fprintf(os.Stderr, "ipfs %s: %v\n", os.Args[1], err) @@ -114,7 +123,7 @@ func getConfigDir(c *commander.Command) (string, error) { } confStr, ok := conf.(string) if !ok { - return "", errors.New("failed to retrieve config flag value.") + return "", errors.New("failed to retrieve config flag value") } if len(confStr) == 0 { return config.PathRoot() diff --git a/cmd/ipfs/publish.go b/cmd/ipfs/publish.go index 649767614da..8e2aa7cc4b3 100644 --- a/cmd/ipfs/publish.go +++ b/cmd/ipfs/publish.go @@ -25,7 +25,6 @@ func init() { } func pubCmd(c *commander.Command, inp []string) error { - u.Debug = true if len(inp) < 1 { u.POut(c.Long) return nil diff --git a/cmd/ipfs/resolve.go b/cmd/ipfs/resolve.go index 18841624b16..3665f2a9177 100644 --- a/cmd/ipfs/resolve.go +++ b/cmd/ipfs/resolve.go @@ -23,7 +23,6 @@ var cmdIpfsResolve = &commander.Command{ } func resolveCmd(c *commander.Command, inp []string) error { - u.Debug = true if len(inp) < 1 { u.POut(c.Long) return nil diff --git a/cmd/ipfs/run.go b/cmd/ipfs/run.go index cc9fe8483c1..0342dceaa43 100644 --- a/cmd/ipfs/run.go +++ b/cmd/ipfs/run.go @@ -7,7 +7,6 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" "github.com/jbenet/go-ipfs/daemon" - u "github.com/jbenet/go-ipfs/util" ) var cmdIpfsRun = &commander.Command{ @@ -20,8 +19,6 @@ var cmdIpfsRun = &commander.Command{ } func runCmd(c *commander.Command, inp []string) error { - u.Debug = true - conf, err := getConfigDir(c.Parent) if err != nil { return err diff --git a/util/util.go b/util/util.go index 4778146c0c4..5b2505a9770 100644 --- a/util/util.go +++ b/util/util.go @@ -11,16 +11,11 @@ import ( ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) -var format = "%{color}%{time:01-02 15:04:05.9999} %{shortfile} %{level}: %{color:reset}%{message}" - -func init() { - backend := logging.NewLogBackend(os.Stderr, "", 0) - logging.SetBackend(backend) - logging.SetFormatter(logging.MustStringFormatter(format)) -} +// LogFormat is the format used for our logger. +var LogFormat = "%{color}%{time:01-02 15:04:05.9999} %{shortfile} %{level}: %{color:reset}%{message}" // Debug is a global flag for debugging. var Debug bool @@ -51,6 +46,7 @@ func Hash(data []byte) (mh.Multihash, error) { return mh.Sum(data, mh.SHA2_256, -1) } +// IsValidHash checks whether a given hash is valid (b58 decodable, len > 0) func IsValidHash(s string) bool { out := b58.Decode(s) if out == nil || len(out) == 0 { @@ -99,6 +95,18 @@ func DOut(format string, a ...interface{}) { } } +// SetupLogging will initialize the logger backend and set the flags. +func SetupLogging() { + backend := logging.NewLogBackend(os.Stderr, "", 0) + logging.SetBackend(backend) + if Debug { + logging.SetLevel(logging.DEBUG, "") + } else { + logging.SetLevel(logging.ERROR, "") + } + logging.SetFormatter(logging.MustStringFormatter(LogFormat)) +} + // ExpandPathnames takes a set of paths and turns them into absolute paths func ExpandPathnames(paths []string) ([]string, error) { var out []string From b6a59c90cf1dec282cd8e7d039ef6005614b21ca Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 2 Oct 2014 06:14:38 +0000 Subject: [PATCH 034/105] add flag for ipfs mountpoint --- cmd/ipfs/mount_unix.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/ipfs/mount_unix.go b/cmd/ipfs/mount_unix.go index 2f7e623f06a..e37400e11b0 100644 --- a/cmd/ipfs/mount_unix.go +++ b/cmd/ipfs/mount_unix.go @@ -32,6 +32,7 @@ var cmdIpfsMount = &commander.Command{ func init() { cmdIpfsMount.Flag.String("n", "", "specify a mountpoint for ipns") + cmdIpfsMount.Flag.String("f", "/ipfs", "specify a mountpoint for ipfs") } func mountCmd(c *commander.Command, inp []string) error { @@ -69,12 +70,12 @@ func mountCmd(c *commander.Command, inp []string) error { go dl.Listen() defer dl.Close() - mp := inp[0] + mp := c.Flag.Lookup("f").Value.Get().(string) fmt.Printf("Mounting at %s\n", mp) var ipnsDone chan struct{} ns, ok := c.Flag.Lookup("n").Value.Get().(string) - if ok { + if ok && ns != "" { ipnsDone = make(chan struct{}) go func() { err = ipns.Mount(n, ns, mp) From 33dfa6f6408b0c38430b88f36280f905890d0b44 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 1 Oct 2014 02:30:19 -0700 Subject: [PATCH 035/105] lint --- config/config.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index cf5cced4477..5c8f202a7b9 100644 --- a/config/config.go +++ b/config/config.go @@ -43,9 +43,16 @@ type Config struct { Bootstrap []*BootstrapPeer // local nodes's bootstrap peers } +// DefaultPathRoot is the path to the default config dir location. const DefaultPathRoot = "~/.go-ipfs" + +// DefaultConfigFile is the filename of the configuration file const DefaultConfigFile = "config" + +// DefaultDataStoreDirectory is the directory to store all the local IPFS data. const DefaultDataStoreDirectory = "datastore" + +// EnvDir is the environment variable used to change the path root. const EnvDir = "IPFS_DIR" // PathRoot returns the default configuration root directory @@ -65,13 +72,11 @@ func Path(configroot, extension string) (string, error) { dir, err := PathRoot() if err != nil { return "", err - } else { - return filepath.Join(dir, extension), nil } + return filepath.Join(dir, extension), nil - } else { - return filepath.Join(configroot, extension), nil } + return filepath.Join(configroot, extension), nil } // DataStorePath returns the default data store path given a configuration root From ee1ba164d0eda69ce7b6d1739e293f14bd67de2b Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 00:09:32 -0700 Subject: [PATCH 036/105] fix arg bug --- cmd/ipfs/mount_unix.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/ipfs/mount_unix.go b/cmd/ipfs/mount_unix.go index e37400e11b0..03515ad5fb7 100644 --- a/cmd/ipfs/mount_unix.go +++ b/cmd/ipfs/mount_unix.go @@ -13,7 +13,6 @@ import ( "github.com/jbenet/go-ipfs/daemon" ipns "github.com/jbenet/go-ipfs/fuse/ipns" rofs "github.com/jbenet/go-ipfs/fuse/readonly" - u "github.com/jbenet/go-ipfs/util" ) var cmdIpfsMount = &commander.Command{ @@ -36,11 +35,6 @@ func init() { } func mountCmd(c *commander.Command, inp []string) error { - if len(inp) < 1 || len(inp[0]) == 0 { - u.POut(c.Long) - return nil - } - conf, err := getConfigDir(c.Parent) if err != nil { fmt.Println("Couldnt get config dir") From 381824477ec92a805dd4011d94eeddf9635a86b3 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 00:04:34 -0700 Subject: [PATCH 037/105] Bugfix: daemon wg sync --- cmd/ipfs/init.go | 2 +- daemon/daemon.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/ipfs/init.go b/cmd/ipfs/init.go index f007972bae5..b868f2b91dc 100644 --- a/cmd/ipfs/init.go +++ b/cmd/ipfs/init.go @@ -2,9 +2,9 @@ package main import ( "encoding/base64" - "path/filepath" "errors" "os" + "path/filepath" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" diff --git a/daemon/daemon.go b/daemon/daemon.go index 2e4681592de..8ec84d8482f 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -94,7 +94,10 @@ func (dl *DaemonListener) Listen() { panic("attempting to listen on a closed daemon Listener") } + // add ourselves to workgroup. and remove ourselves when done. dl.wg.Add(1) + defer dl.wg.Done() + log.Info("daemon listening") for { conn, err := dl.list.Accept() @@ -102,7 +105,6 @@ func (dl *DaemonListener) Listen() { if !dl.closed { log.Warning("DaemonListener Accept: %v", err) } - dl.lk.Close() return } go dl.handleConnection(conn) From 2bd843123e5b3b840a0f975896599862d2861d10 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 02:47:36 -0700 Subject: [PATCH 038/105] added command context initialization --- cmd/ipfs/ipfs.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++ cmd/ipfs/run.go | 32 ++++++++++-------------------- cmd/ipfs/serve.go | 42 +++++++++------------------------------ 3 files changed, 69 insertions(+), 55 deletions(-) diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 32d24ad120d..8e8df2080bd 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -8,9 +8,11 @@ import ( flag "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" commander "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" + ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" config "github.com/jbenet/go-ipfs/config" core "github.com/jbenet/go-ipfs/core" + daemon "github.com/jbenet/go-ipfs/daemon" u "github.com/jbenet/go-ipfs/util" ) @@ -100,6 +102,7 @@ func main() { return } +// localNode constructs a node func localNode(confdir string, online bool) (*core.IpfsNode, error) { filename, err := config.Filename(confdir) if err != nil { @@ -131,3 +134,50 @@ func getConfigDir(c *commander.Command) (string, error) { return u.TildeExpansion(confStr) } + +// cmdContext is a wrapper structure that keeps a node, a daemonlistener, and +// a config directory together. These three are needed for most commands. +type cmdContext struct { + node *core.IpfsNode + daemon *daemon.DaemonListener + configDir string +} + +// setupCmdContext initializes a cmdContext structure from a given command. +func setupCmdContext(c *commander.Command, online bool) (cc cmdContext, err error) { + cc.configDir, err = getConfigDir(c.Parent.Parent) + if err != nil { + return + } + + cc.node, err = localNode(cc.configDir, online) + if err != nil { + return + } + + cc.daemon, err = setupDaemon(cc.configDir, cc.node) + if err != nil { + return + } + + return +} + +// setupDaemon sets up the daemon corresponding to given node. +func setupDaemon(confdir string, node *core.IpfsNode) (*daemon.DaemonListener, error) { + if node.Config.Addresses.API == "" { + return nil, errors.New("no config.Addresses.API endpoint supplied") + } + + maddr, err := ma.NewMultiaddr(node.Config.Addresses.API) + if err != nil { + return nil, err + } + + dl, err := daemon.NewDaemonListener(node, maddr, confdir) + if err != nil { + return nil, err + } + go dl.Listen() + return dl, nil +} diff --git a/cmd/ipfs/run.go b/cmd/ipfs/run.go index 0342dceaa43..b651ca9d9dd 100644 --- a/cmd/ipfs/run.go +++ b/cmd/ipfs/run.go @@ -1,12 +1,12 @@ package main import ( - "errors" + "os" + "os/signal" + "syscall" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" - ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - "github.com/jbenet/go-ipfs/daemon" ) var cmdIpfsRun = &commander.Command{ @@ -19,30 +19,18 @@ var cmdIpfsRun = &commander.Command{ } func runCmd(c *commander.Command, inp []string) error { - conf, err := getConfigDir(c.Parent) - if err != nil { - return err - } - n, err := localNode(conf, true) + cc, err := setupCmdContext(c, true) if err != nil { return err } - // launch the RPC endpoint. - if n.Config.Addresses.API == "" { - return errors.New("no config.Addresses.API endpoint supplied") - } + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT, + syscall.SIGTERM, syscall.SIGQUIT) - maddr, err := ma.NewMultiaddr(n.Config.Addresses.API) - if err != nil { - return err - } + // wait until we get a signal to exit. + <-sigc - dl, err := daemon.NewDaemonListener(n, maddr, conf) - if err != nil { - return err - } - dl.Listen() - dl.Close() + cc.daemon.Close() return nil } diff --git a/cmd/ipfs/serve.go b/cmd/ipfs/serve.go index b4fb48c1db8..fda3b3f22d8 100644 --- a/cmd/ipfs/serve.go +++ b/cmd/ipfs/serve.go @@ -1,14 +1,12 @@ package main import ( - "errors" "fmt" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - "github.com/jbenet/go-ipfs/daemon" h "github.com/jbenet/go-ipfs/server/http" ) @@ -16,58 +14,36 @@ var cmdIpfsServe = &commander.Command{ UsageLine: "serve", Short: "Serve an interface to ipfs", Subcommands: []*commander.Command{ - cmdIpfsServeHttp, + cmdIpfsServeHTTP, }, Flag: *flag.NewFlagSet("ipfs-serve", flag.ExitOnError), } -var cmdIpfsServeHttp = &commander.Command{ +var cmdIpfsServeHTTP = &commander.Command{ UsageLine: "http", Short: "Serve an HTTP API", Long: `ipfs serve http - Serve an http gateway into ipfs.`, - Run: serveHttpCmd, + Run: serveHTTPCmd, Flag: *flag.NewFlagSet("ipfs-serve-http", flag.ExitOnError), } func init() { - cmdIpfsServeHttp.Flag.String("address", "/ip4/127.0.0.1/tcp/8080", "Listen Address") + cmdIpfsServeHTTP.Flag.String("address", "/ip4/127.0.0.1/tcp/8080", "Listen Address") } -func serveHttpCmd(c *commander.Command, _ []string) error { - conf, err := getConfigDir(c.Parent.Parent) +func serveHTTPCmd(c *commander.Command, _ []string) error { + cc, err := setupCmdContext(c, true) if err != nil { return err } - - n, err := localNode(conf, true) - if err != nil { - return err - } - - // launch the API RPC endpoint. - if n.Config.Addresses.API == "" { - return errors.New("no config.RPCAddress endpoint supplied") - } - - maddr, err := ma.NewMultiaddr(n.Config.Addresses.API) - if err != nil { - return err - } - - dl, err := daemon.NewDaemonListener(n, maddr, conf) - if err != nil { - fmt.Println("Failed to create daemon listener.") - return err - } - go dl.Listen() - defer dl.Close() + defer cc.daemon.Close() address := c.Flag.Lookup("address").Value.Get().(string) - maddr, err = ma.NewMultiaddr(address) + maddr, err := ma.NewMultiaddr(address) if err != nil { return err } fmt.Printf("Serving on %s\n", address) - return h.Serve(maddr, n) + return h.Serve(maddr, cc.node) } From 330ee8c79f5bb7b9d39f7eebc6be78f06e3cd140 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 02:50:33 -0700 Subject: [PATCH 039/105] added command struct for makeCommand --- cmd/ipfs/add.go | 7 ++++++- cmd/ipfs/cat.go | 7 ++++++- cmd/ipfs/gen.go | 43 +++++++++++++++++++++---------------------- cmd/ipfs/ls.go | 7 ++++++- cmd/ipfs/pin.go | 7 ++++++- cmd/ipfs/refs.go | 7 ++++++- 6 files changed, 51 insertions(+), 27 deletions(-) diff --git a/cmd/ipfs/add.go b/cmd/ipfs/add.go index 91eb1e6e4c6..5fbe7dd241b 100644 --- a/cmd/ipfs/add.go +++ b/cmd/ipfs/add.go @@ -29,4 +29,9 @@ func init() { cmdIpfsAdd.Flag.Bool("r", false, "add objects recursively") } -var addCmd = MakeCommand("add", []string{"r"}, commands.Add) +var addCmd = makeCommand(command{ + name: "add", + args: 1, + flags: []string{"r"}, + cmdFn: commands.Add, +}) diff --git a/cmd/ipfs/cat.go b/cmd/ipfs/cat.go index d7ab500288a..add0d9b9507 100644 --- a/cmd/ipfs/cat.go +++ b/cmd/ipfs/cat.go @@ -18,7 +18,12 @@ var cmdIpfsCat = &commander.Command{ Flag: *flag.NewFlagSet("ipfs-cat", flag.ExitOnError), } -var catCmd = MakeCommand("cat", nil, commands.Cat) +var catCmd = makeCommand(command{ + name: "cat", + args: 1, + flags: nil, + cmdFn: commands.Cat, +}) /* func catCmd(c *commander.Command, inp []string) error { diff --git a/cmd/ipfs/gen.go b/cmd/ipfs/gen.go index 5c7d1aa40e6..16f89a626fc 100644 --- a/cmd/ipfs/gen.go +++ b/cmd/ipfs/gen.go @@ -1,26 +1,34 @@ package main import ( - "errors" "fmt" "os" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" - "github.com/jbenet/go-ipfs/config" "github.com/jbenet/go-ipfs/core/commands" "github.com/jbenet/go-ipfs/daemon" u "github.com/jbenet/go-ipfs/util" ) -// CommanderFunc is a function that can be passed into the Commander library as +// command is the descriptor of an ipfs daemon command. +// Used with makeCommand to proxy over commands via the daemon. +type command struct { + name string + args int + flags []string + online bool + cmdFn commands.CmdFunc +} + +// commanderFunc is a function that can be passed into the Commander library as // a command handler. Defined here because commander lacks this definition. -type CommanderFunc func(*commander.Command, []string) error +type commanderFunc func(*commander.Command, []string) error -// MakeCommand Wraps a commands.CmdFunc so that it may be safely run by the +// makeCommand Wraps a commands.CmdFunc so that it may be safely run by the // commander library -func MakeCommand(cmdName string, expargs []string, cmdFn commands.CmdFunc) CommanderFunc { +func makeCommand(cmdDesc command) commanderFunc { return func(c *commander.Command, inp []string) error { - if len(inp) < 1 { + if len(inp) < cmdDesc.args { u.POut(c.Long) return nil } @@ -29,34 +37,25 @@ func MakeCommand(cmdName string, expargs []string, cmdFn commands.CmdFunc) Comma return err } - confapi, err := config.ReadConfigKey(confdir+"/config", "Addresses.API") - if err != nil { - return err - } - - apiaddr, ok := confapi.(string) - if !ok { - return errors.New("ApiAddress in config file was not a string") - } - cmd := daemon.NewCommand() - cmd.Command = cmdName + cmd.Command = cmdDesc.name cmd.Args = inp - for _, a := range expargs { + for _, a := range cmdDesc.flags { cmd.Opts[a] = c.Flag.Lookup(a).Value.Get() } - err = daemon.SendCommand(cmd, apiaddr) + + err = daemon.SendCommand(cmd, confdir) if err != nil { fmt.Printf("Executing command locally: %s", err) // Do locally - n, err := localNode(confdir, false) + n, err := localNode(confdir, cmdDesc.online) if err != nil { fmt.Println("Local node creation failed.") return err } - return cmdFn(n, cmd.Args, cmd.Opts, os.Stdout) + return cmdDesc.cmdFn(n, cmd.Args, cmd.Opts, os.Stdout) } return nil } diff --git a/cmd/ipfs/ls.go b/cmd/ipfs/ls.go index aed286deafc..0a2e8aff9cd 100644 --- a/cmd/ipfs/ls.go +++ b/cmd/ipfs/ls.go @@ -21,4 +21,9 @@ var cmdIpfsLs = &commander.Command{ Flag: *flag.NewFlagSet("ipfs-ls", flag.ExitOnError), } -var lsCmd = MakeCommand("ls", nil, commands.Ls) +var lsCmd = makeCommand(command{ + name: "ls", + args: 1, + flags: nil, + cmdFn: commands.Ls, +}) diff --git a/cmd/ipfs/pin.go b/cmd/ipfs/pin.go index 5919168b115..0077597598b 100644 --- a/cmd/ipfs/pin.go +++ b/cmd/ipfs/pin.go @@ -23,4 +23,9 @@ func init() { cmdIpfsPin.Flag.Int("d", 1, "recursive depth") } -var pinCmd = MakeCommand("pin", []string{"r", "d"}, commands.Pin) +var pinCmd = makeCommand(command{ + name: "pin", + args: 1, + flags: []string{"r", "d"}, + cmdFn: commands.Pin, +}) diff --git a/cmd/ipfs/refs.go b/cmd/ipfs/refs.go index f86e51dbf94..b3aaf85fc33 100644 --- a/cmd/ipfs/refs.go +++ b/cmd/ipfs/refs.go @@ -28,4 +28,9 @@ func init() { cmdIpfsRefs.Flag.Bool("u", false, "unique: list each ref only once") } -var refCmd = MakeCommand("refs", []string{"r", "u"}, commands.Refs) +var refCmd = makeCommand(command{ + name: "refs", + args: 1, + flags: []string{"r", "u"}, + cmdFn: commands.Refs, +}) From e91b2b8aeccea78470dc19f0a8fbd19799dd7a55 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 02:50:55 -0700 Subject: [PATCH 040/105] use makeCommand for publish + resolve --- cmd/ipfs/publish.go | 43 +++++++------------------------------------ cmd/ipfs/resolve.go | 42 +++++++----------------------------------- 2 files changed, 14 insertions(+), 71 deletions(-) diff --git a/cmd/ipfs/publish.go b/cmd/ipfs/publish.go index 8e2aa7cc4b3..fe473107718 100644 --- a/cmd/ipfs/publish.go +++ b/cmd/ipfs/publish.go @@ -1,13 +1,9 @@ package main import ( - "os" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" "github.com/jbenet/go-ipfs/core/commands" - "github.com/jbenet/go-ipfs/daemon" - u "github.com/jbenet/go-ipfs/util" ) var cmdIpfsPub = &commander.Command{ @@ -24,35 +20,10 @@ func init() { cmdIpfsPub.Flag.String("k", "", "Specify key to use for publishing.") } -func pubCmd(c *commander.Command, inp []string) error { - if len(inp) < 1 { - u.POut(c.Long) - return nil - } - - conf, err := getConfigDir(c.Parent) - if err != nil { - return err - } - - cmd := daemon.NewCommand() - cmd.Command = "publish" - cmd.Args = inp - cmd.Opts["k"] = c.Flag.Lookup("k").Value.Get() - err = daemon.SendCommand(cmd, conf) - if err != nil { - u.DOut("Executing command locally.\n") - // Do locally - conf, err := getConfigDir(c.Parent) - if err != nil { - return err - } - n, err := localNode(conf, true) - if err != nil { - return err - } - - return commands.Publish(n, cmd.Args, cmd.Opts, os.Stdout) - } - return nil -} +var pubCmd = makeCommand(command{ + name: "publish", + args: 1, + flags: []string{"k"}, + online: true, + cmdFn: commands.Publish, +}) diff --git a/cmd/ipfs/resolve.go b/cmd/ipfs/resolve.go index 3665f2a9177..75b9903b591 100644 --- a/cmd/ipfs/resolve.go +++ b/cmd/ipfs/resolve.go @@ -1,15 +1,9 @@ package main import ( - "fmt" - "os" - "time" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" "github.com/jbenet/go-ipfs/core/commands" - "github.com/jbenet/go-ipfs/daemon" - u "github.com/jbenet/go-ipfs/util" ) var cmdIpfsResolve = &commander.Command{ @@ -22,32 +16,10 @@ var cmdIpfsResolve = &commander.Command{ Flag: *flag.NewFlagSet("ipfs-resolve", flag.ExitOnError), } -func resolveCmd(c *commander.Command, inp []string) error { - if len(inp) < 1 { - u.POut(c.Long) - return nil - } - conf, err := getConfigDir(c.Parent) - if err != nil { - return err - } - - cmd := daemon.NewCommand() - cmd.Command = "resolve" - cmd.Args = inp - err = daemon.SendCommand(cmd, conf) - if err != nil { - now := time.Now() - // Resolve requires working DHT - n, err := localNode(conf, true) - if err != nil { - return err - } - - took := time.Now().Sub(now) - fmt.Printf("localNode creation took %s\n", took.String()) - - return commands.Resolve(n, cmd.Args, cmd.Opts, os.Stdout) - } - return nil -} +var resolveCmd = makeCommand(command{ + name: "resolve", + args: 1, + flags: nil, + online: true, + cmdFn: commands.Resolve, +}) From 7bf7afb7a9cc67c0b91113a735e7210c3df8575f Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 03:04:42 -0700 Subject: [PATCH 041/105] logging on gen --- cmd/ipfs/gen.go | 5 ++--- cmd/ipfs/ipfs.go | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/ipfs/gen.go b/cmd/ipfs/gen.go index 16f89a626fc..cb554990b07 100644 --- a/cmd/ipfs/gen.go +++ b/cmd/ipfs/gen.go @@ -47,12 +47,11 @@ func makeCommand(cmdDesc command) commanderFunc { err = daemon.SendCommand(cmd, confdir) if err != nil { - fmt.Printf("Executing command locally: %s", err) + log.Info("Executing command locally: %s", err) // Do locally n, err := localNode(confdir, cmdDesc.online) if err != nil { - fmt.Println("Local node creation failed.") - return err + return fmt.Errorf("Local node creation failed: %v", err) } return cmdDesc.cmdFn(n, cmd.Args, cmd.Opts, os.Stdout) diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 8e8df2080bd..fb465c0f4cc 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -9,6 +9,7 @@ import ( flag "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" commander "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" config "github.com/jbenet/go-ipfs/config" core "github.com/jbenet/go-ipfs/core" @@ -61,6 +62,9 @@ Use "ipfs help " for more information about a command. Flag: *flag.NewFlagSet("ipfs", flag.ExitOnError), } +// log is the command logger +var log = logging.MustGetLogger("cmd/ipfs") + func init() { config, err := config.PathRoot() if err != nil { From 068c0375f9929e69fafd1589e8db40ba54ed944f Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 03:05:01 -0700 Subject: [PATCH 042/105] bugfix: get to root cmd for config flag --- cmd/ipfs/ipfs.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index fb465c0f4cc..251b70bce3e 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -149,7 +149,11 @@ type cmdContext struct { // setupCmdContext initializes a cmdContext structure from a given command. func setupCmdContext(c *commander.Command, online bool) (cc cmdContext, err error) { - cc.configDir, err = getConfigDir(c.Parent.Parent) + rootCmd := c + for ; rootCmd.Parent != nil; rootCmd = c.Parent { + } + + cc.configDir, err = getConfigDir(rootCmd) if err != nil { return } From bc0769e370e6e271eebbd8c47d4815525096efcc Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 03:05:49 -0700 Subject: [PATCH 043/105] ipfs + ipns mounts with flags + config --- cmd/ipfs/init.go | 6 +++ cmd/ipfs/mount_unix.go | 91 ++++++++++++++++++++++-------------------- config/config.go | 7 ++++ 3 files changed, 60 insertions(+), 44 deletions(-) diff --git a/cmd/ipfs/init.go b/cmd/ipfs/init.go index b868f2b91dc..d70b6f3f9f9 100644 --- a/cmd/ipfs/init.go +++ b/cmd/ipfs/init.go @@ -92,6 +92,12 @@ func initCmd(c *commander.Command, inp []string) error { API: "/ip4/127.0.0.1/tcp/5001", } + // setup the node mount points. + cfg.Mounts = config.Mounts{ + IPFS: "/ipfs", + IPNS: "/ipns", + } + nbits, ok := c.Flag.Lookup("b").Value.Get().(int) if !ok { return errors.New("failed to get bits flag") diff --git a/cmd/ipfs/mount_unix.go b/cmd/ipfs/mount_unix.go index 03515ad5fb7..1b4e8709a42 100644 --- a/cmd/ipfs/mount_unix.go +++ b/cmd/ipfs/mount_unix.go @@ -3,14 +3,12 @@ package main import ( - "errors" "fmt" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" - ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - "github.com/jbenet/go-ipfs/daemon" + core "github.com/jbenet/go-ipfs/core" ipns "github.com/jbenet/go-ipfs/fuse/ipns" rofs "github.com/jbenet/go-ipfs/fuse/readonly" ) @@ -30,59 +28,64 @@ var cmdIpfsMount = &commander.Command{ } func init() { + cmdIpfsMount.Flag.String("f", "", "specify a mountpoint for ipfs") cmdIpfsMount.Flag.String("n", "", "specify a mountpoint for ipns") - cmdIpfsMount.Flag.String("f", "/ipfs", "specify a mountpoint for ipfs") } func mountCmd(c *commander.Command, inp []string) error { - conf, err := getConfigDir(c.Parent) - if err != nil { - fmt.Println("Couldnt get config dir") - return err - } - n, err := localNode(conf, true) + + cc, err := setupCmdContext(c, true) if err != nil { - fmt.Println("Local node creation failed.") return err } + defer cc.daemon.Close() - // launch the API RPC endpoint. - if n.Config.Addresses.API == "" { - return errors.New("no config.RPCAddress endpoint supplied") + // update fsdir with flag. + fsdir := cc.node.Config.Mounts.IPFS + if val, ok := c.Flag.Lookup("f").Value.Get().(string); ok && val != "" { + fsdir = val } + fsdone := mountIpfs(cc.node, fsdir) - maddr, err := ma.NewMultiaddr(n.Config.Addresses.API) - if err != nil { - return err + // get default mount points + nsdir := cc.node.Config.Mounts.IPNS + if val, ok := c.Flag.Lookup("n").Value.Get().(string); ok && val != "" { + nsdir = val } + nsdone := mountIpns(cc.node, nsdir, fsdir) - dl, err := daemon.NewDaemonListener(n, maddr, conf) - if err != nil { - fmt.Println("Failed to create daemon listener.") - return err - } - go dl.Listen() - defer dl.Close() - - mp := c.Flag.Lookup("f").Value.Get().(string) - fmt.Printf("Mounting at %s\n", mp) - - var ipnsDone chan struct{} - ns, ok := c.Flag.Lookup("n").Value.Get().(string) - if ok && ns != "" { - ipnsDone = make(chan struct{}) - go func() { - err = ipns.Mount(n, ns, mp) - if err != nil { - fmt.Printf("Error mounting ipns: %s\n", err) - } - ipnsDone <- struct{}{} - }() - } + // wait till mounts are done. + err1 := <-fsdone + err2 := <-nsdone - err = rofs.Mount(n, mp) - if ipnsDone != nil { - <-ipnsDone + if err1 != nil { + return err1 } - return err + return err2 +} + +func mountIpfs(node *core.IpfsNode, fsdir string) <-chan error { + done := make(chan error) + fmt.Printf("mounting ipfs at %s\n", fsdir) + + go func() { + err := rofs.Mount(node, fsdir) + done <- err + close(done) + }() + + return done +} + +func mountIpns(node *core.IpfsNode, nsdir, fsdir string) <-chan error { + done := make(chan error) + fmt.Printf("mounting ipns at %s\n", nsdir) + + go func() { + err := ipns.Mount(node, nsdir, fsdir) + done <- err + close(done) + }() + + return done } diff --git a/config/config.go b/config/config.go index 5c8f202a7b9..5a25118bdd3 100644 --- a/config/config.go +++ b/config/config.go @@ -29,6 +29,12 @@ type Addresses struct { API string // address for the local API (RPC) } +// Mounts stores the (string) mount points +type Mounts struct { + IPFS string + IPNS string +} + // BootstrapPeer is a peer used to bootstrap the network. type BootstrapPeer struct { Address string @@ -40,6 +46,7 @@ type Config struct { Identity Identity // local node's peer identity Datastore Datastore // local node's storage Addresses Addresses // local node's addresses + Mounts Mounts // local node's mount points Bootstrap []*BootstrapPeer // local nodes's bootstrap peers } From f923652d889f626c672c63ed933c7d1afcb163b8 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 03:07:52 -0700 Subject: [PATCH 044/105] remove extra printf --- core/commands/add.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/commands/add.go b/core/commands/add.go index 1af9de79575..7781812580b 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -94,7 +94,6 @@ func addDir(n *core.IpfsNode, fpath string, depth int) (*dag.Node, error) { } } - fmt.Printf("Adding dir: %s = %s\n", fpath) return tree, addNode(n, tree, fpath) } From a76f6af1dfccf6412aa71c6bcf0843e422b2dfda Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 03:33:20 -0700 Subject: [PATCH 045/105] remove old cat --- cmd/ipfs/cat.go | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/cmd/ipfs/cat.go b/cmd/ipfs/cat.go index add0d9b9507..168a4841e77 100644 --- a/cmd/ipfs/cat.go +++ b/cmd/ipfs/cat.go @@ -24,32 +24,3 @@ var catCmd = makeCommand(command{ flags: nil, cmdFn: commands.Cat, }) - -/* -func catCmd(c *commander.Command, inp []string) error { - if len(inp) < 1 { - u.POut(c.Long) - return nil - } - - conf, err := getConfigDir(c.Parent) - if err != nil { - return err - } - - com := daemon.NewCommand() - com.Command = "cat" - com.Args = inp - - err = daemon.SendCommand(com, conf) - if err != nil { - n, err := localNode(conf, false) - if err != nil { - return err - } - - return commands.Cat(n, com.Args, com.Opts, os.Stdout) - } - return nil -} -*/ From 41bc355be3820ca3147f214c823b2ba84de1abdb Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 2 Oct 2014 03:42:17 -0700 Subject: [PATCH 046/105] /rpcaddr should be storing a multiaddr client expects a multiaddr. server was writing a bsaddr --- daemon/daemon.go | 7 ++++++- daemon/daemon_client.go | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index 8ec84d8482f..3ea4cc5dac8 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -63,7 +63,12 @@ func NewDaemonListener(ipfsnode *core.IpfsNode, addr *ma.Multiaddr, confdir stri return nil, err } - _, err = ofi.Write([]byte(host)) + mstr, err := addr.String() + if err != nil { + return nil, err + } + + _, err = ofi.Write([]byte(mstr)) if err != nil { log.Warning("Could not write to rpcaddress file: %s", err) return nil, err diff --git a/daemon/daemon_client.go b/daemon/daemon_client.go index fe5b1f9efa0..17d1b2c265c 100644 --- a/daemon/daemon_client.go +++ b/daemon/daemon_client.go @@ -67,6 +67,7 @@ func SendCommand(command *Command, confdir string) error { return err } + log.Info("Daemon address: %s", server) maddr, err := ma.NewMultiaddr(server) if err != nil { return err From 80c64ffa6d1e03ebcabed8a116d46daf4cbf6061 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 2 Oct 2014 17:25:08 +0000 Subject: [PATCH 047/105] break ipns into separate files --- fuse/ipns/ipns_unix.go | 132 ---------------------------------------- fuse/ipns/link_unix.go | 24 ++++++++ fuse/ipns/mount_unix.go | 91 +++++++++++++++++++++++++++ fuse/ipns/repub_unix.go | 38 ++++++++++++ 4 files changed, 153 insertions(+), 132 deletions(-) create mode 100644 fuse/ipns/link_unix.go create mode 100644 fuse/ipns/mount_unix.go create mode 100644 fuse/ipns/repub_unix.go diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 56f964b1270..a725c382a3a 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -1,14 +1,9 @@ package ipns import ( - "fmt" "io/ioutil" "os" - "os/exec" - "os/signal" "path/filepath" - "runtime" - "syscall" "time" "bytes" @@ -469,130 +464,3 @@ func (n *Node) Rename(req *fuse.RenameRequest, newDir fs.Node, intr fs.Intr) fus } return nil } - -// Mount mounts an IpfsNode instance at a particular path. It -// serves until the process receives exit signals (to Unmount). -func Mount(ipfs *core.IpfsNode, fpath string, ipfspath string) error { - - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT, - syscall.SIGTERM, syscall.SIGQUIT) - - go func() { - <-sigc - for { - err := Unmount(fpath) - if err == nil { - return - } - time.Sleep(time.Millisecond * 100) - } - ipfs.Network.Close() - }() - - c, err := fuse.Mount(fpath) - if err != nil { - return err - } - defer c.Close() - - fsys, err := NewIpns(ipfs, ipfspath) - if err != nil { - return err - } - - err = fs.Serve(c, fsys) - if err != nil { - return err - } - - // check if the mount process has an error to report - <-c.Ready - if err := c.MountError; err != nil { - return err - } - return nil -} - -// Unmount attempts to unmount the provided FUSE mount point, forcibly -// if necessary. -func Unmount(point string) error { - fmt.Printf("Unmounting %s...\n", point) - - var cmd *exec.Cmd - switch runtime.GOOS { - case "darwin": - cmd = exec.Command("diskutil", "umount", "force", point) - case "linux": - cmd = exec.Command("fusermount", "-u", point) - default: - return fmt.Errorf("unmount: unimplemented") - } - - errc := make(chan error, 1) - go func() { - if err := exec.Command("umount", point).Run(); err == nil { - errc <- err - } - // retry to unmount with the fallback cmd - errc <- cmd.Run() - }() - - select { - case <-time.After(1 * time.Second): - return fmt.Errorf("umount timeout") - case err := <-errc: - return err - } -} - -type Link struct { - Target string -} - -func (l *Link) Attr() fuse.Attr { - log.Debug("Link attr.") - return fuse.Attr{ - Mode: os.ModeSymlink | 0555, - } -} - -func (l *Link) Readlink(req *fuse.ReadlinkRequest, intr fs.Intr) (string, fuse.Error) { - log.Debug("ReadLink: %s", l.Target) - return l.Target, nil -} - -type Republisher struct { - Timeout time.Duration - Publish chan struct{} - node *Node -} - -func NewRepublisher(n *Node, tout time.Duration) *Republisher { - return &Republisher{ - Timeout: tout, - Publish: make(chan struct{}), - node: n, - } -} - -func (np *Republisher) Run() { - for _ = range np.Publish { - timer := time.After(np.Timeout) - for { - select { - case <-timer: - //Do the publish! - log.Info("Publishing Changes!") - err := np.node.updateTree() - if err != nil { - log.Critical("updateTree error: %s", err) - } - goto done - case <-np.Publish: - timer = time.After(np.Timeout) - } - } - done: - } -} diff --git a/fuse/ipns/link_unix.go b/fuse/ipns/link_unix.go new file mode 100644 index 00000000000..e81576c6e66 --- /dev/null +++ b/fuse/ipns/link_unix.go @@ -0,0 +1,24 @@ +package ipns + +import ( + "os" + + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" +) + +type Link struct { + Target string +} + +func (l *Link) Attr() fuse.Attr { + log.Debug("Link attr.") + return fuse.Attr{ + Mode: os.ModeSymlink | 0555, + } +} + +func (l *Link) Readlink(req *fuse.ReadlinkRequest, intr fs.Intr) (string, fuse.Error) { + log.Debug("ReadLink: %s", l.Target) + return l.Target, nil +} diff --git a/fuse/ipns/mount_unix.go b/fuse/ipns/mount_unix.go new file mode 100644 index 00000000000..7df6f140c5d --- /dev/null +++ b/fuse/ipns/mount_unix.go @@ -0,0 +1,91 @@ +package ipns + +import ( + "fmt" + "os" + "os/exec" + "os/signal" + "runtime" + "syscall" + "time" + + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" + "github.com/jbenet/go-ipfs/core" +) + +// Mount mounts an IpfsNode instance at a particular path. It +// serves until the process receives exit signals (to Unmount). +func Mount(ipfs *core.IpfsNode, fpath string, ipfspath string) error { + + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT, + syscall.SIGTERM, syscall.SIGQUIT) + + go func() { + <-sigc + for { + err := Unmount(fpath) + if err == nil { + return + } + time.Sleep(time.Millisecond * 100) + } + ipfs.Network.Close() + }() + + c, err := fuse.Mount(fpath) + if err != nil { + return err + } + defer c.Close() + + fsys, err := NewIpns(ipfs, ipfspath) + if err != nil { + return err + } + + err = fs.Serve(c, fsys) + if err != nil { + return err + } + + // check if the mount process has an error to report + <-c.Ready + if err := c.MountError; err != nil { + return err + } + return nil +} + +// Unmount attempts to unmount the provided FUSE mount point, forcibly +// if necessary. +func Unmount(point string) error { + fmt.Printf("Unmounting %s...\n", point) + + var cmd *exec.Cmd + switch runtime.GOOS { + case "darwin": + cmd = exec.Command("diskutil", "umount", "force", point) + case "linux": + cmd = exec.Command("fusermount", "-u", point) + default: + return fmt.Errorf("unmount: unimplemented") + } + + errc := make(chan error, 1) + go func() { + if err := exec.Command("umount", point).Run(); err == nil { + errc <- err + } + // retry to unmount with the fallback cmd + errc <- cmd.Run() + }() + + select { + case <-time.After(1 * time.Second): + return fmt.Errorf("umount timeout") + case err := <-errc: + return err + } +} diff --git a/fuse/ipns/repub_unix.go b/fuse/ipns/repub_unix.go new file mode 100644 index 00000000000..93d07c92f6e --- /dev/null +++ b/fuse/ipns/repub_unix.go @@ -0,0 +1,38 @@ +package ipns + +import "time" + +type Republisher struct { + Timeout time.Duration + Publish chan struct{} + node *Node +} + +func NewRepublisher(n *Node, tout time.Duration) *Republisher { + return &Republisher{ + Timeout: tout, + Publish: make(chan struct{}), + node: n, + } +} + +func (np *Republisher) Run() { + for _ = range np.Publish { + timer := time.After(np.Timeout) + for { + select { + case <-timer: + //Do the publish! + log.Info("Publishing Changes!") + err := np.node.updateTree() + if err != nil { + log.Critical("updateTree error: %s", err) + } + goto done + case <-np.Publish: + timer = time.After(np.Timeout) + } + } + done: + } +} From d872e0cce522a35eb3db826a5c184bba781b3b20 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 2 Oct 2014 22:30:50 +0000 Subject: [PATCH 048/105] move publish and resolve under the 'name' subcommand --- cmd/ipfs/gen.go | 2 +- cmd/ipfs/ipfs.go | 9 ++++++--- cmd/ipfs/name.go | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 cmd/ipfs/name.go diff --git a/cmd/ipfs/gen.go b/cmd/ipfs/gen.go index cb554990b07..299171550da 100644 --- a/cmd/ipfs/gen.go +++ b/cmd/ipfs/gen.go @@ -32,7 +32,7 @@ func makeCommand(cmdDesc command) commanderFunc { u.POut(c.Long) return nil } - confdir, err := getConfigDir(c.Parent) + confdir, err := getConfigDir(c) if err != nil { return err } diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 251b70bce3e..3d50e1b8de1 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -56,8 +56,7 @@ Use "ipfs help " for more information about a command. cmdIpfsInit, cmdIpfsServe, cmdIpfsRun, - cmdIpfsPub, - cmdIpfsResolve, + cmdIpfsName, }, Flag: *flag.NewFlagSet("ipfs", flag.ExitOnError), } @@ -124,7 +123,11 @@ func localNode(confdir string, online bool) (*core.IpfsNode, error) { // Gets the config "-c" flag from the command, or returns // the default configuration root directory func getConfigDir(c *commander.Command) (string, error) { - conf := c.Flag.Lookup("c").Value.Get() + root := c + for root.Parent != nil { + root = root.Parent + } + conf := root.Flag.Lookup("c").Value.Get() if conf == nil { return config.PathRoot() } diff --git a/cmd/ipfs/name.go b/cmd/ipfs/name.go new file mode 100644 index 00000000000..56d8821ec7d --- /dev/null +++ b/cmd/ipfs/name.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + + flag "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" + commander "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" +) + +var cmdIpfsName = &commander.Command{ + UsageLine: "name", + Short: "Ipfs namespace manipulation tools.", + Long: `ipfs name [publish|resolve] `, + Run: addCmd, + Flag: *flag.NewFlagSet("ipfs-name", flag.ExitOnError), + Subcommands: []*commander.Command{ + cmdIpfsPub, + cmdIpfsResolve, + }, +} + +func nameCmd(c *commander.Command, args []string) error { + fmt.Println(c.Long) + return nil +} From 47275351603c1beb9772781302240728e9a87280 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 3 Oct 2014 19:22:54 +0000 Subject: [PATCH 049/105] fixing mutability issues in ipns --- fuse/ipns/ipns_unix.go | 135 +++++++++++++++++++++++++++++++--------- fuse/ipns/repub_unix.go | 35 +++++++---- merkledag/merkledag.go | 10 +++ util/util.go | 14 +++-- 4 files changed, 149 insertions(+), 45 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index a725c382a3a..22d344e441f 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -1,6 +1,7 @@ package ipns import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -72,7 +73,7 @@ func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, er nd := new(Node) nd.Ipfs = n nd.key = k - nd.repub = NewRepublisher(nd, time.Millisecond*10) + nd.repub = NewRepublisher(nd, time.Millisecond*10, time.Second) go nd.repub.Run() @@ -182,11 +183,15 @@ func (r *Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { // Node is the core object representing a filesystem tree node. type Node struct { + root *Root nsRoot *Node + parent *Node repub *Republisher - // Name really only for logging purposes + // This nodes name in its parent dir. + // NOTE: this strategy wont work well if we allow hard links + // (im all for murdering the thought of hard links) name string // Private keys held by nodes at the root of a keyspace @@ -244,9 +249,10 @@ func (s *Node) Lookup(name string, intr fs.Intr) (fs.Node, fuse.Error) { func (n *Node) makeChild(name string, node *mdag.Node) *Node { child := &Node{ - Ipfs: n.Ipfs, - Nd: node, - name: n.name + "/" + name, + Ipfs: n.Ipfs, + Nd: node, + name: name, + nsRoot: n.nsRoot, } if n.nsRoot == nil { @@ -289,7 +295,7 @@ func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { } func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error { - log.Debug("ipns: Node Write: flags = %s, offset = %d, size = %d", req.Flags.String(), req.Offset, len(req.Data)) + log.Debug("ipns: Node Write [%s]: flags = %s, offset = %d, size = %d", n.name, req.Flags.String(), req.Offset, len(req.Data)) if n.dataBuf == nil { n.dataBuf = new(bytes.Buffer) } @@ -312,19 +318,45 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { // This operation holds everything in memory, // should be changed to stream the block creation/storage // but for now, since the buf is all in memory anyways... - err := imp.NewDagInNode(n.dataBuf, n.Nd) + + //NOTE: + // This should only occur on a file object, if this were to be a + // folder, bad things would happen. + newNode, err := imp.NewDagFromReader(n.dataBuf) if err != nil { - log.Error("ipns: Flush error: %s", err) - // return fuse.EVERYBAD + log.Critical("error creating dag from dataBuf: %s", err) return fuse.ENODATA } + if n.parent != nil { + err := n.parent.update(n.name, newNode) + if err != nil { + log.Critical("error in updating ipns dag tree: %s", err) + // return fuse.ETHISISPRETTYBAD + return fuse.ENOSYS + } + } + n.Nd = newNode - n.sendPublishSignal() + //TEMP + dr, err := mdag.NewDagReader(n.Nd, n.Ipfs.DAG) + if err != nil { + log.Critical("Verification read failed.") + } + b, err := ioutil.ReadAll(dr) + if err != nil { + log.Critical("Verification read failed.") + } + fmt.Println("VERIFICATION READ") + fmt.Printf("READ %d BYTES\n", len(b)) + fmt.Println(string(b)) + // + + n.wasChanged() } return nil } -func (n *Node) sendPublishSignal() { +func (n *Node) wasChanged() { root := n.nsRoot if root == nil { root = n @@ -333,7 +365,8 @@ func (n *Node) sendPublishSignal() { root.repub.Publish <- struct{}{} } -func (n *Node) updateTree() error { +func (n *Node) republishRoot() error { + log.Debug("Republish root") var root *Node if n.nsRoot != nil { root = n.nsRoot @@ -341,19 +374,13 @@ func (n *Node) updateTree() error { root = n } - err := root.Nd.Update() - if err != nil { - log.Error("ipns: dag tree update failed: %s", err) - return err - } - - err = n.Ipfs.DAG.AddRecursive(root.Nd) + // Add any nodes that may be new to the DAG service + err := n.Ipfs.DAG.AddRecursive(root.Nd) if err != nil { log.Critical("ipns: Dag Add Error: %s", err) return err } - n.changed = false n.dataBuf = nil ndkey, err := root.Nd.Key() @@ -380,11 +407,13 @@ func (n *Node) Fsync(req *fuse.FsyncRequest, intr fs.Intr) fuse.Error { func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) { log.Debug("Got mkdir request!") dagnd := &mdag.Node{Data: mdag.FolderPBData()} - n.Nd.AddNodeLink(req.Name, dagnd) + nnode := n.Nd.Copy() + nnode.AddNodeLink(req.Name, dagnd) child := &Node{ Ipfs: n.Ipfs, Nd: dagnd, + name: req.Name, } if n.nsRoot == nil { @@ -393,8 +422,17 @@ func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) child.nsRoot = n.nsRoot } - n.changed = true - n.sendPublishSignal() + if n.parent != nil { + err := n.parent.update(n.name, nnode) + if err != nil { + log.Critical("Error updating node: %s", err) + // Can we panic, please? + return nil, fuse.ENODATA + } + } + n.Nd = nnode + + n.wasChanged() return child, nil } @@ -405,7 +443,7 @@ func (n *Node) Mknod(req *fuse.MknodRequest, intr fs.Intr) (fs.Node, fuse.Error) } func (n *Node) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr) (fs.Handle, fuse.Error) { - log.Debug("[%s] Received open request! flags = %s", n.name, req.Flags.String()) + //log.Debug("[%s] Received open request! flags = %s", n.name, req.Flags.String()) //TODO: check open flags and truncate if necessary return n, nil } @@ -417,23 +455,46 @@ func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr f nd := &mdag.Node{Data: mdag.FilePBData(nil)} child := n.makeChild(req.Name, nd) - err := n.Nd.AddNodeLink(req.Name, nd) + nnode := n.Nd.Copy() + + err := nnode.AddNodeLink(req.Name, nd) if err != nil { log.Error("Error adding child to node: %s", err) return nil, nil, fuse.ENOENT } + if n.parent != nil { + err := n.parent.update(n.name, nnode) + if err != nil { + log.Critical("Error updating node: %s", err) + // Can we panic, please? + return nil, nil, fuse.ENODATA + } + } + n.Nd = nnode + n.wasChanged() + return child, child, nil } func (n *Node) Remove(req *fuse.RemoveRequest, intr fs.Intr) fuse.Error { log.Debug("[%s] Got Remove request: %s", n.name, req.Name) - err := n.Nd.RemoveNodeLink(req.Name) + nnode := n.Nd.Copy() + err := nnode.RemoveNodeLink(req.Name) if err != nil { log.Error("Remove: No such file.") return fuse.ENOENT } - n.changed = true - n.sendPublishSignal() + + if n.parent != nil { + err := n.parent.update(n.name, nnode) + if err != nil { + log.Critical("Error updating node: %s", err) + // Can we panic, please? + return fuse.ENODATA + } + } + n.Nd = nnode + n.wasChanged() return nil } @@ -464,3 +525,21 @@ func (n *Node) Rename(req *fuse.RenameRequest, newDir fs.Node, intr fs.Intr) fus } return nil } + +func (n *Node) update(name string, newnode *mdag.Node) error { + nnode := n.Nd.Copy() + err := nnode.RemoveNodeLink(name) + if err != nil { + return err + } + nnode.AddNodeLink(name, newnode) + + if n.parent != nil { + err := n.parent.update(n.name, newnode) + if err != nil { + return err + } + } + n.Nd = nnode + return nil +} diff --git a/fuse/ipns/repub_unix.go b/fuse/ipns/repub_unix.go index 93d07c92f6e..f1a89828731 100644 --- a/fuse/ipns/repub_unix.go +++ b/fuse/ipns/repub_unix.go @@ -3,34 +3,45 @@ package ipns import "time" type Republisher struct { - Timeout time.Duration - Publish chan struct{} - node *Node + TimeoutLong time.Duration + TimeoutShort time.Duration + Publish chan struct{} + node *Node } -func NewRepublisher(n *Node, tout time.Duration) *Republisher { +func NewRepublisher(n *Node, tshort, tlong time.Duration) *Republisher { return &Republisher{ - Timeout: tout, - Publish: make(chan struct{}), - node: n, + TimeoutShort: tshort, + TimeoutLong: tlong, + Publish: make(chan struct{}), + node: n, } } func (np *Republisher) Run() { for _ = range np.Publish { - timer := time.After(np.Timeout) + quick := time.After(np.TimeoutShort) + longer := time.After(np.TimeoutLong) for { select { - case <-timer: + case <-quick: //Do the publish! log.Info("Publishing Changes!") - err := np.node.updateTree() + err := np.node.republishRoot() if err != nil { - log.Critical("updateTree error: %s", err) + log.Critical("republishRoot error: %s", err) + } + goto done + case <-longer: + //Do the publish! + log.Info("Publishing Changes!") + err := np.node.republishRoot() + if err != nil { + log.Critical("republishRoot error: %s", err) } goto done case <-np.Publish: - timer = time.After(np.Timeout) + quick = time.After(np.TimeoutShort) } } done: diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index b8e96e4651d..31f4e2937f5 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -75,6 +75,16 @@ func (n *Node) RemoveNodeLink(name string) error { return u.ErrNotFound } +func (n *Node) Copy() *Node { + nnode := new(Node) + nnode.Data = make([]byte, len(n.Data)) + copy(nnode.Data, n.Data) + + nnode.Links = make([]*Link, len(n.Links)) + copy(nnode.Links, n.Links) + return nnode +} + // Size returns the total size of the data addressed by node, // including the total sizes of references. func (n *Node) Size() (uint64, error) { diff --git a/util/util.go b/util/util.go index 5b2505a9770..40d04ed734e 100644 --- a/util/util.go +++ b/util/util.go @@ -99,11 +99,15 @@ func DOut(format string, a ...interface{}) { func SetupLogging() { backend := logging.NewLogBackend(os.Stderr, "", 0) logging.SetBackend(backend) - if Debug { - logging.SetLevel(logging.DEBUG, "") - } else { - logging.SetLevel(logging.ERROR, "") - } + /* + if Debug { + logging.SetLevel(logging.DEBUG, "") + } else { + logging.SetLevel(logging.ERROR, "") + } + */ + logging.SetLevel(logging.ERROR, "merkledag") + logging.SetLevel(logging.ERROR, "blockservice") logging.SetFormatter(logging.MustStringFormatter(LogFormat)) } From dc66b699b0dfc7dfa5d032bee7d5b681c318221c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 3 Oct 2014 21:36:29 +0000 Subject: [PATCH 050/105] fix writes zeroing files --- fuse/ipns/ipns_unix.go | 9 +++++---- importer/importer_test.go | 8 +++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 22d344e441f..4a9db2bceda 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -206,7 +206,6 @@ type Node struct { // For writing dataBuf *bytes.Buffer - changed bool } func (s *Node) loadData() error { @@ -302,18 +301,18 @@ func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.I if req.Offset == 0 { n.dataBuf.Reset() n.dataBuf.Write(req.Data) - n.changed = true resp.Size = len(req.Data) } else { log.Error("Unhandled write to offset!") + n.dataBuf = nil } return nil } func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { - log.Debug("Got flush request!") + log.Debug("Got flush request [%s]!", n.name) - if n.changed { + if n.dataBuf != nil { //TODO: // This operation holds everything in memory, // should be changed to stream the block creation/storage @@ -351,6 +350,8 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { fmt.Println(string(b)) // + n.dataBuf = nil + n.wasChanged() } return nil diff --git a/importer/importer_test.go b/importer/importer_test.go index 9fb1afa0809..0a0bf3257de 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -43,10 +43,16 @@ func TestSizeBasedSplit(t *testing.T) { testFileConsistency(t, bs, 31*4095) } +func dup(b []byte) []byte { + o := make([]byte, len(b)) + copy(o, b) + return o +} + func testFileConsistency(t *testing.T, bs BlockSplitter, nbytes int) { buf := new(bytes.Buffer) io.CopyN(buf, rand.Reader, int64(nbytes)) - should := buf.Bytes() + should := dup(buf.Bytes()) nd, err := NewDagFromReaderWithSplitter(buf, bs) if err != nil { t.Fatal(err) From 8a7f6aca99cf6018bff6b0a20b8ad14a59d27bee Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Fri, 3 Oct 2014 14:38:02 -0700 Subject: [PATCH 051/105] godeps: updated datastore --- Godeps/Godeps.json | 2 +- .../jbenet/datastore.go/Godeps/Godeps.json | 43 ++++++ .../jbenet/datastore.go/Godeps/Readme | 5 + .../jbenet/datastore.go/basic_ds.go | 65 +++++++--- .../jbenet/datastore.go/basic_ds_test.go | 13 ++ .../jbenet/datastore.go/datastore.go | 16 ++- .../github.com/jbenet/datastore.go/fs/fs.go | 122 ++++++++++++++++++ .../jbenet/datastore.go/fs/fs_test.go | 65 ++++++++++ .../github.com/jbenet/datastore.go/io/io.go | 44 +++++++ .../jbenet/datastore.go/key_test.go | 5 +- .../jbenet/datastore.go/leveldb/datastore.go | 4 +- .../jbenet/datastore.go/lru/datastore.go | 54 ++++++++ .../jbenet/datastore.go/lru/datastore_test.go | 52 ++++++++ .../github.com/jbenet/datastore.go/query.go | 19 +++ .../jbenet/datastore.go/sync/sync.go | 64 +++++++++ 15 files changed, 546 insertions(+), 27 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index fbe7494b0aa..4e5dfdf27e7 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -64,7 +64,7 @@ }, { "ImportPath": "github.com/jbenet/datastore.go", - "Rev": "e89f0511689bb2d0608496e15491f241842de085" + "Rev": "e7d6f7cb9e3c207a04c5397c449d10a6f9d403a0" }, { "ImportPath": "github.com/jbenet/go-base58", diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json b/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json new file mode 100644 index 00000000000..bebf4a58f25 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json @@ -0,0 +1,43 @@ +{ + "ImportPath": "github.com/jbenet/datastore.go", + "GoVersion": "go1.3.1", + "Packages": [ + "./..." + ], + "Deps": [ + { + "ImportPath": "code.google.com/p/go-uuid/uuid", + "Comment": "null-12", + "Rev": "7dda39b2e7d5e265014674c5af696ba4186679e9" + }, + { + "ImportPath": "code.google.com/p/snappy-go/snappy", + "Comment": "null-15", + "Rev": "12e4b4183793ac4b061921e7980845e750679fd0" + }, + { + "ImportPath": "github.com/codahale/blake2", + "Rev": "3fa823583afba430e8fc7cdbcc670dbf90bfacc4" + }, + { + "ImportPath": "github.com/hashicorp/golang-lru", + "Rev": "4dfff096c4973178c8f35cf6dd1a732a0a139370" + }, + { + "ImportPath": "github.com/mattbaird/elastigo/api", + "Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d" + }, + { + "ImportPath": "github.com/mattbaird/elastigo/core", + "Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d" + }, + { + "ImportPath": "github.com/syndtr/goleveldb/leveldb", + "Rev": "9bca75c48d6c31becfbb127702b425e7226052e3" + }, + { + "ImportPath": "gopkg.in/check.v1", + "Rev": "91ae5f88a67b14891cfd43895b01164f6c120420" + } + ] +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme b/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme new file mode 100644 index 00000000000..4cdaa53d56d --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme @@ -0,0 +1,5 @@ +This directory tree is generated automatically by godep. + +Please do not edit. + +See https://github.com/tools/godep for more information. diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go index 33ea5f3a7bd..782b5de4608 100644 --- a/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go @@ -1,28 +1,30 @@ package datastore -import ( - "log" -) +import "log" // Here are some basic datastore implementations. -// MapDatastore uses a standard Go map for internal storage. type keyMap map[Key]interface{} + +// MapDatastore uses a standard Go map for internal storage. type MapDatastore struct { values keyMap } +// NewMapDatastore constructs a MapDatastore func NewMapDatastore() (d *MapDatastore) { return &MapDatastore{ values: keyMap{}, } } +// Put implements Datastore.Put func (d *MapDatastore) Put(key Key, value interface{}) (err error) { d.values[key] = value return nil } +// Get implements Datastore.Get func (d *MapDatastore) Get(key Key) (value interface{}, err error) { val, found := d.values[key] if !found { @@ -31,19 +33,22 @@ func (d *MapDatastore) Get(key Key) (value interface{}, err error) { return val, nil } +// Has implements Datastore.Has func (d *MapDatastore) Has(key Key) (exists bool, err error) { _, found := d.values[key] return found, nil } +// Delete implements Datastore.Delete func (d *MapDatastore) Delete(key Key) (err error) { delete(d.values, key) return nil } +// KeyList implements Datastore.KeyList func (d *MapDatastore) KeyList() ([]Key, error) { var keys []Key - for k, _ := range d.values { + for k := range d.values { keys = append(keys, k) } return keys, nil @@ -54,26 +59,32 @@ func (d *MapDatastore) KeyList() ([]Key, error) { type NullDatastore struct { } +// NewNullDatastore constructs a null datastoe func NewNullDatastore() *NullDatastore { return &NullDatastore{} } +// Put implements Datastore.Put func (d *NullDatastore) Put(key Key, value interface{}) (err error) { return nil } +// Get implements Datastore.Get func (d *NullDatastore) Get(key Key) (value interface{}, err error) { return nil, nil } +// Has implements Datastore.Has func (d *NullDatastore) Has(key Key) (exists bool, err error) { return false, nil } +// Delete implements Datastore.Delete func (d *NullDatastore) Delete(key Key) (err error) { return nil } +// KeyList implements Datastore.KeyList func (d *NullDatastore) KeyList() ([]Key, error) { return nil, nil } @@ -81,38 +92,56 @@ func (d *NullDatastore) KeyList() ([]Key, error) { // LogDatastore logs all accesses through the datastore. type LogDatastore struct { Name string - Child Datastore + child Datastore +} + +// Shim is a datastore which has a child. +type Shim interface { + Datastore + + Children() []Datastore } -func NewLogDatastore(ds Datastore, name string) *LogDatastore { +// NewLogDatastore constructs a log datastore. +func NewLogDatastore(ds Datastore, name string) Shim { if len(name) < 1 { name = "LogDatastore" } - return &LogDatastore{Name: name, Child: ds} + return &LogDatastore{Name: name, child: ds} +} + +// Children implements Shim +func (d *LogDatastore) Children() []Datastore { + return []Datastore{d.child} } +// Put implements Datastore.Put func (d *LogDatastore) Put(key Key, value interface{}) (err error) { - log.Printf("%s: Put %s", d.Name, key) + log.Printf("%s: Put %s\n", d.Name, key) // log.Printf("%s: Put %s ```%s```", d.Name, key, value) - return d.Child.Put(key, value) + return d.child.Put(key, value) } +// Get implements Datastore.Get func (d *LogDatastore) Get(key Key) (value interface{}, err error) { - log.Printf("%s: Get %s", d.Name, key) - return d.Child.Get(key) + log.Printf("%s: Get %s\n", d.Name, key) + return d.child.Get(key) } +// Has implements Datastore.Has func (d *LogDatastore) Has(key Key) (exists bool, err error) { - log.Printf("%s: Has %s", d.Name, key) - return d.Child.Has(key) + log.Printf("%s: Has %s\n", d.Name, key) + return d.child.Has(key) } +// Delete implements Datastore.Delete func (d *LogDatastore) Delete(key Key) (err error) { - log.Printf("%s: Delete %s", d.Name, key) - return d.Child.Delete(key) + log.Printf("%s: Delete %s\n", d.Name, key) + return d.child.Delete(key) } +// KeyList implements Datastore.KeyList func (d *LogDatastore) KeyList() ([]Key, error) { - log.Printf("%s: Get KeyList.", d.Name) - return d.Child.KeyList() + log.Printf("%s: Get KeyList\n", d.Name) + return d.child.KeyList() } diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go new file mode 100644 index 00000000000..e175d94dab1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go @@ -0,0 +1,13 @@ +package datastore_test + +import ( + . "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + . "launchpad.net/gocheck" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type BasicSuite struct{} + +var _ = Suite(&BasicSuite{}) diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go index 9ff21a6a078..f3260cd7da4 100644 --- a/Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go @@ -5,7 +5,7 @@ import ( ) /* -A Datastore represents storage for any key-value pair. +Datastore represents storage for any key-value pair. Datastores are general enough to be backed by all kinds of different storage: in-memory caches, databases, a remote datastore, flat files on disk, etc. @@ -27,7 +27,6 @@ and thus it should behave predictably and handle exceptional conditions with proper error reporting. Thus, all Datastore calls may return errors, which should be checked by callers. */ - type Datastore interface { // Put stores the object `value` named by `key`. // @@ -53,20 +52,27 @@ type Datastore interface { // Delete removes the value for given `key`. Delete(key Key) (err error) - // Returns a list of keys in the datastore + // KeyList returns a list of keys in the datastore KeyList() ([]Key, error) } +// ThreadSafeDatastore is an interface that all threadsafe datastore should +// implement to leverage type safety checks. +type ThreadSafeDatastore interface { + Datastore + IsThreadSafe() +} + // Errors // ErrNotFound is returned by Get, Has, and Delete when a datastore does not // map the given key to a value. -var ErrNotFound = errors.New("datastore: key not found.") +var ErrNotFound = errors.New("datastore: key not found") // ErrInvalidType is returned by Put when a given value is incopatible with // the type the datastore supports. This means a conversion (or serialization) // is needed beforehand. -var ErrInvalidType = errors.New("datastore: invalid type error.") +var ErrInvalidType = errors.New("datastore: invalid type error") // GetBackedHas provides a default Datastore.Has implementation. // It exists so Datastore.Has implementations can use it, like so: diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go new file mode 100644 index 00000000000..1c1ac791d52 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go @@ -0,0 +1,122 @@ +package fs + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" +) + +// Datastore uses a standard Go map for internal storage. +type Datastore struct { + path string +} + +// NewDatastore returns a new fs Datastore at given `path` +func NewDatastore(path string) (ds.Datastore, error) { + if !isDir(path) { + return nil, fmt.Errorf("Failed to find directory at: %v (file? perms?)", path) + } + + return &Datastore{path: path}, nil +} + +// KeyFilename returns the filename associated with `key` +func (d *Datastore) KeyFilename(key ds.Key) string { + return filepath.Join(d.path, key.String(), ".dsobject") +} + +// Put stores the given value. +func (d *Datastore) Put(key ds.Key, value interface{}) (err error) { + + // TODO: maybe use io.Readers/Writers? + // r, err := dsio.CastAsReader(value) + // if err != nil { + // return err + // } + + val, ok := value.([]byte) + if !ok { + return ds.ErrInvalidType + } + + fn := d.KeyFilename(key) + + // mkdirall above. + err = os.MkdirAll(filepath.Dir(fn), 0755) + if err != nil { + return err + } + + return ioutil.WriteFile(fn, val, 0666) +} + +// Get returns the value for given key +func (d *Datastore) Get(key ds.Key) (value interface{}, err error) { + fn := d.KeyFilename(key) + if !isFile(fn) { + return nil, ds.ErrNotFound + } + + return ioutil.ReadFile(fn) +} + +// Has returns whether the datastore has a value for a given key +func (d *Datastore) Has(key ds.Key) (exists bool, err error) { + return ds.GetBackedHas(d, key) +} + +// Delete removes the value for given key +func (d *Datastore) Delete(key ds.Key) (err error) { + fn := d.KeyFilename(key) + if !isFile(fn) { + return ds.ErrNotFound + } + + return os.Remove(fn) +} + +// KeyList returns a list of all keys in the datastore +func (d *Datastore) KeyList() ([]ds.Key, error) { + + keys := []ds.Key{} + + walkFn := func(path string, info os.FileInfo, err error) error { + // remove ds path prefix + if strings.HasPrefix(path, d.path) { + path = path[len(d.path):] + } + + if !info.IsDir() { + key := ds.NewKey(path) + keys = append(keys, key) + } + return nil + } + + filepath.Walk(d.path, walkFn) + return keys, nil +} + +// isDir returns whether given path is a directory +func isDir(path string) bool { + finfo, err := os.Stat(path) + if err != nil { + return false + } + + return finfo.IsDir() +} + +// isFile returns whether given path is a file +func isFile(path string) bool { + finfo, err := os.Stat(path) + if err != nil { + return false + } + + return !finfo.IsDir() +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go new file mode 100644 index 00000000000..cd69647d0e4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go @@ -0,0 +1,65 @@ +package fs_test + +import ( + "bytes" + "testing" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + fs "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs" + . "launchpad.net/gocheck" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type DSSuite struct { + dir string + ds ds.Datastore +} + +var _ = Suite(&DSSuite{}) + +func (ks *DSSuite) SetUpTest(c *C) { + ks.dir = c.MkDir() + ks.ds, _ = fs.NewDatastore(ks.dir) +} + +func (ks *DSSuite) TestOpen(c *C) { + _, err := fs.NewDatastore("/tmp/foo/bar/baz") + c.Assert(err, Not(Equals), nil) + + // setup ds + _, err = fs.NewDatastore(ks.dir) + c.Assert(err, Equals, nil) +} + +func (ks *DSSuite) TestBasic(c *C) { + + keys := strsToKeys([]string{ + "foo", + "foo/bar", + "foo/bar/baz", + "foo/barb", + "foo/bar/bazb", + "foo/bar/baz/barb", + }) + + for _, k := range keys { + err := ks.ds.Put(k, []byte(k.String())) + c.Check(err, Equals, nil) + } + + for _, k := range keys { + v, err := ks.ds.Get(k) + c.Check(err, Equals, nil) + c.Check(bytes.Equal(v.([]byte), []byte(k.String())), Equals, true) + } +} + +func strsToKeys(strs []string) []ds.Key { + keys := make([]ds.Key, len(strs)) + for i, s := range strs { + keys[i] = ds.NewKey(s) + } + return keys +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go new file mode 100644 index 00000000000..338b2e053d2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go @@ -0,0 +1,44 @@ +package leveldb + +import ( + "bytes" + "io" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" +) + +// CastAsReader does type assertions to find the type of a value and attempts +// to turn it into an io.Reader. If not possible, will return ds.ErrInvalidType +func CastAsReader(value interface{}) (io.Reader, error) { + switch v := value.(type) { + case io.Reader: + return v, nil + + case []byte: + return bytes.NewReader(v), nil + + case string: + return bytes.NewReader([]byte(v)), nil + + default: + return nil, ds.ErrInvalidType + } +} + +// // CastAsWriter does type assertions to find the type of a value and attempts +// // to turn it into an io.Writer. If not possible, will return ds.ErrInvalidType +// func CastAsWriter(value interface{}) (err error) { +// switch v := value.(type) { +// case io.Reader: +// return v, nil +// +// case []byte: +// return bytes.NewReader(v), nil +// +// case string: +// return bytes.NewReader([]byte(v)), nil +// +// default: +// return nil, ds.ErrInvalidType +// } +// } diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go index df210523f57..0fc2ec0542f 100644 --- a/Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go @@ -2,12 +2,13 @@ package datastore_test import ( "bytes" - . "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" - . "launchpad.net/gocheck" "math/rand" "path" "strings" "testing" + + . "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + . "gopkg.in/check.v1" ) // Hook up gocheck into the "go test" runner. diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go index 1b00d8bfd01..6fc2594a178 100644 --- a/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go @@ -13,7 +13,7 @@ type Datastore struct { type Options opt.Options -func NewDatastore(path string, opts *Options) (*Datastore, error) { +func NewDatastore(path string, opts *Options) (ds.ThreadSafeDatastore, error) { var nopts opt.Options if opts != nil { nopts = opt.Options(*opts) @@ -76,3 +76,5 @@ func (d *Datastore) KeyList() ([]ds.Key, error) { func (d *Datastore) Close() (err error) { return d.DB.Close() } + +func (d *Datastore) IsThreadSafe() {} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go new file mode 100644 index 00000000000..2dd74faa9ff --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go @@ -0,0 +1,54 @@ +package lru + +import ( + "errors" + + lru "github.com/hashicorp/golang-lru" + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" +) + +// Datastore uses golang-lru for internal storage. +type Datastore struct { + cache *lru.Cache +} + +// NewDatastore constructs a new LRU Datastore with given capacity. +func NewDatastore(capacity int) (*Datastore, error) { + cache, err := lru.New(capacity) + if err != nil { + return nil, err + } + + return &Datastore{cache: cache}, nil +} + +// Put stores the object `value` named by `key`. +func (d *Datastore) Put(key ds.Key, value interface{}) (err error) { + d.cache.Add(key, value) + return nil +} + +// Get retrieves the object `value` named by `key`. +func (d *Datastore) Get(key ds.Key) (value interface{}, err error) { + val, ok := d.cache.Get(key) + if !ok { + return nil, ds.ErrNotFound + } + return val, nil +} + +// Has returns whether the `key` is mapped to a `value`. +func (d *Datastore) Has(key ds.Key) (exists bool, err error) { + return ds.GetBackedHas(d, key) +} + +// Delete removes the value for given `key`. +func (d *Datastore) Delete(key ds.Key) (err error) { + d.cache.Remove(key) + return nil +} + +// KeyList returns a list of keys in the datastore +func (d *Datastore) KeyList() ([]ds.Key, error) { + return nil, errors.New("KeyList not implemented.") +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go new file mode 100644 index 00000000000..b030df9a240 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go @@ -0,0 +1,52 @@ +package lru_test + +import ( + "strconv" + "testing" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + lru "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru" + . "gopkg.in/check.v1" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type DSSuite struct{} + +var _ = Suite(&DSSuite{}) + +func (ks *DSSuite) TestBasic(c *C) { + var size = 1000 + + d, err := lru.NewDatastore(size) + c.Check(err, Equals, nil) + + for i := 0; i < size; i++ { + err := d.Put(ds.NewKey(strconv.Itoa(i)), i) + c.Check(err, Equals, nil) + } + + for i := 0; i < size; i++ { + j, err := d.Get(ds.NewKey(strconv.Itoa(i))) + c.Check(j, Equals, i) + c.Check(err, Equals, nil) + } + + for i := 0; i < size; i++ { + err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i) + c.Check(err, Equals, nil) + } + + for i := 0; i < size; i++ { + j, err := d.Get(ds.NewKey(strconv.Itoa(i))) + c.Check(j, Equals, nil) + c.Check(err, Equals, ds.ErrNotFound) + } + + for i := 0; i < size; i++ { + j, err := d.Get(ds.NewKey(strconv.Itoa(i + size))) + c.Check(j, Equals, i) + c.Check(err, Equals, nil) + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go new file mode 100644 index 00000000000..2f89def7c96 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go @@ -0,0 +1,19 @@ +package datastore + +// type KeyIterator struct { +// HasNext() bool +// Next() interface{} +// } + +// type Query struct { +// } + +/* +QueryDatastores support a Query interface. Queries are used to support +searching for values (beyond simple key-based `Get`s). +*/ +// type QueryDatastore interface { +// // Query returns an Iterator of Keys whose Values match criteria +// // expressed in `query`. +// Query(Query) (iter Iterator, err error) +// } diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go new file mode 100644 index 00000000000..06a363410b3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go @@ -0,0 +1,64 @@ +package sync + +import ( + "sync" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" +) + +// MutexDatastore contains a child datastire and a mutex. +// used for coarse sync +type MutexDatastore struct { + sync.RWMutex + + child ds.Datastore +} + +// MutexWrap constructs a datastore with a coarse lock around +// the entire datastore, for every single operation +func MutexWrap(d ds.Datastore) ds.ThreadSafeDatastore { + return &MutexDatastore{child: d} +} + +// Children implements Shim +func (d *MutexDatastore) Children() []ds.Datastore { + return []ds.Datastore{d.child} +} + +// IsThreadSafe implements ThreadSafeDatastore +func (d *MutexDatastore) IsThreadSafe() {} + +// Put implements Datastore.Put +func (d *MutexDatastore) Put(key ds.Key, value interface{}) (err error) { + d.Lock() + defer d.Unlock() + return d.child.Put(key, value) +} + +// Get implements Datastore.Get +func (d *MutexDatastore) Get(key ds.Key) (value interface{}, err error) { + d.RLock() + defer d.RUnlock() + return d.child.Get(key) +} + +// Has implements Datastore.Has +func (d *MutexDatastore) Has(key ds.Key) (exists bool, err error) { + d.RLock() + defer d.RUnlock() + return d.child.Has(key) +} + +// Delete implements Datastore.Delete +func (d *MutexDatastore) Delete(key ds.Key) (err error) { + d.Lock() + defer d.Unlock() + return d.child.Delete(key) +} + +// KeyList implements Datastore.KeyList +func (d *MutexDatastore) KeyList() ([]ds.Key, error) { + d.RLock() + defer d.RUnlock() + return d.child.KeyList() +} From 88d804e32a90b7c3ed9e464bdaadab4761d65342 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Fri, 3 Oct 2014 14:45:00 -0700 Subject: [PATCH 052/105] added core logging --- core/core.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/core.go b/core/core.go index dd5887ef54c..decec6307b3 100644 --- a/core/core.go +++ b/core/core.go @@ -9,6 +9,7 @@ import ( ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" bserv "github.com/jbenet/go-ipfs/blockservice" config "github.com/jbenet/go-ipfs/config" @@ -27,6 +28,8 @@ import ( u "github.com/jbenet/go-ipfs/util" ) +var log = logging.MustGetLogger("core") + // IpfsNode is IPFS Core module. It represents an IPFS instance. type IpfsNode struct { From 2ce9415c6934ba882f16302620c2ac5f93c2a9a3 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Fri, 3 Oct 2014 14:45:15 -0700 Subject: [PATCH 053/105] + fs ds + thread safe --- core/core.go | 2 +- core/datastore.go | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/core/core.go b/core/core.go index decec6307b3..8fa209bbae8 100644 --- a/core/core.go +++ b/core/core.go @@ -43,7 +43,7 @@ type IpfsNode struct { Peerstore peer.Peerstore // the local datastore - Datastore ds.Datastore + Datastore ds.ThreadSafeDatastore // the network message stream Network inet.Network diff --git a/core/datastore.go b/core/datastore.go index 9105adaabeb..30f92c8e8b3 100644 --- a/core/datastore.go +++ b/core/datastore.go @@ -4,11 +4,13 @@ import ( "fmt" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + fsds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs" lds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb" + syncds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync" config "github.com/jbenet/go-ipfs/config" ) -func makeDatastore(cfg config.Datastore) (ds.Datastore, error) { +func makeDatastore(cfg config.Datastore) (ds.ThreadSafeDatastore, error) { if len(cfg.Type) == 0 { return nil, fmt.Errorf("config datastore.type required") } @@ -16,14 +18,23 @@ func makeDatastore(cfg config.Datastore) (ds.Datastore, error) { switch cfg.Type { case "leveldb": return makeLevelDBDatastore(cfg) + case "memory": - return ds.NewMapDatastore(), nil + return syncds.MutexWrap(ds.NewMapDatastore()), nil + + case "fs": + log.Warning("using fs.Datastore at .datastore for testing.") + d, err := fsds.NewDatastore(".datastore") // for testing!! + if err != nil { + return nil, err + } + return syncds.MutexWrap(d), nil } return nil, fmt.Errorf("Unknown datastore type: %s", cfg.Type) } -func makeLevelDBDatastore(cfg config.Datastore) (ds.Datastore, error) { +func makeLevelDBDatastore(cfg config.Datastore) (ds.ThreadSafeDatastore, error) { if len(cfg.Path) == 0 { return nil, fmt.Errorf("config datastore.path required for leveldb") } From 612be596438bde0a653e7965686a396b2050cadc Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Fri, 3 Oct 2014 15:34:08 -0700 Subject: [PATCH 054/105] use string datastore keys. --- blockservice/blockservice.go | 8 +++----- blockstore/blockstore.go | 8 ++------ blockstore/blockstore_test.go | 2 +- peer/peerstore.go | 8 ++++---- routing/dht/dht.go | 16 +++++++++++----- routing/dht/handlers.go | 7 ++++--- routing/mock/routing.go | 4 ++-- util/util.go | 12 ++++++++++++ 8 files changed, 39 insertions(+), 26 deletions(-) diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index 36c3ed607b0..369cacc2ac9 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -7,7 +7,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" blocks "github.com/jbenet/go-ipfs/blocks" exchange "github.com/jbenet/go-ipfs/exchange" @@ -37,11 +37,10 @@ func NewBlockService(d ds.Datastore, rem exchange.Interface) (*BlockService, err // AddBlock adds a particular block to the service, Putting it into the datastore. func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { k := b.Key() - dsk := ds.NewKey(string(k)) log.Debug("storing [%s] in datastore", k.Pretty()) // TODO(brian): define a block datastore with a Put method which accepts a // block parameter - err := s.Datastore.Put(dsk, b.Data) + err := s.Datastore.Put(k.DsKey(), b.Data) if err != nil { return k, err } @@ -56,8 +55,7 @@ func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { // Getting it from the datastore using the key (hash). func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) { log.Debug("BlockService GetBlock: '%s'", k.Pretty()) - dsk := ds.NewKey(string(k)) - datai, err := s.Datastore.Get(dsk) + datai, err := s.Datastore.Get(k.DsKey()) if err == nil { log.Debug("Blockservice: Got data in datastore.") bdata, ok := datai.([]byte) diff --git a/blockstore/blockstore.go b/blockstore/blockstore.go index a4fc1f65d85..860d7ba41e0 100644 --- a/blockstore/blockstore.go +++ b/blockstore/blockstore.go @@ -27,7 +27,7 @@ type blockstore struct { } func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) { - maybeData, err := bs.datastore.Get(toDatastoreKey(k)) + maybeData, err := bs.datastore.Get(k.DsKey()) if err != nil { return nil, err } @@ -39,9 +39,5 @@ func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) { } func (bs *blockstore) Put(block blocks.Block) error { - return bs.datastore.Put(toDatastoreKey(block.Key()), block.Data) -} - -func toDatastoreKey(k u.Key) ds.Key { - return ds.NewKey(string(k)) + return bs.datastore.Put(block.Key().DsKey(), block.Data) } diff --git a/blockstore/blockstore_test.go b/blockstore/blockstore_test.go index 4b0909d7547..dfb9783afb2 100644 --- a/blockstore/blockstore_test.go +++ b/blockstore/blockstore_test.go @@ -44,7 +44,7 @@ func TestValueTypeMismatch(t *testing.T) { block := testutil.NewBlockOrFail(t, "some data") datastore := ds.NewMapDatastore() - datastore.Put(toDatastoreKey(block.Key()), "data that isn't a block!") + datastore.Put(block.Key().DsKey(), "data that isn't a block!") blockstore := NewBlockstore(datastore) diff --git a/peer/peerstore.go b/peer/peerstore.go index 9c0f28df316..e8eb0eac00f 100644 --- a/peer/peerstore.go +++ b/peer/peerstore.go @@ -37,7 +37,7 @@ func (p *peerstore) Get(i ID) (*Peer, error) { p.RLock() defer p.RUnlock() - k := ds.NewKey(string(i)) + k := u.Key(i).DsKey() val, err := p.peers.Get(k) if err != nil { return nil, err @@ -54,7 +54,7 @@ func (p *peerstore) Put(peer *Peer) error { p.Lock() defer p.Unlock() - k := ds.NewKey(string(peer.ID)) + k := u.Key(peer.ID).DsKey() return p.peers.Put(k, peer) } @@ -62,7 +62,7 @@ func (p *peerstore) Delete(i ID) error { p.Lock() defer p.Unlock() - k := ds.NewKey(string(i)) + k := u.Key(i).DsKey() return p.peers.Delete(k) } @@ -84,7 +84,7 @@ func (p *peerstore) All() (*Map, error) { pval, ok := val.(*Peer) if ok { - (*ps)[u.Key(k.String())] = pval + (*ps)[u.Key(pval.ID)] = pval } } return ps, nil diff --git a/routing/dht/dht.go b/routing/dht/dht.go index f02be471146..aa7361e53a3 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -13,11 +13,11 @@ import ( peer "github.com/jbenet/go-ipfs/peer" kb "github.com/jbenet/go-ipfs/routing/kbucket" u "github.com/jbenet/go-ipfs/util" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ) @@ -328,7 +328,7 @@ func (dht *IpfsDHT) getFromPeerList(ctx context.Context, key u.Key, func (dht *IpfsDHT) getLocal(key u.Key) ([]byte, error) { dht.dslock.Lock() defer dht.dslock.Unlock() - v, err := dht.datastore.Get(ds.NewKey(string(key))) + v, err := dht.datastore.Get(key.DsKey()) if err != nil { return nil, err } @@ -341,7 +341,7 @@ func (dht *IpfsDHT) getLocal(key u.Key) ([]byte, error) { } func (dht *IpfsDHT) putLocal(key u.Key, value []byte) error { - return dht.datastore.Put(ds.NewKey(string(key)), value) + return dht.datastore.Put(key.DsKey(), value) } // Update signals to all routingTables to Update their last-seen status @@ -494,13 +494,19 @@ func (dht *IpfsDHT) ensureConnectedToPeer(pbp *Message_Peer) (*peer.Peer, error) return p, err } +//TODO: this should be smarter about which keys it selects. func (dht *IpfsDHT) loadProvidableKeys() error { kl, err := dht.datastore.KeyList() if err != nil { return err } - for _, k := range kl { - dht.providers.AddProvider(u.Key(k.Bytes()), dht.self) + for _, dsk := range kl { + k := u.KeyFromDsKey(dsk) + if len(k) == 0 { + log.Error("loadProvidableKeys error: %v", dsk) + } + + dht.providers.AddProvider(k, dht.self) } return nil } diff --git a/routing/dht/handlers.go b/routing/dht/handlers.go index 4301d1e4e97..ac03ed3e8f4 100644 --- a/routing/dht/handlers.go +++ b/routing/dht/handlers.go @@ -51,7 +51,7 @@ func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error // let's first check if we have the value locally. u.DOut("[%s] handleGetValue looking into ds\n", dht.self.ID.Pretty()) - dskey := ds.NewKey(pmes.GetKey()) + dskey := u.Key(pmes.GetKey()).DsKey() iVal, err := dht.datastore.Get(dskey) u.DOut("[%s] handleGetValue looking into ds GOT %v\n", dht.self.ID.Pretty(), iVal) @@ -96,7 +96,7 @@ func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error func (dht *IpfsDHT) handlePutValue(p *peer.Peer, pmes *Message) (*Message, error) { dht.dslock.Lock() defer dht.dslock.Unlock() - dskey := ds.NewKey(pmes.GetKey()) + dskey := u.Key(pmes.GetKey()).DsKey() err := dht.datastore.Put(dskey, pmes.GetValue()) u.DOut("[%s] handlePutValue %v %v\n", dht.self.ID.Pretty(), dskey, pmes.GetValue()) return pmes, err @@ -137,7 +137,8 @@ func (dht *IpfsDHT) handleGetProviders(p *peer.Peer, pmes *Message) (*Message, e resp := newMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel()) // check if we have this value, to add ourselves as provider. - has, err := dht.datastore.Has(ds.NewKey(pmes.GetKey())) + dsk := u.Key(pmes.GetKey()).DsKey() + has, err := dht.datastore.Has(dsk) if err != nil && err != ds.ErrNotFound { u.PErr("unexpected datastore error: %v\n", err) has = false diff --git a/routing/mock/routing.go b/routing/mock/routing.go index e5fdb96fca0..954914c3bed 100644 --- a/routing/mock/routing.go +++ b/routing/mock/routing.go @@ -33,11 +33,11 @@ func (mr *MockRouter) SetRoutingServer(rs RoutingServer) { } func (mr *MockRouter) PutValue(ctx context.Context, key u.Key, val []byte) error { - return mr.datastore.Put(ds.NewKey(string(key)), val) + return mr.datastore.Put(key.DsKey(), val) } func (mr *MockRouter) GetValue(ctx context.Context, key u.Key) ([]byte, error) { - v, err := mr.datastore.Get(ds.NewKey(string(key))) + v, err := mr.datastore.Get(key.DsKey()) if err != nil { return nil, err } diff --git a/util/util.go b/util/util.go index 40d04ed734e..8831c68317f 100644 --- a/util/util.go +++ b/util/util.go @@ -41,6 +41,18 @@ func (k Key) Pretty() string { return b58.Encode([]byte(k)) } +// DsKey returns a Datastore key +func (k Key) DsKey() ds.Key { + return ds.NewKey(k.Pretty()) +} + +// KeyFromDsKey returns a Datastore key +func KeyFromDsKey(dsk ds.Key) Key { + l := dsk.List() + enc := l[len(l)-1] + return Key(b58.Decode(enc)) +} + // Hash is the global IPFS hash function. uses multihash SHA2_256, 256 bits func Hash(data []byte) (mh.Multihash, error) { return mh.Sum(data, mh.SHA2_256, -1) From e7aa1166bcbaa57e20512467cb6661bca2da4aeb Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 3 Oct 2014 23:04:41 +0000 Subject: [PATCH 055/105] add writerAt for fuse writes --- fuse/ipns/ipns_unix.go | 40 ++++++++++++++++++++++------------------ fuse/ipns/writerat.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 18 deletions(-) create mode 100644 fuse/ipns/writerat.go diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 4a9db2bceda..83146c3fd21 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -1,14 +1,13 @@ package ipns import ( + "bytes" "fmt" "io/ioutil" "os" "path/filepath" "time" - "bytes" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" @@ -205,7 +204,7 @@ type Node struct { cached *mdag.PBData // For writing - dataBuf *bytes.Buffer + writerBuf WriteAtBuf } func (s *Node) loadData() error { @@ -290,21 +289,25 @@ func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { } // this is a terrible function... 'ReadAll'? // what if i have a 6TB file? GG RAM. - return ioutil.ReadAll(r) + b, err := ioutil.ReadAll(r) + if err != nil { + log.Error("[%s] Readall error: %s", s.name, err) + return nil, err + } + if len(b) > 4 { + log.Debug("ReadAll trailing bytes: %v", b[len(b)-4:]) + } + return b, nil } func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error { log.Debug("ipns: Node Write [%s]: flags = %s, offset = %d, size = %d", n.name, req.Flags.String(), req.Offset, len(req.Data)) - if n.dataBuf == nil { - n.dataBuf = new(bytes.Buffer) + if n.writerBuf == nil { + n.writerBuf = NewWriterAtFromBytes(nil) } - if req.Offset == 0 { - n.dataBuf.Reset() - n.dataBuf.Write(req.Data) - resp.Size = len(req.Data) - } else { - log.Error("Unhandled write to offset!") - n.dataBuf = nil + _, err := n.writerBuf.WriteAt(req.Data, req.Offset) + if err != nil { + return err } return nil } @@ -312,7 +315,7 @@ func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.I func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { log.Debug("Got flush request [%s]!", n.name) - if n.dataBuf != nil { + if n.writerBuf != nil { //TODO: // This operation holds everything in memory, // should be changed to stream the block creation/storage @@ -321,9 +324,10 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { //NOTE: // This should only occur on a file object, if this were to be a // folder, bad things would happen. - newNode, err := imp.NewDagFromReader(n.dataBuf) + buf := bytes.NewReader(n.writerBuf.Bytes()) + newNode, err := imp.NewDagFromReader(buf) if err != nil { - log.Critical("error creating dag from dataBuf: %s", err) + log.Critical("error creating dag from writerBuf: %s", err) return fuse.ENODATA } if n.parent != nil { @@ -350,7 +354,7 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { fmt.Println(string(b)) // - n.dataBuf = nil + n.writerBuf = nil n.wasChanged() } @@ -382,7 +386,7 @@ func (n *Node) republishRoot() error { return err } - n.dataBuf = nil + n.writerBuf = nil ndkey, err := root.Nd.Key() if err != nil { diff --git a/fuse/ipns/writerat.go b/fuse/ipns/writerat.go new file mode 100644 index 00000000000..c5ddf5c5c4c --- /dev/null +++ b/fuse/ipns/writerat.go @@ -0,0 +1,29 @@ +package ipns + +import "io" + +type WriteAtBuf interface { + io.WriterAt + Bytes() []byte +} + +type writerAt struct { + buf []byte +} + +func NewWriterAtFromBytes(b []byte) WriteAtBuf { + return &writerAt{b} +} + +// TODO: make this better in the future, this is just a quick hack for now +func (wa *writerAt) WriteAt(p []byte, off int64) (int, error) { + if off+int64(len(p)) > int64(len(wa.buf)) { + wa.buf = append(wa.buf, make([]byte, (int(off)+len(p))-len(wa.buf))...) + } + copy(wa.buf[off:], p) + return len(p), nil +} + +func (wa *writerAt) Bytes() []byte { + return wa.buf +} From 7bb2bd6d16504379bc55cc001871fc7e625ed706 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 3 Oct 2014 23:50:05 +0000 Subject: [PATCH 056/105] a little more progress... and some debugging code --- fuse/ipns/ipns_unix.go | 11 ++++++----- merkledag/merkledag.go | 8 ++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 83146c3fd21..18ad323fdc3 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -309,6 +309,7 @@ func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.I if err != nil { return err } + resp.Size = len(req.Data) return nil } @@ -442,17 +443,17 @@ func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) return child, nil } -func (n *Node) Mknod(req *fuse.MknodRequest, intr fs.Intr) (fs.Node, fuse.Error) { - log.Debug("Got mknod request!") - return nil, nil -} - func (n *Node) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr) (fs.Handle, fuse.Error) { //log.Debug("[%s] Received open request! flags = %s", n.name, req.Flags.String()) //TODO: check open flags and truncate if necessary return n, nil } +func (n *Node) Mknod(req *fuse.MknodRequest, intr fs.Intr) (fs.Node, fuse.Error) { + log.Debug("Got mknod request!") + return nil, nil +} + func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr fs.Intr) (fs.Node, fs.Handle, fuse.Error) { log.Debug("Got create request: %s", req.Name) diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 31f4e2937f5..e51a6e3c4e9 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -46,6 +46,14 @@ type Link struct { // AddNodeLink adds a link to another node. func (n *Node) AddNodeLink(name string, that *Node) error { + // DEBUG CODE + for _, l := range n.Links { + if l.Name == name { + panic("Trying to add child that already exists!") + } + } + // + s, err := that.Size() if err != nil { return err From 73b2058f8f5471e3ade0ff1a0f0b772a15519e8f Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Fri, 3 Oct 2014 18:08:21 -0700 Subject: [PATCH 057/105] DNSResolver: use isd.IsDomain this commit dedicated to @whyrusleeping --- namesys/dns.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/namesys/dns.go b/namesys/dns.go index 1154a66fec1..e9c4097d837 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -2,25 +2,29 @@ package namesys import ( "net" - "strings" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" u "github.com/jbenet/go-ipfs/util" + isd "github.com/jbenet/go-is-domain" ) +// DNSResolver implements a Resolver on DNS domains type DNSResolver struct { // TODO: maybe some sort of caching? // cache would need a timeout } +// Matches implements Resolver func (r *DNSResolver) Matches(name string) bool { - return strings.Contains(name, ".") && !strings.HasPrefix(name, ".") + return isd.IsDomain(name) } +// Resolve implements Resolver // TXT records for a given domain name should contain a b58 // encoded multihash. func (r *DNSResolver) Resolve(name string) (string, error) { + log.Info("DNSResolver resolving %v", name) txt, err := net.LookupTXT(name) if err != nil { return "", err From 11332fc73b4538412688a497b38bc151af33af3a Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 4 Oct 2014 06:34:06 +0000 Subject: [PATCH 058/105] make the tests pass! --- fuse/ipns/ipns_unix.go | 8 +++++--- importer/importer.go | 8 ++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 18ad323fdc3..5312d9fff09 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -282,7 +282,7 @@ func (s *Node) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { // ReadAll reads the object data as file data func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { - log.Debug("ipns: ReadAll Node") + log.Debug("ipns: ReadAll [%s]", s.name) r, err := mdag.NewDagReader(s.Nd, s.Ipfs.DAG) if err != nil { return nil, err @@ -297,6 +297,7 @@ func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { if len(b) > 4 { log.Debug("ReadAll trailing bytes: %v", b[len(b)-4:]) } + fmt.Println(b) return b, nil } @@ -329,14 +330,14 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { newNode, err := imp.NewDagFromReader(buf) if err != nil { log.Critical("error creating dag from writerBuf: %s", err) - return fuse.ENODATA + return err } if n.parent != nil { err := n.parent.update(n.name, newNode) if err != nil { log.Critical("error in updating ipns dag tree: %s", err) // return fuse.ETHISISPRETTYBAD - return fuse.ENOSYS + return err } } n.Nd = newNode @@ -353,6 +354,7 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { fmt.Println("VERIFICATION READ") fmt.Printf("READ %d BYTES\n", len(b)) fmt.Println(string(b)) + fmt.Println(b) // n.writerBuf = nil diff --git a/importer/importer.go b/importer/importer.go index b71118317e2..1834ddf3f11 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -30,12 +30,14 @@ func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, er first := <-blkChan root := &dag.Node{Data: dag.FilePBData(first)} + i := 0 for blk := range blkChan { child := &dag.Node{Data: dag.WrapData(blk)} - err := root.AddNodeLink("", child) + err := root.AddNodeLink(fmt.Sprintf("%d", i), child) if err != nil { return nil, err } + i++ } return root, nil @@ -69,12 +71,14 @@ func NewDagInNode(r io.Reader, n *dag.Node) error { first := <-blkChan n.Data = dag.FilePBData(first) + i := 0 for blk := range blkChan { child := &dag.Node{Data: dag.WrapData(blk)} - err := n.AddNodeLink("", child) + err := n.AddNodeLink(fmt.Sprintf("%d", i), child) if err != nil { return err } + i++ } return nil From 0c7ff7b0a820953f49c076651f7a8b02aaf11cfa Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 4 Oct 2014 07:24:57 +0000 Subject: [PATCH 059/105] add skeleton for ipns test, to be finished when there is a mock core.IpfsNode --- fuse/ipns/ipns_test.go | 62 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 fuse/ipns/ipns_test.go diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go new file mode 100644 index 00000000000..4e8d5f43904 --- /dev/null +++ b/fuse/ipns/ipns_test.go @@ -0,0 +1,62 @@ +package ipns + +import ( + "crypto/rand" + "os" + "testing" + + fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" +) + +func randBytes(size int) []byte { + b := make([]byte, size) + rand.Read(b) + return b +} + +func TestIpnsBasicIO(t *testing.T) { + fs, err := NewIpns(nil, "") + if err != nil { + t.Fatal(err) + } + mnt, err := fstest.MountedT(t, fs) + if err != nil { + t.Fatal(err) + } + + data := randBytes(12345) + fi, err := os.Create(mnt.Dir + "/local/testfile") + if err != nil { + t.Fatal(err) + } + + n, err := fi.Write(data) + if err != nil { + t.Fatal(err) + } + + if n != len(data) { + t.Fatal("Didnt write proper amount!") + } + + fi.Close() + + //TODO: maybe wait for the publish to happen? or not, should test both cases + + fi, err = os.Open(mnt.Dir + "/local/testfile") + if err != nil { + t.Fatal(err) + } + + rbuf := make([]byte, len(data)) + n, err = fi.Read(rbuf) + if err != nil { + t.Fatal(err) + } + + if n != len(rbuf) { + t.Fatal("Failed to read correct amount!") + } + + fi.Close() +} From 5423a008942a00be40259b7bf6c522d94cc432a7 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 4 Oct 2014 08:00:13 +0000 Subject: [PATCH 060/105] create test node and make first ipns test pass --- fuse/ipns/ipns_test.go | 9 +++++++- util/testutil/node.go | 49 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 util/testutil/node.go diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index 4e8d5f43904..b35a584b7ef 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -6,6 +6,7 @@ import ( "testing" fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" + testutil "github.com/jbenet/go-ipfs/util/testutil" ) func randBytes(size int) []byte { @@ -15,7 +16,12 @@ func randBytes(size int) []byte { } func TestIpnsBasicIO(t *testing.T) { - fs, err := NewIpns(nil, "") + localnode, err := testutil.NewMockNode() + if err != nil { + t.Fatal(err) + } + + fs, err := NewIpns(localnode, "") if err != nil { t.Fatal(err) } @@ -59,4 +65,5 @@ func TestIpnsBasicIO(t *testing.T) { } fi.Close() + } diff --git a/util/testutil/node.go b/util/testutil/node.go new file mode 100644 index 00000000000..8df1bb2cd22 --- /dev/null +++ b/util/testutil/node.go @@ -0,0 +1,49 @@ +package testutil + +import ( + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + syncds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync" + bs "github.com/jbenet/go-ipfs/blockservice" + core "github.com/jbenet/go-ipfs/core" + ci "github.com/jbenet/go-ipfs/crypto" + mdag "github.com/jbenet/go-ipfs/merkledag" + nsys "github.com/jbenet/go-ipfs/namesys" + "github.com/jbenet/go-ipfs/peer" + mdht "github.com/jbenet/go-ipfs/routing/mock" +) + +var _ = core.IpfsNode{} + +func NewMockNode() (*core.IpfsNode, error) { + nd := new(core.IpfsNode) + + //Generate Identity + nd.Identity = &peer.Peer{ID: []byte("TESTING")} + pk, sk, err := ci.GenerateKeyPair(ci.RSA, 1024) + if err != nil { + return nil, err + } + nd.Identity.PrivKey = pk + nd.Identity.PubKey = sk + + // Temp Datastore + dstore := ds.NewMapDatastore() + nd.Datastore = syncds.MutexWrap(dstore) + + // Routing + dht := mdht.NewMockRouter(nd.Identity, nd.Datastore) + + // Bitswap + //?? + + bserv, err := bs.NewBlockService(nd.Datastore, nil) + if err != nil { + return nil, err + } + + dserv := &mdag.DAGService{bserv} + + // Namespace resolver + nd.Namesys = nsys.NewMasterResolver(dht, dserv) + return nd, nil +} From c64add1959e6d45bcd7b6ec1930bd780356cce55 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 4 Oct 2014 08:30:24 +0000 Subject: [PATCH 061/105] clean up ipns test and add more asserts --- fuse/ipns/ipns_test.go | 13 +++++++------ fuse/ipns/ipns_unix.go | 9 ++------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index b35a584b7ef..330cde546c0 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -1,7 +1,9 @@ package ipns import ( + "bytes" "crypto/rand" + "io/ioutil" "os" "testing" @@ -29,6 +31,7 @@ func TestIpnsBasicIO(t *testing.T) { if err != nil { t.Fatal(err) } + defer mnt.Close() data := randBytes(12345) fi, err := os.Create(mnt.Dir + "/local/testfile") @@ -54,16 +57,14 @@ func TestIpnsBasicIO(t *testing.T) { t.Fatal(err) } - rbuf := make([]byte, len(data)) - n, err = fi.Read(rbuf) + rbuf, err := ioutil.ReadAll(fi) if err != nil { t.Fatal(err) } + fi.Close() - if n != len(rbuf) { - t.Fatal("Failed to read correct amount!") + if !bytes.Equal(rbuf, data) { + t.Fatal("Incorrect Read!") } - fi.Close() - } diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 5312d9fff09..5b2bc8832f9 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -2,7 +2,6 @@ package ipns import ( "bytes" - "fmt" "io/ioutil" "os" "path/filepath" @@ -294,10 +293,6 @@ func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { log.Error("[%s] Readall error: %s", s.name, err) return nil, err } - if len(b) > 4 { - log.Debug("ReadAll trailing bytes: %v", b[len(b)-4:]) - } - fmt.Println(b) return b, nil } @@ -342,7 +337,7 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { } n.Nd = newNode - //TEMP + /*/TEMP dr, err := mdag.NewDagReader(n.Nd, n.Ipfs.DAG) if err != nil { log.Critical("Verification read failed.") @@ -355,7 +350,7 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { fmt.Printf("READ %d BYTES\n", len(b)) fmt.Println(string(b)) fmt.Println(b) - // + //*/ n.writerBuf = nil From 7e1cd592597acea22c198346cdfeebf6233c35ef Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Sat, 4 Oct 2014 03:36:30 -0700 Subject: [PATCH 062/105] initialize loggers at ERROR --- blockservice/blockservice.go | 2 +- cmd/ipfs/ipfs.go | 2 +- core/commands/commands.go | 3 ++- core/core.go | 2 +- daemon/daemon.go | 2 +- fuse/ipns/ipns_unix.go | 2 +- fuse/readonly/readonly_unix.go | 2 +- merkledag/merkledag.go | 2 +- namesys/routing.go | 2 +- path/path.go | 2 +- routing/dht/dht.go | 2 +- util/util.go | 9 +++++++-- 12 files changed, 19 insertions(+), 13 deletions(-) diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index 369cacc2ac9..6970f363b51 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -14,7 +14,7 @@ import ( u "github.com/jbenet/go-ipfs/util" ) -var log = logging.MustGetLogger("blockservice") +var log = u.Logger("blockservice", logging.ERROR) // BlockService is a block datastore. // It uses an internal `datastore.Datastore` instance to store values. diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 3d50e1b8de1..9be051850b0 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -62,7 +62,7 @@ Use "ipfs help " for more information about a command. } // log is the command logger -var log = logging.MustGetLogger("cmd/ipfs") +var log = u.Logger("cmd/ipfs", logging.ERROR) func init() { config, err := config.PathRoot() diff --git a/core/commands/commands.go b/core/commands/commands.go index b356d9bbb91..f6e681b4c0e 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -4,10 +4,11 @@ import ( "io" "github.com/jbenet/go-ipfs/core" + u "github.com/jbenet/go-ipfs/util" logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) -var log = logging.MustGetLogger("commands") +var log = u.Logger("commands", logging.ERROR) type CmdFunc func(*core.IpfsNode, []string, map[string]interface{}, io.Writer) error diff --git a/core/core.go b/core/core.go index 8fa209bbae8..966a8e7103a 100644 --- a/core/core.go +++ b/core/core.go @@ -28,7 +28,7 @@ import ( u "github.com/jbenet/go-ipfs/util" ) -var log = logging.MustGetLogger("core") +var log = u.Logger("core", logging.ERROR) // IpfsNode is IPFS Core module. It represents an IPFS instance. type IpfsNode struct { diff --git a/daemon/daemon.go b/daemon/daemon.go index 3ea4cc5dac8..c40b99cb2b7 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -18,7 +18,7 @@ import ( ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ) -var log = logging.MustGetLogger("daemon") +var log = u.Logger("daemon", logging.ERROR) // LockFile is the filename of the daemon lock, relative to config dir const LockFile = "daemon.lock" diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 5312d9fff09..bfea7c4f768 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -20,7 +20,7 @@ import ( u "github.com/jbenet/go-ipfs/util" ) -var log = logging.MustGetLogger("ipns") +var log = u.Logger("ipns", logging.ERROR) // FileSystem is the readwrite IPNS Fuse Filesystem. type FileSystem struct { diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 8641916e251..ac4b8d7a448 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -24,7 +24,7 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) -var log = logging.MustGetLogger("ipfs") +var log = u.Logger("ipfs", logging.ERROR) // FileSystem is the readonly Ipfs Fuse Filesystem. type FileSystem struct { diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index e51a6e3c4e9..c5db9e00f35 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -12,7 +12,7 @@ import ( u "github.com/jbenet/go-ipfs/util" ) -var log = logging.MustGetLogger("merkledag") +var log = u.Logger("merkledag", logging.ERROR) // NodeMap maps u.Keys to Nodes. // We cannot use []byte/Multihash for keys :( diff --git a/namesys/routing.go b/namesys/routing.go index 4c2b0d81655..ba7c4a7239e 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -14,7 +14,7 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) -var log = logging.MustGetLogger("namesys") +var log = u.Logger("namesys", logging.ERROR) // RoutingResolver implements NSResolver for the main IPFS SFS-like naming type RoutingResolver struct { diff --git a/path/path.go b/path/path.go index ebc9d81d665..83d1aa8b608 100644 --- a/path/path.go +++ b/path/path.go @@ -11,7 +11,7 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) -var log = logging.MustGetLogger("path") +var log = u.Logger("path", logging.ERROR) // Resolver provides path resolution to IPFS // It has a pointer to a DAGService, which is uses to resolve nodes. diff --git a/routing/dht/dht.go b/routing/dht/dht.go index aa7361e53a3..d45c5aabfb1 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -22,7 +22,7 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ) -var log = logging.MustGetLogger("dht") +var log = u.Logger("dht", logging.ERROR) // TODO. SEE https://github.com/jbenet/node-ipfs/blob/master/submodules/ipfs-dht/index.js diff --git a/util/util.go b/util/util.go index 8831c68317f..8a44cb98261 100644 --- a/util/util.go +++ b/util/util.go @@ -118,11 +118,16 @@ func SetupLogging() { logging.SetLevel(logging.ERROR, "") } */ - logging.SetLevel(logging.ERROR, "merkledag") - logging.SetLevel(logging.ERROR, "blockservice") logging.SetFormatter(logging.MustStringFormatter(LogFormat)) } +// Logger retrieves a particular logger + initializes it at a particular level +func Logger(name string, lvl logging.Level) *logging.Logger { + log := logging.MustGetLogger(name) + logging.SetLevel(lvl, name) + return log +} + // ExpandPathnames takes a set of paths and turns them into absolute paths func ExpandPathnames(paths []string) ([]string, error) { var out []string From 0c8ae7674e96dc47f4932b23a13a69d16b0deada Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Sat, 4 Oct 2014 03:53:21 -0700 Subject: [PATCH 063/105] loggers: set level --- blockservice/blockservice.go | 3 +-- cmd/ipfs/ipfs.go | 3 +-- core/commands/commands.go | 4 +--- core/core.go | 3 +-- daemon/daemon.go | 3 +-- fuse/ipns/ipns_unix.go | 3 +-- fuse/readonly/readonly_unix.go | 3 +-- merkledag/merkledag.go | 3 +-- namesys/routing.go | 5 ++--- path/path.go | 3 +-- routing/dht/dht.go | 3 +-- util/util.go | 11 +++++++++-- 12 files changed, 21 insertions(+), 26 deletions(-) diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index 6970f363b51..21bb75c6f36 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -7,14 +7,13 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" blocks "github.com/jbenet/go-ipfs/blocks" exchange "github.com/jbenet/go-ipfs/exchange" u "github.com/jbenet/go-ipfs/util" ) -var log = u.Logger("blockservice", logging.ERROR) +var log = u.Logger("blockservice") // BlockService is a block datastore. // It uses an internal `datastore.Datastore` instance to store values. diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 9be051850b0..7b75ba5c54c 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -9,7 +9,6 @@ import ( flag "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/gonuts/flag" commander "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/commander" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" config "github.com/jbenet/go-ipfs/config" core "github.com/jbenet/go-ipfs/core" @@ -62,7 +61,7 @@ Use "ipfs help " for more information about a command. } // log is the command logger -var log = u.Logger("cmd/ipfs", logging.ERROR) +var log = u.Logger("cmd/ipfs") func init() { config, err := config.PathRoot() diff --git a/core/commands/commands.go b/core/commands/commands.go index f6e681b4c0e..0c2541146b8 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -5,10 +5,8 @@ import ( "github.com/jbenet/go-ipfs/core" u "github.com/jbenet/go-ipfs/util" - - logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) -var log = u.Logger("commands", logging.ERROR) +var log = u.Logger("commands") type CmdFunc func(*core.IpfsNode, []string, map[string]interface{}, io.Writer) error diff --git a/core/core.go b/core/core.go index 966a8e7103a..427c33c2932 100644 --- a/core/core.go +++ b/core/core.go @@ -9,7 +9,6 @@ import ( ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" bserv "github.com/jbenet/go-ipfs/blockservice" config "github.com/jbenet/go-ipfs/config" @@ -28,7 +27,7 @@ import ( u "github.com/jbenet/go-ipfs/util" ) -var log = u.Logger("core", logging.ERROR) +var log = u.Logger("core") // IpfsNode is IPFS Core module. It represents an IPFS instance. type IpfsNode struct { diff --git a/daemon/daemon.go b/daemon/daemon.go index c40b99cb2b7..02fe49023ac 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -9,7 +9,6 @@ import ( "path" "sync" - logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" core "github.com/jbenet/go-ipfs/core" "github.com/jbenet/go-ipfs/core/commands" u "github.com/jbenet/go-ipfs/util" @@ -18,7 +17,7 @@ import ( ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ) -var log = u.Logger("daemon", logging.ERROR) +var log = u.Logger("daemon") // LockFile is the filename of the daemon lock, relative to config dir const LockFile = "daemon.lock" diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index bfea7c4f768..8687fc7259b 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -11,7 +11,6 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" - logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" "github.com/jbenet/go-ipfs/core" ci "github.com/jbenet/go-ipfs/crypto" @@ -20,7 +19,7 @@ import ( u "github.com/jbenet/go-ipfs/util" ) -var log = u.Logger("ipns", logging.ERROR) +var log = u.Logger("ipns") // FileSystem is the readwrite IPNS Fuse Filesystem. type FileSystem struct { diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index ac4b8d7a448..1bd45e8c212 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -21,10 +21,9 @@ import ( core "github.com/jbenet/go-ipfs/core" mdag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) -var log = u.Logger("ipfs", logging.ERROR) +var log = u.Logger("ipfs") // FileSystem is the readonly Ipfs Fuse Filesystem. type FileSystem struct { diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index c5db9e00f35..07612815508 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -4,7 +4,6 @@ import ( "fmt" proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" - logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" blocks "github.com/jbenet/go-ipfs/blocks" @@ -12,7 +11,7 @@ import ( u "github.com/jbenet/go-ipfs/util" ) -var log = u.Logger("merkledag", logging.ERROR) +var log = u.Logger("merkledag") // NodeMap maps u.Keys to Nodes. // We cannot use []byte/Multihash for keys :( diff --git a/namesys/routing.go b/namesys/routing.go index ba7c4a7239e..942263e8efc 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -6,15 +6,14 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" ci "github.com/jbenet/go-ipfs/crypto" mdag "github.com/jbenet/go-ipfs/merkledag" "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" - mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) -var log = u.Logger("namesys", logging.ERROR) +var log = u.Logger("namesys") // RoutingResolver implements NSResolver for the main IPFS SFS-like naming type RoutingResolver struct { diff --git a/path/path.go b/path/path.go index 83d1aa8b608..23920314075 100644 --- a/path/path.go +++ b/path/path.go @@ -8,10 +8,9 @@ import ( mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" merkledag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) -var log = u.Logger("path", logging.ERROR) +var log = u.Logger("path") // Resolver provides path resolution to IPFS // It has a pointer to a DAGService, which is uses to resolve nodes. diff --git a/routing/dht/dht.go b/routing/dht/dht.go index d45c5aabfb1..8cf3a115398 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -17,12 +17,11 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ) -var log = u.Logger("dht", logging.ERROR) +var log = u.Logger("dht") // TODO. SEE https://github.com/jbenet/node-ipfs/blob/master/submodules/ipfs-dht/index.js diff --git a/util/util.go b/util/util.go index 8a44cb98261..1e51064ed0a 100644 --- a/util/util.go +++ b/util/util.go @@ -107,6 +107,8 @@ func DOut(format string, a ...interface{}) { } } +var loggers = []string{} + // SetupLogging will initialize the logger backend and set the flags. func SetupLogging() { backend := logging.NewLogBackend(os.Stderr, "", 0) @@ -119,12 +121,17 @@ func SetupLogging() { } */ logging.SetFormatter(logging.MustStringFormatter(LogFormat)) + + for _, n := range loggers { + logging.SetLevel(logging.ERROR, n) + } } // Logger retrieves a particular logger + initializes it at a particular level -func Logger(name string, lvl logging.Level) *logging.Logger { +func Logger(name string) *logging.Logger { log := logging.MustGetLogger(name) - logging.SetLevel(lvl, name) + // logging.SetLevel(lvl, name) // can't set level here. + loggers = append(loggers, name) return log } From ee1d1ac0786178f9728bc3fccc1175effc0c760b Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Sat, 4 Oct 2014 03:56:12 -0700 Subject: [PATCH 064/105] add: only show added once. --- core/commands/add.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 7781812580b..a85ed4831ef 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -36,7 +36,7 @@ func Add(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Wr } // Add the file - nd, err := AddPath(n, path, depth) + _, err = AddPath(n, path, depth) if err != nil { if err == ErrDepthLimitExceeded && depth == 1 { err = errors.New("use -r to recursively add directories") @@ -45,12 +45,13 @@ func Add(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Wr } // get the key to print it - k, err := nd.Key() - if err != nil { - return fmt.Errorf("addFile error: %v", err) - } - - fmt.Fprintf(out, "added %s %s\n", k.Pretty(), path) + // k, err := nd.Key() + // if err != nil { + // return fmt.Errorf("addFile error: %v", err) + // } + // + // Commenting out of here, because it's already in addNode below. + // fmt.Fprintf(out, "added %s %s\n", k.Pretty(), path) } return nil } From 25b36d10008df4d0a0f6dfde4e1740cfd3ccdf53 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 4 Oct 2014 19:13:56 +0000 Subject: [PATCH 065/105] moved mock core node to core package --- util/testutil/node.go => core/mock.go | 9 +++------ fuse/ipns/ipns_test.go | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) rename util/testutil/node.go => core/mock.go (87%) diff --git a/util/testutil/node.go b/core/mock.go similarity index 87% rename from util/testutil/node.go rename to core/mock.go index 8df1bb2cd22..caaa0c07547 100644 --- a/util/testutil/node.go +++ b/core/mock.go @@ -1,10 +1,9 @@ -package testutil +package core import ( ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" syncds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync" bs "github.com/jbenet/go-ipfs/blockservice" - core "github.com/jbenet/go-ipfs/core" ci "github.com/jbenet/go-ipfs/crypto" mdag "github.com/jbenet/go-ipfs/merkledag" nsys "github.com/jbenet/go-ipfs/namesys" @@ -12,10 +11,8 @@ import ( mdht "github.com/jbenet/go-ipfs/routing/mock" ) -var _ = core.IpfsNode{} - -func NewMockNode() (*core.IpfsNode, error) { - nd := new(core.IpfsNode) +func NewMockNode() (*IpfsNode, error) { + nd := new(IpfsNode) //Generate Identity nd.Identity = &peer.Peer{ID: []byte("TESTING")} diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index 330cde546c0..e452753e8cb 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -8,7 +8,7 @@ import ( "testing" fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" - testutil "github.com/jbenet/go-ipfs/util/testutil" + "github.com/jbenet/go-ipfs/core" ) func randBytes(size int) []byte { @@ -18,7 +18,7 @@ func randBytes(size int) []byte { } func TestIpnsBasicIO(t *testing.T) { - localnode, err := testutil.NewMockNode() + localnode, err := core.NewMockNode() if err != nil { t.Fatal(err) } From a0d77dbf999cb67b3f995ace1666c9c9c08be0ab Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 4 Oct 2014 19:29:08 +0000 Subject: [PATCH 066/105] fixed keyspace tests on 32 bit systems --- routing/keyspace/xor_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routing/keyspace/xor_test.go b/routing/keyspace/xor_test.go index d7d83afa206..7963ea014a8 100644 --- a/routing/keyspace/xor_test.go +++ b/routing/keyspace/xor_test.go @@ -113,8 +113,8 @@ func TestDistancesAndCenterSorting(t *testing.T) { keys[i] = Key{Space: XORKeySpace, Bytes: a} } - cmp := func(a int, b *big.Int) int { - return big.NewInt(int64(a)).Cmp(b) + cmp := func(a int64, b *big.Int) int { + return big.NewInt(a).Cmp(b) } if 0 != cmp(0, keys[2].Distance(keys[3])) { From 16e42f82b06d229b905352e5033b9bf54161455d Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 4 Oct 2014 22:42:03 +0000 Subject: [PATCH 067/105] fixed ipns file persistence bug --- core/mock.go | 13 ++++++-- fuse/ipns/ipns_test.go | 73 +++++++++++++++++++++++++++++++++--------- fuse/ipns/ipns_unix.go | 8 +++-- 3 files changed, 74 insertions(+), 20 deletions(-) diff --git a/core/mock.go b/core/mock.go index caaa0c07547..a0fd475ad3f 100644 --- a/core/mock.go +++ b/core/mock.go @@ -7,6 +7,7 @@ import ( ci "github.com/jbenet/go-ipfs/crypto" mdag "github.com/jbenet/go-ipfs/merkledag" nsys "github.com/jbenet/go-ipfs/namesys" + path "github.com/jbenet/go-ipfs/path" "github.com/jbenet/go-ipfs/peer" mdht "github.com/jbenet/go-ipfs/routing/mock" ) @@ -29,6 +30,7 @@ func NewMockNode() (*IpfsNode, error) { // Routing dht := mdht.NewMockRouter(nd.Identity, nd.Datastore) + nd.Routing = dht // Bitswap //?? @@ -38,9 +40,16 @@ func NewMockNode() (*IpfsNode, error) { return nil, err } - dserv := &mdag.DAGService{bserv} + nd.DAG = &mdag.DAGService{bserv} // Namespace resolver - nd.Namesys = nsys.NewMasterResolver(dht, dserv) + nd.Namesys = nsys.NewMasterResolver(dht, nd.DAG) + + // Publisher + nd.Publisher = nsys.NewPublisher(nd.DAG, dht) + + // Path resolver + nd.Resolver = &path.Resolver{nd.DAG} + return nd, nil } diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index e452753e8cb..940a8ff2abc 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "testing" + "time" fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" "github.com/jbenet/go-ipfs/core" @@ -17,13 +18,40 @@ func randBytes(size int) []byte { return b } -func TestIpnsBasicIO(t *testing.T) { - localnode, err := core.NewMockNode() +func writeFile(t *testing.T, size int, path string) ([]byte, error) { + data := randBytes(size) + fi, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + + n, err := fi.Write(data) + if err != nil { + t.Fatal(err) + } + + if n != len(data) { + t.Fatal("Didnt write proper amount!") + } + + err = fi.Close() if err != nil { t.Fatal(err) } - fs, err := NewIpns(localnode, "") + return data, nil +} + +func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) { + var err error + if node == nil { + node, err = core.NewMockNode() + if err != nil { + t.Fatal(err) + } + } + + fs, err := NewIpns(node, "") if err != nil { t.Fatal(err) } @@ -31,40 +59,53 @@ func TestIpnsBasicIO(t *testing.T) { if err != nil { t.Fatal(err) } + + return node, mnt +} + +func TestIpnsBasicIO(t *testing.T) { + _, mnt := setupIpnsTest(t, nil) defer mnt.Close() - data := randBytes(12345) - fi, err := os.Create(mnt.Dir + "/local/testfile") + fname := mnt.Dir + "/local/testfile" + data, err := writeFile(t, 12345, fname) if err != nil { t.Fatal(err) } - n, err := fi.Write(data) + rbuf, err := ioutil.ReadFile(fname) if err != nil { t.Fatal(err) } - if n != len(data) { - t.Fatal("Didnt write proper amount!") + if !bytes.Equal(rbuf, data) { + t.Fatal("Incorrect Read!") } +} - fi.Close() - - //TODO: maybe wait for the publish to happen? or not, should test both cases +func TestFilePersistence(t *testing.T) { + node, mnt := setupIpnsTest(t, nil) - fi, err = os.Open(mnt.Dir + "/local/testfile") + fname := "/local/atestfile" + data, err := writeFile(t, 127, mnt.Dir+fname) if err != nil { t.Fatal(err) } - rbuf, err := ioutil.ReadAll(fi) + // Wait for publish: TODO: make publish happen faster in tests + time.Sleep(time.Millisecond * 40) + + mnt.Close() + + node, mnt = setupIpnsTest(t, node) + defer mnt.Close() + + rbuf, err := ioutil.ReadFile(mnt.Dir + fname) if err != nil { t.Fatal(err) } - fi.Close() if !bytes.Equal(rbuf, data) { - t.Fatal("Incorrect Read!") + t.Fatalf("File data changed between mounts! sizes differ: %d != %d", len(data), len(rbuf)) } - } diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 61f3fc0791f..d9dee11011f 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -70,7 +70,7 @@ func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, er nd := new(Node) nd.Ipfs = n nd.key = k - nd.repub = NewRepublisher(nd, time.Millisecond*10, time.Second) + nd.repub = NewRepublisher(nd, time.Millisecond*5, time.Millisecond*500) go nd.repub.Run() @@ -249,6 +249,7 @@ func (n *Node) makeChild(name string, node *mdag.Node) *Node { Nd: node, name: name, nsRoot: n.nsRoot, + parent: n, } if n.nsRoot == nil { @@ -322,11 +323,13 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { // folder, bad things would happen. buf := bytes.NewReader(n.writerBuf.Bytes()) newNode, err := imp.NewDagFromReader(buf) + log.Debug("flush: new data = %v", newNode.Data) if err != nil { log.Critical("error creating dag from writerBuf: %s", err) return err } if n.parent != nil { + log.Debug("updating self in parent!") err := n.parent.update(n.name, newNode) if err != nil { log.Critical("error in updating ipns dag tree: %s", err) @@ -529,6 +532,7 @@ func (n *Node) Rename(req *fuse.RenameRequest, newDir fs.Node, intr fs.Intr) fus } func (n *Node) update(name string, newnode *mdag.Node) error { + log.Debug("update '%s' in '%s'", name, n.name) nnode := n.Nd.Copy() err := nnode.RemoveNodeLink(name) if err != nil { @@ -537,7 +541,7 @@ func (n *Node) update(name string, newnode *mdag.Node) error { nnode.AddNodeLink(name, newnode) if n.parent != nil { - err := n.parent.update(n.name, newnode) + err := n.parent.update(n.name, nnode) if err != nil { return err } From a13baff33d6a856fce02f939a6c4ade349884869 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 5 Oct 2014 01:22:01 +0000 Subject: [PATCH 068/105] fixed data size reporting --- fuse/ipns/ipns_test.go | 31 +++++++++++++++++++++---------- fuse/ipns/ipns_unix.go | 11 +++++++---- importer/importer.go | 26 ++++---------------------- merkledag/data.pb.go | 10 +++++++++- merkledag/data.proto | 1 + merkledag/merkledag.go | 23 ++++++++++++++++++++++- util/util.go | 8 +++++--- 7 files changed, 69 insertions(+), 41 deletions(-) diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index 940a8ff2abc..760c530d025 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -18,7 +18,7 @@ func randBytes(size int) []byte { return b } -func writeFile(t *testing.T, size int, path string) ([]byte, error) { +func writeFile(t *testing.T, size int, path string) []byte { data := randBytes(size) fi, err := os.Create(path) if err != nil { @@ -39,7 +39,7 @@ func writeFile(t *testing.T, size int, path string) ([]byte, error) { t.Fatal(err) } - return data, nil + return data } func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) { @@ -68,10 +68,7 @@ func TestIpnsBasicIO(t *testing.T) { defer mnt.Close() fname := mnt.Dir + "/local/testfile" - data, err := writeFile(t, 12345, fname) - if err != nil { - t.Fatal(err) - } + data := writeFile(t, 12345, fname) rbuf, err := ioutil.ReadFile(fname) if err != nil { @@ -87,10 +84,7 @@ func TestFilePersistence(t *testing.T) { node, mnt := setupIpnsTest(t, nil) fname := "/local/atestfile" - data, err := writeFile(t, 127, mnt.Dir+fname) - if err != nil { - t.Fatal(err) - } + data := writeFile(t, 127, mnt.Dir+fname) // Wait for publish: TODO: make publish happen faster in tests time.Sleep(time.Millisecond * 40) @@ -109,3 +103,20 @@ func TestFilePersistence(t *testing.T) { t.Fatalf("File data changed between mounts! sizes differ: %d != %d", len(data), len(rbuf)) } } + +func TestFileSizeReporting(t *testing.T) { + _, mnt := setupIpnsTest(t, nil) + defer mnt.Close() + + fname := mnt.Dir + "/local/sizecheck" + data := writeFile(t, 5555, fname) + + finfo, err := os.Stat(fname) + if err != nil { + t.Fatal(err) + } + + if finfo.Size() != int64(len(data)) { + t.Fatal("Read incorrect size from stat!") + } +} diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index d9dee11011f..f172e4b8477 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -219,10 +219,14 @@ func (s *Node) Attr() fuse.Attr { case mdag.PBData_Directory: return fuse.Attr{Mode: os.ModeDir | 0555} case mdag.PBData_File, mdag.PBData_Raw: - size, _ := s.Nd.Size() + size, err := s.Nd.DataSize() + if err != nil { + log.Error("Error getting size of file: %s", err) + size = 0 + } return fuse.Attr{ Mode: 0666, - Size: uint64(size), + Size: size, Blocks: uint64(len(s.Nd.Links)), } default: @@ -323,7 +327,6 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { // folder, bad things would happen. buf := bytes.NewReader(n.writerBuf.Bytes()) newNode, err := imp.NewDagFromReader(buf) - log.Debug("flush: new data = %v", newNode.Data) if err != nil { log.Critical("error creating dag from writerBuf: %s", err) return err @@ -457,7 +460,7 @@ func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr f log.Debug("Got create request: %s", req.Name) // New 'empty' file - nd := &mdag.Node{Data: mdag.FilePBData(nil)} + nd := &mdag.Node{Data: mdag.FilePBData(nil, 0)} child := n.makeChild(req.Name, nd) nnode := n.Nd.Copy() diff --git a/importer/importer.go b/importer/importer.go index 1834ddf3f11..ed4196e636b 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -28,10 +28,12 @@ func NewDagFromReader(r io.Reader) (*dag.Node, error) { func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, error) { blkChan := spl.Split(r) first := <-blkChan - root := &dag.Node{Data: dag.FilePBData(first)} + root := &dag.Node{} i := 0 + totalsize := uint64(len(first)) for blk := range blkChan { + totalsize += uint64(len(blk)) child := &dag.Node{Data: dag.WrapData(blk)} err := root.AddNodeLink(fmt.Sprintf("%d", i), child) if err != nil { @@ -40,6 +42,7 @@ func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, er i++ } + root.Data = dag.FilePBData(first, totalsize) return root, nil } @@ -62,24 +65,3 @@ func NewDagFromFile(fpath string) (*dag.Node, error) { return NewDagFromReader(f) } - -// TODO: this needs a better name -func NewDagInNode(r io.Reader, n *dag.Node) error { - n.Links = nil - - blkChan := DefaultSplitter.Split(r) - first := <-blkChan - n.Data = dag.FilePBData(first) - - i := 0 - for blk := range blkChan { - child := &dag.Node{Data: dag.WrapData(blk)} - err := n.AddNodeLink(fmt.Sprintf("%d", i), child) - if err != nil { - return err - } - i++ - } - - return nil -} diff --git a/merkledag/data.pb.go b/merkledag/data.pb.go index d2f97d33fd0..3ed88753366 100644 --- a/merkledag/data.pb.go +++ b/merkledag/data.pb.go @@ -13,7 +13,7 @@ It has these top-level messages: */ package merkledag -import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" +import proto "code.google.com/p/goprotobuf/proto" import math "math" // Reference imports to suppress errors if they are not otherwise used. @@ -59,6 +59,7 @@ func (x *PBData_DataType) UnmarshalJSON(data []byte) error { type PBData struct { Type *PBData_DataType `protobuf:"varint,1,req,enum=merkledag.PBData_DataType" json:"Type,omitempty"` Data []byte `protobuf:"bytes,2,opt" json:"Data,omitempty"` + Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -80,6 +81,13 @@ func (m *PBData) GetData() []byte { return nil } +func (m *PBData) GetFilesize() uint64 { + if m != nil && m.Filesize != nil { + return *m.Filesize + } + return 0 +} + func init() { proto.RegisterEnum("merkledag.PBData_DataType", PBData_DataType_name, PBData_DataType_value) } diff --git a/merkledag/data.proto b/merkledag/data.proto index 99c8a224b75..043c9d1f9a3 100644 --- a/merkledag/data.proto +++ b/merkledag/data.proto @@ -9,4 +9,5 @@ message PBData { required DataType Type = 1; optional bytes Data = 2; + optional uint64 filesize = 3; } diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 07612815508..0db22d31cd6 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -1,6 +1,7 @@ package merkledag import ( + "errors" "fmt" proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" @@ -107,6 +108,25 @@ func (n *Node) Size() (uint64, error) { return s, nil } +func (n *Node) DataSize() (uint64, error) { + pbdata := new(PBData) + err := proto.Unmarshal(n.Data, pbdata) + if err != nil { + return 0, err + } + + switch pbdata.GetType() { + case PBData_Directory: + return 0, errors.New("Cant get data size of directory!") + case PBData_File: + return pbdata.GetFilesize(), nil + case PBData_Raw: + return uint64(len(pbdata.GetData())), nil + default: + return 0, errors.New("Unrecognized node data type!") + } +} + // Multihash hashes the encoded data of this node. func (n *Node) Multihash() (mh.Multihash, error) { b, err := n.Encoded(false) @@ -211,11 +231,12 @@ func (n *DAGService) Get(k u.Key) (*Node, error) { return Decoded(b.Data) } -func FilePBData(data []byte) []byte { +func FilePBData(data []byte, totalsize uint64) []byte { pbfile := new(PBData) typ := PBData_File pbfile.Type = &typ pbfile.Data = data + pbfile.Filesize = proto.Uint64(totalsize) data, err := proto.Marshal(pbfile) if err != nil { diff --git a/util/util.go b/util/util.go index 1e51064ed0a..39b635fdbda 100644 --- a/util/util.go +++ b/util/util.go @@ -122,9 +122,11 @@ func SetupLogging() { */ logging.SetFormatter(logging.MustStringFormatter(LogFormat)) - for _, n := range loggers { - logging.SetLevel(logging.ERROR, n) - } + /* + for _, n := range loggers { + logging.SetLevel(logging.ERROR, n) + } + */ } // Logger retrieves a particular logger + initializes it at a particular level From 7e83982247e004fa923e33ee206f6bc43a7ff1ff Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Sun, 5 Oct 2014 14:50:30 -0700 Subject: [PATCH 069/105] print out key if 'malformed' --- crypto/key.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crypto/key.go b/crypto/key.go index 4f702479a46..32c6488ff59 100644 --- a/crypto/key.go +++ b/crypto/key.go @@ -3,6 +3,7 @@ package crypto import ( "bytes" "errors" + "fmt" "crypto/elliptic" "crypto/hmac" @@ -105,7 +106,8 @@ func GenerateEKeyPair(curveName string) ([]byte, GenSharedKey, error) { curveSize := curve.Params().BitSize if len(theirPub) != (curveSize / 4) { - return nil, errors.New("Malformed public key.") + u.PErr("Malformed public key: %v", theirPub) + return nil, fmt.Errorf("Malformed public key: %v != %v", len(theirPub), (curveSize / 4)) } bound := (curveSize / 8) From c0250f3c8e6e3c3eae273d080cf90bd833a1732e Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Sun, 5 Oct 2014 14:50:39 -0700 Subject: [PATCH 070/105] bump logging to ERROR --- util/util.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/util/util.go b/util/util.go index 39b635fdbda..1e51064ed0a 100644 --- a/util/util.go +++ b/util/util.go @@ -122,11 +122,9 @@ func SetupLogging() { */ logging.SetFormatter(logging.MustStringFormatter(LogFormat)) - /* - for _, n := range loggers { - logging.SetLevel(logging.ERROR, n) - } - */ + for _, n := range loggers { + logging.SetLevel(logging.ERROR, n) + } } // Logger retrieves a particular logger + initializes it at a particular level From d7d3af3fafbff0b503e4db9988448ce1283220b6 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Sun, 5 Oct 2014 15:15:49 -0700 Subject: [PATCH 071/105] vendoring protobuf + go-is-domain --- Godeps/Godeps.json | 4 + .../github.com/jbenet/go-is-domain/LICENSE | 21 + .../github.com/jbenet/go-is-domain/README.md | 23 + .../src/github.com/jbenet/go-is-domain/doc.go | 13 + .../jbenet/go-is-domain/domainre.go | 12 + .../jbenet/go-is-domain/is_domain.go | 44 ++ .../jbenet/go-is-domain/is_domain_test.go | 29 + .../go-is-domain/tlds-alpha-by-domain.txt | 727 +++++++++++++++++ .../github.com/jbenet/go-is-domain/tlds.go | 737 ++++++++++++++++++ merkledag/data.pb.go | 2 +- namesys/dns.go | 2 +- 11 files changed, 1612 insertions(+), 2 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-is-domain/LICENSE create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-is-domain/README.md create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-is-domain/doc.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-is-domain/domainre.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-is-domain/is_domain.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-is-domain/is_domain_test.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-is-domain/tlds-alpha-by-domain.txt create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-is-domain/tlds.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 4e5dfdf27e7..1eafde81ebb 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -70,6 +70,10 @@ "ImportPath": "github.com/jbenet/go-base58", "Rev": "568a28d73fd97651d3442392036a658b6976eed5" }, + { + "ImportPath": "github.com/jbenet/go-is-domain", + "Rev": "93b717f2ae17838a265e30277275ee99ee7198d6" + }, { "ImportPath": "github.com/jbenet/go-msgio", "Rev": "c9069ab79c95aa0686347b516972c7329c4391f2" diff --git a/Godeps/_workspace/src/github.com/jbenet/go-is-domain/LICENSE b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/LICENSE new file mode 100644 index 00000000000..c7386b3c940 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Juan Batiz-Benet + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/jbenet/go-is-domain/README.md b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/README.md new file mode 100644 index 00000000000..21af7b39545 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/README.md @@ -0,0 +1,23 @@ +# go-is-domain + +This package is dedicated to [@whyrusleeping](https://github.com/whyrusleeping). + +Docs: https://godoc.org/github.com/jbenet/go-is-domain + + +Check whether something is a domain. + + +```Go + +import ( + isd "github.com/jbenet/go-is-domain" +) + +isd.IsDomain("foo.com") // true +isd.IsDomain("foo.bar.com.") // true +isd.IsDomain("foo.bar.baz") // false + +``` + +MIT Licensed diff --git a/Godeps/_workspace/src/github.com/jbenet/go-is-domain/doc.go b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/doc.go new file mode 100644 index 00000000000..db6855c4ade --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/doc.go @@ -0,0 +1,13 @@ +/* +Package isdomain package allows users to check whether strings represent domain names. + + import ( + isd "github.com/jbenet/go-is-domain" + ) + + isd.IsDomain("foo.com") // true + isd.IsDomain("foo.bar.com.") // true + isd.IsDomain("foo.bar.baz") // false + +*/ +package isdomain diff --git a/Godeps/_workspace/src/github.com/jbenet/go-is-domain/domainre.go b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/domainre.go new file mode 100644 index 00000000000..3b915f8e827 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/domainre.go @@ -0,0 +1,12 @@ +package isdomain + +import "regexp" + +// DomainRegexpStr is a regular expression string to validate domains. +const DomainRegexpStr = "^([a-z0-9]+(-[a-z0-9]+)*\\.)+[a-z]{2,}$" + +var domainRegexp *regexp.Regexp + +func init() { + domainRegexp = regexp.MustCompile(DomainRegexpStr) +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-is-domain/is_domain.go b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/is_domain.go new file mode 100644 index 00000000000..7591c7dc48d --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/is_domain.go @@ -0,0 +1,44 @@ +package isdomain + +import "strings" + +// IsICANNTLD returns whether the given string is a TLD (Top Level Domain), +// according to ICANN. Well, really according to the TLDs listed in this +// package. +func IsICANNTLD(s string) bool { + s = strings.ToUpper(s) + _, found := TLDs[s] + return found +} + +// IsExtendedTLD returns whether the given string is a TLD (Top Level Domain), +// extended with a few other "TLDs": .bit, .onion +func IsExtendedTLD(s string) bool { + s = strings.ToUpper(s) + _, found := ExtendedTLDs[s] + return found +} + +// IsTLD returns whether the given string is a TLD (according to ICANN, or +// in the set of ExtendedTLDs listed in this package. +func IsTLD(s string) bool { + return IsICANNTLD(s) || IsExtendedTLD(s) +} + +// IsDomain returns whether given string is a domain. +// It first checks the TLD, and then uses a regular expression. +func IsDomain(s string) bool { + if strings.HasSuffix(s, ".") { + s = s[:len(s)-1] + } + + split := strings.Split(s, ".") + tld := split[len(split)-1] + + if !IsTLD(tld) { + return false + } + + s = strings.ToLower(s) + return domainRegexp.MatchString(s) +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-is-domain/is_domain_test.go b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/is_domain_test.go new file mode 100644 index 00000000000..6bd224ef424 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/is_domain_test.go @@ -0,0 +1,29 @@ +package isdomain + +import "testing" + +func TestBasic(t *testing.T) { + cases := map[string]bool{ + "foo.bar.baz.com": true, + "foo.bar.baz": false, + "foo.bar.baz.com.": true, + "com": false, // yeah yeah... + ".": false, // yeah yeah... + "..": false, + ".foo.com.": false, + ".foo.com": false, + "fo o.com": false, + "example.com": true, + "fjdoisajfdiosafdsa8fd8saf8dsa8fdsafdsa-fd-sa-fd-saf-dsa.org": true, + "fjdoisajfdiosafdsa8fd8saf8dsa8fdsafdsa-fd-sa-fd-saf-dsa.bit": true, + "fjdoisajfdiosafdsa8fd8saf8dsa8fdsafdsa-fd-sa-fd-saf-dsa.onion": true, + "a.b.c.d.e.f.g.h.i.j.k.l.museum": true, + "a.b.c.d.e.f.g.h.i.j.k.l": false, + } + + for d, ok := range cases { + if IsDomain(d) != ok { + t.Errorf("Misclassification: %v should be %v", d, ok) + } + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-is-domain/tlds-alpha-by-domain.txt b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/tlds-alpha-by-domain.txt new file mode 100644 index 00000000000..521f80e7aba --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/tlds-alpha-by-domain.txt @@ -0,0 +1,727 @@ +# from http://data.iana.org/TLD/tlds-alpha-by-domain.txt +# Version 2014100300, Last Updated Fri Oct 3 07:07:01 2014 UTC +AC +ACADEMY +ACCOUNTANTS +ACTIVE +ACTOR +AD +AE +AERO +AF +AG +AGENCY +AI +AIRFORCE +AL +ALLFINANZ +AM +AN +AO +AQ +AR +ARCHI +ARMY +ARPA +AS +ASIA +ASSOCIATES +AT +ATTORNEY +AU +AUCTION +AUDIO +AUTOS +AW +AX +AXA +AZ +BA +BAR +BARGAINS +BAYERN +BB +BD +BE +BEER +BERLIN +BEST +BF +BG +BH +BI +BID +BIKE +BIO +BIZ +BJ +BLACK +BLACKFRIDAY +BLUE +BM +BMW +BN +BNPPARIBAS +BO +BOO +BOUTIQUE +BR +BRUSSELS +BS +BT +BUDAPEST +BUILD +BUILDERS +BUSINESS +BUZZ +BV +BW +BY +BZ +BZH +CA +CAB +CAL +CAMERA +CAMP +CANCERRESEARCH +CAPETOWN +CAPITAL +CARAVAN +CARDS +CARE +CAREER +CAREERS +CASA +CASH +CAT +CATERING +CC +CD +CENTER +CEO +CERN +CF +CG +CH +CHANNEL +CHEAP +CHRISTMAS +CHROME +CHURCH +CI +CITIC +CITY +CK +CL +CLAIMS +CLEANING +CLICK +CLINIC +CLOTHING +CLUB +CM +CN +CO +CODES +COFFEE +COLLEGE +COLOGNE +COM +COMMUNITY +COMPANY +COMPUTER +CONDOS +CONSTRUCTION +CONSULTING +CONTRACTORS +COOKING +COOL +COOP +COUNTRY +CR +CREDIT +CREDITCARD +CRUISES +CU +CUISINELLA +CV +CW +CX +CY +CYMRU +CZ +DAD +DANCE +DATING +DAY +DE +DEALS +DEGREE +DEMOCRAT +DENTAL +DENTIST +DESI +DIAMONDS +DIET +DIGITAL +DIRECT +DIRECTORY +DISCOUNT +DJ +DK +DM +DNP +DO +DOMAINS +DURBAN +DVAG +DZ +EAT +EC +EDU +EDUCATION +EE +EG +EMAIL +ENGINEER +ENGINEERING +ENTERPRISES +EQUIPMENT +ER +ES +ESQ +ESTATE +ET +EU +EUS +EVENTS +EXCHANGE +EXPERT +EXPOSED +FAIL +FARM +FEEDBACK +FI +FINANCE +FINANCIAL +FISH +FISHING +FITNESS +FJ +FK +FLIGHTS +FLORIST +FLY +FM +FO +FOO +FORSALE +FOUNDATION +FR +FRL +FROGANS +FUND +FURNITURE +FUTBOL +GA +GAL +GALLERY +GB +GBIZ +GD +GE +GENT +GF +GG +GH +GI +GIFT +GIFTS +GIVES +GL +GLASS +GLE +GLOBAL +GLOBO +GM +GMAIL +GMO +GMX +GN +GOOGLE +GOP +GOV +GP +GQ +GR +GRAPHICS +GRATIS +GREEN +GRIPE +GS +GT +GU +GUIDE +GUITARS +GURU +GW +GY +HAMBURG +HAUS +HEALTHCARE +HELP +HERE +HIPHOP +HIV +HK +HM +HN +HOLDINGS +HOLIDAY +HOMES +HORSE +HOST +HOSTING +HOUSE +HOW +HR +HT +HU +IBM +ID +IE +IL +IM +IMMO +IMMOBILIEN +IN +INDUSTRIES +INFO +ING +INK +INSTITUTE +INSURE +INT +INTERNATIONAL +INVESTMENTS +IO +IQ +IR +IS +IT +JE +JETZT +JM +JO +JOBS +JOBURG +JP +JUEGOS +KAUFEN +KE +KG +KH +KI +KIM +KITCHEN +KIWI +KM +KN +KOELN +KP +KR +KRD +KRED +KW +KY +KZ +LA +LACAIXA +LAND +LAWYER +LB +LC +LEASE +LGBT +LI +LIFE +LIGHTING +LIMITED +LIMO +LINK +LK +LOANS +LONDON +LOTTO +LR +LS +LT +LTDA +LU +LUXE +LUXURY +LV +LY +MA +MAISON +MANAGEMENT +MANGO +MARKET +MARKETING +MC +MD +ME +MEDIA +MEET +MELBOURNE +MEME +MENU +MG +MH +MIAMI +MIL +MINI +MK +ML +MM +MN +MO +MOBI +MODA +MOE +MONASH +MORTGAGE +MOSCOW +MOTORCYCLES +MOV +MP +MQ +MR +MS +MT +MU +MUSEUM +MV +MW +MX +MY +MZ +NA +NAGOYA +NAME +NAVY +NC +NE +NET +NETWORK +NEUSTAR +NEW +NEXUS +NF +NG +NGO +NHK +NI +NINJA +NL +NO +NP +NR +NRA +NRW +NU +NYC +NZ +OKINAWA +OM +ONG +ONL +OOO +ORG +ORGANIC +OTSUKA +OVH +PA +PARIS +PARTNERS +PARTS +PE +PF +PG +PH +PHARMACY +PHOTO +PHOTOGRAPHY +PHOTOS +PHYSIO +PICS +PICTURES +PINK +PIZZA +PK +PL +PLACE +PLUMBING +PM +PN +POHL +POST +PR +PRAXI +PRESS +PRO +PROD +PRODUCTIONS +PROF +PROPERTIES +PROPERTY +PS +PT +PUB +PW +PY +QA +QPON +QUEBEC +RE +REALTOR +RECIPES +RED +REHAB +REISE +REISEN +REN +RENTALS +REPAIR +REPORT +REPUBLICAN +REST +RESTAURANT +REVIEWS +RICH +RIO +RO +ROCKS +RODEO +RS +RSVP +RU +RUHR +RW +RYUKYU +SA +SAARLAND +SARL +SB +SC +SCA +SCB +SCHMIDT +SCHULE +SCOT +SD +SE +SERVICES +SEXY +SG +SH +SHIKSHA +SHOES +SI +SINGLES +SJ +SK +SL +SM +SN +SO +SOCIAL +SOFTWARE +SOHU +SOLAR +SOLUTIONS +SOY +SPACE +SPIEGEL +SR +ST +SU +SUPPLIES +SUPPLY +SUPPORT +SURF +SURGERY +SUZUKI +SV +SX +SY +SYSTEMS +SZ +TATAR +TATTOO +TAX +TC +TD +TECHNOLOGY +TEL +TF +TG +TH +TIENDA +TIPS +TIROL +TJ +TK +TL +TM +TN +TO +TODAY +TOKYO +TOOLS +TOP +TOWN +TOYS +TP +TR +TRADE +TRAINING +TRAVEL +TT +TUI +TV +TW +TZ +UA +UG +UK +UNIVERSITY +UNO +UOL +US +UY +UZ +VA +VACATIONS +VC +VE +VEGAS +VENTURES +VERSICHERUNG +VET +VG +VI +VIAJES +VILLAS +VISION +VLAANDEREN +VN +VODKA +VOTE +VOTING +VOTO +VOYAGE +VU +WALES +WANG +WATCH +WEBCAM +WEBSITE +WED +WF +WHOSWHO +WIEN +WIKI +WILLIAMHILL +WME +WORK +WORKS +WORLD +WS +WTC +WTF +XN--1QQW23A +XN--3BST00M +XN--3DS443G +XN--3E0B707E +XN--45BRJ9C +XN--4GBRIM +XN--55QW42G +XN--55QX5D +XN--6FRZ82G +XN--6QQ986B3XL +XN--80ADXHKS +XN--80AO21A +XN--80ASEHDB +XN--80ASWG +XN--90A3AC +XN--C1AVG +XN--CG4BKI +XN--CLCHC0EA0B2G2A9GCD +XN--CZR694B +XN--CZRU2D +XN--D1ACJ3B +XN--FIQ228C5HS +XN--FIQ64B +XN--FIQS8S +XN--FIQZ9S +XN--FPCRJ9C3D +XN--FZC2C9E2C +XN--GECRJ9C +XN--H2BRJ9C +XN--I1B6B1A6A2E +XN--IO0A7I +XN--J1AMH +XN--J6W193G +XN--KPRW13D +XN--KPRY57D +XN--KPUT3I +XN--L1ACC +XN--LGBBAT1AD8J +XN--MGB9AWBF +XN--MGBA3A4F16A +XN--MGBAAM7A8H +XN--MGBAB2BD +XN--MGBAYH7GPA +XN--MGBBH1A71E +XN--MGBC0A9AZCG +XN--MGBERP4A5D4AR +XN--MGBX4CD0AB +XN--NGBC5AZD +XN--NQV7F +XN--NQV7FS00EMA +XN--O3CW4H +XN--OGBPF8FL +XN--P1ACF +XN--P1AI +XN--PGBS0DH +XN--Q9JYB4C +XN--RHQV96G +XN--S9BRJ9C +XN--SES554G +XN--UNUP4Y +XN--VERMGENSBERATER-CTB +XN--VERMGENSBERATUNG-PWB +XN--VHQUV +XN--WGBH1C +XN--WGBL6A +XN--XHQ521B +XN--XKC2AL3HYE2A +XN--XKC2DL3A5EE0H +XN--YFRO4I67O +XN--YGBI2AMMX +XN--ZFR164B +XXX +XYZ +YACHTS +YANDEX +YE +YOKOHAMA +YOUTUBE +YT +ZA +ZIP +ZM +ZONE +ZW diff --git a/Godeps/_workspace/src/github.com/jbenet/go-is-domain/tlds.go b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/tlds.go new file mode 100644 index 00000000000..c4969dbc2b9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-is-domain/tlds.go @@ -0,0 +1,737 @@ +package isdomain + +// TLDs is a set of TLDs, according to ICANN in 2014. +var TLDs = map[string]bool{ + "AC": true, + "ACADEMY": true, + "ACCOUNTANTS": true, + "ACTIVE": true, + "ACTOR": true, + "AD": true, + "AE": true, + "AERO": true, + "AF": true, + "AG": true, + "AGENCY": true, + "AI": true, + "AIRFORCE": true, + "AL": true, + "ALLFINANZ": true, + "AM": true, + "AN": true, + "AO": true, + "AQ": true, + "AR": true, + "ARCHI": true, + "ARMY": true, + "ARPA": true, + "AS": true, + "ASIA": true, + "ASSOCIATES": true, + "AT": true, + "ATTORNEY": true, + "AU": true, + "AUCTION": true, + "AUDIO": true, + "AUTOS": true, + "AW": true, + "AX": true, + "AXA": true, + "AZ": true, + "BA": true, + "BAR": true, + "BARGAINS": true, + "BAYERN": true, + "BB": true, + "BD": true, + "BE": true, + "BEER": true, + "BERLIN": true, + "BEST": true, + "BF": true, + "BG": true, + "BH": true, + "BI": true, + "BID": true, + "BIKE": true, + "BIO": true, + "BIZ": true, + "BJ": true, + "BLACK": true, + "BLACKFRIDAY": true, + "BLUE": true, + "BM": true, + "BMW": true, + "BN": true, + "BNPPARIBAS": true, + "BO": true, + "BOO": true, + "BOUTIQUE": true, + "BR": true, + "BRUSSELS": true, + "BS": true, + "BT": true, + "BUDAPEST": true, + "BUILD": true, + "BUILDERS": true, + "BUSINESS": true, + "BUZZ": true, + "BV": true, + "BW": true, + "BY": true, + "BZ": true, + "BZH": true, + "CA": true, + "CAB": true, + "CAL": true, + "CAMERA": true, + "CAMP": true, + "CANCERRESEARCH": true, + "CAPETOWN": true, + "CAPITAL": true, + "CARAVAN": true, + "CARDS": true, + "CARE": true, + "CAREER": true, + "CAREERS": true, + "CASA": true, + "CASH": true, + "CAT": true, + "CATERING": true, + "CC": true, + "CD": true, + "CENTER": true, + "CEO": true, + "CERN": true, + "CF": true, + "CG": true, + "CH": true, + "CHANNEL": true, + "CHEAP": true, + "CHRISTMAS": true, + "CHROME": true, + "CHURCH": true, + "CI": true, + "CITIC": true, + "CITY": true, + "CK": true, + "CL": true, + "CLAIMS": true, + "CLEANING": true, + "CLICK": true, + "CLINIC": true, + "CLOTHING": true, + "CLUB": true, + "CM": true, + "CN": true, + "CO": true, + "CODES": true, + "COFFEE": true, + "COLLEGE": true, + "COLOGNE": true, + "COM": true, + "COMMUNITY": true, + "COMPANY": true, + "COMPUTER": true, + "CONDOS": true, + "CONSTRUCTION": true, + "CONSULTING": true, + "CONTRACTORS": true, + "COOKING": true, + "COOL": true, + "COOP": true, + "COUNTRY": true, + "CR": true, + "CREDIT": true, + "CREDITCARD": true, + "CRUISES": true, + "CU": true, + "CUISINELLA": true, + "CV": true, + "CW": true, + "CX": true, + "CY": true, + "CYMRU": true, + "CZ": true, + "DAD": true, + "DANCE": true, + "DATING": true, + "DAY": true, + "DE": true, + "DEALS": true, + "DEGREE": true, + "DEMOCRAT": true, + "DENTAL": true, + "DENTIST": true, + "DESI": true, + "DIAMONDS": true, + "DIET": true, + "DIGITAL": true, + "DIRECT": true, + "DIRECTORY": true, + "DISCOUNT": true, + "DJ": true, + "DK": true, + "DM": true, + "DNP": true, + "DO": true, + "DOMAINS": true, + "DURBAN": true, + "DVAG": true, + "DZ": true, + "EAT": true, + "EC": true, + "EDU": true, + "EDUCATION": true, + "EE": true, + "EG": true, + "EMAIL": true, + "ENGINEER": true, + "ENGINEERING": true, + "ENTERPRISES": true, + "EQUIPMENT": true, + "ER": true, + "ES": true, + "ESQ": true, + "ESTATE": true, + "ET": true, + "EU": true, + "EUS": true, + "EVENTS": true, + "EXCHANGE": true, + "EXPERT": true, + "EXPOSED": true, + "FAIL": true, + "FARM": true, + "FEEDBACK": true, + "FI": true, + "FINANCE": true, + "FINANCIAL": true, + "FISH": true, + "FISHING": true, + "FITNESS": true, + "FJ": true, + "FK": true, + "FLIGHTS": true, + "FLORIST": true, + "FLY": true, + "FM": true, + "FO": true, + "FOO": true, + "FORSALE": true, + "FOUNDATION": true, + "FR": true, + "FRL": true, + "FROGANS": true, + "FUND": true, + "FURNITURE": true, + "FUTBOL": true, + "GA": true, + "GAL": true, + "GALLERY": true, + "GB": true, + "GBIZ": true, + "GD": true, + "GE": true, + "GENT": true, + "GF": true, + "GG": true, + "GH": true, + "GI": true, + "GIFT": true, + "GIFTS": true, + "GIVES": true, + "GL": true, + "GLASS": true, + "GLE": true, + "GLOBAL": true, + "GLOBO": true, + "GM": true, + "GMAIL": true, + "GMO": true, + "GMX": true, + "GN": true, + "GOOGLE": true, + "GOP": true, + "GOV": true, + "GP": true, + "GQ": true, + "GR": true, + "GRAPHICS": true, + "GRATIS": true, + "GREEN": true, + "GRIPE": true, + "GS": true, + "GT": true, + "GU": true, + "GUIDE": true, + "GUITARS": true, + "GURU": true, + "GW": true, + "GY": true, + "HAMBURG": true, + "HAUS": true, + "HEALTHCARE": true, + "HELP": true, + "HERE": true, + "HIPHOP": true, + "HIV": true, + "HK": true, + "HM": true, + "HN": true, + "HOLDINGS": true, + "HOLIDAY": true, + "HOMES": true, + "HORSE": true, + "HOST": true, + "HOSTING": true, + "HOUSE": true, + "HOW": true, + "HR": true, + "HT": true, + "HU": true, + "IBM": true, + "ID": true, + "IE": true, + "IL": true, + "IM": true, + "IMMO": true, + "IMMOBILIEN": true, + "IN": true, + "INDUSTRIES": true, + "INFO": true, + "ING": true, + "INK": true, + "INSTITUTE": true, + "INSURE": true, + "INT": true, + "INTERNATIONAL": true, + "INVESTMENTS": true, + "IO": true, + "IQ": true, + "IR": true, + "IS": true, + "IT": true, + "JE": true, + "JETZT": true, + "JM": true, + "JO": true, + "JOBS": true, + "JOBURG": true, + "JP": true, + "JUEGOS": true, + "KAUFEN": true, + "KE": true, + "KG": true, + "KH": true, + "KI": true, + "KIM": true, + "KITCHEN": true, + "KIWI": true, + "KM": true, + "KN": true, + "KOELN": true, + "KP": true, + "KR": true, + "KRD": true, + "KRED": true, + "KW": true, + "KY": true, + "KZ": true, + "LA": true, + "LACAIXA": true, + "LAND": true, + "LAWYER": true, + "LB": true, + "LC": true, + "LEASE": true, + "LGBT": true, + "LI": true, + "LIFE": true, + "LIGHTING": true, + "LIMITED": true, + "LIMO": true, + "LINK": true, + "LK": true, + "LOANS": true, + "LONDON": true, + "LOTTO": true, + "LR": true, + "LS": true, + "LT": true, + "LTDA": true, + "LU": true, + "LUXE": true, + "LUXURY": true, + "LV": true, + "LY": true, + "MA": true, + "MAISON": true, + "MANAGEMENT": true, + "MANGO": true, + "MARKET": true, + "MARKETING": true, + "MC": true, + "MD": true, + "ME": true, + "MEDIA": true, + "MEET": true, + "MELBOURNE": true, + "MEME": true, + "MENU": true, + "MG": true, + "MH": true, + "MIAMI": true, + "MIL": true, + "MINI": true, + "MK": true, + "ML": true, + "MM": true, + "MN": true, + "MO": true, + "MOBI": true, + "MODA": true, + "MOE": true, + "MONASH": true, + "MORTGAGE": true, + "MOSCOW": true, + "MOTORCYCLES": true, + "MOV": true, + "MP": true, + "MQ": true, + "MR": true, + "MS": true, + "MT": true, + "MU": true, + "MUSEUM": true, + "MV": true, + "MW": true, + "MX": true, + "MY": true, + "MZ": true, + "NA": true, + "NAGOYA": true, + "NAME": true, + "NAVY": true, + "NC": true, + "NE": true, + "NET": true, + "NETWORK": true, + "NEUSTAR": true, + "NEW": true, + "NEXUS": true, + "NF": true, + "NG": true, + "NGO": true, + "NHK": true, + "NI": true, + "NINJA": true, + "NL": true, + "NO": true, + "NP": true, + "NR": true, + "NRA": true, + "NRW": true, + "NU": true, + "NYC": true, + "NZ": true, + "OKINAWA": true, + "OM": true, + "ONG": true, + "ONL": true, + "OOO": true, + "ORG": true, + "ORGANIC": true, + "OTSUKA": true, + "OVH": true, + "PA": true, + "PARIS": true, + "PARTNERS": true, + "PARTS": true, + "PE": true, + "PF": true, + "PG": true, + "PH": true, + "PHARMACY": true, + "PHOTO": true, + "PHOTOGRAPHY": true, + "PHOTOS": true, + "PHYSIO": true, + "PICS": true, + "PICTURES": true, + "PINK": true, + "PIZZA": true, + "PK": true, + "PL": true, + "PLACE": true, + "PLUMBING": true, + "PM": true, + "PN": true, + "POHL": true, + "POST": true, + "PR": true, + "PRAXI": true, + "PRESS": true, + "PRO": true, + "PROD": true, + "PRODUCTIONS": true, + "PROF": true, + "PROPERTIES": true, + "PROPERTY": true, + "PS": true, + "PT": true, + "PUB": true, + "PW": true, + "PY": true, + "QA": true, + "QPON": true, + "QUEBEC": true, + "RE": true, + "REALTOR": true, + "RECIPES": true, + "RED": true, + "REHAB": true, + "REISE": true, + "REISEN": true, + "REN": true, + "RENTALS": true, + "REPAIR": true, + "REPORT": true, + "REPUBLICAN": true, + "REST": true, + "RESTAURANT": true, + "REVIEWS": true, + "RICH": true, + "RIO": true, + "RO": true, + "ROCKS": true, + "RODEO": true, + "RS": true, + "RSVP": true, + "RU": true, + "RUHR": true, + "RW": true, + "RYUKYU": true, + "SA": true, + "SAARLAND": true, + "SARL": true, + "SB": true, + "SC": true, + "SCA": true, + "SCB": true, + "SCHMIDT": true, + "SCHULE": true, + "SCOT": true, + "SD": true, + "SE": true, + "SERVICES": true, + "SEXY": true, + "SG": true, + "SH": true, + "SHIKSHA": true, + "SHOES": true, + "SI": true, + "SINGLES": true, + "SJ": true, + "SK": true, + "SL": true, + "SM": true, + "SN": true, + "SO": true, + "SOCIAL": true, + "SOFTWARE": true, + "SOHU": true, + "SOLAR": true, + "SOLUTIONS": true, + "SOY": true, + "SPACE": true, + "SPIEGEL": true, + "SR": true, + "ST": true, + "SU": true, + "SUPPLIES": true, + "SUPPLY": true, + "SUPPORT": true, + "SURF": true, + "SURGERY": true, + "SUZUKI": true, + "SV": true, + "SX": true, + "SY": true, + "SYSTEMS": true, + "SZ": true, + "TATAR": true, + "TATTOO": true, + "TAX": true, + "TC": true, + "TD": true, + "TECHNOLOGY": true, + "TEL": true, + "TF": true, + "TG": true, + "TH": true, + "TIENDA": true, + "TIPS": true, + "TIROL": true, + "TJ": true, + "TK": true, + "TL": true, + "TM": true, + "TN": true, + "TO": true, + "TODAY": true, + "TOKYO": true, + "TOOLS": true, + "TOP": true, + "TOWN": true, + "TOYS": true, + "TP": true, + "TR": true, + "TRADE": true, + "TRAINING": true, + "TRAVEL": true, + "TT": true, + "TUI": true, + "TV": true, + "TW": true, + "TZ": true, + "UA": true, + "UG": true, + "UK": true, + "UNIVERSITY": true, + "UNO": true, + "UOL": true, + "US": true, + "UY": true, + "UZ": true, + "VA": true, + "VACATIONS": true, + "VC": true, + "VE": true, + "VEGAS": true, + "VENTURES": true, + "VERSICHERUNG": true, + "VET": true, + "VG": true, + "VI": true, + "VIAJES": true, + "VILLAS": true, + "VISION": true, + "VLAANDEREN": true, + "VN": true, + "VODKA": true, + "VOTE": true, + "VOTING": true, + "VOTO": true, + "VOYAGE": true, + "VU": true, + "WALES": true, + "WANG": true, + "WATCH": true, + "WEBCAM": true, + "WEBSITE": true, + "WED": true, + "WF": true, + "WHOSWHO": true, + "WIEN": true, + "WIKI": true, + "WILLIAMHILL": true, + "WME": true, + "WORK": true, + "WORKS": true, + "WORLD": true, + "WS": true, + "WTC": true, + "WTF": true, + "XN--1QQW23A": true, + "XN--3BST00M": true, + "XN--3DS443G": true, + "XN--3E0B707E": true, + "XN--45BRJ9C": true, + "XN--4GBRIM": true, + "XN--55QW42G": true, + "XN--55QX5D": true, + "XN--6FRZ82G": true, + "XN--6QQ986B3XL": true, + "XN--80ADXHKS": true, + "XN--80AO21A": true, + "XN--80ASEHDB": true, + "XN--80ASWG": true, + "XN--90A3AC": true, + "XN--C1AVG": true, + "XN--CG4BKI": true, + "XN--CLCHC0EA0B2G2A9GCD": true, + "XN--CZR694B": true, + "XN--CZRU2D": true, + "XN--D1ACJ3B": true, + "XN--FIQ228C5HS": true, + "XN--FIQ64B": true, + "XN--FIQS8S": true, + "XN--FIQZ9S": true, + "XN--FPCRJ9C3D": true, + "XN--FZC2C9E2C": true, + "XN--GECRJ9C": true, + "XN--H2BRJ9C": true, + "XN--I1B6B1A6A2E": true, + "XN--IO0A7I": true, + "XN--J1AMH": true, + "XN--J6W193G": true, + "XN--KPRW13D": true, + "XN--KPRY57D": true, + "XN--KPUT3I": true, + "XN--L1ACC": true, + "XN--LGBBAT1AD8J": true, + "XN--MGB9AWBF": true, + "XN--MGBA3A4F16A": true, + "XN--MGBAAM7A8H": true, + "XN--MGBAB2BD": true, + "XN--MGBAYH7GPA": true, + "XN--MGBBH1A71E": true, + "XN--MGBC0A9AZCG": true, + "XN--MGBERP4A5D4AR": true, + "XN--MGBX4CD0AB": true, + "XN--NGBC5AZD": true, + "XN--NQV7F": true, + "XN--NQV7FS00EMA": true, + "XN--O3CW4H": true, + "XN--OGBPF8FL": true, + "XN--P1ACF": true, + "XN--P1AI": true, + "XN--PGBS0DH": true, + "XN--Q9JYB4C": true, + "XN--RHQV96G": true, + "XN--S9BRJ9C": true, + "XN--SES554G": true, + "XN--UNUP4Y": true, + "XN--VERMGENSBERATER-CTB": true, + "XN--VERMGENSBERATUNG-PWB": true, + "XN--VHQUV": true, + "XN--WGBH1C": true, + "XN--WGBL6A": true, + "XN--XHQ521B": true, + "XN--XKC2AL3HYE2A": true, + "XN--XKC2DL3A5EE0H": true, + "XN--YFRO4I67O": true, + "XN--YGBI2AMMX": true, + "XN--ZFR164B": true, + "XXX": true, + "XYZ": true, + "YACHTS": true, + "YANDEX": true, + "YE": true, + "YOKOHAMA": true, + "YOUTUBE": true, + "YT": true, + "ZA": true, + "ZIP": true, + "ZM": true, + "ZONE": true, + "ZW": true, +} + +// ExtendedTLDs is a set of additional "TLDs", allowing decentralized name +// systems, like TOR and Namecoin. +var ExtendedTLDs = map[string]bool{ + "BIT": true, + "ONION": true, +} diff --git a/merkledag/data.pb.go b/merkledag/data.pb.go index 3ed88753366..bd098edf269 100644 --- a/merkledag/data.pb.go +++ b/merkledag/data.pb.go @@ -13,7 +13,7 @@ It has these top-level messages: */ package merkledag -import proto "code.google.com/p/goprotobuf/proto" +import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" import math "math" // Reference imports to suppress errors if they are not otherwise used. diff --git a/namesys/dns.go b/namesys/dns.go index e9c4097d837..8dda6cb51ed 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -6,7 +6,7 @@ import ( b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" u "github.com/jbenet/go-ipfs/util" - isd "github.com/jbenet/go-is-domain" + isd "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-is-domain" ) // DNSResolver implements a Resolver on DNS domains From 5136c786e51c7e95a6db6d55e5527dca53e2e945 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Sun, 5 Oct 2014 15:56:52 -0700 Subject: [PATCH 072/105] Bugfix: "Malformed Public Key" Error This commit fixed the notoriously annoying "Malformed Public Key" problem. The issue was that sometimes the byte representation of the points (x,y in big.Int) generated would be one less byte than expected. This is simply because (* big.Int) Write uses the least amount of bytes needed for the int. I instead changed the marshalling/unmarshalling to do exactly what stdlib crypto/tls does: use `ellipctic.Marshal` which marshals according to the ANSI X9.62 standard. http://golang.org/pkg/crypto/elliptic/#Marshal http://golang.org/src/pkg/crypto/tls/key_agreement.go#L214 ```Go // crypto/tls ka.privateKey, x, y, err = elliptic.GenerateKey(ka.curve, config.rand()) ecdhePublic := elliptic.Marshal(ka.curve, x, y) // ipfs/crypto priv, x, y, err := elliptic.GenerateKey(curve, rand.Reader) pubKey := elliptic.Marshal(curve, x, y) ``` ((Warning: we're using `rand.Reader` directly, which we shouldn't do, as it can be seeded. We should use a configured source, as crypto/tls. Flagged in #143)) This makes me think we should re-use a lot of their datastructures and functions directly (e.g. ecdheKeyAgreement) Fixed: #135 cc @bren2010 @whyrusleeping --- crypto/key.go | 23 ++++++----------------- crypto/spipe/handshake.go | 4 ++-- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/crypto/key.go b/crypto/key.go index 32c6488ff59..8fcc13e7de2 100644 --- a/crypto/key.go +++ b/crypto/key.go @@ -13,7 +13,6 @@ import ( "crypto/sha256" "crypto/sha512" "hash" - "math/big" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" @@ -97,26 +96,16 @@ func GenerateEKeyPair(curveName string) ([]byte, GenSharedKey, error) { return nil, nil, err } - var pubKey bytes.Buffer - pubKey.Write(x.Bytes()) - pubKey.Write(y.Bytes()) + pubKey := elliptic.Marshal(curve, x, y) + u.PErr("GenerateEKeyPair %d\n", len(pubKey)) done := func(theirPub []byte) ([]byte, error) { // Verify and unpack node's public key. - curveSize := curve.Params().BitSize - - if len(theirPub) != (curveSize / 4) { - u.PErr("Malformed public key: %v", theirPub) - return nil, fmt.Errorf("Malformed public key: %v != %v", len(theirPub), (curveSize / 4)) + x, y := elliptic.Unmarshal(curve, theirPub) + if x == nil { + return nil, fmt.Errorf("Malformed public key: %d %v", len(theirPub), theirPub) } - bound := (curveSize / 8) - x := big.NewInt(0) - y := big.NewInt(0) - - x.SetBytes(theirPub[0:bound]) - y.SetBytes(theirPub[bound : bound*2]) - if !curve.IsOnCurve(x, y) { return nil, errors.New("Invalid public key.") } @@ -127,7 +116,7 @@ func GenerateEKeyPair(curveName string) ([]byte, GenSharedKey, error) { return secret.Bytes(), nil } - return pubKey.Bytes(), done, nil + return pubKey, done, nil } // Generates a set of keys for each party by stretching the shared key. diff --git a/crypto/spipe/handshake.go b/crypto/spipe/handshake.go index f42f13aaa63..a29d3c1d2b8 100644 --- a/crypto/spipe/handshake.go +++ b/crypto/spipe/handshake.go @@ -119,7 +119,7 @@ func (s *SecurePipe) handshake() error { } // u.POut("Selected %s %s %s\n", exchange, cipherType, hashType) - epubkey, done, err := ci.GenerateEKeyPair(exchange) // Generate EphemeralPubKey + epubkey, genSharedKey, err := ci.GenerateEKeyPair(exchange) // Generate EphemeralPubKey var handshake bytes.Buffer // Gather corpus to sign. handshake.Write(encoded) @@ -173,7 +173,7 @@ func (s *SecurePipe) handshake() error { return errors.New("Bad signature!") } - secret, err := done(exchangeResp.GetEpubkey()) + secret, err := genSharedKey(exchangeResp.GetEpubkey()) if err != nil { return err } From 5592030ed39dbeaefddbaeddb90abbce104891e7 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 6 Oct 2014 03:42:59 +0000 Subject: [PATCH 073/105] working on dag modification structures, factored out the data format into an importer subpackage and added more ipns tests --- core/commands/add.go | 3 +- fuse/ipns/ipns_test.go | 91 +++++++++++++++++++ fuse/ipns/ipns_unix.go | 23 +++-- fuse/readonly/readonly_unix.go | 9 +- importer/dagwriter/dagwriter.go | 80 +++++++++++++++++ importer/dagwriter/dagwriter_test.go | 102 ++++++++++++++++++++++ importer/format/Makefile | 5 ++ {merkledag => importer/format}/data.pb.go | 16 +++- {merkledag => importer/format}/data.proto | 3 +- importer/format/format.go | 71 +++++++++++++++ importer/importer.go | 5 +- importer/splitting.go | 29 ++++++ merkledag/Makefile | 5 +- merkledag/dagreader.go | 17 ++-- merkledag/merkledag.go | 97 ++++++-------------- merkledag/merkledag_test.go | 3 +- 16 files changed, 454 insertions(+), 105 deletions(-) create mode 100644 importer/dagwriter/dagwriter.go create mode 100644 importer/dagwriter/dagwriter_test.go create mode 100644 importer/format/Makefile rename {merkledag => importer/format}/data.pb.go (82%) rename {merkledag => importer/format}/data.proto (77%) create mode 100644 importer/format/format.go diff --git a/core/commands/add.go b/core/commands/add.go index a85ed4831ef..006c8c956b0 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -10,6 +10,7 @@ import ( "github.com/jbenet/go-ipfs/core" "github.com/jbenet/go-ipfs/importer" + ft "github.com/jbenet/go-ipfs/importer/format" dag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" ) @@ -75,7 +76,7 @@ func AddPath(n *core.IpfsNode, fpath string, depth int) (*dag.Node, error) { } func addDir(n *core.IpfsNode, fpath string, depth int) (*dag.Node, error) { - tree := &dag.Node{Data: dag.FolderPBData()} + tree := &dag.Node{Data: ft.FolderPBData()} files, err := ioutil.ReadDir(fpath) if err != nil { diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index 760c530d025..c311c522526 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -63,6 +63,7 @@ func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.M return node, mnt } +// Test writing a file and reading it back func TestIpnsBasicIO(t *testing.T) { _, mnt := setupIpnsTest(t, nil) defer mnt.Close() @@ -80,6 +81,7 @@ func TestIpnsBasicIO(t *testing.T) { } } +// Test to make sure file changes persist over mounts of ipns func TestFilePersistence(t *testing.T) { node, mnt := setupIpnsTest(t, nil) @@ -104,6 +106,7 @@ func TestFilePersistence(t *testing.T) { } } +// Test to make sure the filesystem reports file sizes correctly func TestFileSizeReporting(t *testing.T) { _, mnt := setupIpnsTest(t, nil) defer mnt.Close() @@ -120,3 +123,91 @@ func TestFileSizeReporting(t *testing.T) { t.Fatal("Read incorrect size from stat!") } } + +// Test to make sure you cant create multiple entries with the same name +func TestDoubleEntryFailure(t *testing.T) { + _, mnt := setupIpnsTest(t, nil) + defer mnt.Close() + + dname := mnt.Dir + "/local/thisisadir" + err := os.Mkdir(dname, 0777) + if err != nil { + t.Fatal(err) + } + + err = os.Mkdir(dname, 0777) + if err == nil { + t.Fatal("Should have gotten error one creating new directory.") + } +} + +func TestAppendFile(t *testing.T) { + _, mnt := setupIpnsTest(t, nil) + defer mnt.Close() + + fname := mnt.Dir + "/local/file" + data := writeFile(t, 1300, fname) + + fi, err := os.OpenFile(fname, os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + t.Fatal(err) + } + + nudata := randBytes(500) + + n, err := fi.Write(nudata) + if err != nil { + t.Fatal(err) + } + err = fi.Close() + if err != nil { + t.Fatal(err) + } + + if n != len(nudata) { + t.Fatal("Failed to write enough bytes.") + } + + data = append(data, nudata...) + + rbuf, err := ioutil.ReadFile(fname) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(rbuf, data) { + t.Fatal("Data inconsistent!") + } +} + +// Test writing a medium sized file one byte at a time +func TestMultiWrite(t *testing.T) { + _, mnt := setupIpnsTest(t, nil) + defer mnt.Close() + + fpath := mnt.Dir + "/local/file" + fi, err := os.Create(fpath) + if err != nil { + t.Fatal(err) + } + + data := randBytes(1001) + for i := 0; i < len(data); i++ { + n, err := fi.Write(data[i : i+1]) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") + } + } + fi.Close() + + rbuf, err := ioutil.ReadFile(fpath) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(rbuf, data) { + t.Fatal("File on disk did not match bytes written") + } +} diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index f172e4b8477..7e6b1b33e35 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -14,6 +14,7 @@ import ( "github.com/jbenet/go-ipfs/core" ci "github.com/jbenet/go-ipfs/crypto" imp "github.com/jbenet/go-ipfs/importer" + ft "github.com/jbenet/go-ipfs/importer/format" mdag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" ) @@ -77,7 +78,7 @@ func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, er pointsTo, err := n.Namesys.Resolve(name) if err != nil { log.Warning("Could not resolve value for local ipns entry, providing empty dir") - nd.Nd = &mdag.Node{Data: mdag.FolderPBData()} + nd.Nd = &mdag.Node{Data: ft.FolderPBData()} root.LocalDirs[name] = nd continue } @@ -199,14 +200,14 @@ type Node struct { Ipfs *core.IpfsNode Nd *mdag.Node fd *mdag.DagReader - cached *mdag.PBData + cached *ft.PBData // For writing writerBuf WriteAtBuf } func (s *Node) loadData() error { - s.cached = new(mdag.PBData) + s.cached = new(ft.PBData) return proto.Unmarshal(s.Nd.Data, s.cached) } @@ -216,10 +217,10 @@ func (s *Node) Attr() fuse.Attr { s.loadData() } switch s.cached.GetType() { - case mdag.PBData_Directory: + case ft.PBData_Directory: return fuse.Attr{Mode: os.ModeDir | 0555} - case mdag.PBData_File, mdag.PBData_Raw: - size, err := s.Nd.DataSize() + case ft.PBData_File, ft.PBData_Raw: + size, err := ft.DataSize(s.Nd.Data) if err != nil { log.Error("Error getting size of file: %s", err) size = 0 @@ -414,7 +415,7 @@ func (n *Node) Fsync(req *fuse.FsyncRequest, intr fs.Intr) fuse.Error { func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) { log.Debug("Got mkdir request!") - dagnd := &mdag.Node{Data: mdag.FolderPBData()} + dagnd := &mdag.Node{Data: ft.FolderPBData()} nnode := n.Nd.Copy() nnode.AddNodeLink(req.Name, dagnd) @@ -448,6 +449,12 @@ func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) func (n *Node) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr) (fs.Handle, fuse.Error) { //log.Debug("[%s] Received open request! flags = %s", n.name, req.Flags.String()) //TODO: check open flags and truncate if necessary + if req.Flags&fuse.OpenTruncate != 0 { + log.Warning("Need to truncate file!") + } + if req.Flags&fuse.OpenAppend != 0 { + log.Warning("Need to append to file!") + } return n, nil } @@ -460,7 +467,7 @@ func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr f log.Debug("Got create request: %s", req.Name) // New 'empty' file - nd := &mdag.Node{Data: mdag.FilePBData(nil, 0)} + nd := &mdag.Node{Data: ft.FilePBData(nil, 0)} child := n.makeChild(req.Name, nd) nnode := n.Nd.Copy() diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 1bd45e8c212..e7550bee794 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -19,6 +19,7 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" core "github.com/jbenet/go-ipfs/core" + ft "github.com/jbenet/go-ipfs/importer/format" mdag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" ) @@ -79,11 +80,11 @@ type Node struct { Ipfs *core.IpfsNode Nd *mdag.Node fd *mdag.DagReader - cached *mdag.PBData + cached *ft.PBData } func (s *Node) loadData() error { - s.cached = new(mdag.PBData) + s.cached = new(ft.PBData) return proto.Unmarshal(s.Nd.Data, s.cached) } @@ -94,9 +95,9 @@ func (s *Node) Attr() fuse.Attr { s.loadData() } switch s.cached.GetType() { - case mdag.PBData_Directory: + case ft.PBData_Directory: return fuse.Attr{Mode: os.ModeDir | 0555} - case mdag.PBData_File, mdag.PBData_Raw: + case ft.PBData_File, ft.PBData_Raw: size, _ := s.Nd.Size() return fuse.Attr{ Mode: 0444, diff --git a/importer/dagwriter/dagwriter.go b/importer/dagwriter/dagwriter.go new file mode 100644 index 00000000000..2c2bbafb711 --- /dev/null +++ b/importer/dagwriter/dagwriter.go @@ -0,0 +1,80 @@ +package dagwriter + +import ( + imp "github.com/jbenet/go-ipfs/importer" + ft "github.com/jbenet/go-ipfs/importer/format" + dag "github.com/jbenet/go-ipfs/merkledag" + "github.com/jbenet/go-ipfs/util" +) + +var log = util.Logger("dagwriter") + +type DagWriter struct { + dagserv *dag.DAGService + node *dag.Node + totalSize int64 + splChan chan []byte + done chan struct{} + splitter imp.StreamSplitter + seterr error +} + +func NewDagWriter(ds *dag.DAGService, splitter imp.StreamSplitter) *DagWriter { + dw := new(DagWriter) + dw.dagserv = ds + dw.splChan = make(chan []byte, 8) + dw.splitter = splitter + dw.done = make(chan struct{}) + go dw.startSplitter() + return dw +} + +func (dw *DagWriter) startSplitter() { + blkchan := dw.splitter.Split(dw.splChan) + first := <-blkchan + root := new(dag.Node) + fileSize := uint64(0) + for blkData := range blkchan { + fileSize += uint64(len(blkData)) + node := &dag.Node{Data: ft.WrapData(blkData)} + _, err := dw.dagserv.Add(node) + if err != nil { + dw.seterr = err + log.Critical("Got error adding created node to dagservice: %s", err) + return + } + err = root.AddNodeLinkClean("", node) + if err != nil { + dw.seterr = err + log.Critical("Got error adding created node to root node: %s", err) + return + } + } + root.Data = ft.FilePBData(first, fileSize) + _, err := dw.dagserv.Add(root) + if err != nil { + dw.seterr = err + log.Critical("Got error adding created node to dagservice: %s", err) + return + } + dw.node = root + dw.done <- struct{}{} +} + +func (dw *DagWriter) Write(b []byte) (int, error) { + if dw.seterr != nil { + return 0, dw.seterr + } + dw.splChan <- b + return len(b), nil +} + +func (dw *DagWriter) Close() error { + close(dw.splChan) + <-dw.done + return nil +} + +func (dw *DagWriter) GetNode() *dag.Node { + return dw.node +} diff --git a/importer/dagwriter/dagwriter_test.go b/importer/dagwriter/dagwriter_test.go new file mode 100644 index 00000000000..114e363bf02 --- /dev/null +++ b/importer/dagwriter/dagwriter_test.go @@ -0,0 +1,102 @@ +package dagwriter + +import ( + "testing" + + "io" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + bs "github.com/jbenet/go-ipfs/blockservice" + imp "github.com/jbenet/go-ipfs/importer" + mdag "github.com/jbenet/go-ipfs/merkledag" +) + +type datasource struct { + i int +} + +func (d *datasource) Read(b []byte) (int, error) { + for i, _ := range b { + b[i] = byte(d.i % 256) + d.i++ + } + return len(b), nil +} + +func (d *datasource) Matches(t *testing.T, r io.Reader, length int) bool { + b := make([]byte, 100) + i := 0 + for { + n, err := r.Read(b) + if err != nil && err != io.EOF { + t.Fatal(err) + } + for _, v := range b[:n] { + if v != byte(i%256) { + t.Fatalf("Buffers differed at byte: %d (%d != %d)", i, v, (i % 256)) + } + i++ + } + if err == io.EOF { + break + } + } + if i != length { + t.Fatalf("Incorrect length. (%d != %d)", i, length) + } + return true +} + +func TestDagWriter(t *testing.T) { + dstore := ds.NewMapDatastore() + bserv, err := bs.NewBlockService(dstore, nil) + if err != nil { + t.Fatal(err) + } + dag := &mdag.DAGService{bserv} + dw := NewDagWriter(dag, &imp.SizeSplitter2{4096}) + + nbytes := int64(1024 * 1024 * 2) + n, err := io.CopyN(dw, &datasource{}, nbytes) + if err != nil { + t.Fatal(err) + } + + if n != nbytes { + t.Fatal("Copied incorrect amount of bytes!") + } + + dw.Close() + + node := dw.GetNode() + read, err := mdag.NewDagReader(node, dag) + if err != nil { + t.Fatal(err) + } + + d := &datasource{} + if !d.Matches(t, read, int(nbytes)) { + t.Fatal("Failed to validate!") + } +} + +func TestMassiveWrite(t *testing.T) { + t.SkipNow() + dstore := ds.NewNullDatastore() + bserv, err := bs.NewBlockService(dstore, nil) + if err != nil { + t.Fatal(err) + } + dag := &mdag.DAGService{bserv} + dw := NewDagWriter(dag, &imp.SizeSplitter2{4096}) + + nbytes := int64(1024 * 1024 * 1024 * 16) + n, err := io.CopyN(dw, &datasource{}, nbytes) + if err != nil { + t.Fatal(err) + } + if n != nbytes { + t.Fatal("Incorrect copy size.") + } + dw.Close() +} diff --git a/importer/format/Makefile b/importer/format/Makefile new file mode 100644 index 00000000000..87f182fe5bf --- /dev/null +++ b/importer/format/Makefile @@ -0,0 +1,5 @@ +all: data.pb.go + +data.pb.go: data.proto + protoc --go_out=. data.proto + diff --git a/merkledag/data.pb.go b/importer/format/data.pb.go similarity index 82% rename from merkledag/data.pb.go rename to importer/format/data.pb.go index 3ed88753366..9b8589d336f 100644 --- a/merkledag/data.pb.go +++ b/importer/format/data.pb.go @@ -3,7 +3,7 @@ // DO NOT EDIT! /* -Package merkledag is a generated protocol buffer package. +Package format is a generated protocol buffer package. It is generated from these files: data.proto @@ -11,7 +11,7 @@ It is generated from these files: It has these top-level messages: PBData */ -package merkledag +package format import proto "code.google.com/p/goprotobuf/proto" import math "math" @@ -57,9 +57,10 @@ func (x *PBData_DataType) UnmarshalJSON(data []byte) error { } type PBData struct { - Type *PBData_DataType `protobuf:"varint,1,req,enum=merkledag.PBData_DataType" json:"Type,omitempty"` + Type *PBData_DataType `protobuf:"varint,1,req,enum=format.PBData_DataType" json:"Type,omitempty"` Data []byte `protobuf:"bytes,2,opt" json:"Data,omitempty"` Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"` + Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -88,6 +89,13 @@ func (m *PBData) GetFilesize() uint64 { return 0 } +func (m *PBData) GetBlocksizes() []uint64 { + if m != nil { + return m.Blocksizes + } + return nil +} + func init() { - proto.RegisterEnum("merkledag.PBData_DataType", PBData_DataType_name, PBData_DataType_value) + proto.RegisterEnum("format.PBData_DataType", PBData_DataType_name, PBData_DataType_value) } diff --git a/merkledag/data.proto b/importer/format/data.proto similarity index 77% rename from merkledag/data.proto rename to importer/format/data.proto index 043c9d1f9a3..9538c7c15d0 100644 --- a/merkledag/data.proto +++ b/importer/format/data.proto @@ -1,4 +1,4 @@ -package merkledag; +package format; message PBData { enum DataType { @@ -10,4 +10,5 @@ message PBData { required DataType Type = 1; optional bytes Data = 2; optional uint64 filesize = 3; + repeated uint64 blocksizes = 4; } diff --git a/importer/format/format.go b/importer/format/format.go new file mode 100644 index 00000000000..f6dc14a394c --- /dev/null +++ b/importer/format/format.go @@ -0,0 +1,71 @@ +// Package format implements a data format for files in the ipfs filesystem +// It is not the only format in ipfs, but it is the one that the filesystem assumes +package format + +import ( + "errors" + + "code.google.com/p/goprotobuf/proto" +) + +func FilePBData(data []byte, totalsize uint64) []byte { + pbfile := new(PBData) + typ := PBData_File + pbfile.Type = &typ + pbfile.Data = data + pbfile.Filesize = proto.Uint64(totalsize) + + data, err := proto.Marshal(pbfile) + if err != nil { + //this really shouldnt happen, i promise + panic(err) + } + return data +} + +func FolderPBData() []byte { + pbfile := new(PBData) + typ := PBData_Directory + pbfile.Type = &typ + + data, err := proto.Marshal(pbfile) + if err != nil { + //this really shouldnt happen, i promise + panic(err) + } + return data +} + +func WrapData(b []byte) []byte { + pbdata := new(PBData) + typ := PBData_Raw + pbdata.Data = b + pbdata.Type = &typ + + out, err := proto.Marshal(pbdata) + if err != nil { + // This shouldnt happen. seriously. + panic(err) + } + + return out +} + +func DataSize(data []byte) (uint64, error) { + pbdata := new(PBData) + err := proto.Unmarshal(data, pbdata) + if err != nil { + return 0, err + } + + switch pbdata.GetType() { + case PBData_Directory: + return 0, errors.New("Cant get data size of directory!") + case PBData_File: + return pbdata.GetFilesize(), nil + case PBData_Raw: + return uint64(len(pbdata.GetData())), nil + default: + return 0, errors.New("Unrecognized node data type!") + } +} diff --git a/importer/importer.go b/importer/importer.go index ed4196e636b..bbf6b51372c 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -5,6 +5,7 @@ import ( "io" "os" + ft "github.com/jbenet/go-ipfs/importer/format" dag "github.com/jbenet/go-ipfs/merkledag" ) @@ -34,7 +35,7 @@ func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, er totalsize := uint64(len(first)) for blk := range blkChan { totalsize += uint64(len(blk)) - child := &dag.Node{Data: dag.WrapData(blk)} + child := &dag.Node{Data: ft.WrapData(blk)} err := root.AddNodeLink(fmt.Sprintf("%d", i), child) if err != nil { return nil, err @@ -42,7 +43,7 @@ func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, er i++ } - root.Data = dag.FilePBData(first, totalsize) + root.Data = ft.FilePBData(first, totalsize) return root, nil } diff --git a/importer/splitting.go b/importer/splitting.go index 05eaec15ae5..65b67248682 100644 --- a/importer/splitting.go +++ b/importer/splitting.go @@ -6,10 +6,16 @@ import ( u "github.com/jbenet/go-ipfs/util" ) +// OLD type BlockSplitter interface { Split(io.Reader) chan []byte } +// NEW +type StreamSplitter interface { + Split(chan []byte) chan []byte +} + type SizeSplitter struct { Size int } @@ -39,3 +45,26 @@ func (ss *SizeSplitter) Split(r io.Reader) chan []byte { }() return out } + +type SizeSplitter2 struct { + Size int +} + +func (ss *SizeSplitter2) Split(in chan []byte) chan []byte { + out := make(chan []byte) + go func() { + defer close(out) + var buf []byte + for b := range in { + buf = append(buf, b...) + for len(buf) > ss.Size { + out <- buf[:ss.Size] + buf = buf[ss.Size:] + } + } + if len(buf) > 0 { + out <- buf + } + }() + return out +} diff --git a/merkledag/Makefile b/merkledag/Makefile index 2524ed3ba48..711f34bdac2 100644 --- a/merkledag/Makefile +++ b/merkledag/Makefile @@ -1,11 +1,8 @@ -all: node.pb.go data.pb.go +all: node.pb.go node.pb.go: node.proto protoc --gogo_out=. --proto_path=../../../../:/usr/local/opt/protobuf/include:. $< -data.pb.go: data.proto - protoc --go_out=. data.proto - clean: rm node.pb.go diff --git a/merkledag/dagreader.go b/merkledag/dagreader.go index 1e8a0c8b9ad..badc661fdf8 100644 --- a/merkledag/dagreader.go +++ b/merkledag/dagreader.go @@ -6,6 +6,7 @@ import ( "io" proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + ft "github.com/jbenet/go-ipfs/importer/format" u "github.com/jbenet/go-ipfs/util" ) @@ -20,21 +21,21 @@ type DagReader struct { } func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) { - pb := new(PBData) + pb := new(ft.PBData) err := proto.Unmarshal(n.Data, pb) if err != nil { return nil, err } switch pb.GetType() { - case PBData_Directory: + case ft.PBData_Directory: return nil, ErrIsDir - case PBData_File: + case ft.PBData_File: return &DagReader{ node: n, serv: serv, buf: bytes.NewBuffer(pb.GetData()), }, nil - case PBData_Raw: + case ft.PBData_Raw: return bytes.NewBuffer(pb.GetData()), nil default: panic("Unrecognized node type!") @@ -54,7 +55,7 @@ func (dr *DagReader) precalcNextBuf() error { } nxt = nxtNode } - pb := new(PBData) + pb := new(ft.PBData) err := proto.Unmarshal(nxt.Data, pb) if err != nil { return err @@ -62,13 +63,13 @@ func (dr *DagReader) precalcNextBuf() error { dr.position++ switch pb.GetType() { - case PBData_Directory: + case ft.PBData_Directory: panic("Why is there a directory under a file?") - case PBData_File: + case ft.PBData_File: //TODO: this *should* work, needs testing first //return NewDagReader(nxt, dr.serv) panic("Not yet handling different layers of indirection!") - case PBData_Raw: + case ft.PBData_Raw: dr.buf = bytes.NewBuffer(pb.GetData()) return nil default: diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 0db22d31cd6..675c8ccdda9 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -1,11 +1,8 @@ package merkledag import ( - "errors" "fmt" - proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" - mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" blocks "github.com/jbenet/go-ipfs/blocks" bserv "github.com/jbenet/go-ipfs/blockservice" @@ -37,6 +34,9 @@ type Link struct { // cumulative size of target object Size uint64 + // cumulative size of data stored in object + DataSize uint64 + // multihash of the target object Hash mh.Multihash @@ -46,14 +46,28 @@ type Link struct { // AddNodeLink adds a link to another node. func (n *Node) AddNodeLink(name string, that *Node) error { - // DEBUG CODE - for _, l := range n.Links { - if l.Name == name { - panic("Trying to add child that already exists!") - } + s, err := that.Size() + if err != nil { + return err } - // + h, err := that.Multihash() + if err != nil { + return err + } + + n.Links = append(n.Links, &Link{ + Name: name, + Size: s, + Hash: h, + Node: that, + }) + return nil +} + +// AddNodeLink adds a link to another node. without keeping a reference to +// the child node +func (n *Node) AddNodeLinkClean(name string, that *Node) error { s, err := that.Size() if err != nil { return err @@ -68,7 +82,6 @@ func (n *Node) AddNodeLink(name string, that *Node) error { Name: name, Size: s, Hash: h, - Node: that, }) return nil } @@ -83,6 +96,8 @@ func (n *Node) RemoveNodeLink(name string) error { return u.ErrNotFound } +// Copy returns a copy of the node. +// NOTE: does not make copies of Node objects in the links. func (n *Node) Copy() *Node { nnode := new(Node) nnode.Data = make([]byte, len(n.Data)) @@ -108,25 +123,6 @@ func (n *Node) Size() (uint64, error) { return s, nil } -func (n *Node) DataSize() (uint64, error) { - pbdata := new(PBData) - err := proto.Unmarshal(n.Data, pbdata) - if err != nil { - return 0, err - } - - switch pbdata.GetType() { - case PBData_Directory: - return 0, errors.New("Cant get data size of directory!") - case PBData_File: - return pbdata.GetFilesize(), nil - case PBData_Raw: - return uint64(len(pbdata.GetData())), nil - default: - return 0, errors.New("Unrecognized node data type!") - } -} - // Multihash hashes the encoded data of this node. func (n *Node) Multihash() (mh.Multihash, error) { b, err := n.Encoded(false) @@ -230,46 +226,3 @@ func (n *DAGService) Get(k u.Key) (*Node, error) { return Decoded(b.Data) } - -func FilePBData(data []byte, totalsize uint64) []byte { - pbfile := new(PBData) - typ := PBData_File - pbfile.Type = &typ - pbfile.Data = data - pbfile.Filesize = proto.Uint64(totalsize) - - data, err := proto.Marshal(pbfile) - if err != nil { - //this really shouldnt happen, i promise - panic(err) - } - return data -} - -func FolderPBData() []byte { - pbfile := new(PBData) - typ := PBData_Directory - pbfile.Type = &typ - - data, err := proto.Marshal(pbfile) - if err != nil { - //this really shouldnt happen, i promise - panic(err) - } - return data -} - -func WrapData(b []byte) []byte { - pbdata := new(PBData) - typ := PBData_Raw - pbdata.Data = b - pbdata.Type = &typ - - out, err := proto.Marshal(pbdata) - if err != nil { - // This shouldnt happen. seriously. - panic(err) - } - - return out -} diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 7cd1649e27a..2db166beb3e 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -2,8 +2,9 @@ package merkledag import ( "fmt" - u "github.com/jbenet/go-ipfs/util" "testing" + + u "github.com/jbenet/go-ipfs/util" ) func TestNode(t *testing.T) { From f3ae0e8e1b0663cec133b00ef7c0b805200c4b65 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Mon, 6 Oct 2014 02:26:50 -0700 Subject: [PATCH 074/105] u.Hash - error the u.Hash error can be safely ignored (panic) because multihash only fails from the selection of hash function. If the fn + length are valid, it won't error. cc @whyrusleeping --- blocks/blocks.go | 6 +----- blockservice/blocks_test.go | 7 +------ crypto/key.go | 2 +- crypto/spipe/handshake.go | 16 +++------------- merkledag/merkledag.go | 2 +- namesys/publisher.go | 11 ++--------- namesys/resolve_test.go | 6 +----- namesys/routing.go | 5 +---- peer/queue/queue_test.go | 2 +- util/util.go | 11 +++++++++-- 10 files changed, 21 insertions(+), 47 deletions(-) diff --git a/blocks/blocks.go b/blocks/blocks.go index c58ab9f209e..b45c1d1fb7f 100644 --- a/blocks/blocks.go +++ b/blocks/blocks.go @@ -13,11 +13,7 @@ type Block struct { // NewBlock creates a Block object from opaque data. It will hash the data. func NewBlock(data []byte) (*Block, error) { - h, err := u.Hash(data) - if err != nil { - return nil, err - } - return &Block{Data: data, Multihash: h}, nil + return &Block{Data: data, Multihash: u.Hash(data)}, nil } // Key returns the block's Multihash as a Key value. diff --git a/blockservice/blocks_test.go b/blockservice/blocks_test.go index c610fbd2a17..bfffc37b8cc 100644 --- a/blockservice/blocks_test.go +++ b/blockservice/blocks_test.go @@ -23,12 +23,7 @@ func TestBlocks(t *testing.T) { return } - h, err := u.Hash([]byte("beep boop")) - if err != nil { - t.Error("failed to hash data", err) - return - } - + h := u.Hash([]byte("beep boop")) if !bytes.Equal(b.Multihash, h) { t.Error("Block Multihash and data multihash not equal") } diff --git a/crypto/key.go b/crypto/key.go index 8fcc13e7de2..0ed39c741a1 100644 --- a/crypto/key.go +++ b/crypto/key.go @@ -249,5 +249,5 @@ func KeyHash(k Key) ([]byte, error) { if err != nil { return nil, err } - return u.Hash(kb) + return u.Hash(kb), nil } diff --git a/crypto/spipe/handshake.go b/crypto/spipe/handshake.go index a29d3c1d2b8..975c3d2e431 100644 --- a/crypto/spipe/handshake.go +++ b/crypto/spipe/handshake.go @@ -292,25 +292,15 @@ func IDFromPubKey(pk ci.PubKey) (peer.ID, error) { if err != nil { return nil, err } - hash, err := u.Hash(b) - if err != nil { - return nil, err - } + hash := u.Hash(b) return peer.ID(hash), nil } // Determines which algorithm to use. Note: f(a, b) = f(b, a) func selectBest(myPrefs, theirPrefs string) (string, error) { // Person with greatest hash gets first choice. - myHash, err := u.Hash([]byte(myPrefs)) - if err != nil { - return "", err - } - - theirHash, err := u.Hash([]byte(theirPrefs)) - if err != nil { - return "", err - } + myHash := u.Hash([]byte(myPrefs)) + theirHash := u.Hash([]byte(theirPrefs)) cmp := bytes.Compare(myHash, theirHash) var firstChoiceArr, secChoiceArr []string diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 675c8ccdda9..e7c13873c41 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -130,7 +130,7 @@ func (n *Node) Multihash() (mh.Multihash, error) { return nil, err } - return u.Hash(b) + return u.Hash(b), nil } // Key returns the Multihash as a key, for maps. diff --git a/namesys/publisher.go b/namesys/publisher.go index 76eb6a55b01..0c605301c98 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -42,16 +42,9 @@ func (p *ipnsPublisher) Publish(k ci.PrivKey, value string) error { return nil } - nameb, err := u.Hash(pkbytes) - if err != nil { - return nil - } + nameb := u.Hash(pkbytes) namekey := u.Key(nameb).Pretty() - - ipnskey, err := u.Hash([]byte("/ipns/" + namekey)) - if err != nil { - return err - } + ipnskey := u.Hash([]byte("/ipns/" + namekey)) // Store associated public key timectx, _ := context.WithDeadline(ctx, time.Now().Add(time.Second*4)) diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index 35898b50f23..ff5292224f3 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -48,11 +48,7 @@ func TestRoutingResolve(t *testing.T) { t.Fatal(err) } - pkhash, err := u.Hash(pubkb) - if err != nil { - t.Fatal(err) - } - + pkhash := u.Hash(pubkb) res, err := resolve.Resolve(u.Key(pkhash).Pretty()) if err != nil { t.Fatal(err) diff --git a/namesys/routing.go b/namesys/routing.go index 942263e8efc..abacb22d4f0 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -45,10 +45,7 @@ func (r *RoutingResolver) Resolve(name string) (string, error) { // use the routing system to get the name. // /ipns/ - h, err := u.Hash([]byte("/ipns/" + name)) - if err != nil { - return "", err - } + h := u.Hash([]byte("/ipns/" + name)) ipnsKey := u.Key(h) val, err := r.routing.GetValue(ctx, ipnsKey) diff --git a/peer/queue/queue_test.go b/peer/queue/queue_test.go index 8a7d22189fc..a4812befab8 100644 --- a/peer/queue/queue_test.go +++ b/peer/queue/queue_test.go @@ -68,7 +68,7 @@ func TestQueue(t *testing.T) { func newPeerTime(t time.Time) *peer.Peer { s := fmt.Sprintf("hmmm time: %v", t) - h, _ := u.Hash([]byte(s)) + h := u.Hash([]byte(s)) return &peer.Peer{ID: peer.ID(h)} } diff --git a/util/util.go b/util/util.go index 1e51064ed0a..3a55188dc41 100644 --- a/util/util.go +++ b/util/util.go @@ -54,8 +54,15 @@ func KeyFromDsKey(dsk ds.Key) Key { } // Hash is the global IPFS hash function. uses multihash SHA2_256, 256 bits -func Hash(data []byte) (mh.Multihash, error) { - return mh.Sum(data, mh.SHA2_256, -1) +func Hash(data []byte) mh.Multihash { + h, err := mh.Sum(data, mh.SHA2_256, -1) + if err != nil { + // this error can be safely ignored (panic) because multihash only fails + // from the selection of hash function. If the fn + length are valid, it + // won't error. + panic("multihash failed to hash using SHA2_256.") + } + return h } // IsValidHash checks whether a given hash is valid (b58 decodable, len > 0) From 20a20c9e1a42cff5a45ef02a3529793998d33320 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Mon, 6 Oct 2014 04:07:22 -0700 Subject: [PATCH 075/105] updated multiaddr --- Godeps/Godeps.json | 4 +- .../github.com/jbenet/go-multiaddr/LICENSE | 21 ++++ .../github.com/jbenet/go-multiaddr/README.md | 38 +++--- .../src/github.com/jbenet/go-multiaddr/doc.go | 36 ++++++ .../github.com/jbenet/go-multiaddr/index.go | 117 ------------------ .../jbenet/go-multiaddr/interface.go | 42 +++++++ .../jbenet/go-multiaddr/multiaddr.go | 110 ++++++++++++++++ .../jbenet/go-multiaddr/multiaddr_test.go | 46 ++----- .../src/github.com/jbenet/go-multiaddr/net.go | 40 +++++- .../jbenet/go-multiaddr/net_test.go | 32 ++++- 10 files changed, 302 insertions(+), 184 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-multiaddr/LICENSE create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-multiaddr/doc.go delete mode 100644 Godeps/_workspace/src/github.com/jbenet/go-multiaddr/index.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-multiaddr/interface.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/go-multiaddr/multiaddr.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1eafde81ebb..1744a65aff1 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -80,8 +80,8 @@ }, { "ImportPath": "github.com/jbenet/go-multiaddr", - "Comment": "0.1.2-3-g74443fc", - "Rev": "74443fca319c4c2f5e9968b8e268c30a4a74dc64" + "Comment": "0.1.2-9-g1ec9436", + "Rev": "1ec9436b1d642f4f04c0d9e21a0719cda3d659ee" }, { "ImportPath": "github.com/jbenet/go-multihash", diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/LICENSE b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/LICENSE new file mode 100644 index 00000000000..c7386b3c940 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Juan Batiz-Benet + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/README.md b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/README.md index b63ffa2dad0..7ff9854f18f 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/README.md +++ b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/README.md @@ -7,18 +7,20 @@ ### Simple ```go -import "github.com/jbenet/go-multiaddr" +import ma "github.com/jbenet/go-multiaddr" -m := multiaddr.NewMultiaddr("/ip4/127.0.0.1/udp/1234") -// -m.buffer -// -m.String() -// /ip4/127.0.0.1/udp/1234 +// construct from a string (err signals parse failure) +m1, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/1234") -// construct with Buffer -m = multiaddr.Multiaddr{ Bytes: m.Bytes } -// +// construct from bytes (err signals parse failure) +m2, err := ma.NewMultiaddrBytes(m1.Bytes()) + +// true +strings.Equal(m1.String(), "/ip4/127.0.0.1/udp/1234") +strings.Equal(m1.String(), m2.String()) +bytes.Equal(m1.Bytes(), m2.Bytes()) +m1.Equal(m2) +m2.Equal(m1) ``` ### Protocols @@ -36,7 +38,7 @@ addr.Protocols() ```go // handles the stupid url version too -m = multiaddr.NewUrl("udp4://127.0.0.1:1234") +m = ma.NewUrl("udp4://127.0.0.1:1234") // m.Url(buf) // udp4://127.0.0.1:1234 @@ -45,9 +47,9 @@ m.Url(buf) ### En/decapsulate ```go -m.Encapsulate(m.NewMultiaddr("/sctp/5678")) +m.Encapsulate(ma.NewMultiaddr("/sctp/5678")) // -m.Decapsulate(m.NewMultiaddr("/udp")) // up to + inc last occurrence of subaddr +m.Decapsulate(ma.NewMultiaddr("/udp")) // up to + inc last occurrence of subaddr // ``` @@ -56,11 +58,11 @@ m.Decapsulate(m.NewMultiaddr("/udp")) // up to + inc last occurrence of subaddr Multiaddr allows expressing tunnels very nicely. ```js -printer := multiaddr.NewMultiaddr("/ip4/192.168.0.13/tcp/80") -proxy := multiaddr.NewMultiaddr("/ip4/10.20.30.40/tcp/443") +printer, _ := ma.NewMultiaddr("/ip4/192.168.0.13/tcp/80") +proxy, _ := ma.NewMultiaddr("/ip4/10.20.30.40/tcp/443") printerOverProxy := proxy.Encapsulate(printer) -// +// /ip4/10.20.30.40/tcp/443/ip4/192.168.0.13/tcp/80 -proxyAgain := printerOverProxy.Decapsulate(multiaddr.NewMultiaddr("/ip4")) -// +proxyAgain := printerOverProxy.Decapsulate(printer) +// /ip4/10.20.30.40/tcp/443 ``` diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/doc.go b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/doc.go new file mode 100644 index 00000000000..c26e443cbd2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/doc.go @@ -0,0 +1,36 @@ +/* +Package multiaddr provides an implementation of the Multiaddr network +address format. Multiaddr emphasizes explicitness, self-description, and +portability. It allows applications to treat addresses as opaque tokens, +and to avoid making assumptions about the address representation (e.g. length). +Learn more at https://github.com/jbenet/multiaddr + +Basic Use: + + import ( + "bytes" + "strings" + ma "github.com/jbenet/go-multiaddr" + ) + + // construct from a string (err signals parse failure) + m1, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/1234") + + // construct from bytes (err signals parse failure) + m2, err := ma.NewMultiaddrBytes(m1.Bytes()) + + // true + strings.Equal(m1.String(), "/ip4/127.0.0.1/udp/1234") + strings.Equal(m1.String(), m2.String()) + bytes.Equal(m1.Bytes(), m2.Bytes()) + m1.Equal(m2) + m2.Equal(m1) + + // tunneling (en/decap) + printer, _ := ma.NewMultiaddr("/ip4/192.168.0.13/tcp/80") + proxy, _ := ma.NewMultiaddr("/ip4/10.20.30.40/tcp/443") + printerOverProxy := proxy.Encapsulate(printer) + proxyAgain := printerOverProxy.Decapsulate(printer) + +*/ +package multiaddr diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/index.go b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/index.go deleted file mode 100644 index a55fa669ec7..00000000000 --- a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/index.go +++ /dev/null @@ -1,117 +0,0 @@ -package multiaddr - -import ( - "bytes" - "fmt" - "strings" -) - -// Multiaddr is the data structure representing a multiaddr -type Multiaddr struct { - Bytes []byte -} - -// NewMultiaddr parses and validates an input string, returning a *Multiaddr -func NewMultiaddr(s string) (*Multiaddr, error) { - b, err := stringToBytes(s) - if err != nil { - return nil, err - } - return &Multiaddr{Bytes: b}, nil -} - -// Equal tests whether two multiaddrs are equal -func (m *Multiaddr) Equal(m2 *Multiaddr) bool { - return bytes.Equal(m.Bytes, m2.Bytes) -} - -// String returns the string representation of a Multiaddr -func (m *Multiaddr) String() (string, error) { - return bytesToString(m.Bytes) -} - -// Protocols returns the list of protocols this Multiaddr has. -func (m *Multiaddr) Protocols() (ret []*Protocol, err error) { - - // panic handler, in case we try accessing bytes incorrectly. - defer func() { - if e := recover(); e != nil { - ret = nil - err = e.(error) - } - }() - - ps := []*Protocol{} - b := m.Bytes[:] - for len(b) > 0 { - p := ProtocolWithCode(int(b[0])) - if p == nil { - return nil, fmt.Errorf("no protocol with code %d", b[0]) - } - ps = append(ps, p) - b = b[1+(p.Size/8):] - } - return ps, nil -} - -// Encapsulate wraps a given Multiaddr, returning the resulting joined Multiaddr -func (m *Multiaddr) Encapsulate(o *Multiaddr) *Multiaddr { - b := make([]byte, len(m.Bytes)+len(o.Bytes)) - b = append(m.Bytes, o.Bytes...) - return &Multiaddr{Bytes: b} -} - -// Decapsulate unwraps Multiaddr up until the given Multiaddr is found. -func (m *Multiaddr) Decapsulate(o *Multiaddr) (*Multiaddr, error) { - s1, err := m.String() - if err != nil { - return nil, err - } - - s2, err := o.String() - if err != nil { - return nil, err - } - - i := strings.LastIndex(s1, s2) - if i < 0 { - return nil, fmt.Errorf("%s not contained in %s", s2, s1) - } - return NewMultiaddr(s1[:i]) -} - -// DialArgs is a convenience function returning arguments for use in net.Dial -func (m *Multiaddr) DialArgs() (string, string, error) { - if !m.IsThinWaist() { - return "", "", fmt.Errorf("%s is not a 'thin waist' address", m) - } - - str, err := m.String() - if err != nil { - return "", "", err - } - - parts := strings.Split(str, "/")[1:] - network := parts[2] - host := strings.Join([]string{parts[1], parts[3]}, ":") - return network, host, nil -} - -// IsThinWaist returns whether this multiaddr includes "Thin Waist" Protocols. -// This means: /{IP4, IP6}/{TCP, UDP} -func (m *Multiaddr) IsThinWaist() bool { - p, err := m.Protocols() - if err != nil { - return false - } - - if p[0].Code != P_IP4 && p[0].Code != P_IP6 { - return false - } - - if p[1].Code != P_TCP && p[1].Code != P_UDP { - return false - } - - return true -} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/interface.go b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/interface.go new file mode 100644 index 00000000000..6f57625a3c9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/interface.go @@ -0,0 +1,42 @@ +package multiaddr + +/* +Multiaddr is a cross-protocol, cross-platform format for representing +internet addresses. It emphasizes explicitness and self-description. +Learn more here: https://github.com/jbenet/multiaddr + +Multiaddrs have both a binary and string representation. + + import ma "github.com/jbenet/go-multiaddr" + + addr, err := ma.NewMultiaddr("/ip4/1.2.3.4/tcp/80") + // err non-nil when parsing failed. + +*/ +type Multiaddr interface { + // Equal returns whether two Multiaddrs are exactly equal + Equal(Multiaddr) bool + + // Bytes returns the []byte representation of this Multiaddr + Bytes() []byte + + // String returns the string representation of this Multiaddr + // (may panic if internal state is corrupted) + String() string + + // Protocols returns the list of Protocols this Multiaddr includes + // will panic if protocol code incorrect (and bytes accessed incorrectly) + Protocols() []*Protocol + + // Encapsulate wraps this Multiaddr around another. For example: + // + // /ip4/1.2.3.4 encapsulate /tcp/80 = /ip4/1.2.3.4/tcp/80 + // + Encapsulate(Multiaddr) Multiaddr + + // Decapsultate removes a Multiaddr wrapping. For example: + // + // /ip4/1.2.3.4/tcp/80 decapsulate /ip4/1.2.3.4 = /tcp/80 + // + Decapsulate(Multiaddr) Multiaddr +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/multiaddr.go b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/multiaddr.go new file mode 100644 index 00000000000..4ee63ca43bb --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/multiaddr.go @@ -0,0 +1,110 @@ +package multiaddr + +import ( + "bytes" + "fmt" + "strings" +) + +// multiaddr is the data structure representing a Multiaddr +type multiaddr struct { + bytes []byte +} + +// NewMultiaddr parses and validates an input string, returning a *Multiaddr +func NewMultiaddr(s string) (Multiaddr, error) { + b, err := stringToBytes(s) + if err != nil { + return nil, err + } + return &multiaddr{bytes: b}, nil +} + +// NewMultiaddrBytes initializes a Multiaddr from a byte representation. +// It validates it as an input string. +func NewMultiaddrBytes(b []byte) (Multiaddr, error) { + s, err := bytesToString(b) + if err != nil { + return nil, err + } + return NewMultiaddr(s) +} + +// Equal tests whether two multiaddrs are equal +func (m *multiaddr) Equal(m2 Multiaddr) bool { + return bytes.Equal(m.bytes, m2.Bytes()) +} + +// Bytes returns the []byte representation of this Multiaddr +func (m *multiaddr) Bytes() []byte { + // consider returning copy to prevent changing underneath us? + cpy := make([]byte, len(m.bytes)) + copy(cpy, m.bytes) + return cpy +} + +// String returns the string representation of a Multiaddr +func (m *multiaddr) String() string { + s, err := bytesToString(m.bytes) + if err != nil { + panic("multiaddr failed to convert back to string. corrupted?") + } + return s +} + +// Protocols returns the list of protocols this Multiaddr has. +// will panic in case we access bytes incorrectly. +func (m *multiaddr) Protocols() []*Protocol { + + // panic handler, in case we try accessing bytes incorrectly. + defer func() { + if e := recover(); e != nil { + err := e.(error) + panic("Multiaddr.Protocols error: " + err.Error()) + } + }() + + ps := []*Protocol{} + b := m.bytes[:] + for len(b) > 0 { + p := ProtocolWithCode(int(b[0])) + if p == nil { + // this is a panic (and not returning err) because this should've been + // caught on constructing the Multiaddr + panic(fmt.Errorf("no protocol with code %d", b[0])) + } + ps = append(ps, p) + b = b[1+(p.Size/8):] + } + return ps +} + +// Encapsulate wraps a given Multiaddr, returning the resulting joined Multiaddr +func (m *multiaddr) Encapsulate(o Multiaddr) Multiaddr { + mb := m.bytes + ob := o.Bytes() + + var b bytes.Buffer + b.Write(mb) + b.Write(ob) + return &multiaddr{bytes: b.Bytes()} +} + +// Decapsulate unwraps Multiaddr up until the given Multiaddr is found. +func (m *multiaddr) Decapsulate(o Multiaddr) Multiaddr { + s1 := m.String() + s2 := o.String() + i := strings.LastIndex(s1, s2) + if i < 0 { + // if multiaddr not contained, returns a copy. + cpy := make([]byte, len(m.bytes)) + copy(cpy, m.bytes) + return &multiaddr{bytes: cpy} + } + + ma, err := NewMultiaddr(s1[:i]) + if err != nil { + panic("Multiaddr.Decapsulate incorrect byte boundaries.") + } + return ma +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/multiaddr_test.go b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/multiaddr_test.go index 7bc2e92bfce..12d6214630e 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/multiaddr_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/multiaddr_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -func newMultiaddr(t *testing.T, a string) *Multiaddr { +func newMultiaddr(t *testing.T, a string) Multiaddr { m, err := NewMultiaddr(a) if err != nil { t.Error(err) @@ -88,11 +88,7 @@ func TestProtocols(t *testing.T) { t.Error("failed to construct", "/ip4/127.0.0.1/udp/1234") } - ps, err := m.Protocols() - if err != nil { - t.Error("failed to get protocols", "/ip4/127.0.0.1/udp/1234") - } - + ps := m.Protocols() if ps[0] != ProtocolWithName("ip4") { t.Error(ps[0], ProtocolWithName("ip4")) t.Error("failed to get ip4 protocol") @@ -117,47 +113,19 @@ func TestEncapsulate(t *testing.T) { } b := m.Encapsulate(m2) - if s, _ := b.String(); s != "/ip4/127.0.0.1/udp/1234/udp/5678" { + if s := b.String(); s != "/ip4/127.0.0.1/udp/1234/udp/5678" { t.Error("encapsulate /ip4/127.0.0.1/udp/1234/udp/5678 failed.", s) } m3, _ := NewMultiaddr("/udp/5678") - c, err := b.Decapsulate(m3) - if err != nil { - t.Error("decapsulate /udp failed.", err) - } - - if s, _ := c.String(); s != "/ip4/127.0.0.1/udp/1234" { + c := b.Decapsulate(m3) + if s := c.String(); s != "/ip4/127.0.0.1/udp/1234" { t.Error("decapsulate /udp failed.", "/ip4/127.0.0.1/udp/1234", s) } m4, _ := NewMultiaddr("/ip4/127.0.0.1") - d, err := c.Decapsulate(m4) - if err != nil { - t.Error("decapsulate /ip4 failed.", err) - } - - if s, _ := d.String(); s != "" { + d := c.Decapsulate(m4) + if s := d.String(); s != "" { t.Error("decapsulate /ip4 failed.", "/", s) } } - -func TestDialArgs(t *testing.T) { - m, err := NewMultiaddr("/ip4/127.0.0.1/udp/1234") - if err != nil { - t.Fatal("failed to construct", "/ip4/127.0.0.1/udp/1234") - } - - nw, host, err := m.DialArgs() - if err != nil { - t.Fatal("failed to get dial args", "/ip4/127.0.0.1/udp/1234", err) - } - - if nw != "udp" { - t.Error("failed to get udp network Dial Arg") - } - - if host != "127.0.0.1:1234" { - t.Error("failed to get host:port Dial Arg") - } -} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net.go b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net.go index 516fe8392f2..ed91dc2e14b 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net.go @@ -3,12 +3,13 @@ package multiaddr import ( "fmt" "net" + "strings" ) var errIncorrectNetAddr = fmt.Errorf("incorrect network addr conversion") // FromNetAddr converts a net.Addr type to a Multiaddr. -func FromNetAddr(a net.Addr) (*Multiaddr, error) { +func FromNetAddr(a net.Addr) (Multiaddr, error) { switch a.Network() { case "tcp", "tcp4", "tcp6": ac, ok := a.(*net.TCPAddr) @@ -65,7 +66,7 @@ func FromNetAddr(a net.Addr) (*Multiaddr, error) { } // FromIP converts a net.IP type to a Multiaddr. -func FromIP(ip net.IP) (*Multiaddr, error) { +func FromIP(ip net.IP) (Multiaddr, error) { switch { case ip.To4() != nil: return NewMultiaddr("/ip4/" + ip.String()) @@ -75,3 +76,38 @@ func FromIP(ip net.IP) (*Multiaddr, error) { return nil, errIncorrectNetAddr } } + +// DialArgs is a convenience function returning arguments for use in net.Dial +func DialArgs(m Multiaddr) (string, string, error) { + if !IsThinWaist(m) { + return "", "", fmt.Errorf("%s is not a 'thin waist' address", m) + } + + str := m.String() + parts := strings.Split(str, "/")[1:] + network := parts[2] + + var host string + switch parts[0] { + case "ip4": + host = strings.Join([]string{parts[1], parts[3]}, ":") + case "ip6": + host = fmt.Sprintf("[%s]:%s", parts[1], parts[3]) + } + return network, host, nil +} + +// IsThinWaist returns whether a Multiaddr starts with "Thin Waist" Protocols. +// This means: /{IP4, IP6}/{TCP, UDP} +func IsThinWaist(m Multiaddr) bool { + p := m.Protocols() + if p[0].Code != P_IP4 && p[0].Code != P_IP6 { + return false + } + + if p[1].Code != P_TCP && p[1].Code != P_UDP { + return false + } + + return true +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net_test.go b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net_test.go index fd1ede1f1c3..c9cb4b4ac8f 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-multiaddr/net_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -type GenFunc func() (*Multiaddr, error) +type GenFunc func() (Multiaddr, error) func testConvert(t *testing.T, s string, gen GenFunc) { m, err := gen() @@ -13,25 +13,25 @@ func testConvert(t *testing.T, s string, gen GenFunc) { t.Fatal("failed to generate.") } - if s2, _ := m.String(); err != nil || s2 != s { + if s2 := m.String(); err != nil || s2 != s { t.Fatal("failed to convert: " + s + " != " + s2) } } func TestFromIP4(t *testing.T) { - testConvert(t, "/ip4/10.20.30.40", func() (*Multiaddr, error) { + testConvert(t, "/ip4/10.20.30.40", func() (Multiaddr, error) { return FromIP(net.ParseIP("10.20.30.40")) }) } func TestFromIP6(t *testing.T) { - testConvert(t, "/ip6/2001:4860:0:2001::68", func() (*Multiaddr, error) { + testConvert(t, "/ip6/2001:4860:0:2001::68", func() (Multiaddr, error) { return FromIP(net.ParseIP("2001:4860:0:2001::68")) }) } func TestFromTCP(t *testing.T) { - testConvert(t, "/ip4/10.20.30.40/tcp/1234", func() (*Multiaddr, error) { + testConvert(t, "/ip4/10.20.30.40/tcp/1234", func() (Multiaddr, error) { return FromNetAddr(&net.TCPAddr{ IP: net.ParseIP("10.20.30.40"), Port: 1234, @@ -40,10 +40,30 @@ func TestFromTCP(t *testing.T) { } func TestFromUDP(t *testing.T) { - testConvert(t, "/ip4/10.20.30.40/udp/1234", func() (*Multiaddr, error) { + testConvert(t, "/ip4/10.20.30.40/udp/1234", func() (Multiaddr, error) { return FromNetAddr(&net.UDPAddr{ IP: net.ParseIP("10.20.30.40"), Port: 1234, }) }) } + +func TestDialArgs(t *testing.T) { + m, err := NewMultiaddr("/ip4/127.0.0.1/udp/1234") + if err != nil { + t.Fatal("failed to construct", "/ip4/127.0.0.1/udp/1234") + } + + nw, host, err := DialArgs(m) + if err != nil { + t.Fatal("failed to get dial args", "/ip4/127.0.0.1/udp/1234", err) + } + + if nw != "udp" { + t.Error("failed to get udp network Dial Arg") + } + + if host != "127.0.0.1:1234" { + t.Error("failed to get host:port Dial Arg") + } +} From 910a76e220a6db720e9dd434e168d2e67bd6a249 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Mon, 6 Oct 2014 04:13:39 -0700 Subject: [PATCH 076/105] updated multiaddr use across codebase --- core/core.go | 4 ++-- daemon/daemon.go | 11 +++-------- daemon/daemon_client.go | 2 +- net/conn/conn.go | 8 ++++---- net/swarm/conn.go | 4 ++-- net/swarm/swarm.go | 2 +- net/swarm/swarm_test.go | 2 +- peer/peer.go | 13 ++++--------- routing/dht/Message.go | 6 +----- routing/dht/dht_test.go | 6 +++--- routing/dht/ext_test.go | 2 +- server/http/http.go | 4 ++-- 12 files changed, 25 insertions(+), 39 deletions(-) diff --git a/core/core.go b/core/core.go index 427c33c2932..9aaa6e50406 100644 --- a/core/core.go +++ b/core/core.go @@ -177,14 +177,14 @@ func initIdentity(cfg *config.Config, online bool) (*peer.Peer, error) { } // address is optional - var addresses []*ma.Multiaddr + var addresses []ma.Multiaddr if len(cfg.Addresses.Swarm) > 0 { maddr, err := ma.NewMultiaddr(cfg.Addresses.Swarm) if err != nil { return nil, err } - addresses = []*ma.Multiaddr{maddr} + addresses = []ma.Multiaddr{maddr} } var ( diff --git a/daemon/daemon.go b/daemon/daemon.go index 02fe49023ac..45ac49e681d 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -39,7 +39,7 @@ type Command struct { Opts map[string]interface{} } -func NewDaemonListener(ipfsnode *core.IpfsNode, addr *ma.Multiaddr, confdir string) (*DaemonListener, error) { +func NewDaemonListener(ipfsnode *core.IpfsNode, addr ma.Multiaddr, confdir string) (*DaemonListener, error) { var err error confdir, err = u.TildeExpansion(confdir) if err != nil { @@ -51,7 +51,7 @@ func NewDaemonListener(ipfsnode *core.IpfsNode, addr *ma.Multiaddr, confdir stri return nil, err } - network, host, err := addr.DialArgs() + network, host, err := ma.DialArgs(addr) if err != nil { return nil, err } @@ -62,12 +62,7 @@ func NewDaemonListener(ipfsnode *core.IpfsNode, addr *ma.Multiaddr, confdir stri return nil, err } - mstr, err := addr.String() - if err != nil { - return nil, err - } - - _, err = ofi.Write([]byte(mstr)) + _, err = ofi.Write([]byte(addr.String())) if err != nil { log.Warning("Could not write to rpcaddress file: %s", err) return nil, err diff --git a/daemon/daemon_client.go b/daemon/daemon_client.go index 17d1b2c265c..478b7fd7d33 100644 --- a/daemon/daemon_client.go +++ b/daemon/daemon_client.go @@ -73,7 +73,7 @@ func SendCommand(command *Command, confdir string) error { return err } - network, host, err := maddr.DialArgs() + network, host, err := ma.DialArgs(maddr) conn, err := net.Dial(network, host) if err != nil { diff --git a/net/conn/conn.go b/net/conn/conn.go index 645264b8da4..25ecc44d684 100644 --- a/net/conn/conn.go +++ b/net/conn/conn.go @@ -21,7 +21,7 @@ const MaxMessageSize = 1 << 20 // Conn represents a connection to another Peer (IPFS Node). type Conn struct { Peer *peer.Peer - Addr *ma.Multiaddr + Addr ma.Multiaddr Conn net.Conn Closed chan bool @@ -34,7 +34,7 @@ type Conn struct { type Map map[u.Key]*Conn // NewConn constructs a new connection -func NewConn(peer *peer.Peer, addr *ma.Multiaddr, nconn net.Conn) (*Conn, error) { +func NewConn(peer *peer.Peer, addr ma.Multiaddr, nconn net.Conn) (*Conn, error) { conn := &Conn{ Peer: peer, Addr: addr, @@ -56,7 +56,7 @@ func Dial(network string, peer *peer.Peer) (*Conn, error) { return nil, fmt.Errorf("No address for network %s", network) } - network, host, err := addr.DialArgs() + network, host, err := ma.DialArgs(addr) if err != nil { return nil, err } @@ -104,6 +104,6 @@ func (c *Conn) Close() error { // NetConnMultiaddr returns the net.Conn's address, recast as a multiaddr. // (consider moving this directly into the multiaddr package) -func NetConnMultiaddr(nconn net.Conn) (*ma.Multiaddr, error) { +func NetConnMultiaddr(nconn net.Conn) (ma.Multiaddr, error) { return ma.FromNetAddr(nconn.RemoteAddr()) } diff --git a/net/swarm/conn.go b/net/swarm/conn.go index 0713ccf0b8d..6b2dce32bec 100644 --- a/net/swarm/conn.go +++ b/net/swarm/conn.go @@ -37,8 +37,8 @@ func (s *Swarm) listen() error { } // Listen for new connections on the given multiaddr -func (s *Swarm) connListen(maddr *ma.Multiaddr) error { - netstr, addr, err := maddr.DialArgs() +func (s *Swarm) connListen(maddr ma.Multiaddr) error { + netstr, addr, err := ma.DialArgs(maddr) if err != nil { return err } diff --git a/net/swarm/swarm.go b/net/swarm/swarm.go index df84e5a94ce..9941c708608 100644 --- a/net/swarm/swarm.go +++ b/net/swarm/swarm.go @@ -143,7 +143,7 @@ func (s *Swarm) Dial(peer *peer.Peer) (*conn.Conn, error) { // DialAddr is for connecting to a peer when you know their addr but not their ID. // Should only be used when sure that not connected to peer in question // TODO(jbenet) merge with Dial? need way to patch back. -func (s *Swarm) DialAddr(addr *ma.Multiaddr) (*conn.Conn, error) { +func (s *Swarm) DialAddr(addr ma.Multiaddr) (*conn.Conn, error) { if addr == nil { return nil, errors.New("addr must be a non-nil Multiaddr") } diff --git a/net/swarm/swarm_test.go b/net/swarm/swarm_test.go index b2747481c9e..7d7c138025c 100644 --- a/net/swarm/swarm_test.go +++ b/net/swarm/swarm_test.go @@ -95,7 +95,7 @@ func TestSwarm(t *testing.T) { if a == nil { t.Fatal("error setting up peer (addr is nil)", peer) } - n, h, err := a.DialArgs() + n, h, err := ma.DialArgs(a) if err != nil { t.Fatal("error getting dial args from addr") } diff --git a/peer/peer.go b/peer/peer.go index 69d73c2d4af..8b0bda8aae9 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -38,7 +38,7 @@ type Map map[u.Key]*Peer // ID, and relevant Addresses. type Peer struct { ID ID - Addresses []*ma.Multiaddr + Addresses []ma.Multiaddr PrivKey ic.PrivKey PubKey ic.PubKey @@ -54,7 +54,7 @@ func (p *Peer) Key() u.Key { } // AddAddress adds the given Multiaddr address to Peer's addresses. -func (p *Peer) AddAddress(a *ma.Multiaddr) { +func (p *Peer) AddAddress(a ma.Multiaddr) { p.Lock() defer p.Unlock() @@ -67,17 +67,12 @@ func (p *Peer) AddAddress(a *ma.Multiaddr) { } // NetAddress returns the first Multiaddr found for a given network. -func (p *Peer) NetAddress(n string) *ma.Multiaddr { +func (p *Peer) NetAddress(n string) ma.Multiaddr { p.RLock() defer p.RUnlock() for _, a := range p.Addresses { - ps, err := a.Protocols() - if err != nil { - continue // invalid addr - } - - for _, p := range ps { + for _, p := range a.Protocols() { if p.Name == n { return a } diff --git a/routing/dht/Message.go b/routing/dht/Message.go index 1be9a3b801e..84d323c37b9 100644 --- a/routing/dht/Message.go +++ b/routing/dht/Message.go @@ -20,11 +20,7 @@ func peerToPBPeer(p *peer.Peer) *Message_Peer { if len(p.Addresses) == 0 || p.Addresses[0] == nil { pbp.Addr = proto.String("") } else { - addr, err := p.Addresses[0].String() - if err != nil { - //Temp: what situations could cause this? - panic(err) - } + addr := p.Addresses[0].String() pbp.Addr = &addr } pid := string(p.ID) diff --git a/routing/dht/dht_test.go b/routing/dht/dht_test.go index 1bbc62cdc0f..f5d391387b5 100644 --- a/routing/dht/dht_test.go +++ b/routing/dht/dht_test.go @@ -43,8 +43,8 @@ func setupDHT(t *testing.T, p *peer.Peer) *IpfsDHT { return d } -func setupDHTS(n int, t *testing.T) ([]*ma.Multiaddr, []*peer.Peer, []*IpfsDHT) { - var addrs []*ma.Multiaddr +func setupDHTS(n int, t *testing.T) ([]ma.Multiaddr, []*peer.Peer, []*IpfsDHT) { + var addrs []ma.Multiaddr for i := 0; i < n; i++ { a, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 5000+i)) if err != nil { @@ -67,7 +67,7 @@ func setupDHTS(n int, t *testing.T) ([]*ma.Multiaddr, []*peer.Peer, []*IpfsDHT) return addrs, peers, dhts } -func makePeer(addr *ma.Multiaddr) *peer.Peer { +func makePeer(addr ma.Multiaddr) *peer.Peer { p := new(peer.Peer) p.AddAddress(addr) sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512) diff --git a/routing/dht/ext_test.go b/routing/dht/ext_test.go index f8b9293a870..df8f26ff3d7 100644 --- a/routing/dht/ext_test.go +++ b/routing/dht/ext_test.go @@ -184,7 +184,7 @@ func TestGetFailures(t *testing.T) { func _randPeer() *peer.Peer { p := new(peer.Peer) p.ID = make(peer.ID, 16) - p.Addresses = []*ma.Multiaddr{nil} + p.Addresses = []ma.Multiaddr{nil} crand.Read(p.ID) return p } diff --git a/server/http/http.go b/server/http/http.go index 75623d5f2ab..14eff9cd204 100644 --- a/server/http/http.go +++ b/server/http/http.go @@ -17,14 +17,14 @@ type handler struct { } // Serve starts the http server -func Serve(address *ma.Multiaddr, node *core.IpfsNode) error { +func Serve(address ma.Multiaddr, node *core.IpfsNode) error { r := mux.NewRouter() handler := &handler{&ipfsHandler{node}} r.HandleFunc("/ipfs/", handler.postHandler).Methods("POST") r.PathPrefix("/ipfs/").Handler(handler).Methods("GET") http.Handle("/", r) - _, host, err := address.DialArgs() + _, host, err := ma.DialArgs(address) if err != nil { return err } From 77fccaa2f3129e823fb4b9bc7c4c396448372335 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Mon, 6 Oct 2014 04:23:55 -0700 Subject: [PATCH 077/105] Obviated need for `.ID.Pretty()` all over the place. --- blockservice/blockservice.go | 4 ++-- core/commands/add.go | 6 +++--- core/commands/publish.go | 2 +- crypto/spipe/handshake.go | 16 ++++++++-------- exchange/bitswap/bitswap.go | 10 +++++----- exchange/bitswap/bitswap_test.go | 20 ++++++++++---------- net/service/service.go | 2 +- net/swarm/conn.go | 12 +++++------- peer/peer.go | 10 ++++++++++ routing/dht/dht.go | 13 ++++++------- 10 files changed, 51 insertions(+), 44 deletions(-) diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index 21bb75c6f36..9b150142445 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -36,7 +36,7 @@ func NewBlockService(d ds.Datastore, rem exchange.Interface) (*BlockService, err // AddBlock adds a particular block to the service, Putting it into the datastore. func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { k := b.Key() - log.Debug("storing [%s] in datastore", k.Pretty()) + log.Debug("storing [%s] in datastore", k) // TODO(brian): define a block datastore with a Put method which accepts a // block parameter err := s.Datastore.Put(k.DsKey(), b.Data) @@ -53,7 +53,7 @@ func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { // GetBlock retrieves a particular block from the service, // Getting it from the datastore using the key (hash). func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) { - log.Debug("BlockService GetBlock: '%s'", k.Pretty()) + log.Debug("BlockService GetBlock: '%s'", k) datai, err := s.Datastore.Get(k.DsKey()) if err == nil { log.Debug("Blockservice: Got data in datastore.") diff --git a/core/commands/add.go b/core/commands/add.go index 006c8c956b0..4a3a0e81c0b 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -52,7 +52,7 @@ func Add(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Wr // } // // Commenting out of here, because it's already in addNode below. - // fmt.Fprintf(out, "added %s %s\n", k.Pretty(), path) + // fmt.Fprintf(out, "added %s %s\n", k, path) } return nil } @@ -110,7 +110,7 @@ func addFile(n *core.IpfsNode, fpath string, depth int) (*dag.Node, error) { return nil, err } - log.Info("Adding file: %s = %s\n", fpath, k.Pretty()) + log.Info("Adding file: %s = %s\n", fpath, k) for _, l := range root.Links { log.Info("SubBlock: %s\n", l.Hash.B58String()) } @@ -131,7 +131,7 @@ func addNode(n *core.IpfsNode, nd *dag.Node, fpath string) error { return err } - u.POut("added %s %s\n", k.Pretty(), fpath) + u.POut("added %s %s\n", k, fpath) // ensure we keep it. atm no-op return n.PinDagNodeRecursively(nd, -1) diff --git a/core/commands/publish.go b/core/commands/publish.go index b292361cd26..09b41f43b22 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -29,7 +29,7 @@ func Publish(n *core.IpfsNode, args []string, opts map[string]interface{}, out i if err != nil { return err } - fmt.Fprintf(out, "published mapping %s to %s\n", u.Key(hash).Pretty(), args[0]) + fmt.Fprintf(out, "published mapping %s to %s\n", u.Key(hash), args[0]) return nil } diff --git a/crypto/spipe/handshake.go b/crypto/spipe/handshake.go index 975c3d2e431..21a06d548e6 100644 --- a/crypto/spipe/handshake.go +++ b/crypto/spipe/handshake.go @@ -49,7 +49,7 @@ func (s *SecurePipe) handshake() error { return err } - // u.DOut("handshake: %s <--> %s\n", s.local.ID.Pretty(), s.remote.ID.Pretty()) + // u.DOut("handshake: %s <--> %s\n", s.local, s.remote) myPubKey, err := s.local.PubKey.Bytes() if err != nil { return err @@ -101,7 +101,7 @@ func (s *SecurePipe) handshake() error { if err != nil { return err } - u.DOut("[%s] Remote Peer Identified as %s\n", s.local.ID.Pretty(), s.remote.ID.Pretty()) + u.DOut("[%s] Remote Peer Identified as %s\n", s.local, s.remote) exchange, err := selectBest(SupportedExchanges, proposeResp.GetExchanges()) if err != nil { @@ -163,7 +163,7 @@ func (s *SecurePipe) handshake() error { theirHandshake.Write(encoded) theirHandshake.Write(exchangeResp.GetEpubkey()) - // u.POut("Remote Peer Identified as %s\n", s.remote.ID.Pretty()) + // u.POut("Remote Peer Identified as %s\n", s.remote) ok, err := s.remote.PubKey.Verify(theirHandshake.Bytes(), exchangeResp.GetSignature()) if err != nil { return err @@ -205,7 +205,7 @@ func (s *SecurePipe) handshake() error { return errors.New("Negotiation failed.") } - u.DOut("[%s] handshake: Got node id: %s\n", s.local.ID.Pretty(), s.remote.ID.Pretty()) + u.DOut("[%s] handshake: Got node id: %s\n", s.local, s.remote) return nil } @@ -233,7 +233,7 @@ func (s *SecurePipe) handleSecureIn(hashType string, tIV, tCKey, tMKey []byte) { return } - // u.DOut("[peer %s] secure in [from = %s] %d\n", s.local.ID.Pretty(), s.remote.ID.Pretty(), len(data)) + // u.DOut("[peer %s] secure in [from = %s] %d\n", s.local, s.remote, len(data)) if len(data) <= macSize { continue } @@ -281,7 +281,7 @@ func (s *SecurePipe) handleSecureOut(hashType string, mIV, mCKey, mMKey []byte) copy(buff[len(data):], myMac.Sum(nil)) myMac.Reset() - // u.DOut("[peer %s] secure out [to = %s] %d\n", s.local.ID.Pretty(), s.remote.ID.Pretty(), len(buff)) + // u.DOut("[peer %s] secure out [to = %s] %d\n", s.local, s.remote, len(buff)) s.insecure.Out <- buff } } @@ -358,7 +358,7 @@ func getOrConstructPeer(peers peer.Peerstore, rpk ci.PubKey) (*peer.Peer, error) // let's verify ID if !npeer.ID.Equal(rid) { e := "Expected peer.ID does not match sent pubkey's hash: %v - %v" - return nil, fmt.Errorf(e, npeer.ID.Pretty(), rid.Pretty()) + return nil, fmt.Errorf(e, npeer, rid) } if npeer.PubKey == nil { @@ -371,7 +371,7 @@ func getOrConstructPeer(peers peer.Peerstore, rpk ci.PubKey) (*peer.Peer, error) // this shouldn't ever happen, given we hashed, etc, but it could mean // expected code (or protocol) invariants violated. if !npeer.PubKey.Equals(rpk) { - return nil, fmt.Errorf("WARNING: PubKey mismatch: %v", npeer.ID.Pretty()) + return nil, fmt.Errorf("WARNING: PubKey mismatch: %v", npeer) } return npeer, nil } diff --git a/exchange/bitswap/bitswap.go b/exchange/bitswap/bitswap.go index 4ba9e179fa4..e4eaeb4a4f4 100644 --- a/exchange/bitswap/bitswap.go +++ b/exchange/bitswap/bitswap.go @@ -61,7 +61,7 @@ type bitswap struct { // // TODO ensure only one active request per key func (bs *bitswap) Block(parent context.Context, k u.Key) (*blocks.Block, error) { - u.DOut("Get Block %v\n", k.Pretty()) + u.DOut("Get Block %v\n", k) ctx, cancelFunc := context.WithCancel(parent) bs.wantlist.Add(k) @@ -110,7 +110,7 @@ func (bs *bitswap) Block(parent context.Context, k u.Key) (*blocks.Block, error) // HasBlock announces the existance of a block to bitswap, potentially sending // it to peers (Partners) whose WantLists include it. func (bs *bitswap) HasBlock(ctx context.Context, blk blocks.Block) error { - u.DOut("Has Block %v\n", blk.Key().Pretty()) + u.DOut("Has Block %v\n", blk.Key()) bs.wantlist.Remove(blk.Key()) bs.sendToPeersThatWant(ctx, blk) return bs.routing.Provide(ctx, blk.Key()) @@ -119,7 +119,7 @@ func (bs *bitswap) HasBlock(ctx context.Context, blk blocks.Block) error { // TODO(brian): handle errors func (bs *bitswap) ReceiveMessage(ctx context.Context, p *peer.Peer, incoming bsmsg.BitSwapMessage) ( *peer.Peer, bsmsg.BitSwapMessage) { - u.DOut("ReceiveMessage from %v\n", p.Key().Pretty()) + u.DOut("ReceiveMessage from %v\n", p.Key()) if p == nil { // TODO propagate the error upward @@ -173,10 +173,10 @@ func (bs *bitswap) send(ctx context.Context, p *peer.Peer, m bsmsg.BitSwapMessag } func (bs *bitswap) sendToPeersThatWant(ctx context.Context, block blocks.Block) { - u.DOut("Sending %v to peers that want it\n", block.Key().Pretty()) + u.DOut("Sending %v to peers that want it\n", block.Key()) for _, p := range bs.strategy.Peers() { if bs.strategy.BlockIsWantedByPeer(block.Key(), p) { - u.DOut("%v wants %v\n", p.Key().Pretty(), block.Key().Pretty()) + u.DOut("%v wants %v\n", p, block.Key()) if bs.strategy.ShouldSendBlockToPeer(block.Key(), p) { message := bsmsg.New() message.AppendBlock(block) diff --git a/exchange/bitswap/bitswap_test.go b/exchange/bitswap/bitswap_test.go index a9fc11f8210..3a9bed97c20 100644 --- a/exchange/bitswap/bitswap_test.go +++ b/exchange/bitswap/bitswap_test.go @@ -160,49 +160,49 @@ func TestSendToWantingPeer(t *testing.T) { w := sg.Next() o := sg.Next() - t.Logf("Session %v\n", me.peer.Key().Pretty()) - t.Logf("Session %v\n", w.peer.Key().Pretty()) - t.Logf("Session %v\n", o.peer.Key().Pretty()) + t.Logf("Session %v\n", me.peer) + t.Logf("Session %v\n", w.peer) + t.Logf("Session %v\n", o.peer) alpha := bg.Next() const timeout = 1 * time.Millisecond // FIXME don't depend on time - t.Logf("Peer %v attempts to get %v. NB: not available\n", w.peer.Key().Pretty(), alpha.Key().Pretty()) + t.Logf("Peer %v attempts to get %v. NB: not available\n", w.peer, alpha.Key()) ctx, _ := context.WithTimeout(context.Background(), timeout) _, err := w.exchange.Block(ctx, alpha.Key()) if err == nil { - t.Fatalf("Expected %v to NOT be available", alpha.Key().Pretty()) + t.Fatalf("Expected %v to NOT be available", alpha.Key()) } beta := bg.Next() - t.Logf("Peer %v announes availability of %v\n", w.peer.Key().Pretty(), beta.Key().Pretty()) + t.Logf("Peer %v announes availability of %v\n", w.peer, beta.Key()) ctx, _ = context.WithTimeout(context.Background(), timeout) if err := w.blockstore.Put(beta); err != nil { t.Fatal(err) } w.exchange.HasBlock(ctx, beta) - t.Logf("%v gets %v from %v and discovers it wants %v\n", me.peer.Key().Pretty(), beta.Key().Pretty(), w.peer.Key().Pretty(), alpha.Key().Pretty()) + t.Logf("%v gets %v from %v and discovers it wants %v\n", me.peer, beta.Key(), w.peer, alpha.Key()) ctx, _ = context.WithTimeout(context.Background(), timeout) if _, err := me.exchange.Block(ctx, beta.Key()); err != nil { t.Fatal(err) } - t.Logf("%v announces availability of %v\n", o.peer.Key().Pretty(), alpha.Key().Pretty()) + t.Logf("%v announces availability of %v\n", o.peer, alpha.Key()) ctx, _ = context.WithTimeout(context.Background(), timeout) if err := o.blockstore.Put(alpha); err != nil { t.Fatal(err) } o.exchange.HasBlock(ctx, alpha) - t.Logf("%v requests %v\n", me.peer.Key().Pretty(), alpha.Key().Pretty()) + t.Logf("%v requests %v\n", me.peer, alpha.Key()) ctx, _ = context.WithTimeout(context.Background(), timeout) if _, err := me.exchange.Block(ctx, alpha.Key()); err != nil { t.Fatal(err) } - t.Logf("%v should now have %v\n", w.peer.Key().Pretty(), alpha.Key().Pretty()) + t.Logf("%v should now have %v\n", w.peer, alpha.Key()) block, err := w.blockstore.Get(alpha.Key()) if err != nil { t.Fatal("Should not have received an error") diff --git a/net/service/service.go b/net/service/service.go index f3d4ba5aa5d..db7cb82a832 100644 --- a/net/service/service.go +++ b/net/service/service.go @@ -82,7 +82,7 @@ func (s *Service) sendMessage(ctx context.Context, m msg.NetMessage, rid Request return err } - // u.DOut("Service send message [to = %s]\n", m.Peer().ID.Pretty()) + // u.DOut("Service send message [to = %s]\n", m.Peer()) // send message m2 := msg.New(m.Peer(), data) diff --git a/net/swarm/conn.go b/net/swarm/conn.go index 6b2dce32bec..e4b980834a9 100644 --- a/net/swarm/conn.go +++ b/net/swarm/conn.go @@ -106,7 +106,7 @@ func (s *Swarm) connSetup(c *conn.Conn) error { } if c.Peer != nil { - u.DOut("Starting connection: %s\n", c.Peer.Key().Pretty()) + u.DOut("Starting connection: %s\n", c.Peer) } else { u.DOut("Starting connection: [unknown peer]\n") } @@ -115,7 +115,7 @@ func (s *Swarm) connSetup(c *conn.Conn) error { return fmt.Errorf("Conn securing error: %v", err) } - u.DOut("Secured connection: %s\n", c.Peer.Key().Pretty()) + u.DOut("Secured connection: %s\n", c.Peer) // add address of connection to Peer. Maybe it should happen in connSecure. c.Peer.AddAddress(c.Addr) @@ -184,8 +184,7 @@ func (s *Swarm) fanOut() { continue } - // u.DOut("[peer: %s] Sent message [to = %s]\n", - // s.local.ID.Pretty(), msg.Peer().ID.Pretty()) + // u.DOut("[peer: %s] Sent message [to = %s]\n", s.local, msg.Peer()) // queue it in the connection's buffer conn.Secure.Out <- msg.Data() @@ -208,13 +207,12 @@ func (s *Swarm) fanIn(c *conn.Conn) { case data, ok := <-c.Secure.In: if !ok { - e := fmt.Errorf("Error retrieving from conn: %v", c.Peer.Key().Pretty()) + e := fmt.Errorf("Error retrieving from conn: %v", c.Peer) s.errChan <- e goto out } - // u.DOut("[peer: %s] Received message [from = %s]\n", - // s.local.ID.Pretty(), c.Peer.ID.Pretty()) + // u.DOut("[peer: %s] Received message [from = %s]\n", s.local, c.Peer) msg := msg.New(c.Peer, data) s.Incoming <- msg diff --git a/peer/peer.go b/peer/peer.go index 8b0bda8aae9..2803a8f009b 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -16,6 +16,11 @@ import ( // ID is a byte slice representing the identity of a peer. type ID mh.Multihash +// String is utililty function for printing out peer ID strings. +func (id ID) String() string { + return id.Pretty() +} + // Equal is utililty function for comparing two peer ID's func (id ID) Equal(other ID) bool { return bytes.Equal(id, other) @@ -48,6 +53,11 @@ type Peer struct { sync.RWMutex } +// String prints out the peer. +func (p *Peer) String() string { + return "[Peer " + p.ID.Pretty() + "]" +} + // Key returns the ID as a Key (string) for maps. func (p *Peer) Key() u.Key { return u.Key(p.ID) diff --git a/routing/dht/dht.go b/routing/dht/dht.go index 8cf3a115398..1f1fdd3e5b1 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -76,7 +76,7 @@ func NewDHT(p *peer.Peer, ps peer.Peerstore, net inet.Network, sender inet.Sende // Connect to a new peer at the given address, ping and add to the routing table func (dht *IpfsDHT) Connect(ctx context.Context, npeer *peer.Peer) (*peer.Peer, error) { - log.Debug("Connect to new peer: %s\n", npeer.ID.Pretty()) + log.Debug("Connect to new peer: %s\n", npeer) // TODO(jbenet,whyrusleeping) // @@ -132,8 +132,7 @@ func (dht *IpfsDHT) HandleMessage(ctx context.Context, mes msg.NetMessage) msg.N // Print out diagnostic log.Debug("[peer: %s] Got message type: '%s' [from = %s]\n", - dht.self.ID.Pretty(), - Message_MessageType_name[int32(pmes.GetType())], mPeer.ID.Pretty()) + dht.self, Message_MessageType_name[int32(pmes.GetType())], mPeer) // get handler for this msg type. handler := dht.handlerForMsgType(pmes.GetType()) @@ -177,7 +176,7 @@ func (dht *IpfsDHT) sendRequest(ctx context.Context, p *peer.Peer, pmes *Message // Print out diagnostic log.Debug("Sent message type: '%s' [to = %s]", - Message_MessageType_name[int32(pmes.GetType())], p.ID.Pretty()) + Message_MessageType_name[int32(pmes.GetType())], p) rmes, err := dht.sender.SendRequest(ctx, mes) if err != nil { @@ -222,7 +221,7 @@ func (dht *IpfsDHT) putProvider(ctx context.Context, p *peer.Peer, key string) e return err } - log.Debug("[%s] putProvider: %s for %s", dht.self.ID.Pretty(), p.ID.Pretty(), key) + log.Debug("[%s] putProvider: %s for %s", dht.self, p, key) if *rpmes.Key != *pmes.Key { return errors.New("provider not added correctly") } @@ -346,7 +345,7 @@ func (dht *IpfsDHT) putLocal(key u.Key, value []byte) error { // Update signals to all routingTables to Update their last-seen status // on the given peer. func (dht *IpfsDHT) Update(p *peer.Peer) { - log.Debug("updating peer: [%s] latency = %f\n", p.ID.Pretty(), p.GetLatency().Seconds()) + log.Debug("updating peer: [%s] latency = %f\n", p, p.GetLatency().Seconds()) removedCount := 0 for _, route := range dht.routingTables { removed := route.Update(p) @@ -402,7 +401,7 @@ func (dht *IpfsDHT) addProviders(key u.Key, peers []*Message_Peer) []*peer.Peer continue } - log.Debug("[%s] adding provider: %s for %s", dht.self.ID.Pretty(), p, key) + log.Debug("[%s] adding provider: %s for %s", dht.self, p, key) // Dont add outselves to the list if p.ID.Equal(dht.self.ID) { From 3591e10b2e4a9f40ea8c33984933d27bd0354388 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 6 Oct 2014 23:49:45 +0000 Subject: [PATCH 078/105] implement dagmodifier and tests. --- .gitignore | 1 + blockservice/blockservice.go | 4 +- importer/dagwriter/dagmodifier.go | 190 +++++++++++++++++++++++++ importer/dagwriter/dagmodifier_test.go | 187 ++++++++++++++++++++++++ importer/dagwriter/dagwriter.go | 17 ++- importer/dagwriter/dagwriter_test.go | 25 ++++ importer/format/format.go | 39 +++++ importer/importer.go | 16 ++- merkledag/merkledag.go | 47 +++--- 9 files changed, 489 insertions(+), 37 deletions(-) create mode 100644 importer/dagwriter/dagmodifier.go create mode 100644 importer/dagwriter/dagmodifier_test.go diff --git a/.gitignore b/.gitignore index a14ae54c8ac..fcbdfe803ec 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ *.out *.test *.orig +*~ diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index 9b150142445..b8e445568ed 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -36,7 +36,7 @@ func NewBlockService(d ds.Datastore, rem exchange.Interface) (*BlockService, err // AddBlock adds a particular block to the service, Putting it into the datastore. func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { k := b.Key() - log.Debug("storing [%s] in datastore", k) + log.Debug("blockservice: storing [%s] in datastore", k.Pretty()) // TODO(brian): define a block datastore with a Put method which accepts a // block parameter err := s.Datastore.Put(k.DsKey(), b.Data) @@ -53,7 +53,7 @@ func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { // GetBlock retrieves a particular block from the service, // Getting it from the datastore using the key (hash). func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) { - log.Debug("BlockService GetBlock: '%s'", k) + log.Debug("BlockService GetBlock: '%s'", k.Pretty()) datai, err := s.Datastore.Get(k.DsKey()) if err == nil { log.Debug("Blockservice: Got data in datastore.") diff --git a/importer/dagwriter/dagmodifier.go b/importer/dagwriter/dagmodifier.go new file mode 100644 index 00000000000..f1fa5b3482f --- /dev/null +++ b/importer/dagwriter/dagmodifier.go @@ -0,0 +1,190 @@ +package dagwriter + +import ( + "errors" + + "code.google.com/p/goprotobuf/proto" + + imp "github.com/jbenet/go-ipfs/importer" + ft "github.com/jbenet/go-ipfs/importer/format" + mdag "github.com/jbenet/go-ipfs/merkledag" + u "github.com/jbenet/go-ipfs/util" +) + +// DagModifier is the only struct licensed and able to correctly +// perform surgery on a DAG 'file' +// Dear god, please rename this to something more pleasant +type DagModifier struct { + dagserv *mdag.DAGService + curNode *mdag.Node + + pbdata *ft.PBData +} + +func NewDagModifier(from *mdag.Node, serv *mdag.DAGService) (*DagModifier, error) { + pbd, err := ft.FromBytes(from.Data) + if err != nil { + return nil, err + } + + return &DagModifier{ + curNode: from.Copy(), + dagserv: serv, + pbdata: pbd, + }, nil +} + +// WriteAt will modify a dag file in place +// NOTE: it currently assumes only a single level of indirection +func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { + + // Check bounds + if dm.pbdata.GetFilesize() < offset { + return 0, errors.New("Attempted to perform write starting past end of file") + } + + // This shouldnt be necessary if we do subblocks sizes properly + newsize := dm.pbdata.GetFilesize() + if uint64(len(b))+offset > dm.pbdata.GetFilesize() { + newsize = uint64(len(b)) + offset + } + + // First need to find where we are writing at + end := uint64(len(b)) + offset + zeroblocklen := uint64(len(dm.pbdata.Data)) + origlen := len(b) + + if end <= zeroblocklen { + log.Debug("Writing into zero block.") + // Replacing zeroeth data block (embedded in the root node) + //TODO: check chunking here + copy(dm.pbdata.Data[offset:], b) + return len(b), nil + } + + // Find where write should start + var traversed uint64 + startsubblk := len(dm.pbdata.Blocksizes) + if offset < zeroblocklen { + dm.pbdata.Data = dm.pbdata.Data[:offset] + startsubblk = 0 + } else { + traversed = uint64(zeroblocklen) + for i, size := range dm.pbdata.Blocksizes { + if uint64(offset) < traversed+size { + log.Debug("Starting mod at block %d. [%d < %d + %d]", i, offset, traversed, size) + // Here is where we start + startsubblk = i + lnk := dm.curNode.Links[i] + node, err := dm.dagserv.Get(u.Key(lnk.Hash)) + if err != nil { + return 0, err + } + data, err := ft.UnwrapData(node.Data) + if err != nil { + return 0, err + } + b = append(data[:offset-traversed], b...) + break + } + traversed += size + } + if startsubblk == len(dm.pbdata.Blocksizes) { + // TODO: something? + /* + if traversed < offset { + return 0, errors.New("Tried to start write outside bounds of file.") + } + */ + } + } + + // Find blocks that need to be overwritten + var changed []int + mid := -1 + var midoff uint64 + for i, size := range dm.pbdata.Blocksizes[startsubblk:] { + if end > traversed { + changed = append(changed, i+startsubblk) + } else if end == traversed { + break + } else { + break + } + traversed += size + if end < traversed { + mid = i + startsubblk + midoff = end - (traversed - size) + break + } + } + + var midlnk *mdag.Link + if mid >= 0 { + midlnk = dm.curNode.Links[mid] + midnode, err := dm.dagserv.Get(u.Key(midlnk.Hash)) + if err != nil { + return 0, err + } + + // NOTE: this may have to be changed later when we have multiple + // layers of indirection + data, err := ft.UnwrapData(midnode.Data) + if err != nil { + return 0, err + } + b = append(b, data[midoff:]...) + } + + // TODO: dont assume a splitting func here + subblocks := splitBytes(b, &imp.SizeSplitter2{512}) + var links []*mdag.Link + var sizes []uint64 + for _, sb := range subblocks { + n := &mdag.Node{Data: ft.WrapData(sb)} + _, err := dm.dagserv.Add(n) + if err != nil { + log.Error("Failed adding node to DAG service: %s", err) + return 0, err + } + lnk, err := mdag.MakeLink(n) + if err != nil { + return 0, err + } + links = append(links, lnk) + sizes = append(sizes, uint64(len(sb))) + } + + // This is disgusting + if len(changed) > 0 { + dm.curNode.Links = append(dm.curNode.Links[:changed[0]], append(links, dm.curNode.Links[changed[len(changed)-1]+1:]...)...) + dm.pbdata.Blocksizes = append(dm.pbdata.Blocksizes[:changed[0]], append(sizes, dm.pbdata.Blocksizes[changed[len(changed)-1]+1:]...)...) + } else { + dm.curNode.Links = append(dm.curNode.Links, links...) + dm.pbdata.Blocksizes = append(dm.pbdata.Blocksizes, sizes...) + } + dm.pbdata.Filesize = proto.Uint64(newsize) + + return origlen, nil +} + +func splitBytes(b []byte, spl imp.StreamSplitter) [][]byte { + ch := make(chan []byte) + out := spl.Split(ch) + ch <- b + close(ch) + var arr [][]byte + for blk := range out { + arr = append(arr, blk) + } + return arr +} + +func (dm *DagModifier) GetNode() (*mdag.Node, error) { + b, err := proto.Marshal(dm.pbdata) + if err != nil { + return nil, err + } + dm.curNode.Data = b + return dm.curNode.Copy(), nil +} diff --git a/importer/dagwriter/dagmodifier_test.go b/importer/dagwriter/dagmodifier_test.go new file mode 100644 index 00000000000..7d5497258ee --- /dev/null +++ b/importer/dagwriter/dagmodifier_test.go @@ -0,0 +1,187 @@ +package dagwriter + +import ( + "fmt" + "io" + "io/ioutil" + "math/rand" + "testing" + "time" + + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" + bs "github.com/jbenet/go-ipfs/blockservice" + imp "github.com/jbenet/go-ipfs/importer" + ft "github.com/jbenet/go-ipfs/importer/format" + mdag "github.com/jbenet/go-ipfs/merkledag" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" +) + +type randGen struct { + src rand.Source +} + +func newRand() *randGen { + return &randGen{rand.NewSource(time.Now().UnixNano())} +} + +func (r *randGen) Read(p []byte) (n int, err error) { + todo := len(p) + offset := 0 + for { + val := int64(r.src.Int63()) + for i := 0; i < 8; i++ { + p[offset] = byte(val & 0xff) + todo-- + if todo == 0 { + return len(p), nil + } + offset++ + val >>= 8 + } + } + + panic("unreachable") +} + +func getMockDagServ(t *testing.T) *mdag.DAGService { + dstore := ds.NewMapDatastore() + bserv, err := bs.NewBlockService(dstore, nil) + if err != nil { + t.Fatal(err) + } + return &mdag.DAGService{bserv} +} + +func getNode(t *testing.T, dserv *mdag.DAGService, size int64) ([]byte, *mdag.Node) { + dw := NewDagWriter(dserv, &imp.SizeSplitter2{500}) + + n, err := io.CopyN(dw, newRand(), size) + if err != nil { + t.Fatal(err) + } + if n != size { + t.Fatal("Incorrect copy amount!") + } + + dw.Close() + node := dw.GetNode() + + dr, err := mdag.NewDagReader(node, dserv) + if err != nil { + t.Fatal(err) + } + + b, err := ioutil.ReadAll(dr) + if err != nil { + t.Fatal(err) + } + + return b, node +} + +func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier) []byte { + newdata := make([]byte, size) + r := newRand() + r.Read(newdata) + + if size+beg > uint64(len(orig)) { + orig = append(orig, make([]byte, (size+beg)-uint64(len(orig)))...) + } + copy(orig[beg:], newdata) + + nmod, err := dm.WriteAt(newdata, uint64(beg)) + if err != nil { + t.Fatal(err) + } + + if nmod != int(size) { + t.Fatalf("Mod length not correct! %d != %d", nmod, size) + } + + nd, err := dm.GetNode() + if err != nil { + t.Fatal(err) + } + + rd, err := mdag.NewDagReader(nd, dm.dagserv) + if err != nil { + t.Fatal(err) + } + + after, err := ioutil.ReadAll(rd) + if err != nil { + t.Fatal(err) + } + + err = arrComp(after, orig) + if err != nil { + t.Fatal(err) + } + return orig +} + +func TestDagModifierBasic(t *testing.T) { + logging.SetLevel(logging.CRITICAL, "blockservice") + logging.SetLevel(logging.CRITICAL, "merkledag") + dserv := getMockDagServ(t) + b, n := getNode(t, dserv, 50000) + + dagmod, err := NewDagModifier(n, dserv) + if err != nil { + t.Fatal(err) + } + + // Within zero block + beg := uint64(15) + length := uint64(60) + + t.Log("Testing mod within zero block") + b = testModWrite(t, beg, length, b, dagmod) + + // Within bounds of existing file + beg = 1000 + length = 4000 + t.Log("Testing mod within bounds of existing file.") + b = testModWrite(t, beg, length, b, dagmod) + + // Extend bounds + beg = 49500 + length = 4000 + + t.Log("Testing mod that extends file.") + b = testModWrite(t, beg, length, b, dagmod) + + // "Append" + beg = uint64(len(b)) + length = 3000 + b = testModWrite(t, beg, length, b, dagmod) + + // Verify reported length + node, err := dagmod.GetNode() + if err != nil { + t.Fatal(err) + } + + size, err := ft.DataSize(node.Data) + if err != nil { + t.Fatal(err) + } + + expected := uint64(50000 + 3500 + 3000) + if size != expected { + t.Fatal("Final reported size is incorrect [%d != %d]", size, expected) + } +} + +func arrComp(a, b []byte) error { + if len(a) != len(b) { + return fmt.Errorf("Arrays differ in length. %d != %d", len(a), len(b)) + } + for i, v := range a { + if v != b[i] { + return fmt.Errorf("Arrays differ at index: %d", i) + } + } + return nil +} diff --git a/importer/dagwriter/dagwriter.go b/importer/dagwriter/dagwriter.go index 2c2bbafb711..23c33a1440c 100644 --- a/importer/dagwriter/dagwriter.go +++ b/importer/dagwriter/dagwriter.go @@ -32,10 +32,11 @@ func NewDagWriter(ds *dag.DAGService, splitter imp.StreamSplitter) *DagWriter { func (dw *DagWriter) startSplitter() { blkchan := dw.splitter.Split(dw.splChan) first := <-blkchan + mbf := new(ft.MultiBlock) root := new(dag.Node) - fileSize := uint64(0) + for blkData := range blkchan { - fileSize += uint64(len(blkData)) + mbf.AddBlockSize(uint64(len(blkData))) node := &dag.Node{Data: ft.WrapData(blkData)} _, err := dw.dagserv.Add(node) if err != nil { @@ -50,8 +51,16 @@ func (dw *DagWriter) startSplitter() { return } } - root.Data = ft.FilePBData(first, fileSize) - _, err := dw.dagserv.Add(root) + mbf.Data = first + data, err := mbf.GetBytes() + if err != nil { + dw.seterr = err + log.Critical("Failed generating bytes for multiblock file: %s", err) + return + } + root.Data = data + + _, err = dw.dagserv.Add(root) if err != nil { dw.seterr = err log.Critical("Got error adding created node to dagservice: %s", err) diff --git a/importer/dagwriter/dagwriter_test.go b/importer/dagwriter/dagwriter_test.go index 114e363bf02..b7b6cc21bd2 100644 --- a/importer/dagwriter/dagwriter_test.go +++ b/importer/dagwriter/dagwriter_test.go @@ -100,3 +100,28 @@ func TestMassiveWrite(t *testing.T) { } dw.Close() } + +func BenchmarkDagWriter(b *testing.B) { + dstore := ds.NewNullDatastore() + bserv, err := bs.NewBlockService(dstore, nil) + if err != nil { + b.Fatal(err) + } + dag := &mdag.DAGService{bserv} + + b.ResetTimer() + nbytes := int64(b.N) + for i := 0; i < b.N; i++ { + b.SetBytes(nbytes) + dw := NewDagWriter(dag, &imp.SizeSplitter2{4096}) + n, err := io.CopyN(dw, &datasource{}, nbytes) + if err != nil { + b.Fatal(err) + } + if n != nbytes { + b.Fatal("Incorrect copy size.") + } + dw.Close() + } + +} diff --git a/importer/format/format.go b/importer/format/format.go index f6dc14a394c..1fd5fe0f521 100644 --- a/importer/format/format.go +++ b/importer/format/format.go @@ -8,6 +8,15 @@ import ( "code.google.com/p/goprotobuf/proto" ) +func FromBytes(data []byte) (*PBData, error) { + pbdata := new(PBData) + err := proto.Unmarshal(data, pbdata) + if err != nil { + return nil, err + } + return pbdata, nil +} + func FilePBData(data []byte, totalsize uint64) []byte { pbfile := new(PBData) typ := PBData_File @@ -51,6 +60,15 @@ func WrapData(b []byte) []byte { return out } +func UnwrapData(data []byte) ([]byte, error) { + pbdata := new(PBData) + err := proto.Unmarshal(data, pbdata) + if err != nil { + return nil, err + } + return pbdata.GetData(), nil +} + func DataSize(data []byte) (uint64, error) { pbdata := new(PBData) err := proto.Unmarshal(data, pbdata) @@ -69,3 +87,24 @@ func DataSize(data []byte) (uint64, error) { return 0, errors.New("Unrecognized node data type!") } } + +type MultiBlock struct { + Data []byte + blocksizes []uint64 + subtotal uint64 +} + +func (mb *MultiBlock) AddBlockSize(s uint64) { + mb.subtotal += s + mb.blocksizes = append(mb.blocksizes, s) +} + +func (mb *MultiBlock) GetBytes() ([]byte, error) { + pbn := new(PBData) + t := PBData_File + pbn.Type = &t + pbn.Filesize = proto.Uint64(uint64(len(mb.Data)) + mb.subtotal) + pbn.Blocksizes = mb.blocksizes + pbn.Data = mb.Data + return proto.Marshal(pbn) +} diff --git a/importer/importer.go b/importer/importer.go index bbf6b51372c..e64f440b580 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -31,19 +31,23 @@ func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, er first := <-blkChan root := &dag.Node{} - i := 0 - totalsize := uint64(len(first)) + mbf := new(ft.MultiBlock) for blk := range blkChan { - totalsize += uint64(len(blk)) + mbf.AddBlockSize(uint64(len(blk))) child := &dag.Node{Data: ft.WrapData(blk)} - err := root.AddNodeLink(fmt.Sprintf("%d", i), child) + err := root.AddNodeLink("", child) if err != nil { return nil, err } - i++ } - root.Data = ft.FilePBData(first, totalsize) + mbf.Data = first + data, err := mbf.GetBytes() + if err != nil { + return nil, err + } + + root.Data = data return root, nil } diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index e7c13873c41..ba22f56e74d 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -34,9 +34,6 @@ type Link struct { // cumulative size of target object Size uint64 - // cumulative size of data stored in object - DataSize uint64 - // multihash of the target object Hash mh.Multihash @@ -44,45 +41,45 @@ type Link struct { Node *Node } -// AddNodeLink adds a link to another node. -func (n *Node) AddNodeLink(name string, that *Node) error { - s, err := that.Size() +func MakeLink(n *Node) (*Link, error) { + s, err := n.Size() if err != nil { - return err + return nil, err } - h, err := that.Multihash() + h, err := n.Multihash() if err != nil { - return err + return nil, err } - - n.Links = append(n.Links, &Link{ - Name: name, + return &Link{ Size: s, Hash: h, - Node: that, - }) - return nil + }, nil } -// AddNodeLink adds a link to another node. without keeping a reference to -// the child node -func (n *Node) AddNodeLinkClean(name string, that *Node) error { - s, err := that.Size() +// AddNodeLink adds a link to another node. +func (n *Node) AddNodeLink(name string, that *Node) error { + lnk, err := MakeLink(that) if err != nil { return err } + lnk.Name = name + lnk.Node = that + + n.Links = append(n.Links, lnk) + return nil +} - h, err := that.Multihash() +// AddNodeLink adds a link to another node. without keeping a reference to +// the child node +func (n *Node) AddNodeLinkClean(name string, that *Node) error { + lnk, err := MakeLink(that) if err != nil { return err } + lnk.Name = name - n.Links = append(n.Links, &Link{ - Name: name, - Size: s, - Hash: h, - }) + n.Links = append(n.Links, lnk) return nil } From 5c802ae8524004bf37db96e98ae518bf1cf90252 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 7 Oct 2014 05:55:28 +0000 Subject: [PATCH 079/105] add more tests and rework a lot of utility structures --- importer/dagwriter/dagmodifier.go | 23 ++++---- importer/dagwriter/dagmodifier_test.go | 38 ++----------- importer/dagwriter/dagwriter.go | 7 ++- importer/dagwriter/dagwriter_test.go | 6 +- importer/format/format_test.go | 36 ++++++++++++ importer/importer.go | 3 + importer/rabin.go | 40 -------------- importer/splitting.go | 39 +------------ util/util.go | 76 ++++++++++++++++++++++++++ util/util_test.go | 35 +++++++++++- 10 files changed, 175 insertions(+), 128 deletions(-) create mode 100644 importer/format/format_test.go diff --git a/importer/dagwriter/dagmodifier.go b/importer/dagwriter/dagmodifier.go index f1fa5b3482f..b749da22e06 100644 --- a/importer/dagwriter/dagmodifier.go +++ b/importer/dagwriter/dagmodifier.go @@ -1,6 +1,7 @@ package dagwriter import ( + "bytes" "errors" "code.google.com/p/goprotobuf/proto" @@ -18,19 +19,21 @@ type DagModifier struct { dagserv *mdag.DAGService curNode *mdag.Node - pbdata *ft.PBData + pbdata *ft.PBData + splitter imp.BlockSplitter } -func NewDagModifier(from *mdag.Node, serv *mdag.DAGService) (*DagModifier, error) { +func NewDagModifier(from *mdag.Node, serv *mdag.DAGService, spl imp.BlockSplitter) (*DagModifier, error) { pbd, err := ft.FromBytes(from.Data) if err != nil { return nil, err } return &DagModifier{ - curNode: from.Copy(), - dagserv: serv, - pbdata: pbd, + curNode: from.Copy(), + dagserv: serv, + pbdata: pbd, + splitter: spl, }, nil } @@ -136,8 +139,7 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { b = append(b, data[midoff:]...) } - // TODO: dont assume a splitting func here - subblocks := splitBytes(b, &imp.SizeSplitter2{512}) + subblocks := splitBytes(b, dm.splitter) var links []*mdag.Link var sizes []uint64 for _, sb := range subblocks { @@ -168,11 +170,8 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { return origlen, nil } -func splitBytes(b []byte, spl imp.StreamSplitter) [][]byte { - ch := make(chan []byte) - out := spl.Split(ch) - ch <- b - close(ch) +func splitBytes(b []byte, spl imp.BlockSplitter) [][]byte { + out := spl.Split(bytes.NewReader(b)) var arr [][]byte for blk := range out { arr = append(arr, blk) diff --git a/importer/dagwriter/dagmodifier_test.go b/importer/dagwriter/dagmodifier_test.go index 7d5497258ee..129d836d5b6 100644 --- a/importer/dagwriter/dagmodifier_test.go +++ b/importer/dagwriter/dagmodifier_test.go @@ -4,46 +4,18 @@ import ( "fmt" "io" "io/ioutil" - "math/rand" "testing" - "time" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" bs "github.com/jbenet/go-ipfs/blockservice" imp "github.com/jbenet/go-ipfs/importer" ft "github.com/jbenet/go-ipfs/importer/format" mdag "github.com/jbenet/go-ipfs/merkledag" + u "github.com/jbenet/go-ipfs/util" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" ) -type randGen struct { - src rand.Source -} - -func newRand() *randGen { - return &randGen{rand.NewSource(time.Now().UnixNano())} -} - -func (r *randGen) Read(p []byte) (n int, err error) { - todo := len(p) - offset := 0 - for { - val := int64(r.src.Int63()) - for i := 0; i < 8; i++ { - p[offset] = byte(val & 0xff) - todo-- - if todo == 0 { - return len(p), nil - } - offset++ - val >>= 8 - } - } - - panic("unreachable") -} - func getMockDagServ(t *testing.T) *mdag.DAGService { dstore := ds.NewMapDatastore() bserv, err := bs.NewBlockService(dstore, nil) @@ -54,9 +26,9 @@ func getMockDagServ(t *testing.T) *mdag.DAGService { } func getNode(t *testing.T, dserv *mdag.DAGService, size int64) ([]byte, *mdag.Node) { - dw := NewDagWriter(dserv, &imp.SizeSplitter2{500}) + dw := NewDagWriter(dserv, &imp.SizeSplitter{500}) - n, err := io.CopyN(dw, newRand(), size) + n, err := io.CopyN(dw, u.NewFastRand(), size) if err != nil { t.Fatal(err) } @@ -82,7 +54,7 @@ func getNode(t *testing.T, dserv *mdag.DAGService, size int64) ([]byte, *mdag.No func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier) []byte { newdata := make([]byte, size) - r := newRand() + r := u.NewFastRand() r.Read(newdata) if size+beg > uint64(len(orig)) { @@ -127,7 +99,7 @@ func TestDagModifierBasic(t *testing.T) { dserv := getMockDagServ(t) b, n := getNode(t, dserv, 50000) - dagmod, err := NewDagModifier(n, dserv) + dagmod, err := NewDagModifier(n, dserv, &imp.SizeSplitter{512}) if err != nil { t.Fatal(err) } diff --git a/importer/dagwriter/dagwriter.go b/importer/dagwriter/dagwriter.go index 23c33a1440c..2396181360c 100644 --- a/importer/dagwriter/dagwriter.go +++ b/importer/dagwriter/dagwriter.go @@ -15,11 +15,11 @@ type DagWriter struct { totalSize int64 splChan chan []byte done chan struct{} - splitter imp.StreamSplitter + splitter imp.BlockSplitter seterr error } -func NewDagWriter(ds *dag.DAGService, splitter imp.StreamSplitter) *DagWriter { +func NewDagWriter(ds *dag.DAGService, splitter imp.BlockSplitter) *DagWriter { dw := new(DagWriter) dw.dagserv = ds dw.splChan = make(chan []byte, 8) @@ -30,7 +30,8 @@ func NewDagWriter(ds *dag.DAGService, splitter imp.StreamSplitter) *DagWriter { } func (dw *DagWriter) startSplitter() { - blkchan := dw.splitter.Split(dw.splChan) + r := util.NewByteChanReader(dw.splChan) + blkchan := dw.splitter.Split(r) first := <-blkchan mbf := new(ft.MultiBlock) root := new(dag.Node) diff --git a/importer/dagwriter/dagwriter_test.go b/importer/dagwriter/dagwriter_test.go index b7b6cc21bd2..ec6d5644946 100644 --- a/importer/dagwriter/dagwriter_test.go +++ b/importer/dagwriter/dagwriter_test.go @@ -54,7 +54,7 @@ func TestDagWriter(t *testing.T) { t.Fatal(err) } dag := &mdag.DAGService{bserv} - dw := NewDagWriter(dag, &imp.SizeSplitter2{4096}) + dw := NewDagWriter(dag, &imp.SizeSplitter{4096}) nbytes := int64(1024 * 1024 * 2) n, err := io.CopyN(dw, &datasource{}, nbytes) @@ -88,7 +88,7 @@ func TestMassiveWrite(t *testing.T) { t.Fatal(err) } dag := &mdag.DAGService{bserv} - dw := NewDagWriter(dag, &imp.SizeSplitter2{4096}) + dw := NewDagWriter(dag, &imp.SizeSplitter{4096}) nbytes := int64(1024 * 1024 * 1024 * 16) n, err := io.CopyN(dw, &datasource{}, nbytes) @@ -113,7 +113,7 @@ func BenchmarkDagWriter(b *testing.B) { nbytes := int64(b.N) for i := 0; i < b.N; i++ { b.SetBytes(nbytes) - dw := NewDagWriter(dag, &imp.SizeSplitter2{4096}) + dw := NewDagWriter(dag, &imp.SizeSplitter{4096}) n, err := io.CopyN(dw, &datasource{}, nbytes) if err != nil { b.Fatal(err) diff --git a/importer/format/format_test.go b/importer/format/format_test.go new file mode 100644 index 00000000000..06194aefc4d --- /dev/null +++ b/importer/format/format_test.go @@ -0,0 +1,36 @@ +package format + +import ( + "testing" + + "code.google.com/p/goprotobuf/proto" +) + +func TestMultiBlock(t *testing.T) { + mbf := new(MultiBlock) + for i := 0; i < 15; i++ { + mbf.AddBlockSize(100) + } + + mbf.Data = make([]byte, 128) + + b, err := mbf.GetBytes() + if err != nil { + t.Fatal(err) + } + + pbn := new(PBData) + err = proto.Unmarshal(b, pbn) + if err != nil { + t.Fatal(err) + } + + ds, err := DataSize(b) + if err != nil { + t.Fatal(err) + } + + if ds != (100*15)+128 { + t.Fatal("Datasize calculations incorrect!") + } +} diff --git a/importer/importer.go b/importer/importer.go index e64f440b580..3c92eb3ab13 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -7,8 +7,11 @@ import ( ft "github.com/jbenet/go-ipfs/importer/format" dag "github.com/jbenet/go-ipfs/merkledag" + "github.com/jbenet/go-ipfs/util" ) +var log = util.Logger("importer") + // BlockSizeLimit specifies the maximum size an imported block can have. var BlockSizeLimit = int64(1048576) // 1 MB diff --git a/importer/rabin.go b/importer/rabin.go index 3eab5bc9cf0..64b6f8d3f31 100644 --- a/importer/rabin.go +++ b/importer/rabin.go @@ -92,43 +92,3 @@ func (mr *MaybeRabin) Split(r io.Reader) chan []byte { }() return out } - -/* -func WhyrusleepingCantImplementRabin(r io.Reader) chan []byte { - out := make(chan []byte, 4) - go func() { - buf := bufio.NewReader(r) - blkbuf := new(bytes.Buffer) - window := make([]byte, 16) - var val uint64 - prime := uint64(61) - - get := func(i int) uint64 { - return uint64(window[i%len(window)]) - } - - set := func(i int, val byte) { - window[i%len(window)] = val - } - - for i := 0; ; i++ { - curb, err := buf.ReadByte() - if err != nil { - break - } - set(i, curb) - blkbuf.WriteByte(curb) - - hash := md5.Sum(window) - if hash[0] == 0 && hash[1] == 0 { - out <- blkbuf.Bytes() - blkbuf.Reset() - } - } - out <- blkbuf.Bytes() - close(out) - }() - - return out -} -*/ diff --git a/importer/splitting.go b/importer/splitting.go index 65b67248682..7b6e56ceee7 100644 --- a/importer/splitting.go +++ b/importer/splitting.go @@ -1,19 +1,9 @@ package importer -import ( - "io" +import "io" - u "github.com/jbenet/go-ipfs/util" -) - -// OLD type BlockSplitter interface { - Split(io.Reader) chan []byte -} - -// NEW -type StreamSplitter interface { - Split(chan []byte) chan []byte + Split(r io.Reader) chan []byte } type SizeSplitter struct { @@ -34,7 +24,7 @@ func (ss *SizeSplitter) Split(r io.Reader) chan []byte { } return } - u.PErr("block split error: %v\n", err) + log.Error("Block split error: %s", err) return } if nread < ss.Size { @@ -45,26 +35,3 @@ func (ss *SizeSplitter) Split(r io.Reader) chan []byte { }() return out } - -type SizeSplitter2 struct { - Size int -} - -func (ss *SizeSplitter2) Split(in chan []byte) chan []byte { - out := make(chan []byte) - go func() { - defer close(out) - var buf []byte - for b := range in { - buf = append(buf, b...) - for len(buf) > ss.Size { - out <- buf[:ss.Size] - buf = buf[ss.Size:] - } - } - if len(buf) > 0 { - out <- buf - } - }() - return out -} diff --git a/util/util.go b/util/util.go index 3a55188dc41..5a9d21e815b 100644 --- a/util/util.go +++ b/util/util.go @@ -3,10 +3,13 @@ package util import ( "errors" "fmt" + "io" + "math/rand" "os" "os/user" "path/filepath" "strings" + "time" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" @@ -154,3 +157,76 @@ func ExpandPathnames(paths []string) ([]string, error) { } return out, nil } + +// byteChanReader wraps a byte chan in a reader +type byteChanReader struct { + in chan []byte + buf []byte +} + +func NewByteChanReader(in chan []byte) io.Reader { + return &byteChanReader{in: in} +} + +func (bcr *byteChanReader) Read(b []byte) (int, error) { + if len(bcr.buf) == 0 { + data, ok := <-bcr.in + if !ok { + return 0, io.EOF + } + bcr.buf = data + } + + if len(bcr.buf) >= len(b) { + copy(b, bcr.buf) + bcr.buf = bcr.buf[len(b):] + return len(b), nil + } + + copy(b, bcr.buf) + b = b[len(bcr.buf):] + totread := len(bcr.buf) + + for data := range bcr.in { + if len(data) > len(b) { + totread += len(b) + copy(b, data[:len(b)]) + bcr.buf = data[len(b):] + return totread, nil + } + copy(b, data) + totread += len(data) + b = b[len(data):] + if len(b) == 0 { + return totread, nil + } + } + return totread, io.EOF +} + +type randGen struct { + src rand.Source +} + +func NewFastRand() io.Reader { + return &randGen{rand.NewSource(time.Now().UnixNano())} +} + +func (r *randGen) Read(p []byte) (n int, err error) { + todo := len(p) + offset := 0 + for { + val := int64(r.src.Int63()) + for i := 0; i < 8; i++ { + p[offset] = byte(val & 0xff) + todo-- + if todo == 0 { + return len(p), nil + } + offset++ + val >>= 8 + } + } + + panic("unreachable") +} diff --git a/util/util_test.go b/util/util_test.go index 821a97e66e4..a85c492feeb 100644 --- a/util/util_test.go +++ b/util/util_test.go @@ -2,8 +2,11 @@ package util import ( "bytes" - mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" + "io/ioutil" + "math/rand" "testing" + + mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" ) func TestKey(t *testing.T) { @@ -25,3 +28,33 @@ func TestKey(t *testing.T) { t.Error("Keys not equal.") } } + +func TestByteChanReader(t *testing.T) { + data := make([]byte, 1024*1024) + r := NewFastRand() + r.Read(data) + dch := make(chan []byte, 8) + + go func() { + beg := 0 + for i := 0; i < len(data); { + i += rand.Intn(100) + 1 + if i > len(data) { + i = len(data) + } + dch <- data[beg:i] + beg = i + } + close(dch) + }() + + read := NewByteChanReader(dch) + out, err := ioutil.ReadAll(read) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(out, data) { + t.Fatal("Reader failed to stream correct bytes") + } +} From 98cde1578d975fd655747643299e1db55e9ba54b Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 7 Oct 2014 07:23:30 +0000 Subject: [PATCH 080/105] integrate dagmodifier into ipns --- fuse/ipns/ipns_unix.go | 45 +++++++++++++------------------ importer/dagwriter/dagmodifier.go | 10 ++++--- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 7e6b1b33e35..dc03d51138e 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -1,7 +1,6 @@ package ipns import ( - "bytes" "io/ioutil" "os" "path/filepath" @@ -14,6 +13,7 @@ import ( "github.com/jbenet/go-ipfs/core" ci "github.com/jbenet/go-ipfs/crypto" imp "github.com/jbenet/go-ipfs/importer" + dt "github.com/jbenet/go-ipfs/importer/dagwriter" ft "github.com/jbenet/go-ipfs/importer/format" mdag "github.com/jbenet/go-ipfs/merkledag" u "github.com/jbenet/go-ipfs/util" @@ -199,11 +199,8 @@ type Node struct { Ipfs *core.IpfsNode Nd *mdag.Node - fd *mdag.DagReader + dagMod *dt.DagModifier cached *ft.PBData - - // For writing - writerBuf WriteAtBuf } func (s *Node) loadData() error { @@ -303,35 +300,32 @@ func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error { log.Debug("ipns: Node Write [%s]: flags = %s, offset = %d, size = %d", n.name, req.Flags.String(), req.Offset, len(req.Data)) - if n.writerBuf == nil { - n.writerBuf = NewWriterAtFromBytes(nil) + if n.dagMod == nil { + dmod, err := dt.NewDagModifier(n.Nd, n.Ipfs.DAG, imp.DefaultSplitter) + if err != nil { + log.Error("Error creating dag modifier: %s", err) + return err + } + n.dagMod = dmod } - _, err := n.writerBuf.WriteAt(req.Data, req.Offset) + wrote, err := n.dagMod.WriteAt(req.Data, uint64(req.Offset)) if err != nil { return err } - resp.Size = len(req.Data) + resp.Size = wrote return nil } func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { log.Debug("Got flush request [%s]!", n.name) - if n.writerBuf != nil { - //TODO: - // This operation holds everything in memory, - // should be changed to stream the block creation/storage - // but for now, since the buf is all in memory anyways... - - //NOTE: - // This should only occur on a file object, if this were to be a - // folder, bad things would happen. - buf := bytes.NewReader(n.writerBuf.Bytes()) - newNode, err := imp.NewDagFromReader(buf) + if n.dagMod != nil { + newNode, err := n.dagMod.GetNode() if err != nil { - log.Critical("error creating dag from writerBuf: %s", err) + log.Error("Error getting dag node from dagMod: %s", err) return err } + if n.parent != nil { log.Debug("updating self in parent!") err := n.parent.update(n.name, newNode) @@ -358,7 +352,7 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { fmt.Println(b) //*/ - n.writerBuf = nil + n.dagMod = nil n.wasChanged() } @@ -390,8 +384,6 @@ func (n *Node) republishRoot() error { return err } - n.writerBuf = nil - ndkey, err := root.Nd.Key() if err != nil { log.Error("getKey error: %s", err) @@ -451,8 +443,9 @@ func (n *Node) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr //TODO: check open flags and truncate if necessary if req.Flags&fuse.OpenTruncate != 0 { log.Warning("Need to truncate file!") - } - if req.Flags&fuse.OpenAppend != 0 { + n.cached = nil + n.Nd = &mdag.Node{Data: ft.FilePBData(nil, 0)} + } else if req.Flags&fuse.OpenAppend != 0 { log.Warning("Need to append to file!") } return n, nil diff --git a/importer/dagwriter/dagmodifier.go b/importer/dagwriter/dagmodifier.go index b749da22e06..cdcfa17edfd 100644 --- a/importer/dagwriter/dagmodifier.go +++ b/importer/dagwriter/dagmodifier.go @@ -109,8 +109,6 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { for i, size := range dm.pbdata.Blocksizes[startsubblk:] { if end > traversed { changed = append(changed, i+startsubblk) - } else if end == traversed { - break } else { break } @@ -122,6 +120,7 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { } } + // If our write starts in the middle of a block... var midlnk *mdag.Link if mid >= 0 { midlnk = dm.curNode.Links[mid] @@ -139,6 +138,7 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { b = append(b, data[midoff:]...) } + // Generate new sub-blocks, and sizes subblocks := splitBytes(b, dm.splitter) var links []*mdag.Link var sizes []uint64 @@ -159,8 +159,10 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { // This is disgusting if len(changed) > 0 { - dm.curNode.Links = append(dm.curNode.Links[:changed[0]], append(links, dm.curNode.Links[changed[len(changed)-1]+1:]...)...) - dm.pbdata.Blocksizes = append(dm.pbdata.Blocksizes[:changed[0]], append(sizes, dm.pbdata.Blocksizes[changed[len(changed)-1]+1:]...)...) + sechalflink := append(links, dm.curNode.Links[changed[len(changed)-1]+1:]...) + dm.curNode.Links = append(dm.curNode.Links[:changed[0]], sechalflink...) + sechalfblks := append(sizes, dm.pbdata.Blocksizes[changed[len(changed)-1]+1:]...) + dm.pbdata.Blocksizes = append(dm.pbdata.Blocksizes[:changed[0]], sechalfblks...) } else { dm.curNode.Links = append(dm.curNode.Links, links...) dm.pbdata.Blocksizes = append(dm.pbdata.Blocksizes, sizes...) From 6e0cfb327309266c156aae497625d90048ffdec8 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 7 Oct 2014 20:46:01 +0000 Subject: [PATCH 081/105] removed error from return type of blocks.NewBlock() --- blocks/blocks.go | 4 +-- blocks/blocks_test.go | 15 ++-------- blockstore/blockstore.go | 2 +- blockstore/blockstore_test.go | 6 ++-- exchange/bitswap/bitswap_test.go | 28 ++++++++----------- exchange/bitswap/message/message.go | 14 +++------- exchange/bitswap/message/message_test.go | 19 ++++++------- .../notifications/notifications_test.go | 10 +++---- exchange/bitswap/strategy/strategy_test.go | 6 ++-- exchange/bitswap/testnet/network_test.go | 10 +++---- exchange/offline/offline_test.go | 6 ++-- importer/dagwriter/dagmodifier.go | 10 +++---- importer/dagwriter/dagwriter_test.go | 2 +- merkledag/coding.go | 3 ++ merkledag/merkledag.go | 10 +++++-- util/testutil/blocks.go | 22 --------------- 16 files changed, 64 insertions(+), 103 deletions(-) delete mode 100644 util/testutil/blocks.go diff --git a/blocks/blocks.go b/blocks/blocks.go index b45c1d1fb7f..696c774ab4d 100644 --- a/blocks/blocks.go +++ b/blocks/blocks.go @@ -12,8 +12,8 @@ type Block struct { } // NewBlock creates a Block object from opaque data. It will hash the data. -func NewBlock(data []byte) (*Block, error) { - return &Block{Data: data, Multihash: u.Hash(data)}, nil +func NewBlock(data []byte) *Block { + return &Block{Data: data, Multihash: u.Hash(data)} } // Key returns the block's Multihash as a Key value. diff --git a/blocks/blocks_test.go b/blocks/blocks_test.go index 915d84c023f..53a85227575 100644 --- a/blocks/blocks_test.go +++ b/blocks/blocks_test.go @@ -6,20 +6,11 @@ func TestBlocksBasic(t *testing.T) { // Test empty data empty := []byte{} - _, err := NewBlock(empty) - if err != nil { - t.Fatal(err) - } + NewBlock(empty) // Test nil case - _, err = NewBlock(nil) - if err != nil { - t.Fatal(err) - } + NewBlock(nil) // Test some data - _, err = NewBlock([]byte("Hello world!")) - if err != nil { - t.Fatal(err) - } + NewBlock([]byte("Hello world!")) } diff --git a/blockstore/blockstore.go b/blockstore/blockstore.go index 860d7ba41e0..d597288768f 100644 --- a/blockstore/blockstore.go +++ b/blockstore/blockstore.go @@ -35,7 +35,7 @@ func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) { if !ok { return nil, ValueTypeMismatch } - return blocks.NewBlock(bdata) + return blocks.NewBlock(bdata), nil } func (bs *blockstore) Put(block blocks.Block) error { diff --git a/blockstore/blockstore_test.go b/blockstore/blockstore_test.go index dfb9783afb2..e64424c49df 100644 --- a/blockstore/blockstore_test.go +++ b/blockstore/blockstore_test.go @@ -5,8 +5,8 @@ import ( "testing" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + blocks "github.com/jbenet/go-ipfs/blocks" u "github.com/jbenet/go-ipfs/util" - testutil "github.com/jbenet/go-ipfs/util/testutil" ) // TODO(brian): TestGetReturnsNil @@ -24,7 +24,7 @@ func TestGetWhenKeyNotPresent(t *testing.T) { func TestPutThenGetBlock(t *testing.T) { bs := NewBlockstore(ds.NewMapDatastore()) - block := testutil.NewBlockOrFail(t, "some data") + block := blocks.NewBlock("some data") err := bs.Put(block) if err != nil { @@ -41,7 +41,7 @@ func TestPutThenGetBlock(t *testing.T) { } func TestValueTypeMismatch(t *testing.T) { - block := testutil.NewBlockOrFail(t, "some data") + block := blocks.NewBlock("some data") datastore := ds.NewMapDatastore() datastore.Put(block.Key().DsKey(), "data that isn't a block!") diff --git a/exchange/bitswap/bitswap_test.go b/exchange/bitswap/bitswap_test.go index 3a9bed97c20..fd01aacd9c1 100644 --- a/exchange/bitswap/bitswap_test.go +++ b/exchange/bitswap/bitswap_test.go @@ -9,7 +9,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" - "github.com/jbenet/go-ipfs/blocks" + blocks "github.com/jbenet/go-ipfs/blocks" bstore "github.com/jbenet/go-ipfs/blockstore" exchange "github.com/jbenet/go-ipfs/exchange" notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications" @@ -18,7 +18,6 @@ import ( peer "github.com/jbenet/go-ipfs/peer" mock "github.com/jbenet/go-ipfs/routing/mock" util "github.com/jbenet/go-ipfs/util" - testutil "github.com/jbenet/go-ipfs/util/testutil" ) func TestGetBlockTimeout(t *testing.T) { @@ -30,7 +29,7 @@ func TestGetBlockTimeout(t *testing.T) { self := g.Next() ctx, _ := context.WithTimeout(context.Background(), time.Nanosecond) - block := testutil.NewBlockOrFail(t, "block") + block := blocks.NewBlock([]byte("block")) _, err := self.exchange.Block(ctx, block.Key()) if err != context.DeadlineExceeded { @@ -44,7 +43,7 @@ func TestProviderForKeyButNetworkCannotFind(t *testing.T) { rs := mock.VirtualRoutingServer() g := NewSessionGenerator(net, rs) - block := testutil.NewBlockOrFail(t, "block") + block := blocks.NewBlock([]byte("block")) rs.Announce(&peer.Peer{}, block.Key()) // but not on network solo := g.Next() @@ -63,15 +62,15 @@ func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) { net := tn.VirtualNetwork() rs := mock.VirtualRoutingServer() - block := testutil.NewBlockOrFail(t, "block") + block := blocks.NewBlock([]byte("block")) g := NewSessionGenerator(net, rs) hasBlock := g.Next() - if err := hasBlock.blockstore.Put(block); err != nil { + if err := hasBlock.blockstore.Put(*block); err != nil { t.Fatal(err) } - if err := hasBlock.exchange.HasBlock(context.Background(), block); err != nil { + if err := hasBlock.exchange.HasBlock(context.Background(), *block); err != nil { t.Fatal(err) } @@ -93,7 +92,7 @@ func TestSwarm(t *testing.T) { net := tn.VirtualNetwork() rs := mock.VirtualRoutingServer() sg := NewSessionGenerator(net, rs) - bg := NewBlockGenerator(t) + bg := NewBlockGenerator() t.Log("Create a ton of instances, and just a few blocks") @@ -154,7 +153,7 @@ func TestSendToWantingPeer(t *testing.T) { net := tn.VirtualNetwork() rs := mock.VirtualRoutingServer() sg := NewSessionGenerator(net, rs) - bg := NewBlockGenerator(t) + bg := NewBlockGenerator() me := sg.Next() w := sg.Next() @@ -212,20 +211,17 @@ func TestSendToWantingPeer(t *testing.T) { } } -func NewBlockGenerator(t *testing.T) BlockGenerator { - return BlockGenerator{ - T: t, - } +func NewBlockGenerator() BlockGenerator { + return BlockGenerator{} } type BlockGenerator struct { - *testing.T // b/c block generation can fail - seq int + seq int } func (bg *BlockGenerator) Next() blocks.Block { bg.seq++ - return testutil.NewBlockOrFail(bg.T, string(bg.seq)) + return *blocks.NewBlock([]byte(string(bg.seq))) } func (bg *BlockGenerator) Blocks(n int) []*blocks.Block { diff --git a/exchange/bitswap/message/message.go b/exchange/bitswap/message/message.go index 22258e17fe9..a724f7cc752 100644 --- a/exchange/bitswap/message/message.go +++ b/exchange/bitswap/message/message.go @@ -32,19 +32,16 @@ func New() *message { return new(message) } -func newMessageFromProto(pbm PBMessage) (BitSwapMessage, error) { +func newMessageFromProto(pbm PBMessage) BitSwapMessage { m := New() for _, s := range pbm.GetWantlist() { m.AppendWanted(u.Key(s)) } for _, d := range pbm.GetBlocks() { - b, err := blocks.NewBlock(d) - if err != nil { - return nil, err - } + b := blocks.NewBlock(d) m.AppendBlock(*b) } - return m, nil + return m } // TODO(brian): convert these into keys @@ -70,10 +67,7 @@ func FromNet(nmsg netmsg.NetMessage) (BitSwapMessage, error) { if err := proto.Unmarshal(nmsg.Data(), pb); err != nil { return nil, err } - m, err := newMessageFromProto(*pb) - if err != nil { - return nil, err - } + m := newMessageFromProto(*pb) return m, nil } diff --git a/exchange/bitswap/message/message_test.go b/exchange/bitswap/message/message_test.go index 9590f1ff1a3..b5954eba8ed 100644 --- a/exchange/bitswap/message/message_test.go +++ b/exchange/bitswap/message/message_test.go @@ -4,9 +4,9 @@ import ( "bytes" "testing" + "github.com/jbenet/go-ipfs/blocks" peer "github.com/jbenet/go-ipfs/peer" u "github.com/jbenet/go-ipfs/util" - testutil "github.com/jbenet/go-ipfs/util/testutil" ) func TestAppendWanted(t *testing.T) { @@ -26,10 +26,7 @@ func TestNewMessageFromProto(t *testing.T) { if !contains(protoMessage.Wantlist, str) { t.Fail() } - m, err := newMessageFromProto(*protoMessage) - if err != nil { - t.Fatal(err) - } + m := newMessageFromProto(*protoMessage) if !contains(m.ToProto().GetWantlist(), str) { t.Fail() } @@ -43,8 +40,8 @@ func TestAppendBlock(t *testing.T) { m := New() for _, str := range strs { - block := testutil.NewBlockOrFail(t, str) - m.AppendBlock(block) + block := blocks.NewBlock([]byte(str)) + m.AppendBlock(*block) } // assert strings are in proto message @@ -134,10 +131,10 @@ func TestToNetFromNetPreservesWantList(t *testing.T) { func TestToAndFromNetMessage(t *testing.T) { original := New() - original.AppendBlock(testutil.NewBlockOrFail(t, "W")) - original.AppendBlock(testutil.NewBlockOrFail(t, "E")) - original.AppendBlock(testutil.NewBlockOrFail(t, "F")) - original.AppendBlock(testutil.NewBlockOrFail(t, "M")) + original.AppendBlock(*blocks.NewBlock([]byte("W"))) + original.AppendBlock(*blocks.NewBlock([]byte("E"))) + original.AppendBlock(*blocks.NewBlock([]byte("F"))) + original.AppendBlock(*blocks.NewBlock([]byte("M"))) p := &peer.Peer{ID: []byte("X")} netmsg, err := original.ToNet(p) diff --git a/exchange/bitswap/notifications/notifications_test.go b/exchange/bitswap/notifications/notifications_test.go index b12cc7d83ad..063634f61e9 100644 --- a/exchange/bitswap/notifications/notifications_test.go +++ b/exchange/bitswap/notifications/notifications_test.go @@ -6,25 +6,23 @@ import ( "time" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - testutil "github.com/jbenet/go-ipfs/util/testutil" - blocks "github.com/jbenet/go-ipfs/blocks" ) func TestPublishSubscribe(t *testing.T) { - blockSent := testutil.NewBlockOrFail(t, "Greetings from The Interval") + blockSent := blocks.NewBlock([]byte("Greetings from The Interval")) n := New() defer n.Shutdown() ch := n.Subscribe(context.Background(), blockSent.Key()) - n.Publish(blockSent) + n.Publish(*blockSent) blockRecvd, ok := <-ch if !ok { t.Fail() } - assertBlocksEqual(t, blockRecvd, blockSent) + assertBlocksEqual(t, blockRecvd, *blockSent) } @@ -35,7 +33,7 @@ func TestCarryOnWhenDeadlineExpires(t *testing.T) { n := New() defer n.Shutdown() - block := testutil.NewBlockOrFail(t, "A Missed Connection") + block := blocks.NewBlock([]byte("A Missed Connection")) blockChannel := n.Subscribe(fastExpiringCtx, block.Key()) assertBlockChannelNil(t, blockChannel) diff --git a/exchange/bitswap/strategy/strategy_test.go b/exchange/bitswap/strategy/strategy_test.go index 21f293c1c25..dccc4a37465 100644 --- a/exchange/bitswap/strategy/strategy_test.go +++ b/exchange/bitswap/strategy/strategy_test.go @@ -4,9 +4,9 @@ import ( "strings" "testing" + blocks "github.com/jbenet/go-ipfs/blocks" message "github.com/jbenet/go-ipfs/exchange/bitswap/message" peer "github.com/jbenet/go-ipfs/peer" - testutil "github.com/jbenet/go-ipfs/util/testutil" ) type peerAndStrategist struct { @@ -30,7 +30,7 @@ func TestConsistentAccounting(t *testing.T) { m := message.New() content := []string{"this", "is", "message", "i"} - m.AppendBlock(testutil.NewBlockOrFail(t, strings.Join(content, " "))) + m.AppendBlock(*blocks.NewBlock([]byte(strings.Join(content, " ")))) sender.MessageSent(receiver.Peer, m) receiver.MessageReceived(sender.Peer, m) @@ -57,7 +57,7 @@ func TestBlockRecordedAsWantedAfterMessageReceived(t *testing.T) { beggar := newPeerAndStrategist("can't be chooser") chooser := newPeerAndStrategist("chooses JIF") - block := testutil.NewBlockOrFail(t, "data wanted by beggar") + block := blocks.NewBlock([]byte("data wanted by beggar")) messageFromBeggarToChooser := message.New() messageFromBeggarToChooser.AppendWanted(block.Key()) diff --git a/exchange/bitswap/testnet/network_test.go b/exchange/bitswap/testnet/network_test.go index 15502783eac..fbd7c88936b 100644 --- a/exchange/bitswap/testnet/network_test.go +++ b/exchange/bitswap/testnet/network_test.go @@ -5,10 +5,10 @@ import ( "testing" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + blocks "github.com/jbenet/go-ipfs/blocks" bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" peer "github.com/jbenet/go-ipfs/peer" - testutil "github.com/jbenet/go-ipfs/util/testutil" ) func TestSendRequestToCooperativePeer(t *testing.T) { @@ -33,7 +33,7 @@ func TestSendRequestToCooperativePeer(t *testing.T) { // TODO test contents of incoming message m := bsmsg.New() - m.AppendBlock(testutil.NewBlockOrFail(t, expectedStr)) + m.AppendBlock(*blocks.NewBlock([]byte(expectedStr))) return from, m })) @@ -41,7 +41,7 @@ func TestSendRequestToCooperativePeer(t *testing.T) { t.Log("Build a message and send a synchronous request to recipient") message := bsmsg.New() - message.AppendBlock(testutil.NewBlockOrFail(t, "data")) + message.AppendBlock(*blocks.NewBlock([]byte("data"))) response, err := initiator.SendRequest( context.Background(), &peer.Peer{ID: idOfRecipient}, message) if err != nil { @@ -77,7 +77,7 @@ func TestSendMessageAsyncButWaitForResponse(t *testing.T) { *peer.Peer, bsmsg.BitSwapMessage) { msgToWaiter := bsmsg.New() - msgToWaiter.AppendBlock(testutil.NewBlockOrFail(t, expectedStr)) + msgToWaiter.AppendBlock(*blocks.NewBlock([]byte(expectedStr))) return fromWaiter, msgToWaiter })) @@ -105,7 +105,7 @@ func TestSendMessageAsyncButWaitForResponse(t *testing.T) { })) messageSentAsync := bsmsg.New() - messageSentAsync.AppendBlock(testutil.NewBlockOrFail(t, "data")) + messageSentAsync.AppendBlock(*blocks.NewBlock([]byte("data"))) errSending := waiter.SendMessage( context.Background(), &peer.Peer{ID: idOfResponder}, messageSentAsync) if errSending != nil { diff --git a/exchange/offline/offline_test.go b/exchange/offline/offline_test.go index 26821f2c8a3..b759a61ca37 100644 --- a/exchange/offline/offline_test.go +++ b/exchange/offline/offline_test.go @@ -5,8 +5,8 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + blocks "github.com/jbenet/go-ipfs/blocks" u "github.com/jbenet/go-ipfs/util" - testutil "github.com/jbenet/go-ipfs/util/testutil" ) func TestBlockReturnsErr(t *testing.T) { @@ -20,8 +20,8 @@ func TestBlockReturnsErr(t *testing.T) { func TestHasBlockReturnsNil(t *testing.T) { off := NewOfflineExchange() - block := testutil.NewBlockOrFail(t, "data") - err := off.HasBlock(context.Background(), block) + block := blocks.NewBlock([]byte("data")) + err := off.HasBlock(context.Background(), *block) if err != nil { t.Fatal("") } diff --git a/importer/dagwriter/dagmodifier.go b/importer/dagwriter/dagmodifier.go index cdcfa17edfd..c8a5bbc4479 100644 --- a/importer/dagwriter/dagmodifier.go +++ b/importer/dagwriter/dagmodifier.go @@ -46,14 +46,14 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { return 0, errors.New("Attempted to perform write starting past end of file") } + // First need to find where we are writing at + end := uint64(len(b)) + offset + // This shouldnt be necessary if we do subblocks sizes properly newsize := dm.pbdata.GetFilesize() - if uint64(len(b))+offset > dm.pbdata.GetFilesize() { - newsize = uint64(len(b)) + offset + if end > dm.pbdata.GetFilesize() { + newsize = end } - - // First need to find where we are writing at - end := uint64(len(b)) + offset zeroblocklen := uint64(len(dm.pbdata.Data)) origlen := len(b) diff --git a/importer/dagwriter/dagwriter_test.go b/importer/dagwriter/dagwriter_test.go index ec6d5644946..8f0bc26866e 100644 --- a/importer/dagwriter/dagwriter_test.go +++ b/importer/dagwriter/dagwriter_test.go @@ -110,7 +110,7 @@ func BenchmarkDagWriter(b *testing.B) { dag := &mdag.DAGService{bserv} b.ResetTimer() - nbytes := int64(b.N) + nbytes := int64(100000) for i := 0; i < b.N; i++ { b.SetBytes(nbytes) dw := NewDagWriter(dag, &imp.SizeSplitter{4096}) diff --git a/merkledag/coding.go b/merkledag/coding.go index 45142ac4724..1d83f32ef0f 100644 --- a/merkledag/coding.go +++ b/merkledag/coding.go @@ -3,6 +3,8 @@ package merkledag import ( "fmt" + u "github.com/jbenet/go-ipfs/util" + mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" ) @@ -76,6 +78,7 @@ func (n *Node) Encoded(force bool) ([]byte, error) { if err != nil { return []byte{}, err } + n.cached = u.Hash(n.encoded) } return n.encoded, nil diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index ba22f56e74d..05440cd6e50 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -24,6 +24,8 @@ type Node struct { // cache encoded/marshaled value encoded []byte + + cached mh.Multihash } // Link represents an IPFS Merkle DAG Link between Nodes. @@ -122,12 +124,12 @@ func (n *Node) Size() (uint64, error) { // Multihash hashes the encoded data of this node. func (n *Node) Multihash() (mh.Multihash, error) { - b, err := n.Encoded(false) + _, err := n.Encoded(false) if err != nil { return nil, err } - return u.Hash(b), nil + return n.cached, nil } // Key returns the Multihash as a key, for maps. @@ -183,7 +185,9 @@ func (n *DAGService) Add(nd *Node) (u.Key, error) { return "", err } - b, err := blocks.NewBlock(d) + b := new(blocks.Block) + b.Data = d + b.Multihash, err = nd.Multihash() if err != nil { return "", err } diff --git a/util/testutil/blocks.go b/util/testutil/blocks.go deleted file mode 100644 index 05b69239998..00000000000 --- a/util/testutil/blocks.go +++ /dev/null @@ -1,22 +0,0 @@ -package testutil - -import ( - "testing" - - blocks "github.com/jbenet/go-ipfs/blocks" -) - -// NewBlockOrFail returns a block created from msgData. Signals test failure if -// creation fails. -// -// NB: NewBlockOrFail accepts a msgData parameter to avoid non-determinism in -// tests. Generating random block data could potentially result in unexpected -// behavior in tests. Thus, it is left up to the caller to select the msgData -// that will determine the blocks key. -func NewBlockOrFail(t *testing.T, msgData string) blocks.Block { - block, blockCreationErr := blocks.NewBlock([]byte(msgData)) - if blockCreationErr != nil { - t.Fatal(blockCreationErr) - } - return *block -} From bdaee9bcb272f812ded1461a6befbb4ce32444e7 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 8 Oct 2014 03:42:29 +0000 Subject: [PATCH 082/105] some performance tweaks for the dagwriter write path --- blockservice/blockservice.go | 4 ++-- merkledag/merkledag.go | 2 +- util/util.go | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index b8e445568ed..dcf15ce95b9 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -36,7 +36,7 @@ func NewBlockService(d ds.Datastore, rem exchange.Interface) (*BlockService, err // AddBlock adds a particular block to the service, Putting it into the datastore. func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { k := b.Key() - log.Debug("blockservice: storing [%s] in datastore", k.Pretty()) + log.Debug("blockservice: storing [%s] in datastore", k) // TODO(brian): define a block datastore with a Put method which accepts a // block parameter err := s.Datastore.Put(k.DsKey(), b.Data) @@ -53,7 +53,7 @@ func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) { // GetBlock retrieves a particular block from the service, // Getting it from the datastore using the key (hash). func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) { - log.Debug("BlockService GetBlock: '%s'", k.Pretty()) + log.Debug("BlockService GetBlock: '%s'", k) datai, err := s.Datastore.Get(k.DsKey()) if err == nil { log.Debug("Blockservice: Got data in datastore.") diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 05440cd6e50..d7e8148edcb 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -175,7 +175,7 @@ type DAGService struct { // Add adds a node to the DAGService, storing the block in the BlockService func (n *DAGService) Add(nd *Node) (u.Key, error) { k, _ := nd.Key() - log.Debug("DagService Add [%s]\n", k.Pretty()) + log.Debug("DagService Add [%s]", k) if n == nil { return "", fmt.Errorf("DAGService is nil") } diff --git a/util/util.go b/util/util.go index 5a9d21e815b..d82df4a6fdc 100644 --- a/util/util.go +++ b/util/util.go @@ -39,7 +39,11 @@ var ErrNotFound = ds.ErrNotFound // Key is a string representation of multihash for use with maps. type Key string -// Pretty returns Key in a b58 encoded string +// String returns Key in a b58 encoded string +func (k Key) String() string { + return b58.Encode([]byte(k)) +} + func (k Key) Pretty() string { return b58.Encode([]byte(k)) } From 5ba2efc2b83e55cb12bb82919d75e78f0ef7a064 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 8 Oct 2014 04:25:51 +0000 Subject: [PATCH 083/105] make tests pass --- blockservice/blocks_test.go | 7 +------ blockstore/blockstore_test.go | 6 +++--- cmd/ipfs/mount_unix.go | 3 +++ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/blockservice/blocks_test.go b/blockservice/blocks_test.go index bfffc37b8cc..764d2d400eb 100644 --- a/blockservice/blocks_test.go +++ b/blockservice/blocks_test.go @@ -17,12 +17,7 @@ func TestBlocks(t *testing.T) { return } - b, err := blocks.NewBlock([]byte("beep boop")) - if err != nil { - t.Error("failed to construct block", err) - return - } - + b := blocks.NewBlock([]byte("beep boop")) h := u.Hash([]byte("beep boop")) if !bytes.Equal(b.Multihash, h) { t.Error("Block Multihash and data multihash not equal") diff --git a/blockstore/blockstore_test.go b/blockstore/blockstore_test.go index e64424c49df..c9e7275b157 100644 --- a/blockstore/blockstore_test.go +++ b/blockstore/blockstore_test.go @@ -24,9 +24,9 @@ func TestGetWhenKeyNotPresent(t *testing.T) { func TestPutThenGetBlock(t *testing.T) { bs := NewBlockstore(ds.NewMapDatastore()) - block := blocks.NewBlock("some data") + block := blocks.NewBlock([]byte("some data")) - err := bs.Put(block) + err := bs.Put(*block) if err != nil { t.Fatal(err) } @@ -41,7 +41,7 @@ func TestPutThenGetBlock(t *testing.T) { } func TestValueTypeMismatch(t *testing.T) { - block := blocks.NewBlock("some data") + block := blocks.NewBlock([]byte("some data")) datastore := ds.NewMapDatastore() datastore.Put(block.Key().DsKey(), "data that isn't a block!") diff --git a/cmd/ipfs/mount_unix.go b/cmd/ipfs/mount_unix.go index 1b4e8709a42..84fbc9cf61a 100644 --- a/cmd/ipfs/mount_unix.go +++ b/cmd/ipfs/mount_unix.go @@ -78,6 +78,9 @@ func mountIpfs(node *core.IpfsNode, fsdir string) <-chan error { } func mountIpns(node *core.IpfsNode, nsdir, fsdir string) <-chan error { + if nsdir == "" { + return nil + } done := make(chan error) fmt.Printf("mounting ipns at %s\n", nsdir) From b98e48836f9d3b561e1958614cd0304ee43bc5f8 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Tue, 7 Oct 2014 05:00:19 -0700 Subject: [PATCH 084/105] added u.Key.String --- util/util.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util/util.go b/util/util.go index d82df4a6fdc..0d703fd4cc6 100644 --- a/util/util.go +++ b/util/util.go @@ -39,11 +39,12 @@ var ErrNotFound = ds.ErrNotFound // Key is a string representation of multihash for use with maps. type Key string -// String returns Key in a b58 encoded string +// String is utililty function for printing out keys as strings (Pretty). func (k Key) String() string { - return b58.Encode([]byte(k)) + return key.Pretty() } +// Pretty returns Key in a b58 encoded string func (k Key) Pretty() string { return b58.Encode([]byte(k)) } From 7c5679536c547f9f6ab0c120ae5192cfc572465e Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Tue, 7 Oct 2014 21:27:47 -0700 Subject: [PATCH 085/105] bugfix: use consistent interface We'll want a `type blocks.Block interface {}` later, but for now, make sure Blockstore uses ptrs for both Get and Put. + fix NewBlock output compile error --- blockstore/blockstore.go | 4 ++-- exchange/bitswap/bitswap.go | 2 +- exchange/bitswap/bitswap_test.go | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/blockstore/blockstore.go b/blockstore/blockstore.go index d597288768f..6a0d10b74c8 100644 --- a/blockstore/blockstore.go +++ b/blockstore/blockstore.go @@ -13,7 +13,7 @@ var ValueTypeMismatch = errors.New("The retrieved value is not a Block") type Blockstore interface { Get(u.Key) (*blocks.Block, error) - Put(blocks.Block) error + Put(*blocks.Block) error } func NewBlockstore(d ds.Datastore) Blockstore { @@ -38,6 +38,6 @@ func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) { return blocks.NewBlock(bdata), nil } -func (bs *blockstore) Put(block blocks.Block) error { +func (bs *blockstore) Put(block *blocks.Block) error { return bs.datastore.Put(block.Key().DsKey(), block.Data) } diff --git a/exchange/bitswap/bitswap.go b/exchange/bitswap/bitswap.go index e4eaeb4a4f4..20f9d234c06 100644 --- a/exchange/bitswap/bitswap.go +++ b/exchange/bitswap/bitswap.go @@ -134,7 +134,7 @@ func (bs *bitswap) ReceiveMessage(ctx context.Context, p *peer.Peer, incoming bs for _, block := range incoming.Blocks() { // TODO verify blocks? - if err := bs.blockstore.Put(block); err != nil { + if err := bs.blockstore.Put(&block); err != nil { continue // FIXME(brian): err ignored } go bs.notifications.Publish(block) diff --git a/exchange/bitswap/bitswap_test.go b/exchange/bitswap/bitswap_test.go index fd01aacd9c1..d1c92d8d0fe 100644 --- a/exchange/bitswap/bitswap_test.go +++ b/exchange/bitswap/bitswap_test.go @@ -67,7 +67,7 @@ func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) { hasBlock := g.Next() - if err := hasBlock.blockstore.Put(*block); err != nil { + if err := hasBlock.blockstore.Put(block); err != nil { t.Fatal(err) } if err := hasBlock.exchange.HasBlock(context.Background(), *block); err != nil { @@ -106,7 +106,7 @@ func TestSwarm(t *testing.T) { first := instances[0] for _, b := range blocks { - first.blockstore.Put(*b) + first.blockstore.Put(b) first.exchange.HasBlock(context.Background(), *b) rs.Announce(first.peer, b.Key()) } @@ -177,7 +177,7 @@ func TestSendToWantingPeer(t *testing.T) { beta := bg.Next() t.Logf("Peer %v announes availability of %v\n", w.peer, beta.Key()) ctx, _ = context.WithTimeout(context.Background(), timeout) - if err := w.blockstore.Put(beta); err != nil { + if err := w.blockstore.Put(&beta); err != nil { t.Fatal(err) } w.exchange.HasBlock(ctx, beta) @@ -190,7 +190,7 @@ func TestSendToWantingPeer(t *testing.T) { t.Logf("%v announces availability of %v\n", o.peer, alpha.Key()) ctx, _ = context.WithTimeout(context.Background(), timeout) - if err := o.blockstore.Put(alpha); err != nil { + if err := o.blockstore.Put(&alpha); err != nil { t.Fatal(err) } o.exchange.HasBlock(ctx, alpha) From 66839fa1dea9628c55e226b48c76f1a7b2789c02 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Tue, 7 Oct 2014 21:29:03 -0700 Subject: [PATCH 086/105] changed logging, in dht and elsewhere - use log.* instead of u.* - use automatic type conversions to .String() (Peer.String() prints nicely, and avoids calling b58 encoding until needed) --- crypto/spipe/handshake.go | 4 ++-- peer/peer.go | 2 +- routing/dht/dht.go | 6 +++--- routing/dht/dht_test.go | 2 +- routing/dht/handlers.go | 29 ++++++++++++++--------------- routing/dht/query.go | 18 +++++++++--------- routing/dht/routing.go | 12 ++++++------ routing/kbucket/table_test.go | 4 ++-- util/util.go | 2 +- 9 files changed, 39 insertions(+), 40 deletions(-) diff --git a/crypto/spipe/handshake.go b/crypto/spipe/handshake.go index 21a06d548e6..2d6f2ea9f19 100644 --- a/crypto/spipe/handshake.go +++ b/crypto/spipe/handshake.go @@ -101,7 +101,7 @@ func (s *SecurePipe) handshake() error { if err != nil { return err } - u.DOut("[%s] Remote Peer Identified as %s\n", s.local, s.remote) + u.DOut("%s Remote Peer Identified as %s\n", s.local, s.remote) exchange, err := selectBest(SupportedExchanges, proposeResp.GetExchanges()) if err != nil { @@ -205,7 +205,7 @@ func (s *SecurePipe) handshake() error { return errors.New("Negotiation failed.") } - u.DOut("[%s] handshake: Got node id: %s\n", s.local, s.remote) + u.DOut("%s handshake: Got node id: %s\n", s.local, s.remote) return nil } diff --git a/peer/peer.go b/peer/peer.go index 2803a8f009b..ab071ea52dc 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -55,7 +55,7 @@ type Peer struct { // String prints out the peer. func (p *Peer) String() string { - return "[Peer " + p.ID.Pretty() + "]" + return "[Peer " + p.ID.String() + "]" } // Key returns the ID as a Key (string) for maps. diff --git a/routing/dht/dht.go b/routing/dht/dht.go index 1f1fdd3e5b1..cfa500ee28f 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -221,7 +221,7 @@ func (dht *IpfsDHT) putProvider(ctx context.Context, p *peer.Peer, key string) e return err } - log.Debug("[%s] putProvider: %s for %s", dht.self, p, key) + log.Debug("%s putProvider: %s for %s", dht.self, p, key) if *rpmes.Key != *pmes.Key { return errors.New("provider not added correctly") } @@ -345,7 +345,7 @@ func (dht *IpfsDHT) putLocal(key u.Key, value []byte) error { // Update signals to all routingTables to Update their last-seen status // on the given peer. func (dht *IpfsDHT) Update(p *peer.Peer) { - log.Debug("updating peer: [%s] latency = %f\n", p, p.GetLatency().Seconds()) + log.Debug("updating peer: %s latency = %f\n", p, p.GetLatency().Seconds()) removedCount := 0 for _, route := range dht.routingTables { removed := route.Update(p) @@ -401,7 +401,7 @@ func (dht *IpfsDHT) addProviders(key u.Key, peers []*Message_Peer) []*peer.Peer continue } - log.Debug("[%s] adding provider: %s for %s", dht.self, p, key) + log.Debug("%s adding provider: %s for %s", dht.self, p, key) // Dont add outselves to the list if p.ID.Equal(dht.self.ID) { diff --git a/routing/dht/dht_test.go b/routing/dht/dht_test.go index f5d391387b5..23bdb88e72b 100644 --- a/routing/dht/dht_test.go +++ b/routing/dht/dht_test.go @@ -287,7 +287,7 @@ func TestProvidesAsync(t *testing.T) { select { case p := <-provs: if !p.ID.Equal(dhts[3].self.ID) { - t.Fatalf("got a provider, but not the right one. %v", p.ID.Pretty()) + t.Fatalf("got a provider, but not the right one. %s", p) } case <-ctx.Done(): t.Fatal("Didnt get back providers") diff --git a/routing/dht/handlers.go b/routing/dht/handlers.go index ac03ed3e8f4..49e3eb75080 100644 --- a/routing/dht/handlers.go +++ b/routing/dht/handlers.go @@ -38,7 +38,7 @@ func (dht *IpfsDHT) handlerForMsgType(t Message_MessageType) dhtHandler { } func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error) { - u.DOut("[%s] handleGetValue for key: %s\n", dht.self.ID.Pretty(), pmes.GetKey()) + log.Debug("%s handleGetValue for key: %s\n", dht.self, pmes.GetKey()) // setup response resp := newMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel()) @@ -50,10 +50,10 @@ func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error } // let's first check if we have the value locally. - u.DOut("[%s] handleGetValue looking into ds\n", dht.self.ID.Pretty()) + log.Debug("%s handleGetValue looking into ds\n", dht.self) dskey := u.Key(pmes.GetKey()).DsKey() iVal, err := dht.datastore.Get(dskey) - u.DOut("[%s] handleGetValue looking into ds GOT %v\n", dht.self.ID.Pretty(), iVal) + log.Debug("%s handleGetValue looking into ds GOT %v\n", dht.self, iVal) // if we got an unexpected error, bail. if err != nil && err != ds.ErrNotFound { @@ -65,7 +65,7 @@ func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error // if we have the value, send it back if err == nil { - u.DOut("[%s] handleGetValue success!\n", dht.self.ID.Pretty()) + log.Debug("%s handleGetValue success!\n", dht.self) byts, ok := iVal.([]byte) if !ok { @@ -78,14 +78,14 @@ func (dht *IpfsDHT) handleGetValue(p *peer.Peer, pmes *Message) (*Message, error // if we know any providers for the requested value, return those. provs := dht.providers.GetProviders(u.Key(pmes.GetKey())) if len(provs) > 0 { - u.DOut("handleGetValue returning %d provider[s]\n", len(provs)) + log.Debug("handleGetValue returning %d provider[s]\n", len(provs)) resp.ProviderPeers = peersToPBPeers(provs) } // Find closest peer on given cluster to desired key and reply with that info closer := dht.betterPeerToQuery(pmes) if closer != nil { - u.DOut("handleGetValue returning a closer peer: '%s'\n", closer.ID.Pretty()) + log.Debug("handleGetValue returning a closer peer: '%s'\n", closer) resp.CloserPeers = peersToPBPeers([]*peer.Peer{closer}) } @@ -98,12 +98,12 @@ func (dht *IpfsDHT) handlePutValue(p *peer.Peer, pmes *Message) (*Message, error defer dht.dslock.Unlock() dskey := u.Key(pmes.GetKey()).DsKey() err := dht.datastore.Put(dskey, pmes.GetValue()) - u.DOut("[%s] handlePutValue %v %v\n", dht.self.ID.Pretty(), dskey, pmes.GetValue()) + log.Debug("%s handlePutValue %v %v\n", dht.self, dskey, pmes.GetValue()) return pmes, err } func (dht *IpfsDHT) handlePing(p *peer.Peer, pmes *Message) (*Message, error) { - u.DOut("[%s] Responding to ping from [%s]!\n", dht.self.ID.Pretty(), p.ID.Pretty()) + log.Debug("%s Responding to ping from %s!\n", dht.self, p) return pmes, nil } @@ -119,16 +119,16 @@ func (dht *IpfsDHT) handleFindPeer(p *peer.Peer, pmes *Message) (*Message, error } if closest == nil { - u.PErr("handleFindPeer: could not find anything.\n") + log.Error("handleFindPeer: could not find anything.\n") return resp, nil } if len(closest.Addresses) == 0 { - u.PErr("handleFindPeer: no addresses for connected peer...\n") + log.Error("handleFindPeer: no addresses for connected peer...\n") return resp, nil } - u.DOut("handleFindPeer: sending back '%s'\n", closest.ID.Pretty()) + log.Debug("handleFindPeer: sending back '%s'\n", closest) resp.CloserPeers = peersToPBPeers([]*peer.Peer{closest}) return resp, nil } @@ -140,7 +140,7 @@ func (dht *IpfsDHT) handleGetProviders(p *peer.Peer, pmes *Message) (*Message, e dsk := u.Key(pmes.GetKey()).DsKey() has, err := dht.datastore.Has(dsk) if err != nil && err != ds.ErrNotFound { - u.PErr("unexpected datastore error: %v\n", err) + log.Error("unexpected datastore error: %v\n", err) has = false } @@ -172,8 +172,7 @@ type providerInfo struct { func (dht *IpfsDHT) handleAddProvider(p *peer.Peer, pmes *Message) (*Message, error) { key := u.Key(pmes.GetKey()) - u.DOut("[%s] Adding [%s] as a provider for '%s'\n", - dht.self.ID.Pretty(), p.ID.Pretty(), peer.ID(key).Pretty()) + log.Debug("%s adding %s as a provider for '%s'\n", dht.self, p, peer.ID(key)) dht.providers.AddProvider(key, p) return pmes, nil // send back same msg as confirmation. @@ -192,7 +191,7 @@ func (dht *IpfsDHT) handleDiagnostic(p *peer.Peer, pmes *Message) (*Message, err for _, ps := range seq { _, err := msg.FromObject(ps, pmes) if err != nil { - u.PErr("handleDiagnostics error creating message: %v\n", err) + log.Error("handleDiagnostics error creating message: %v\n", err) continue } // dht.sender.SendRequest(context.TODO(), mes) diff --git a/routing/dht/query.go b/routing/dht/query.go index a62646f05b2..0a9ca0bd830 100644 --- a/routing/dht/query.go +++ b/routing/dht/query.go @@ -151,7 +151,7 @@ func (r *dhtQueryRunner) Run(peers []*peer.Peer) (*dhtQueryResult, error) { func (r *dhtQueryRunner) addPeerToQuery(next *peer.Peer, benchmark *peer.Peer) { if next == nil { // wtf why are peers nil?!? - u.PErr("Query getting nil peers!!!\n") + log.Error("Query getting nil peers!!!\n") return } @@ -170,7 +170,7 @@ func (r *dhtQueryRunner) addPeerToQuery(next *peer.Peer, benchmark *peer.Peer) { r.peersSeen[next.Key()] = next r.Unlock() - log.Debug("adding peer to query: %v\n", next.ID.Pretty()) + log.Debug("adding peer to query: %v\n", next) // do this after unlocking to prevent possible deadlocks. r.peersRemaining.Increment(1) @@ -194,14 +194,14 @@ func (r *dhtQueryRunner) spawnWorkers() { if !more { return // channel closed. } - u.DOut("spawning worker for: %v\n", p.ID.Pretty()) + log.Debug("spawning worker for: %v\n", p) go r.queryPeer(p) } } } func (r *dhtQueryRunner) queryPeer(p *peer.Peer) { - u.DOut("spawned worker for: %v\n", p.ID.Pretty()) + log.Debug("spawned worker for: %v\n", p) // make sure we rate limit concurrency. select { @@ -211,33 +211,33 @@ func (r *dhtQueryRunner) queryPeer(p *peer.Peer) { return } - u.DOut("running worker for: %v\n", p.ID.Pretty()) + log.Debug("running worker for: %v\n", p) // finally, run the query against this peer res, err := r.query.qfunc(r.ctx, p) if err != nil { - u.DOut("ERROR worker for: %v %v\n", p.ID.Pretty(), err) + log.Debug("ERROR worker for: %v %v\n", p, err) r.Lock() r.errs = append(r.errs, err) r.Unlock() } else if res.success { - u.DOut("SUCCESS worker for: %v\n", p.ID.Pretty(), res) + log.Debug("SUCCESS worker for: %v\n", p, res) r.Lock() r.result = res r.Unlock() r.cancel() // signal to everyone that we're done. } else if res.closerPeers != nil { - u.DOut("PEERS CLOSER -- worker for: %v\n", p.ID.Pretty()) + log.Debug("PEERS CLOSER -- worker for: %v\n", p) for _, next := range res.closerPeers { r.addPeerToQuery(next, p) } } // signal we're done proccessing peer p - u.DOut("completing worker for: %v\n", p.ID.Pretty()) + log.Debug("completing worker for: %v\n", p) r.peersRemaining.Decrement(1) r.rateLimit <- struct{}{} } diff --git a/routing/dht/routing.go b/routing/dht/routing.go index 4fa6c8c9466..25567038cdb 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -18,7 +18,7 @@ import ( // PutValue adds value corresponding to given Key. // This is the top level "Store" operation of the DHT func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error { - log.Debug("PutValue %s", key.Pretty()) + log.Debug("PutValue %s", key) err := dht.putLocal(key, value) if err != nil { return err @@ -31,7 +31,7 @@ func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error } query := newQuery(key, func(ctx context.Context, p *peer.Peer) (*dhtQueryResult, error) { - log.Debug("[%s] PutValue qry part %v", dht.self.ID.Pretty(), p.ID.Pretty()) + log.Debug("%s PutValue qry part %v", dht.self, p) err := dht.putValueToNetwork(ctx, p, string(key), value) if err != nil { return nil, err @@ -47,7 +47,7 @@ func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error // If the search does not succeed, a multiaddr string of a closer peer is // returned along with util.ErrSearchIncomplete func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) { - log.Debug("Get Value [%s]", key.Pretty()) + log.Debug("Get Value [%s]", key) // If we have it local, dont bother doing an RPC! // NOTE: this might not be what we want to do... @@ -189,7 +189,7 @@ func (dht *IpfsDHT) addPeerListAsync(k u.Key, peers []*Message_Peer, ps *peerSet // FindProviders searches for peers who can provide the value for given key. func (dht *IpfsDHT) FindProviders(ctx context.Context, key u.Key) ([]*peer.Peer, error) { // get closest peer - log.Debug("Find providers for: '%s'", key.Pretty()) + log.Debug("Find providers for: '%s'", key) p := dht.routingTables[0].NearestPeer(kb.ConvertKey(key)) if p == nil { return nil, nil @@ -333,11 +333,11 @@ func (dht *IpfsDHT) findPeerMultiple(ctx context.Context, id peer.ID) (*peer.Pee // Ping a peer, log the time it took func (dht *IpfsDHT) Ping(ctx context.Context, p *peer.Peer) error { // Thoughts: maybe this should accept an ID and do a peer lookup? - log.Info("ping %s start", p.ID.Pretty()) + log.Info("ping %s start", p) pmes := newMessage(Message_PING, "", 0) _, err := dht.sendRequest(ctx, p, pmes) - log.Info("ping %s end (err = %s)", p.ID.Pretty(), err) + log.Info("ping %s end (err = %s)", p, err) return err } diff --git a/routing/kbucket/table_test.go b/routing/kbucket/table_test.go index 49be52c6553..cc1cdfba1da 100644 --- a/routing/kbucket/table_test.go +++ b/routing/kbucket/table_test.go @@ -101,7 +101,7 @@ func TestTableFind(t *testing.T) { rt.Update(peers[i]) } - t.Logf("Searching for peer: '%s'", peers[2].ID.Pretty()) + t.Logf("Searching for peer: '%s'", peers[2]) found := rt.NearestPeer(ConvertPeerID(peers[2].ID)) if !found.ID.Equal(peers[2].ID) { t.Fatalf("Failed to lookup known node...") @@ -118,7 +118,7 @@ func TestTableFindMultiple(t *testing.T) { rt.Update(peers[i]) } - t.Logf("Searching for peer: '%s'", peers[2].ID.Pretty()) + t.Logf("Searching for peer: '%s'", peers[2]) found := rt.NearestPeers(ConvertPeerID(peers[2].ID), 15) if len(found) != 15 { t.Fatalf("Got back different number of peers than we expected.") diff --git a/util/util.go b/util/util.go index 0d703fd4cc6..7a1a34b3926 100644 --- a/util/util.go +++ b/util/util.go @@ -41,7 +41,7 @@ type Key string // String is utililty function for printing out keys as strings (Pretty). func (k Key) String() string { - return key.Pretty() + return k.Pretty() } // Pretty returns Key in a b58 encoded string From 8e9b10f9d1ede2480e687d56c7a2c7b3c8ce0f75 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Tue, 7 Oct 2014 21:35:33 -0700 Subject: [PATCH 087/105] one last failed test --- blockstore/blockstore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockstore/blockstore_test.go b/blockstore/blockstore_test.go index c9e7275b157..98c00a9faba 100644 --- a/blockstore/blockstore_test.go +++ b/blockstore/blockstore_test.go @@ -26,7 +26,7 @@ func TestPutThenGetBlock(t *testing.T) { bs := NewBlockstore(ds.NewMapDatastore()) block := blocks.NewBlock([]byte("some data")) - err := bs.Put(*block) + err := bs.Put(block) if err != nil { t.Fatal(err) } From ffe2bdceef3a56d365b528e490a18f9f1b710bf3 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 8 Oct 2014 02:50:36 -0700 Subject: [PATCH 088/105] init SetupLoggers this is useful so that loggers are all setup during tests --- cmd/ipfs/ipfs.go | 2 +- util/util.go | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 7b75ba5c54c..8031f2754de 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -80,7 +80,7 @@ func main() { u.Debug = false // setup logging - u.SetupLogging() + // u.SetupLogging() done in an init() block now. // if debugging, setup profiling. if u.Debug { diff --git a/util/util.go b/util/util.go index 7a1a34b3926..a22488372f1 100644 --- a/util/util.go +++ b/util/util.go @@ -17,8 +17,12 @@ import ( logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" ) +func init() { + SetupLogging() +} + // LogFormat is the format used for our logger. -var LogFormat = "%{color}%{time:01-02 15:04:05.9999} %{shortfile} %{level}: %{color:reset}%{message}" +var LogFormat = "%{color}%{time:2006-01-02 15:04:05.999999} %{shortfile} %{level}: %{color:reset}%{message}" // Debug is a global flag for debugging. var Debug bool @@ -122,12 +126,14 @@ func DOut(format string, a ...interface{}) { } } -var loggers = []string{} +var loggers = map[string]*logging.Logger{} // SetupLogging will initialize the logger backend and set the flags. func SetupLogging() { backend := logging.NewLogBackend(os.Stderr, "", 0) logging.SetBackend(backend) + logging.SetFormatter(logging.MustStringFormatter(LogFormat)) + /* if Debug { logging.SetLevel(logging.DEBUG, "") @@ -135,10 +141,10 @@ func SetupLogging() { logging.SetLevel(logging.ERROR, "") } */ - logging.SetFormatter(logging.MustStringFormatter(LogFormat)) - for _, n := range loggers { + for n, log := range loggers { logging.SetLevel(logging.ERROR, n) + log.Error("setting logger: %s to %v\n", n, logging.ERROR) } } @@ -146,7 +152,7 @@ func SetupLogging() { func Logger(name string) *logging.Logger { log := logging.MustGetLogger(name) // logging.SetLevel(lvl, name) // can't set level here. - loggers = append(loggers, name) + loggers[name] = log return log } From 459e0d53737b97a2f3f3ee38982c4adf863791df Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 8 Oct 2014 02:52:16 -0700 Subject: [PATCH 089/105] ipns TestFastRepublish --- fuse/ipns/ipns_test.go | 108 +++++++++++++++++++++++++++++++++++++++- fuse/ipns/ipns_unix.go | 7 ++- fuse/ipns/repub_unix.go | 37 ++++++-------- 3 files changed, 127 insertions(+), 25 deletions(-) diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index c311c522526..8daf1476998 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -9,7 +9,8 @@ import ( "time" fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" - "github.com/jbenet/go-ipfs/core" + core "github.com/jbenet/go-ipfs/core" + u "github.com/jbenet/go-ipfs/util" ) func randBytes(size int) []byte { @@ -19,7 +20,10 @@ func randBytes(size int) []byte { } func writeFile(t *testing.T, size int, path string) []byte { - data := randBytes(size) + return writeFileData(t, randBytes(size), path) +} + +func writeFileData(t *testing.T, data []byte, path string) []byte { fi, err := os.Create(path) if err != nil { t.Fatal(err) @@ -179,6 +183,106 @@ func TestAppendFile(t *testing.T) { } } +func TestFastRepublish(t *testing.T) { + + // make timeout noticeable. + osrt := shortRepublishTimeout + shortRepublishTimeout = time.Millisecond * 100 + + olrt := longRepublishTimeout + longRepublishTimeout = time.Second + + node, mnt := setupIpnsTest(t, nil) + + h, err := node.Identity.PrivKey.GetPublic().Hash() + if err != nil { + t.Fatal(err) + } + pubkeyHash := u.Key(h).Pretty() + + // set them back + defer func() { + shortRepublishTimeout = osrt + longRepublishTimeout = olrt + mnt.Close() + }() + + closed := make(chan struct{}) + dataA := []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + dataB := []byte("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + + fname := mnt.Dir + "/local/file" + + // get first resolved hash + log.Debug("publishing first hash") + writeFileData(t, dataA, fname) // random + <-time.After(shortRepublishTimeout * 11 / 10) + log.Debug("resolving first hash") + resolvedHash, err := node.Namesys.Resolve(pubkeyHash) + if err != nil { + t.Fatal("resolve err:", pubkeyHash, err) + } + + // constantly keep writing to the file + go func() { + for { + select { + case <-closed: + return + + case <-time.After(shortRepublishTimeout * 8 / 10): + writeFileData(t, dataB, fname) + } + } + }() + + hasPublished := func() bool { + res, err := node.Namesys.Resolve(pubkeyHash) + if err != nil { + t.Fatal("resolve err: %v", err) + } + return res != resolvedHash + } + + // test things + + // at this point, should not have written dataA and not have written dataB + rbuf, err := ioutil.ReadFile(fname) + if err != nil || !bytes.Equal(rbuf, dataA) { + t.Fatal("Data inconsistent! %v %v", err, string(rbuf)) + } + + if hasPublished() { + t.Fatal("published (wrote)") + } + + <-time.After(shortRepublishTimeout * 11 / 10) + + // at this point, should have written written dataB, but not published it + rbuf, err = ioutil.ReadFile(fname) + if err != nil || !bytes.Equal(rbuf, dataB) { + t.Fatal("Data inconsistent! %v %v", err, string(rbuf)) + } + + if hasPublished() { + t.Fatal("published (wrote)") + } + + <-time.After(longRepublishTimeout * 11 / 10) + + // at this point, should have written written dataB, and published it + rbuf, err = ioutil.ReadFile(fname) + if err != nil || !bytes.Equal(rbuf, dataB) { + t.Fatal("Data inconsistent! %v %v", err, string(rbuf)) + } + + if !hasPublished() { + t.Fatal("not published") + } + + close(closed) +} + // Test writing a medium sized file one byte at a time func TestMultiWrite(t *testing.T) { _, mnt := setupIpnsTest(t, nil) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index dc03d51138e..a17f68649eb 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -21,6 +21,11 @@ import ( var log = u.Logger("ipns") +var ( + shortRepublishTimeout = time.Millisecond * 5 + longRepublishTimeout = time.Millisecond * 500 +) + // FileSystem is the readwrite IPNS Fuse Filesystem. type FileSystem struct { Ipfs *core.IpfsNode @@ -71,7 +76,7 @@ func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, er nd := new(Node) nd.Ipfs = n nd.key = k - nd.repub = NewRepublisher(nd, time.Millisecond*5, time.Millisecond*500) + nd.repub = NewRepublisher(nd, shortRepublishTimeout, longRepublishTimeout) go nd.repub.Run() diff --git a/fuse/ipns/repub_unix.go b/fuse/ipns/repub_unix.go index f1a89828731..4e807578d3a 100644 --- a/fuse/ipns/repub_unix.go +++ b/fuse/ipns/repub_unix.go @@ -22,28 +22,21 @@ func (np *Republisher) Run() { for _ = range np.Publish { quick := time.After(np.TimeoutShort) longer := time.After(np.TimeoutLong) - for { - select { - case <-quick: - //Do the publish! - log.Info("Publishing Changes!") - err := np.node.republishRoot() - if err != nil { - log.Critical("republishRoot error: %s", err) - } - goto done - case <-longer: - //Do the publish! - log.Info("Publishing Changes!") - err := np.node.republishRoot() - if err != nil { - log.Critical("republishRoot error: %s", err) - } - goto done - case <-np.Publish: - quick = time.After(np.TimeoutShort) - } + + wait: + select { + case <-quick: + case <-longer: + case <-np.Publish: + quick = time.After(np.TimeoutShort) + goto wait + } + + log.Info("Publishing Changes!") + err := np.node.republishRoot() + if err != nil { + log.Critical("republishRoot error: %s", err) } - done: + } } From cf6efc7a09f3c5cfa6f9dae258edd46a88d33327 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 8 Oct 2014 03:11:29 -0700 Subject: [PATCH 090/105] deprecate merkledag.Node.Update --- merkledag/dagreader.go | 1 + merkledag/merkledag.go | 25 ------------------------- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/merkledag/dagreader.go b/merkledag/dagreader.go index badc661fdf8..bc66a600166 100644 --- a/merkledag/dagreader.go +++ b/merkledag/dagreader.go @@ -26,6 +26,7 @@ func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) { if err != nil { return nil, err } + switch pb.GetType() { case ft.PBData_Directory: return nil, ErrIsDir diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index d7e8148edcb..f0c93ad63d6 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -138,31 +138,6 @@ func (n *Node) Key() (u.Key, error) { return u.Key(h), err } -// Recursively update all hash links and size values in the tree -func (n *Node) Update() error { - log.Debug("node update") - for _, l := range n.Links { - if l.Node != nil { - err := l.Node.Update() - if err != nil { - return err - } - nhash, err := l.Node.Multihash() - if err != nil { - return err - } - l.Hash = nhash - size, err := l.Node.Size() - if err != nil { - return err - } - l.Size = size - } - } - _, err := n.Encoded(true) - return err -} - // DAGService is an IPFS Merkle DAG service. // - the root is virtual (like a forest) // - stores nodes' data in a BlockService From 2944360f5c2f667a887e98cccf2162475f5be7f3 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 8 Oct 2014 04:14:50 -0700 Subject: [PATCH 091/105] New NameSystem interface type NameSystem interface { Resolver Publisher } should say it all. cc @whyrusleeping --- core/commands/publish.go | 2 +- core/core.go | 11 +++----- core/mock.go | 5 +--- fuse/ipns/ipns_unix.go | 2 +- namesys/dns.go | 7 ++--- namesys/interface.go | 43 ++++++++++++++++++++++++++++++ namesys/namesys.go | 57 ++++++++++++++++++++++++++++++++++++++++ namesys/nsresolver.go | 8 ------ namesys/proquint.go | 6 +++-- namesys/publisher.go | 22 ++++++---------- namesys/resolve_test.go | 21 +++------------ namesys/resolver.go | 42 ----------------------------- namesys/routing.go | 24 ++++++++--------- 13 files changed, 138 insertions(+), 112 deletions(-) create mode 100644 namesys/interface.go create mode 100644 namesys/namesys.go delete mode 100644 namesys/nsresolver.go delete mode 100644 namesys/resolver.go diff --git a/core/commands/publish.go b/core/commands/publish.go index 09b41f43b22..5794372b7ae 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -19,7 +19,7 @@ func Publish(n *core.IpfsNode, args []string, opts map[string]interface{}, out i k := n.Identity.PrivKey - pub := nsys.NewPublisher(n.DAG, n.Routing) + pub := nsys.NewRoutingPublisher(n.Routing) err := pub.Publish(k, args[0]) if err != nil { return err diff --git a/core/core.go b/core/core.go index 9aaa6e50406..4e764649b16 100644 --- a/core/core.go +++ b/core/core.go @@ -63,10 +63,7 @@ type IpfsNode struct { Resolver *path.Resolver // the name system, resolves paths to hashes - Namesys namesys.Resolver - - // the routing publisher - Publisher namesys.Publisher + Namesys namesys.NameSystem } // NewIpfsNode constructs a new IpfsNode based on the given config. @@ -148,8 +145,7 @@ func NewIpfsNode(cfg *config.Config, online bool) (*IpfsNode, error) { } dag := &merkledag.DAGService{Blocks: bs} - resolve := namesys.NewMasterResolver(route, dag) - publisher := namesys.NewPublisher(dag, route) + ns := namesys.NewNameSystem(route) success = true return &IpfsNode{ @@ -162,8 +158,7 @@ func NewIpfsNode(cfg *config.Config, online bool) (*IpfsNode, error) { Exchange: exchangeSession, Identity: local, Routing: route, - Namesys: resolve, - Publisher: publisher, + Namesys: ns, }, nil } diff --git a/core/mock.go b/core/mock.go index a0fd475ad3f..9eececa3dac 100644 --- a/core/mock.go +++ b/core/mock.go @@ -43,10 +43,7 @@ func NewMockNode() (*IpfsNode, error) { nd.DAG = &mdag.DAGService{bserv} // Namespace resolver - nd.Namesys = nsys.NewMasterResolver(dht, nd.DAG) - - // Publisher - nd.Publisher = nsys.NewPublisher(nd.DAG, dht) + nd.Namesys = nsys.NewNameSystem(dht) // Path resolver nd.Resolver = &path.Resolver{nd.DAG} diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index a17f68649eb..9a243f70f42 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -397,7 +397,7 @@ func (n *Node) republishRoot() error { } log.Debug("Publishing changes!") - err = n.Ipfs.Publisher.Publish(root.key, ndkey.Pretty()) + err = n.Ipfs.Namesys.Publish(root.key, ndkey.Pretty()) if err != nil { log.Error("ipns: Publish Failed: %s", err) return err diff --git a/namesys/dns.go b/namesys/dns.go index 8dda6cb51ed..66448511f85 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -4,9 +4,10 @@ import ( "net" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" + isd "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-is-domain" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" + u "github.com/jbenet/go-ipfs/util" - isd "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-is-domain" ) // DNSResolver implements a Resolver on DNS domains @@ -15,8 +16,8 @@ type DNSResolver struct { // cache would need a timeout } -// Matches implements Resolver -func (r *DNSResolver) Matches(name string) bool { +// CanResolve implements Resolver +func (r *DNSResolver) CanResolve(name string) bool { return isd.IsDomain(name) } diff --git a/namesys/interface.go b/namesys/interface.go new file mode 100644 index 00000000000..eef1fc32b24 --- /dev/null +++ b/namesys/interface.go @@ -0,0 +1,43 @@ +package namesys + +import ( + "errors" + + ci "github.com/jbenet/go-ipfs/crypto" +) + +// ErrResolveFailed signals an error when attempting to resolve. +var ErrResolveFailed = errors.New("could not resolve name.") + +// ErrPublishFailed signals an error when attempting to publish. +var ErrPublishFailed = errors.New("could not publish name.") + +// Namesys represents a cohesive name publishing and resolving system. +// +// Publishing a name is the process of establishing a mapping, a key-value +// pair, according to naming rules and databases. +// +// Resolving a name is the process of looking up the value associated with the +// key (name). +type NameSystem interface { + Resolver + Publisher +} + +// Resolver is an object capable of resolving names. +type Resolver interface { + + // Resolve looks up a name, and returns the value previously published. + Resolve(name string) (value string, err error) + + // CanResolve checks whether this Resolver can resolve a name + CanResolve(name string) bool +} + +// Publisher is an object capable of publishing particular names. +type Publisher interface { + + // Publish establishes a name-value mapping. + // TODO make this not PrivKey specific. + Publish(name ci.PrivKey, value string) error +} diff --git a/namesys/namesys.go b/namesys/namesys.go new file mode 100644 index 00000000000..2ea9a30bdfc --- /dev/null +++ b/namesys/namesys.go @@ -0,0 +1,57 @@ +package namesys + +import ( + ci "github.com/jbenet/go-ipfs/crypto" + routing "github.com/jbenet/go-ipfs/routing" +) + +// ipnsNameSystem implements IPNS naming. +// +// Uses three Resolvers: +// (a) ipfs routing naming: SFS-like PKI names. +// (b) dns domains: resolves using links in DNS TXT records +// (c) proquints: interprets string as the raw byte data. +// +// It can only publish to: (a) ipfs routing naming. +// +type ipns struct { + resolvers []Resolver + publisher Publisher +} + +// NewNameSystem will construct the IPFS naming system based on Routing +func NewNameSystem(r routing.IpfsRouting) NameSystem { + return &ipns{ + resolvers: []Resolver{ + new(DNSResolver), + new(ProquintResolver), + NewRoutingResolver(r), + }, + publisher: NewRoutingPublisher(r), + } +} + +// Resolve implements Resolver +func (ns *ipns) Resolve(name string) (string, error) { + for _, r := range ns.resolvers { + if r.CanResolve(name) { + return r.Resolve(name) + } + } + return "", ErrResolveFailed +} + +// CanResolve implements Resolver +func (ns *ipns) CanResolve(name string) bool { + for _, r := range ns.resolvers { + if r.CanResolve(name) { + return true + } + } + return false +} + +// Publish implements Publisher +func (ns *ipns) Publish(name ci.PrivKey, value string) error { + return ns.publisher.Publish(name, value) +} diff --git a/namesys/nsresolver.go b/namesys/nsresolver.go deleted file mode 100644 index 89ef9ff5a87..00000000000 --- a/namesys/nsresolver.go +++ /dev/null @@ -1,8 +0,0 @@ -package namesys - -type Resolver interface { - // Resolve returns a base58 encoded string - Resolve(string) (string, error) - - Matches(string) bool -} diff --git a/namesys/proquint.go b/namesys/proquint.go index bf34c3b6c99..89bbc4a44c4 100644 --- a/namesys/proquint.go +++ b/namesys/proquint.go @@ -8,13 +8,15 @@ import ( type ProquintResolver struct{} -func (r *ProquintResolver) Matches(name string) bool { +// CanResolve implements Resolver. Checks whether the name is a proquint string. +func (r *ProquintResolver) CanResolve(name string) bool { ok, err := proquint.IsProquint(name) return err == nil && ok } +// Resolve implements Resolver. Decodes the proquint string. func (r *ProquintResolver) Resolve(name string) (string, error) { - ok := r.Matches(name) + ok := r.CanResolve(name) if !ok { return "", errors.New("not a valid proquint string") } diff --git a/namesys/publisher.go b/namesys/publisher.go index 0c605301c98..0828f5e0869 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -7,32 +7,26 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ci "github.com/jbenet/go-ipfs/crypto" - mdag "github.com/jbenet/go-ipfs/merkledag" routing "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" ) +// ipnsPublisher is capable of publishing and resolving names to the IPFS +// routing system. type ipnsPublisher struct { - dag *mdag.DAGService routing routing.IpfsRouting } -type Publisher interface { - Publish(ci.PrivKey, string) error +// NewRoutingPublisher constructs a publisher for the IPFS Routing name system. +func NewRoutingPublisher(route routing.IpfsRouting) Publisher { + return &ipnsPublisher{routing: route} } -func NewPublisher(dag *mdag.DAGService, route routing.IpfsRouting) Publisher { - return &ipnsPublisher{ - dag: dag, - routing: route, - } -} - -// Publish accepts a keypair and a value, +// Publish implements Publisher. Accepts a keypair and a value, func (p *ipnsPublisher) Publish(k ci.PrivKey, value string) error { log.Debug("namesys: Publish %s", value) ctx := context.TODO() - data, err := CreateEntryData(k, value) + data, err := createRoutingEntryData(k, value) if err != nil { return err } @@ -63,7 +57,7 @@ func (p *ipnsPublisher) Publish(k ci.PrivKey, value string) error { return nil } -func CreateEntryData(pk ci.PrivKey, val string) ([]byte, error) { +func createRoutingEntryData(pk ci.PrivKey, val string) ([]byte, error) { entry := new(IpnsEntry) sig, err := pk.Sign([]byte(val)) if err != nil { diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index ff5292224f3..30b996647d8 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -4,9 +4,7 @@ import ( "testing" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" - bs "github.com/jbenet/go-ipfs/blockservice" ci "github.com/jbenet/go-ipfs/crypto" - mdag "github.com/jbenet/go-ipfs/merkledag" "github.com/jbenet/go-ipfs/peer" mock "github.com/jbenet/go-ipfs/routing/mock" u "github.com/jbenet/go-ipfs/util" @@ -19,26 +17,15 @@ func TestRoutingResolve(t *testing.T) { lds := ds.NewMapDatastore() d := mock.NewMockRouter(local, lds) - bserv, err := bs.NewBlockService(lds, nil) - if err != nil { - t.Fatal(err) - } - - dag := &mdag.DAGService{Blocks: bserv} - - resolve := NewMasterResolver(d, dag) - - pub := ipnsPublisher{ - dag: dag, - routing: d, - } + resolver := NewRoutingResolver(d) + publisher := NewRoutingPublisher(d) privk, pubk, err := ci.GenerateKeyPair(ci.RSA, 512) if err != nil { t.Fatal(err) } - err = pub.Publish(privk, "Hello") + err = publisher.Publish(privk, "Hello") if err != nil { t.Fatal(err) } @@ -49,7 +36,7 @@ func TestRoutingResolve(t *testing.T) { } pkhash := u.Hash(pubkb) - res, err := resolve.Resolve(u.Key(pkhash).Pretty()) + res, err := resolver.Resolve(u.Key(pkhash).Pretty()) if err != nil { t.Fatal(err) } diff --git a/namesys/resolver.go b/namesys/resolver.go deleted file mode 100644 index 7765a4ba074..00000000000 --- a/namesys/resolver.go +++ /dev/null @@ -1,42 +0,0 @@ -package namesys - -import ( - "errors" - - mdag "github.com/jbenet/go-ipfs/merkledag" - "github.com/jbenet/go-ipfs/routing" -) - -var ErrCouldntResolve = errors.New("could not resolve name.") - -type masterResolver struct { - res []Resolver -} - -func NewMasterResolver(r routing.IpfsRouting, dag *mdag.DAGService) Resolver { - mr := new(masterResolver) - mr.res = []Resolver{ - new(DNSResolver), - new(ProquintResolver), - NewRoutingResolver(r, dag), - } - return mr -} - -func (mr *masterResolver) Resolve(name string) (string, error) { - for _, r := range mr.res { - if r.Matches(name) { - return r.Resolve(name) - } - } - return "", ErrCouldntResolve -} - -func (mr *masterResolver) Matches(name string) bool { - for _, r := range mr.res { - if r.Matches(name) { - return true - } - } - return false -} diff --git a/namesys/routing.go b/namesys/routing.go index abacb22d4f0..da1c05d0e81 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -8,32 +8,32 @@ import ( mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" ci "github.com/jbenet/go-ipfs/crypto" - mdag "github.com/jbenet/go-ipfs/merkledag" - "github.com/jbenet/go-ipfs/routing" + routing "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" ) var log = u.Logger("namesys") -// RoutingResolver implements NSResolver for the main IPFS SFS-like naming -type RoutingResolver struct { +// routingResolver implements NSResolver for the main IPFS SFS-like naming +type routingResolver struct { routing routing.IpfsRouting - dag *mdag.DAGService } -func NewRoutingResolver(route routing.IpfsRouting, dagservice *mdag.DAGService) *RoutingResolver { - return &RoutingResolver{ - routing: route, - dag: dagservice, - } +// NewRoutingResolver constructs a name resolver using the IPFS Routing system +// to implement SFS-like naming on top. +func NewRoutingResolver(route routing.IpfsRouting) Resolver { + return &routingResolver{routing: route} } -func (r *RoutingResolver) Matches(name string) bool { +// CanResolve implements Resolver. Checks whether name is a b58 encoded string. +func (r *routingResolver) CanResolve(name string) bool { _, err := mh.FromB58String(name) return err == nil } -func (r *RoutingResolver) Resolve(name string) (string, error) { +// Resolve implements Resolver. Uses the IPFS routing system to resolve SFS-like +// names. +func (r *routingResolver) Resolve(name string) (string, error) { log.Debug("RoutingResolve: '%s'", name) ctx := context.TODO() hash, err := mh.FromB58String(name) From dc2dbdbb0235938cea6413d61d7632df5b9dd952 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 8 Oct 2014 04:17:49 -0700 Subject: [PATCH 092/105] vendor things cc @whyrusleeping --- importer/dagwriter/dagmodifier.go | 2 +- importer/format/format.go | 2 +- importer/format/format_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/importer/dagwriter/dagmodifier.go b/importer/dagwriter/dagmodifier.go index c8a5bbc4479..5ceb067d886 100644 --- a/importer/dagwriter/dagmodifier.go +++ b/importer/dagwriter/dagmodifier.go @@ -4,7 +4,7 @@ import ( "bytes" "errors" - "code.google.com/p/goprotobuf/proto" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" imp "github.com/jbenet/go-ipfs/importer" ft "github.com/jbenet/go-ipfs/importer/format" diff --git a/importer/format/format.go b/importer/format/format.go index 1fd5fe0f521..19be4e556c8 100644 --- a/importer/format/format.go +++ b/importer/format/format.go @@ -5,7 +5,7 @@ package format import ( "errors" - "code.google.com/p/goprotobuf/proto" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ) func FromBytes(data []byte) (*PBData, error) { diff --git a/importer/format/format_test.go b/importer/format/format_test.go index 06194aefc4d..bd234d7a6d2 100644 --- a/importer/format/format_test.go +++ b/importer/format/format_test.go @@ -3,7 +3,7 @@ package format import ( "testing" - "code.google.com/p/goprotobuf/proto" + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ) func TestMultiBlock(t *testing.T) { From 916fc54633fcf028ab03396690556293e156e0ce Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 8 Oct 2014 16:51:53 +0000 Subject: [PATCH 093/105] add more comments! --- importer/dagwriter/dagmodifier.go | 15 ++++++++------- importer/dagwriter/dagwriter.go | 17 +++++++++++++++++ importer/format/format.go | 11 ++++++++++- merkledag/dagreader.go | 19 ++++++++++++++++--- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/importer/dagwriter/dagmodifier.go b/importer/dagwriter/dagmodifier.go index 5ceb067d886..c178ecbe121 100644 --- a/importer/dagwriter/dagmodifier.go +++ b/importer/dagwriter/dagmodifier.go @@ -87,18 +87,16 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { if err != nil { return 0, err } + + // We have to rewrite the data before our write in this block. b = append(data[:offset-traversed], b...) break } traversed += size } if startsubblk == len(dm.pbdata.Blocksizes) { - // TODO: something? - /* - if traversed < offset { - return 0, errors.New("Tried to start write outside bounds of file.") - } - */ + // TODO: Im not sure if theres any case that isnt being handled here. + // leaving this note here as a future reference in case something breaks } } @@ -157,7 +155,7 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { sizes = append(sizes, uint64(len(sb))) } - // This is disgusting + // This is disgusting (and can be rewritten if performance demands) if len(changed) > 0 { sechalflink := append(links, dm.curNode.Links[changed[len(changed)-1]+1:]...) dm.curNode.Links = append(dm.curNode.Links[:changed[0]], sechalflink...) @@ -172,6 +170,8 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { return origlen, nil } +// splitBytes uses a splitterFunc to turn a large array of bytes +// into many smaller arrays of bytes func splitBytes(b []byte, spl imp.BlockSplitter) [][]byte { out := spl.Split(bytes.NewReader(b)) var arr [][]byte @@ -181,6 +181,7 @@ func splitBytes(b []byte, spl imp.BlockSplitter) [][]byte { return arr } +// GetNode gets the modified DAG Node func (dm *DagModifier) GetNode() (*mdag.Node, error) { b, err := proto.Marshal(dm.pbdata) if err != nil { diff --git a/importer/dagwriter/dagwriter.go b/importer/dagwriter/dagwriter.go index 2396181360c..e6ab1ed03a7 100644 --- a/importer/dagwriter/dagwriter.go +++ b/importer/dagwriter/dagwriter.go @@ -29,14 +29,23 @@ func NewDagWriter(ds *dag.DAGService, splitter imp.BlockSplitter) *DagWriter { return dw } +// startSplitter manages splitting incoming bytes and +// creating dag nodes from them. Created nodes are stored +// in the DAGService and then released to the GC. func (dw *DagWriter) startSplitter() { + + // Since the splitter functions take a reader (and should!) + // we wrap our byte chan input in a reader r := util.NewByteChanReader(dw.splChan) blkchan := dw.splitter.Split(r) + + // First data block is reserved for storage in the root node first := <-blkchan mbf := new(ft.MultiBlock) root := new(dag.Node) for blkData := range blkchan { + // Store the block size in the root node mbf.AddBlockSize(uint64(len(blkData))) node := &dag.Node{Data: ft.WrapData(blkData)} _, err := dw.dagserv.Add(node) @@ -45,6 +54,8 @@ func (dw *DagWriter) startSplitter() { log.Critical("Got error adding created node to dagservice: %s", err) return } + + // Add a link to this node without storing a reference to the memory err = root.AddNodeLinkClean("", node) if err != nil { dw.seterr = err @@ -52,6 +63,8 @@ func (dw *DagWriter) startSplitter() { return } } + + // Generate the root node data mbf.Data = first data, err := mbf.GetBytes() if err != nil { @@ -61,6 +74,7 @@ func (dw *DagWriter) startSplitter() { } root.Data = data + // Add root node to the dagservice _, err = dw.dagserv.Add(root) if err != nil { dw.seterr = err @@ -79,6 +93,9 @@ func (dw *DagWriter) Write(b []byte) (int, error) { return len(b), nil } +// Close the splitters input channel and wait for it to finish +// Must be called to finish up splitting, otherwise split method +// will never halt func (dw *DagWriter) Close() error { close(dw.splChan) <-dw.done diff --git a/importer/format/format.go b/importer/format/format.go index 19be4e556c8..5dbe76648d9 100644 --- a/importer/format/format.go +++ b/importer/format/format.go @@ -8,6 +8,10 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ) +var ErrMalformedFileFormat = errors.New("malformed data in file format") +var ErrInvalidDirLocation = errors.New("found directory node in unexpected place") +var ErrUnrecognizedType = errors.New("unrecognized node type") + func FromBytes(data []byte) (*PBData, error) { pbdata := new(PBData) err := proto.Unmarshal(data, pbdata) @@ -26,12 +30,17 @@ func FilePBData(data []byte, totalsize uint64) []byte { data, err := proto.Marshal(pbfile) if err != nil { - //this really shouldnt happen, i promise + // This really shouldnt happen, i promise + // The only failure case for marshal is if required fields + // are not filled out, and they all are. If the proto object + // gets changed and nobody updates this function, the code + // should panic due to programmer error panic(err) } return data } +// Returns Bytes that represent a Directory func FolderPBData() []byte { pbfile := new(PBData) typ := PBData_Directory diff --git a/merkledag/dagreader.go b/merkledag/dagreader.go index bc66a600166..3010ca89d29 100644 --- a/merkledag/dagreader.go +++ b/merkledag/dagreader.go @@ -20,6 +20,8 @@ type DagReader struct { buf *bytes.Buffer } +// NewDagReader creates a new reader object that reads the data represented by the given +// node, using the passed in DAGService for data retreival func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) { pb := new(ft.PBData) err := proto.Unmarshal(n.Data, pb) @@ -29,6 +31,7 @@ func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) { switch pb.GetType() { case ft.PBData_Directory: + // Dont allow reading directories return nil, ErrIsDir case ft.PBData_File: return &DagReader{ @@ -37,12 +40,15 @@ func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) { buf: bytes.NewBuffer(pb.GetData()), }, nil case ft.PBData_Raw: + // Raw block will just be a single level, return a byte buffer return bytes.NewBuffer(pb.GetData()), nil default: - panic("Unrecognized node type!") + return nil, ft.ErrUnrecognizedType } } +// Follows the next link in line and loads it from the DAGService, +// setting the next buffer to read from func (dr *DagReader) precalcNextBuf() error { if dr.position >= len(dr.node.Links) { return io.EOF @@ -65,7 +71,7 @@ func (dr *DagReader) precalcNextBuf() error { switch pb.GetType() { case ft.PBData_Directory: - panic("Why is there a directory under a file?") + return ft.ErrInvalidDirLocation case ft.PBData_File: //TODO: this *should* work, needs testing first //return NewDagReader(nxt, dr.serv) @@ -74,11 +80,12 @@ func (dr *DagReader) precalcNextBuf() error { dr.buf = bytes.NewBuffer(pb.GetData()) return nil default: - panic("Unrecognized node type!") + return ft.ErrUnrecognizedType } } func (dr *DagReader) Read(b []byte) (int, error) { + // If no cached buffer, load one if dr.buf == nil { err := dr.precalcNextBuf() if err != nil { @@ -87,16 +94,22 @@ func (dr *DagReader) Read(b []byte) (int, error) { } total := 0 for { + // Attempt to fill bytes from cached buffer n, err := dr.buf.Read(b[total:]) total += n if err != nil { + // EOF is expected if err != io.EOF { return total, err } } + + // If weve read enough bytes, return if total == len(b) { return total, nil } + + // Otherwise, load up the next block err = dr.precalcNextBuf() if err != nil { return total, err From 15a470100380d313730fdab83ed833b791163406 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 8 Oct 2014 17:29:46 +0000 Subject: [PATCH 094/105] update error handling in ipns --- fuse/ipns/ipns_unix.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 9a243f70f42..84e4083f274 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -216,7 +216,10 @@ func (s *Node) loadData() error { // Attr returns the attributes of a given node. func (s *Node) Attr() fuse.Attr { if s.cached == nil { - s.loadData() + err := s.loadData() + if err != nil { + log.Error("Error loading PBData for file: '%s'", s.name) + } } switch s.cached.GetType() { case ft.PBData_Directory: @@ -259,6 +262,7 @@ func (n *Node) makeChild(name string, node *mdag.Node) *Node { parent: n, } + // Always ensure that each child knows where the root is if n.nsRoot == nil { child.nsRoot = n } else { @@ -305,7 +309,9 @@ func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.Intr) fuse.Error { log.Debug("ipns: Node Write [%s]: flags = %s, offset = %d, size = %d", n.name, req.Flags.String(), req.Offset, len(req.Data)) + if n.dagMod == nil { + // Create a DagModifier to allow us to change the existing dag node dmod, err := dt.NewDagModifier(n.Nd, n.Ipfs.DAG, imp.DefaultSplitter) if err != nil { log.Error("Error creating dag modifier: %s", err) @@ -324,6 +330,7 @@ func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.I func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { log.Debug("Got flush request [%s]!", n.name) + // If a write has happened if n.dagMod != nil { newNode, err := n.dagMod.GetNode() if err != nil { @@ -364,6 +371,7 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error { return nil } +// Signal that a node in this tree was changed so the root can republish func (n *Node) wasChanged() { root := n.nsRoot if root == nil { @@ -375,6 +383,8 @@ func (n *Node) wasChanged() { func (n *Node) republishRoot() error { log.Debug("Republish root") + + // We should already be the root, this is just a sanity check var root *Node if n.nsRoot != nil { root = n.nsRoot @@ -392,7 +402,6 @@ func (n *Node) republishRoot() error { ndkey, err := root.Nd.Key() if err != nil { log.Error("getKey error: %s", err) - // return fuse.ETHISREALLYSUCKS return err } log.Debug("Publishing changes!") @@ -432,8 +441,7 @@ func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) err := n.parent.update(n.name, nnode) if err != nil { log.Critical("Error updating node: %s", err) - // Can we panic, please? - return nil, fuse.ENODATA + return nil, err } } n.Nd = nnode @@ -473,14 +481,14 @@ func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr f err := nnode.AddNodeLink(req.Name, nd) if err != nil { log.Error("Error adding child to node: %s", err) - return nil, nil, fuse.ENOENT + return nil, nil, err } if n.parent != nil { err := n.parent.update(n.name, nnode) if err != nil { log.Critical("Error updating node: %s", err) // Can we panic, please? - return nil, nil, fuse.ENODATA + return nil, nil, err } } n.Nd = nnode @@ -502,8 +510,7 @@ func (n *Node) Remove(req *fuse.RemoveRequest, intr fs.Intr) fuse.Error { err := n.parent.update(n.name, nnode) if err != nil { log.Critical("Error updating node: %s", err) - // Can we panic, please? - return fuse.ENODATA + return err } } n.Nd = nnode @@ -530,15 +537,16 @@ func (n *Node) Rename(req *fuse.RenameRequest, newDir fs.Node, intr fs.Intr) fus err := newDir.Nd.AddNodeLink(req.NewName, mdn) if err != nil { log.Error("Error adding node to new dir on rename: %s", err) - return fuse.ENOENT + return err } default: log.Critical("Unknown node type for rename target dir!") - return fuse.ENOENT + return err } return nil } +// Updates the child of this node, specified by name to the given newnode func (n *Node) update(name string, newnode *mdag.Node) error { log.Debug("update '%s' in '%s'", name, n.name) nnode := n.Nd.Copy() From 093c8fb04d0f272af661f4ba8c383a082306770f Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 8 Oct 2014 21:14:18 +0000 Subject: [PATCH 095/105] Rework package structure for unixfs and subpackage cc @jbenet --- core/commands/add.go | 2 +- core/commands/cat.go | 4 ++-- fuse/ipns/ipns_unix.go | 15 ++++++++------- fuse/readonly/readonly_unix.go | 7 ++++--- importer/{ => chunk}/rabin.go | 2 +- importer/{ => chunk}/splitting.go | 12 ++++++++++-- importer/importer.go | 9 ++++----- importer/importer_test.go | 18 +++++++++++------- server/http/ipfs.go | 3 ++- {importer/format => unixfs}/Makefile | 0 {importer/format => unixfs}/data.pb.go | 10 +++++----- {importer/format => unixfs}/data.proto | 2 +- {importer/format => unixfs}/format.go | 2 +- {importer/format => unixfs}/format_test.go | 2 +- .../dagwriter => unixfs/io}/dagmodifier.go | 12 ++++++------ .../io}/dagmodifier_test.go | 14 +++++++------- {merkledag => unixfs/io}/dagreader.go | 11 ++++++----- {importer/dagwriter => unixfs/io}/dagwriter.go | 10 +++++----- .../dagwriter => unixfs/io}/dagwriter_test.go | 12 ++++++------ 19 files changed, 81 insertions(+), 66 deletions(-) rename importer/{ => chunk}/rabin.go (99%) rename importer/{ => chunk}/splitting.go (79%) rename {importer/format => unixfs}/Makefile (100%) rename {importer/format => unixfs}/data.pb.go (88%) rename {importer/format => unixfs}/data.proto (92%) rename {importer/format => unixfs}/format.go (99%) rename {importer/format => unixfs}/format_test.go (97%) rename {importer/dagwriter => unixfs/io}/dagmodifier.go (95%) rename {importer/dagwriter => unixfs/io}/dagmodifier_test.go (90%) rename {merkledag => unixfs/io}/dagreader.go (93%) rename {importer/dagwriter => unixfs/io}/dagwriter.go (91%) rename {importer/dagwriter => unixfs/io}/dagwriter_test.go (89%) diff --git a/core/commands/add.go b/core/commands/add.go index 4a3a0e81c0b..e51c4b7982c 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -10,8 +10,8 @@ import ( "github.com/jbenet/go-ipfs/core" "github.com/jbenet/go-ipfs/importer" - ft "github.com/jbenet/go-ipfs/importer/format" dag "github.com/jbenet/go-ipfs/merkledag" + ft "github.com/jbenet/go-ipfs/unixfs" u "github.com/jbenet/go-ipfs/util" ) diff --git a/core/commands/cat.go b/core/commands/cat.go index 5cb3f9651c4..7ebae801c9b 100644 --- a/core/commands/cat.go +++ b/core/commands/cat.go @@ -5,7 +5,7 @@ import ( "io" "github.com/jbenet/go-ipfs/core" - mdag "github.com/jbenet/go-ipfs/merkledag" + uio "github.com/jbenet/go-ipfs/unixfs/io" ) func Cat(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error { @@ -15,7 +15,7 @@ func Cat(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Wr return fmt.Errorf("catFile error: %v", err) } - read, err := mdag.NewDagReader(dagnode, n.DAG) + read, err := uio.NewDagReader(dagnode, n.DAG) if err != nil { return fmt.Errorf("cat error: %v", err) } diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 84e4083f274..3f56dbb6821 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -1,6 +1,7 @@ package ipns import ( + "errors" "io/ioutil" "os" "path/filepath" @@ -12,10 +13,10 @@ import ( "github.com/jbenet/go-ipfs/core" ci "github.com/jbenet/go-ipfs/crypto" - imp "github.com/jbenet/go-ipfs/importer" - dt "github.com/jbenet/go-ipfs/importer/dagwriter" - ft "github.com/jbenet/go-ipfs/importer/format" + "github.com/jbenet/go-ipfs/importer/chunk" mdag "github.com/jbenet/go-ipfs/merkledag" + ft "github.com/jbenet/go-ipfs/unixfs" + uio "github.com/jbenet/go-ipfs/unixfs/io" u "github.com/jbenet/go-ipfs/util" ) @@ -204,7 +205,7 @@ type Node struct { Ipfs *core.IpfsNode Nd *mdag.Node - dagMod *dt.DagModifier + dagMod *uio.DagModifier cached *ft.PBData } @@ -293,7 +294,7 @@ func (s *Node) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { // ReadAll reads the object data as file data func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { log.Debug("ipns: ReadAll [%s]", s.name) - r, err := mdag.NewDagReader(s.Nd, s.Ipfs.DAG) + r, err := uio.NewDagReader(s.Nd, s.Ipfs.DAG) if err != nil { return nil, err } @@ -312,7 +313,7 @@ func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.I if n.dagMod == nil { // Create a DagModifier to allow us to change the existing dag node - dmod, err := dt.NewDagModifier(n.Nd, n.Ipfs.DAG, imp.DefaultSplitter) + dmod, err := uio.NewDagModifier(n.Nd, n.Ipfs.DAG, chunk.DefaultSplitter) if err != nil { log.Error("Error creating dag modifier: %s", err) return err @@ -541,7 +542,7 @@ func (n *Node) Rename(req *fuse.RenameRequest, newDir fs.Node, intr fs.Intr) fus } default: log.Critical("Unknown node type for rename target dir!") - return err + return errors.New("Unknown fs node type!") } return nil } diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index e7550bee794..2091495f196 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -19,8 +19,9 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" core "github.com/jbenet/go-ipfs/core" - ft "github.com/jbenet/go-ipfs/importer/format" mdag "github.com/jbenet/go-ipfs/merkledag" + ft "github.com/jbenet/go-ipfs/unixfs" + uio "github.com/jbenet/go-ipfs/unixfs/io" u "github.com/jbenet/go-ipfs/util" ) @@ -79,7 +80,7 @@ func (*Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { type Node struct { Ipfs *core.IpfsNode Nd *mdag.Node - fd *mdag.DagReader + fd *uio.DagReader cached *ft.PBData } @@ -143,7 +144,7 @@ func (s *Node) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { // ReadAll reads the object data as file data func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) { u.DOut("Read node.\n") - r, err := mdag.NewDagReader(s.Nd, s.Ipfs.DAG) + r, err := uio.NewDagReader(s.Nd, s.Ipfs.DAG) if err != nil { return nil, err } diff --git a/importer/rabin.go b/importer/chunk/rabin.go similarity index 99% rename from importer/rabin.go rename to importer/chunk/rabin.go index 64b6f8d3f31..fbfb4cec41e 100644 --- a/importer/rabin.go +++ b/importer/chunk/rabin.go @@ -1,4 +1,4 @@ -package importer +package chunk import ( "bufio" diff --git a/importer/splitting.go b/importer/chunk/splitting.go similarity index 79% rename from importer/splitting.go rename to importer/chunk/splitting.go index 7b6e56ceee7..0b5717eaf6e 100644 --- a/importer/splitting.go +++ b/importer/chunk/splitting.go @@ -1,6 +1,14 @@ -package importer +package chunk -import "io" +import ( + "io" + + "github.com/jbenet/go-ipfs/util" +) + +var log = util.Logger("chunk") + +var DefaultSplitter = &SizeSplitter{1024 * 512} type BlockSplitter interface { Split(r io.Reader) chan []byte diff --git a/importer/importer.go b/importer/importer.go index 3c92eb3ab13..0a4d9848e4e 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -5,8 +5,9 @@ import ( "io" "os" - ft "github.com/jbenet/go-ipfs/importer/format" + "github.com/jbenet/go-ipfs/importer/chunk" dag "github.com/jbenet/go-ipfs/merkledag" + ft "github.com/jbenet/go-ipfs/unixfs" "github.com/jbenet/go-ipfs/util" ) @@ -18,18 +19,16 @@ var BlockSizeLimit = int64(1048576) // 1 MB // ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit. var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded") -var DefaultSplitter = &SizeSplitter{1024 * 512} - // todo: incremental construction with an ipfs node. dumping constructed // objects into the datastore, to avoid buffering all in memory // NewDagFromReader constructs a Merkle DAG from the given io.Reader. // size required for block construction. func NewDagFromReader(r io.Reader) (*dag.Node, error) { - return NewDagFromReaderWithSplitter(r, DefaultSplitter) + return NewDagFromReaderWithSplitter(r, chunk.DefaultSplitter) } -func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, error) { +func NewDagFromReaderWithSplitter(r io.Reader, spl chunk.BlockSplitter) (*dag.Node, error) { blkChan := spl.Split(r) first := <-blkChan root := &dag.Node{} diff --git a/importer/importer_test.go b/importer/importer_test.go index 0a0bf3257de..dd52f9e1fc3 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -9,9 +9,13 @@ import ( "os" "testing" - dag "github.com/jbenet/go-ipfs/merkledag" + "github.com/jbenet/go-ipfs/importer/chunk" + uio "github.com/jbenet/go-ipfs/unixfs/io" ) +// NOTE: +// These tests tests a combination of unixfs/io/dagreader and importer/chunk. +// Maybe split them up somehow? func TestBuildDag(t *testing.T) { td := os.TempDir() fi, err := os.Create(td + "/tmpfi") @@ -34,9 +38,9 @@ func TestBuildDag(t *testing.T) { //Test where calls to read are smaller than the chunk size func TestSizeBasedSplit(t *testing.T) { - bs := &SizeSplitter{512} + bs := &chunk.SizeSplitter{512} testFileConsistency(t, bs, 32*512) - bs = &SizeSplitter{4096} + bs = &chunk.SizeSplitter{4096} testFileConsistency(t, bs, 32*4096) // Uneven offset @@ -49,7 +53,7 @@ func dup(b []byte) []byte { return o } -func testFileConsistency(t *testing.T, bs BlockSplitter, nbytes int) { +func testFileConsistency(t *testing.T, bs chunk.BlockSplitter, nbytes int) { buf := new(bytes.Buffer) io.CopyN(buf, rand.Reader, int64(nbytes)) should := dup(buf.Bytes()) @@ -57,7 +61,7 @@ func testFileConsistency(t *testing.T, bs BlockSplitter, nbytes int) { if err != nil { t.Fatal(err) } - r, err := dag.NewDagReader(nd, nil) + r, err := uio.NewDagReader(nd, nil) if err != nil { t.Fatal(err) } @@ -86,14 +90,14 @@ func arrComp(a, b []byte) error { } func TestMaybeRabinConsistency(t *testing.T) { - testFileConsistency(t, NewMaybeRabin(4096), 256*4096) + testFileConsistency(t, chunk.NewMaybeRabin(4096), 256*4096) } func TestRabinBlockSize(t *testing.T) { buf := new(bytes.Buffer) nbytes := 1024 * 1024 io.CopyN(buf, rand.Reader, int64(nbytes)) - rab := NewMaybeRabin(4096) + rab := chunk.NewMaybeRabin(4096) blkch := rab.Split(buf) var blocks [][]byte diff --git a/server/http/ipfs.go b/server/http/ipfs.go index 6a177023f8d..bd094bada9e 100644 --- a/server/http/ipfs.go +++ b/server/http/ipfs.go @@ -6,6 +6,7 @@ import ( core "github.com/jbenet/go-ipfs/core" "github.com/jbenet/go-ipfs/importer" dag "github.com/jbenet/go-ipfs/merkledag" + uio "github.com/jbenet/go-ipfs/unixfs/io" u "github.com/jbenet/go-ipfs/util" ) @@ -33,5 +34,5 @@ func (i *ipfsHandler) AddNodeToDAG(nd *dag.Node) (u.Key, error) { } func (i *ipfsHandler) NewDagReader(nd *dag.Node) (io.Reader, error) { - return dag.NewDagReader(nd, i.node.DAG) + return uio.NewDagReader(nd, i.node.DAG) } diff --git a/importer/format/Makefile b/unixfs/Makefile similarity index 100% rename from importer/format/Makefile rename to unixfs/Makefile diff --git a/importer/format/data.pb.go b/unixfs/data.pb.go similarity index 88% rename from importer/format/data.pb.go rename to unixfs/data.pb.go index ffc942296d1..89e5f808447 100644 --- a/importer/format/data.pb.go +++ b/unixfs/data.pb.go @@ -3,7 +3,7 @@ // DO NOT EDIT! /* -Package format is a generated protocol buffer package. +Package unixfs is a generated protocol buffer package. It is generated from these files: data.proto @@ -11,9 +11,9 @@ It is generated from these files: It has these top-level messages: PBData */ -package format +package unixfs -import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" +import proto "code.google.com/p/goprotobuf/proto" import math "math" // Reference imports to suppress errors if they are not otherwise used. @@ -57,7 +57,7 @@ func (x *PBData_DataType) UnmarshalJSON(data []byte) error { } type PBData struct { - Type *PBData_DataType `protobuf:"varint,1,req,enum=format.PBData_DataType" json:"Type,omitempty"` + Type *PBData_DataType `protobuf:"varint,1,req,enum=unixfs.PBData_DataType" json:"Type,omitempty"` Data []byte `protobuf:"bytes,2,opt" json:"Data,omitempty"` Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"` Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"` @@ -97,5 +97,5 @@ func (m *PBData) GetBlocksizes() []uint64 { } func init() { - proto.RegisterEnum("format.PBData_DataType", PBData_DataType_name, PBData_DataType_value) + proto.RegisterEnum("unixfs.PBData_DataType", PBData_DataType_name, PBData_DataType_value) } diff --git a/importer/format/data.proto b/unixfs/data.proto similarity index 92% rename from importer/format/data.proto rename to unixfs/data.proto index 9538c7c15d0..b9504b0c324 100644 --- a/importer/format/data.proto +++ b/unixfs/data.proto @@ -1,4 +1,4 @@ -package format; +package unixfs; message PBData { enum DataType { diff --git a/importer/format/format.go b/unixfs/format.go similarity index 99% rename from importer/format/format.go rename to unixfs/format.go index 5dbe76648d9..6ba8e3aa475 100644 --- a/importer/format/format.go +++ b/unixfs/format.go @@ -1,6 +1,6 @@ // Package format implements a data format for files in the ipfs filesystem // It is not the only format in ipfs, but it is the one that the filesystem assumes -package format +package unixfs import ( "errors" diff --git a/importer/format/format_test.go b/unixfs/format_test.go similarity index 97% rename from importer/format/format_test.go rename to unixfs/format_test.go index bd234d7a6d2..eca926e9f18 100644 --- a/importer/format/format_test.go +++ b/unixfs/format_test.go @@ -1,4 +1,4 @@ -package format +package unixfs import ( "testing" diff --git a/importer/dagwriter/dagmodifier.go b/unixfs/io/dagmodifier.go similarity index 95% rename from importer/dagwriter/dagmodifier.go rename to unixfs/io/dagmodifier.go index c178ecbe121..8680da46a27 100644 --- a/importer/dagwriter/dagmodifier.go +++ b/unixfs/io/dagmodifier.go @@ -1,4 +1,4 @@ -package dagwriter +package io import ( "bytes" @@ -6,9 +6,9 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" - imp "github.com/jbenet/go-ipfs/importer" - ft "github.com/jbenet/go-ipfs/importer/format" + "github.com/jbenet/go-ipfs/importer/chunk" mdag "github.com/jbenet/go-ipfs/merkledag" + ft "github.com/jbenet/go-ipfs/unixfs" u "github.com/jbenet/go-ipfs/util" ) @@ -20,10 +20,10 @@ type DagModifier struct { curNode *mdag.Node pbdata *ft.PBData - splitter imp.BlockSplitter + splitter chunk.BlockSplitter } -func NewDagModifier(from *mdag.Node, serv *mdag.DAGService, spl imp.BlockSplitter) (*DagModifier, error) { +func NewDagModifier(from *mdag.Node, serv *mdag.DAGService, spl chunk.BlockSplitter) (*DagModifier, error) { pbd, err := ft.FromBytes(from.Data) if err != nil { return nil, err @@ -172,7 +172,7 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) { // splitBytes uses a splitterFunc to turn a large array of bytes // into many smaller arrays of bytes -func splitBytes(b []byte, spl imp.BlockSplitter) [][]byte { +func splitBytes(b []byte, spl chunk.BlockSplitter) [][]byte { out := spl.Split(bytes.NewReader(b)) var arr [][]byte for blk := range out { diff --git a/importer/dagwriter/dagmodifier_test.go b/unixfs/io/dagmodifier_test.go similarity index 90% rename from importer/dagwriter/dagmodifier_test.go rename to unixfs/io/dagmodifier_test.go index 129d836d5b6..e3ea8e4f76b 100644 --- a/importer/dagwriter/dagmodifier_test.go +++ b/unixfs/io/dagmodifier_test.go @@ -1,4 +1,4 @@ -package dagwriter +package io import ( "fmt" @@ -8,9 +8,9 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" bs "github.com/jbenet/go-ipfs/blockservice" - imp "github.com/jbenet/go-ipfs/importer" - ft "github.com/jbenet/go-ipfs/importer/format" + "github.com/jbenet/go-ipfs/importer/chunk" mdag "github.com/jbenet/go-ipfs/merkledag" + ft "github.com/jbenet/go-ipfs/unixfs" u "github.com/jbenet/go-ipfs/util" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" @@ -26,7 +26,7 @@ func getMockDagServ(t *testing.T) *mdag.DAGService { } func getNode(t *testing.T, dserv *mdag.DAGService, size int64) ([]byte, *mdag.Node) { - dw := NewDagWriter(dserv, &imp.SizeSplitter{500}) + dw := NewDagWriter(dserv, &chunk.SizeSplitter{500}) n, err := io.CopyN(dw, u.NewFastRand(), size) if err != nil { @@ -39,7 +39,7 @@ func getNode(t *testing.T, dserv *mdag.DAGService, size int64) ([]byte, *mdag.No dw.Close() node := dw.GetNode() - dr, err := mdag.NewDagReader(node, dserv) + dr, err := NewDagReader(node, dserv) if err != nil { t.Fatal(err) } @@ -76,7 +76,7 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier) t.Fatal(err) } - rd, err := mdag.NewDagReader(nd, dm.dagserv) + rd, err := NewDagReader(nd, dm.dagserv) if err != nil { t.Fatal(err) } @@ -99,7 +99,7 @@ func TestDagModifierBasic(t *testing.T) { dserv := getMockDagServ(t) b, n := getNode(t, dserv, 50000) - dagmod, err := NewDagModifier(n, dserv, &imp.SizeSplitter{512}) + dagmod, err := NewDagModifier(n, dserv, &chunk.SizeSplitter{512}) if err != nil { t.Fatal(err) } diff --git a/merkledag/dagreader.go b/unixfs/io/dagreader.go similarity index 93% rename from merkledag/dagreader.go rename to unixfs/io/dagreader.go index 3010ca89d29..29196a1e3f1 100644 --- a/merkledag/dagreader.go +++ b/unixfs/io/dagreader.go @@ -1,4 +1,4 @@ -package merkledag +package io import ( "bytes" @@ -6,7 +6,8 @@ import ( "io" proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" - ft "github.com/jbenet/go-ipfs/importer/format" + mdag "github.com/jbenet/go-ipfs/merkledag" + ft "github.com/jbenet/go-ipfs/unixfs" u "github.com/jbenet/go-ipfs/util" ) @@ -14,15 +15,15 @@ var ErrIsDir = errors.New("this dag node is a directory") // DagReader provides a way to easily read the data contained in a dag. type DagReader struct { - serv *DAGService - node *Node + serv *mdag.DAGService + node *mdag.Node position int buf *bytes.Buffer } // NewDagReader creates a new reader object that reads the data represented by the given // node, using the passed in DAGService for data retreival -func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) { +func NewDagReader(n *mdag.Node, serv *mdag.DAGService) (io.Reader, error) { pb := new(ft.PBData) err := proto.Unmarshal(n.Data, pb) if err != nil { diff --git a/importer/dagwriter/dagwriter.go b/unixfs/io/dagwriter.go similarity index 91% rename from importer/dagwriter/dagwriter.go rename to unixfs/io/dagwriter.go index e6ab1ed03a7..4abb1b36c76 100644 --- a/importer/dagwriter/dagwriter.go +++ b/unixfs/io/dagwriter.go @@ -1,9 +1,9 @@ -package dagwriter +package io import ( - imp "github.com/jbenet/go-ipfs/importer" - ft "github.com/jbenet/go-ipfs/importer/format" + "github.com/jbenet/go-ipfs/importer/chunk" dag "github.com/jbenet/go-ipfs/merkledag" + ft "github.com/jbenet/go-ipfs/unixfs" "github.com/jbenet/go-ipfs/util" ) @@ -15,11 +15,11 @@ type DagWriter struct { totalSize int64 splChan chan []byte done chan struct{} - splitter imp.BlockSplitter + splitter chunk.BlockSplitter seterr error } -func NewDagWriter(ds *dag.DAGService, splitter imp.BlockSplitter) *DagWriter { +func NewDagWriter(ds *dag.DAGService, splitter chunk.BlockSplitter) *DagWriter { dw := new(DagWriter) dw.dagserv = ds dw.splChan = make(chan []byte, 8) diff --git a/importer/dagwriter/dagwriter_test.go b/unixfs/io/dagwriter_test.go similarity index 89% rename from importer/dagwriter/dagwriter_test.go rename to unixfs/io/dagwriter_test.go index 8f0bc26866e..73ba5c4e910 100644 --- a/importer/dagwriter/dagwriter_test.go +++ b/unixfs/io/dagwriter_test.go @@ -1,4 +1,4 @@ -package dagwriter +package io import ( "testing" @@ -7,7 +7,7 @@ import ( ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" bs "github.com/jbenet/go-ipfs/blockservice" - imp "github.com/jbenet/go-ipfs/importer" + chunk "github.com/jbenet/go-ipfs/importer/chunk" mdag "github.com/jbenet/go-ipfs/merkledag" ) @@ -54,7 +54,7 @@ func TestDagWriter(t *testing.T) { t.Fatal(err) } dag := &mdag.DAGService{bserv} - dw := NewDagWriter(dag, &imp.SizeSplitter{4096}) + dw := NewDagWriter(dag, &chunk.SizeSplitter{4096}) nbytes := int64(1024 * 1024 * 2) n, err := io.CopyN(dw, &datasource{}, nbytes) @@ -69,7 +69,7 @@ func TestDagWriter(t *testing.T) { dw.Close() node := dw.GetNode() - read, err := mdag.NewDagReader(node, dag) + read, err := NewDagReader(node, dag) if err != nil { t.Fatal(err) } @@ -88,7 +88,7 @@ func TestMassiveWrite(t *testing.T) { t.Fatal(err) } dag := &mdag.DAGService{bserv} - dw := NewDagWriter(dag, &imp.SizeSplitter{4096}) + dw := NewDagWriter(dag, &chunk.SizeSplitter{4096}) nbytes := int64(1024 * 1024 * 1024 * 16) n, err := io.CopyN(dw, &datasource{}, nbytes) @@ -113,7 +113,7 @@ func BenchmarkDagWriter(b *testing.B) { nbytes := int64(100000) for i := 0; i < b.N; i++ { b.SetBytes(nbytes) - dw := NewDagWriter(dag, &imp.SizeSplitter{4096}) + dw := NewDagWriter(dag, &chunk.SizeSplitter{4096}) n, err := io.CopyN(dw, &datasource{}, nbytes) if err != nil { b.Fatal(err) From 9ebf3ae64081a943dadc7f361f7b27e4eced67a7 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 8 Oct 2014 14:33:31 -0700 Subject: [PATCH 096/105] keytransform ds --- Godeps/Godeps.json | 2 +- .../datastore.go/keytransform/keytransform.go | 88 +++++++++++++++++++ .../keytransform/keytransform_test.go | 60 +++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/keytransform.go create mode 100644 Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/keytransform_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1744a65aff1..818b74451f2 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -64,7 +64,7 @@ }, { "ImportPath": "github.com/jbenet/datastore.go", - "Rev": "e7d6f7cb9e3c207a04c5397c449d10a6f9d403a0" + "Rev": "60ebc56447b5a8264cfed3ae3ff48deb984d7cf1" }, { "ImportPath": "github.com/jbenet/go-base58", diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/keytransform.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/keytransform.go new file mode 100644 index 00000000000..3be41db6533 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/keytransform.go @@ -0,0 +1,88 @@ +package keytransform + +import ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + +// KeyTransform is a function that transforms one key into another. +type KeyTransform func(ds.Key) ds.Key + +// Datastore is a keytransform.Datastore +type Datastore interface { + ds.Shim + + // Transform runs the transformation function + Transform(ds.Key) ds.Key + + // TransformFunc returns the KeyTransform function + TransformFunc() KeyTransform +} + +// ktds keeps a KeyTransform function +type ktds struct { + child ds.Datastore + xform KeyTransform +} + +// WrapDatastore wraps a given datastore with a KeyTransform function. +// The resulting wrapped datastore will use the transform on all Datastore +// operations. +func WrapDatastore(child ds.Datastore, f KeyTransform) Datastore { + if f == nil { + panic("f (KeyTransform) is nil") + } + + if child == nil { + panic("child (ds.Datastore) is nil") + } + + return &ktds{child, f} +} + +// TransformFunc returns the KeyTransform function +func (d *ktds) TransformFunc() KeyTransform { + return d.xform +} + +// Transform runs the KeyTransform function +func (d *ktds) Transform(k ds.Key) ds.Key { + return d.xform(k) +} + +// Children implements ds.Shim +func (d *ktds) Children() []ds.Datastore { + return []ds.Datastore{d.child} +} + +// Put stores the given value, transforming the key first. +func (d *ktds) Put(key ds.Key, value interface{}) (err error) { + return d.child.Put(d.Transform(key), value) +} + +// Get returns the value for given key, transforming the key first. +func (d *ktds) Get(key ds.Key) (value interface{}, err error) { + return d.child.Get(d.Transform(key)) +} + +// Has returns whether the datastore has a value for a given key, transforming +// the key first. +func (d *ktds) Has(key ds.Key) (exists bool, err error) { + return d.child.Has(d.Transform(key)) +} + +// Delete removes the value for given key +func (d *ktds) Delete(key ds.Key) (err error) { + return d.child.Delete(d.Transform(key)) +} + +// KeyList returns a list of all keys in the datastore, transforming keys out. +func (d *ktds) KeyList() ([]ds.Key, error) { + + keys, err := d.child.KeyList() + if err != nil { + return nil, err + } + + for i, k := range keys { + keys[i] = d.Transform(k) + } + return keys, nil +} diff --git a/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/keytransform_test.go b/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/keytransform_test.go new file mode 100644 index 00000000000..ad752d27424 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/keytransform_test.go @@ -0,0 +1,60 @@ +package keytransform_test + +import ( + "bytes" + "testing" + + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" + kt "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform" + . "launchpad.net/gocheck" +) + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { TestingT(t) } + +type DSSuite struct { + dir string + ds ds.Datastore +} + +var _ = Suite(&DSSuite{}) + +func (ks *DSSuite) TestBasic(c *C) { + + mpds := ds.NewMapDatastore() + ktds := kt.WrapDatastore(mpds, func(k ds.Key) ds.Key { + return k.Reverse() + }) + + keys := strsToKeys([]string{ + "foo", + "foo/bar", + "foo/bar/baz", + "foo/barb", + "foo/bar/bazb", + "foo/bar/baz/barb", + }) + + for _, k := range keys { + err := ktds.Put(k, []byte(k.String())) + c.Check(err, Equals, nil) + } + + for _, k := range keys { + v1, err := ktds.Get(k) + c.Check(err, Equals, nil) + c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true) + + v2, err := mpds.Get(k.Reverse()) + c.Check(err, Equals, nil) + c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true) + } +} + +func strsToKeys(strs []string) []ds.Key { + keys := make([]ds.Key, len(strs)) + for i, s := range strs { + keys[i] = ds.NewKey(s) + } + return keys +} From 0ffc20384e80f829b1dc9ba40a1671c5291a4934 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 8 Oct 2014 14:48:32 -0700 Subject: [PATCH 097/105] make vendor is your friend cc @whyrusleeping --- unixfs/data.pb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unixfs/data.pb.go b/unixfs/data.pb.go index 89e5f808447..2efdd8a4ca4 100644 --- a/unixfs/data.pb.go +++ b/unixfs/data.pb.go @@ -13,7 +13,7 @@ It has these top-level messages: */ package unixfs -import proto "code.google.com/p/goprotobuf/proto" +import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" import math "math" // Reference imports to suppress errors if they are not otherwise used. From dc6fdd39c58264fd206c54a9fc793a0a01ebe19b Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 8 Oct 2014 14:49:02 -0700 Subject: [PATCH 098/105] use encoded (pretty) keys only on fs ds --- core/datastore.go | 6 +++++- util/util.go | 28 ++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/core/datastore.go b/core/datastore.go index 30f92c8e8b3..da73285f1ea 100644 --- a/core/datastore.go +++ b/core/datastore.go @@ -5,9 +5,12 @@ import ( ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" fsds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs" + ktds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform" lds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb" syncds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/sync" + config "github.com/jbenet/go-ipfs/config" + u "github.com/jbenet/go-ipfs/util" ) func makeDatastore(cfg config.Datastore) (ds.ThreadSafeDatastore, error) { @@ -28,7 +31,8 @@ func makeDatastore(cfg config.Datastore) (ds.ThreadSafeDatastore, error) { if err != nil { return nil, err } - return syncds.MutexWrap(d), nil + ktd := ktds.WrapDatastore(d, u.DsKeyB58Encode) + return syncds.MutexWrap(ktd), nil } return nil, fmt.Errorf("Unknown datastore type: %s", cfg.Type) diff --git a/util/util.go b/util/util.go index a22488372f1..767f05d409a 100644 --- a/util/util.go +++ b/util/util.go @@ -55,14 +55,34 @@ func (k Key) Pretty() string { // DsKey returns a Datastore key func (k Key) DsKey() ds.Key { - return ds.NewKey(k.Pretty()) + return ds.NewKey(string(k)) } // KeyFromDsKey returns a Datastore key func KeyFromDsKey(dsk ds.Key) Key { - l := dsk.List() - enc := l[len(l)-1] - return Key(b58.Decode(enc)) + return Key(dsk.BaseNamespace()) +} + +// DsKeyB58Encode returns a B58 encoded Datastore key +// TODO: this is hacky because it encodes every path component. some +// path components may be proper strings already... +func DsKeyB58Encode(dsk ds.Key) ds.Key { + k := ds.NewKey("/") + for _, n := range dsk.Namespaces() { + k = k.Child(b58.Encode([]byte(n))) + } + return k +} + +// DsKeyB58Decode returns a b58 decoded Datastore key +// TODO: this is hacky because it encodes every path component. some +// path components may be proper strings already... +func DsKeyB58Decode(dsk ds.Key) ds.Key { + k := ds.NewKey("/") + for _, n := range dsk.Namespaces() { + k = k.Child(string(b58.Decode(n))) + } + return k } // Hash is the global IPFS hash function. uses multihash SHA2_256, 256 bits From 31cc605c3ac9c188e6ed411e398d17ec2c1413e8 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 8 Oct 2014 14:49:21 -0700 Subject: [PATCH 099/105] comment out debug msg --- util/util.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/util/util.go b/util/util.go index 767f05d409a..dfc0836beb3 100644 --- a/util/util.go +++ b/util/util.go @@ -154,13 +154,14 @@ func SetupLogging() { logging.SetBackend(backend) logging.SetFormatter(logging.MustStringFormatter(LogFormat)) - /* - if Debug { - logging.SetLevel(logging.DEBUG, "") - } else { - logging.SetLevel(logging.ERROR, "") - } - */ + // just uncomment Debug = True right here for all logging. + // but please don't commit that. + // Debug = True + if Debug { + logging.SetLevel(logging.DEBUG, "") + } else { + logging.SetLevel(logging.ERROR, "") + } for n, log := range loggers { logging.SetLevel(logging.ERROR, n) From 1aafd285b545521bdbd0ada6bd8fe48fa598002c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 8 Oct 2014 21:55:50 +0000 Subject: [PATCH 100/105] add in some extra debug logging, and increase routing table latencies --- routing/dht/dht.go | 4 ++-- routing/dht/handlers.go | 1 + routing/dht/routing.go | 1 + routing/kbucket/table.go | 4 ++++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/routing/dht/dht.go b/routing/dht/dht.go index cfa500ee28f..c95e0751136 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -67,8 +67,8 @@ func NewDHT(p *peer.Peer, ps peer.Peerstore, net inet.Network, sender inet.Sende dht.providers = NewProviderManager(p.ID) dht.routingTables = make([]*kb.RoutingTable, 3) - dht.routingTables[0] = kb.NewRoutingTable(20, kb.ConvertPeerID(p.ID), time.Millisecond*30) - dht.routingTables[1] = kb.NewRoutingTable(20, kb.ConvertPeerID(p.ID), time.Millisecond*100) + dht.routingTables[0] = kb.NewRoutingTable(20, kb.ConvertPeerID(p.ID), time.Millisecond*1000) + dht.routingTables[1] = kb.NewRoutingTable(20, kb.ConvertPeerID(p.ID), time.Millisecond*1000) dht.routingTables[2] = kb.NewRoutingTable(20, kb.ConvertPeerID(p.ID), time.Hour) dht.birth = time.Now() return dht diff --git a/routing/dht/handlers.go b/routing/dht/handlers.go index 49e3eb75080..417dd0918f1 100644 --- a/routing/dht/handlers.go +++ b/routing/dht/handlers.go @@ -137,6 +137,7 @@ func (dht *IpfsDHT) handleGetProviders(p *peer.Peer, pmes *Message) (*Message, e resp := newMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel()) // check if we have this value, to add ourselves as provider. + log.Debug("handling GetProviders: '%s'", pmes.GetKey()) dsk := u.Key(pmes.GetKey()).DsKey() has, err := dht.datastore.Has(dsk) if err != nil && err != ds.ErrNotFound { diff --git a/routing/dht/routing.go b/routing/dht/routing.go index 25567038cdb..d29a46fef0e 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -192,6 +192,7 @@ func (dht *IpfsDHT) FindProviders(ctx context.Context, key u.Key) ([]*peer.Peer, log.Debug("Find providers for: '%s'", key) p := dht.routingTables[0].NearestPeer(kb.ConvertKey(key)) if p == nil { + log.Warning("Got no nearest peer for find providers: '%s'", key) return nil, nil } diff --git a/routing/kbucket/table.go b/routing/kbucket/table.go index 2a0f16d1a0b..242546ba403 100644 --- a/routing/kbucket/table.go +++ b/routing/kbucket/table.go @@ -11,6 +11,8 @@ import ( u "github.com/jbenet/go-ipfs/util" ) +var log = u.Logger("table") + // RoutingTable defines the routing table. type RoutingTable struct { @@ -138,6 +140,8 @@ func (rt *RoutingTable) NearestPeer(id ID) *peer.Peer { if len(peers) > 0 { return peers[0] } + + log.Error("NearestPeer: Returning nil, table size = %d", rt.Size()) return nil } From 38177b489d1139a4f7ad7d5694b58584f031d650 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 8 Oct 2014 22:38:33 +0000 Subject: [PATCH 101/105] add another test to try and reproduce data loss issue --- unixfs/io/dagmodifier_test.go | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/unixfs/io/dagmodifier_test.go b/unixfs/io/dagmodifier_test.go index e3ea8e4f76b..32d9a84b5da 100644 --- a/unixfs/io/dagmodifier_test.go +++ b/unixfs/io/dagmodifier_test.go @@ -146,6 +146,47 @@ func TestDagModifierBasic(t *testing.T) { } } +func TestMultiWrite(t *testing.T) { + dserv := getMockDagServ(t) + _, n := getNode(t, dserv, 0) + + dagmod, err := NewDagModifier(n, dserv, &chunk.SizeSplitter{512}) + if err != nil { + t.Fatal(err) + } + + data := make([]byte, 4000) + u.NewFastRand().Read(data) + + for i := 0; i < len(data); i++ { + n, err := dagmod.WriteAt(data[i:i+1], uint64(i)) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)") + } + } + nd, err := dagmod.GetNode() + if err != nil { + t.Fatal(err) + } + + read, err := NewDagReader(nd, dserv) + if err != nil { + t.Fatal(err) + } + rbuf, err := ioutil.ReadAll(read) + if err != nil { + t.Fatal(err) + } + + err = arrComp(rbuf, data) + if err != nil { + t.Fatal(err) + } +} + func arrComp(a, b []byte) error { if len(a) != len(b) { return fmt.Errorf("Arrays differ in length. %d != %d", len(a), len(b)) From c565309a2153303ca820ce44864917e0991a610c Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 9 Oct 2014 02:59:56 -0700 Subject: [PATCH 102/105] skip ipns fuse tests in travis --- .travis.yml | 2 ++ fuse/ipns/ipns_test.go | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/.travis.yml b/.travis.yml index 32ce5a78639..f5bf5732a28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,5 @@ go: script: - go test -v ./... + +env: TEST_NO_FUSE=1 diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index 8daf1476998..108a01c0b6f 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -13,6 +13,17 @@ import ( u "github.com/jbenet/go-ipfs/util" ) +func maybeSkipFuseTests(t *testing.T) bool { + v := "TEST_NO_FUSE" + n := strings.ToLower(os.Getenv(v)) + skip := n != "" && n != "false" && n != "f" + + if skip { + t.Skipf("Skipping FUSE tests (%s=%s)", v, n) + } + return skip +} + func randBytes(size int) []byte { b := make([]byte, size) rand.Read(b) @@ -47,6 +58,8 @@ func writeFileData(t *testing.T, data []byte, path string) []byte { } func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) { + maybeSkipFuseTests(t) + var err error if node == nil { node, err = core.NewMockNode() From 6ac11702098a7649cb8908b458f676d01975c9d0 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 9 Oct 2014 03:03:57 -0700 Subject: [PATCH 103/105] Skip ipns_test.TestMultiWrite in darwin --- fuse/ipns/ipns_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index 108a01c0b6f..0d3ed7a842e 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -5,6 +5,8 @@ import ( "crypto/rand" "io/ioutil" "os" + "runtime" + "strings" "testing" "time" @@ -298,6 +300,11 @@ func TestFastRepublish(t *testing.T) { // Test writing a medium sized file one byte at a time func TestMultiWrite(t *testing.T) { + if runtime.GOOS == "darwin" { + link := "https://github.com/jbenet/go-ipfs/issues/147" + t.Skipf("Skipping as is broken in OSX. See %s", link) + } + _, mnt := setupIpnsTest(t, nil) defer mnt.Close() From 2fa43ce46c58533d4de5cae8565ba917117211c0 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 9 Oct 2014 03:39:45 -0700 Subject: [PATCH 104/105] ipfs name cmd improvements - cleaned up cmd help - ipfs name publish [] - ipfs name resolve [] - publish validates - both validate n args --- cmd/ipfs/name.go | 42 +++++++++++++++++++++++++++++++++++----- cmd/ipfs/publish.go | 26 ++++++++++++++++++------- cmd/ipfs/resolve.go | 23 +++++++++++++++++++--- core/commands/publish.go | 22 +++++++++++++++++++-- core/commands/resolve.go | 19 +++++++++++++++++- namesys/publisher.go | 13 +++++++++++-- 6 files changed, 125 insertions(+), 20 deletions(-) diff --git a/cmd/ipfs/name.go b/cmd/ipfs/name.go index 56d8821ec7d..a2ef78e5067 100644 --- a/cmd/ipfs/name.go +++ b/cmd/ipfs/name.go @@ -8,11 +8,43 @@ import ( ) var cmdIpfsName = &commander.Command{ - UsageLine: "name", - Short: "Ipfs namespace manipulation tools.", - Long: `ipfs name [publish|resolve] `, - Run: addCmd, - Flag: *flag.NewFlagSet("ipfs-name", flag.ExitOnError), + UsageLine: "name [publish | resolve]", + Short: "ipfs namespace (ipns) tool", + Long: `ipfs name - Get/Set ipfs config values. + + ipfs name publish [] - Assign the to + ipfs name resolve [] - Resolve the value of + +IPNS is a PKI namespace, where names are the hashes of public keys, and +the private key enables publishing new (signed) values. In both publish +and resolve, the default value of is your own identity public key. + + +Examples: + +Publish a to your identity name: + + > ipfs name publish QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + published name QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n to QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + +Publish a to another public key: + + > ipfs name publish QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + published name QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n to QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + +Resolve the value of your identity: + + > ipfs name resolve + QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + +Resolve te value of another name: + + > ipfs name resolve QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n + QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + +`, + Run: addCmd, + Flag: *flag.NewFlagSet("ipfs-name", flag.ExitOnError), Subcommands: []*commander.Command{ cmdIpfsPub, cmdIpfsResolve, diff --git a/cmd/ipfs/publish.go b/cmd/ipfs/publish.go index fe473107718..041da0028b2 100644 --- a/cmd/ipfs/publish.go +++ b/cmd/ipfs/publish.go @@ -8,22 +8,34 @@ import ( var cmdIpfsPub = &commander.Command{ UsageLine: "publish", - Short: "Publish an object to ipns under your key.", - Long: `ipfs publish - Publish object to ipns. + Short: "publish a to ipns.", + Long: `ipfs publish [] - publish a to ipns. + +IPNS is a PKI namespace, where names are the hashes of public keys, and +the private key enables publishing new (signed) values. In publish, the +default value of is your own identity public key. + +Examples: + +Publish a to your identity name: + + > ipfs name publish QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + published name QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n to QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + +Publish a to another public key: + + > ipfs name publish QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + published name QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n to QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy `, Run: pubCmd, Flag: *flag.NewFlagSet("ipfs-publish", flag.ExitOnError), } -func init() { - cmdIpfsPub.Flag.String("k", "", "Specify key to use for publishing.") -} - var pubCmd = makeCommand(command{ name: "publish", args: 1, - flags: []string{"k"}, + flags: nil, online: true, cmdFn: commands.Publish, }) diff --git a/cmd/ipfs/resolve.go b/cmd/ipfs/resolve.go index 75b9903b591..9f5107ff867 100644 --- a/cmd/ipfs/resolve.go +++ b/cmd/ipfs/resolve.go @@ -8,8 +8,25 @@ import ( var cmdIpfsResolve = &commander.Command{ UsageLine: "resolve", - Short: "resolve an ipns link to a hash", - Long: `ipfs resolve ... - Resolve hash. + Short: "resolve an ipns name to a ", + Long: `ipfs resolve [] - Resolve an ipns name to a . + +IPNS is a PKI namespace, where names are the hashes of public keys, and +the private key enables publishing new (signed) values. In resolve, the +default value of is your own identity public key. + + +Examples: + +Resolve the value of your identity: + + > ipfs name resolve + QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + +Resolve te value of another name: + + > ipfs name resolve QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n + QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy `, Run: resolveCmd, @@ -18,7 +35,7 @@ var cmdIpfsResolve = &commander.Command{ var resolveCmd = makeCommand(command{ name: "resolve", - args: 1, + args: 0, flags: nil, online: true, cmdFn: commands.Resolve, diff --git a/core/commands/publish.go b/core/commands/publish.go index 5794372b7ae..3d58ab43127 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -13,14 +13,32 @@ import ( func Publish(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error { log.Debug("Begin Publish") + if n.Identity == nil { return errors.New("Identity not loaded!") } + // name := "" + ref := "" + + switch len(args) { + case 2: + // name = args[0] + ref = args[1] + return errors.New("keychains not yet implemented") + case 1: + // name = n.Identity.ID.String() + ref = args[0] + + default: + return fmt.Errorf("Publish expects 1 or 2 args; got %d.", len(args)) + } + + // later, n.Keychain.Get(name).PrivKey k := n.Identity.PrivKey pub := nsys.NewRoutingPublisher(n.Routing) - err := pub.Publish(k, args[0]) + err := pub.Publish(k, ref) if err != nil { return err } @@ -29,7 +47,7 @@ func Publish(n *core.IpfsNode, args []string, opts map[string]interface{}, out i if err != nil { return err } - fmt.Fprintf(out, "published mapping %s to %s\n", u.Key(hash), args[0]) + fmt.Fprintf(out, "published name %s to %s\n", u.Key(hash), ref) return nil } diff --git a/core/commands/resolve.go b/core/commands/resolve.go index f4bd0c5462d..7307dc2654d 100644 --- a/core/commands/resolve.go +++ b/core/commands/resolve.go @@ -1,6 +1,7 @@ package commands import ( + "errors" "fmt" "io" @@ -8,7 +9,23 @@ import ( ) func Resolve(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error { - res, err := n.Namesys.Resolve(args[0]) + + name := "" + + switch len(args) { + case 1: + name = args[0] + case 0: + if n.Identity == nil { + return errors.New("Identity not loaded!") + } + name = n.Identity.ID.String() + + default: + return fmt.Errorf("Publish expects 1 or 2 args; got %d.", len(args)) + } + + res, err := n.Namesys.Resolve(name) if err != nil { return err } diff --git a/namesys/publisher.go b/namesys/publisher.go index 0828f5e0869..88533f8a089 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -1,10 +1,12 @@ package namesys import ( + "fmt" "time" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" ci "github.com/jbenet/go-ipfs/crypto" routing "github.com/jbenet/go-ipfs/routing" @@ -25,6 +27,13 @@ func NewRoutingPublisher(route routing.IpfsRouting) Publisher { // Publish implements Publisher. Accepts a keypair and a value, func (p *ipnsPublisher) Publish(k ci.PrivKey, value string) error { log.Debug("namesys: Publish %s", value) + + // validate `value` is a ref (multihash) + _, err := mh.FromB58String(value) + if err != nil { + return fmt.Errorf("publish value must be str multihash. %v", err) + } + ctx := context.TODO() data, err := createRoutingEntryData(k, value) if err != nil { From a53813c8fa1efac6509f4aa7c5ca49a0d2cfc705 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Thu, 9 Oct 2014 03:50:52 -0700 Subject: [PATCH 105/105] fixed resolver test --- namesys/resolve_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index 30b996647d8..5e652f42f8d 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -26,6 +26,12 @@ func TestRoutingResolve(t *testing.T) { } err = publisher.Publish(privk, "Hello") + if err == nil { + t.Fatal("should have errored out when publishing a non-multihash val") + } + + h := u.Key(u.Hash([]byte("Hello"))).Pretty() + err = publisher.Publish(privk, h) if err != nil { t.Fatal(err) } @@ -41,7 +47,7 @@ func TestRoutingResolve(t *testing.T) { t.Fatal(err) } - if res != "Hello" { + if res != h { t.Fatal("Got back incorrect value.") } }