Merge pull request #4896 from ipfs/feat/ai-mirror

URL store

core/builder.go

@@ -213,7 +213,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
 	n.GCLocker = bstore.NewGCLocker()
 	n.Blockstore = bstore.NewGCBlockstore(cbs, n.GCLocker)
 
-	if conf.Experimental.FilestoreEnabled {
+	if conf.Experimental.FilestoreEnabled || conf.Experimental.UrlstoreEnabled {
 		// hash security
 		n.Filestore = filestore.NewFilestore(cbs, n.Repo.FileManager())
 		n.Blockstore = bstore.NewGCBlockstore(n.Filestore, n.GCLocker)

core/commands/add.go

@@ -10,6 +10,7 @@ import (
 	blockservice "github.com/ipfs/go-ipfs/blockservice"
 	core "github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/core/coreunix"
+	filestore "github.com/ipfs/go-ipfs/filestore"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
 	mfs "github.com/ipfs/go-ipfs/mfs"
@@ -183,8 +184,7 @@ You can now check what blocks have been created by:
 
 		// nocopy -> filestoreEnabled
 		if nocopy && !cfg.Experimental.FilestoreEnabled {
-			res.SetError(errors.New("filestore is not enabled, see https://git.io/vNItf"),
-				cmdkit.ErrClient)
+			res.SetError(filestore.ErrFilestoreNotEnabled, cmdkit.ErrClient)
 			return
 		}
 

core/commands/commands_test.go

@@ -210,6 +210,8 @@ func TestCommands(t *testing.T) {
 		"/tar/add",
 		"/tar/cat",
 		"/update",
+		"/urlstore",
+		"/urlstore/add",
 		"/version",
 	}
 

core/commands/filestore.go

@@ -237,7 +237,7 @@ func getFilestore(env interface{}) (*core.IpfsNode, *filestore.Filestore, error)
 	}
 	fs := n.Filestore
 	if fs == nil {
-		return n, nil, fmt.Errorf("filestore not enabled")
+		return n, nil, filestore.ErrFilestoreNotEnabled
 	}
 	return n, fs, err
 }

core/commands/root.go

@@ -5,12 +5,12 @@ import (
 	"strings"
 
 	oldcmds "github.com/ipfs/go-ipfs/commands"
+	lgc "github.com/ipfs/go-ipfs/commands/legacy"
 	dag "github.com/ipfs/go-ipfs/core/commands/dag"
 	e "github.com/ipfs/go-ipfs/core/commands/e"
 	ocmd "github.com/ipfs/go-ipfs/core/commands/object"
 	unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs"
 
-	lgc "github.com/ipfs/go-ipfs/commands/legacy"
 	"gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds"
 	logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
 	"gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit"
@@ -136,6 +136,7 @@ var rootSubcommands = map[string]*cmds.Command{
 	"tar":       lgc.NewCommand(TarCmd),
 	"file":      lgc.NewCommand(unixfs.UnixFSCmd),
 	"update":    lgc.NewCommand(ExternalBinary()),
+	"urlstore":  lgc.NewCommand(urlStoreCmd),
 	"version":   lgc.NewCommand(VersionCmd),
 	"shutdown":  lgc.NewCommand(daemonShutdownCmd),
 }

core/commands/urlstore.go

@@ -0,0 +1,122 @@
+package commands
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	cmds "github.com/ipfs/go-ipfs/commands"
+	filestore "github.com/ipfs/go-ipfs/filestore"
+	balanced "github.com/ipfs/go-ipfs/importer/balanced"
+	ihelper "github.com/ipfs/go-ipfs/importer/helpers"
+
+	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
+	chunk "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker"
+	cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
+	cmdkit "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit"
+)
+
+var urlStoreCmd = &cmds.Command{
+
+	Subcommands: map[string]*cmds.Command{
+		"add": urlAdd,
+	},
+}
+
+var urlAdd = &cmds.Command{
+	Helptext: cmdkit.HelpText{
+		Tagline: "Add URL via urlstore.",
+		LongDescription: `
+Add URLs to ipfs without storing the data locally.
+
+The URL provided must be stable and ideally on a web server under your
+control.
+
+The file is added using raw-leaves but otherwise using the default
+settings for 'ipfs add'.
+
+The file is not pinned, so this command should be followed by an 'ipfs
+pin add'.
+
+This command is considered temporary until a better solution can be
+found.  It may disappear or the semantics can change at any
+time.
+`,
+	},
+	Arguments: []cmdkit.Argument{
+		cmdkit.StringArg("url", true, false, "URL to add to IPFS"),
+	},
+	Type: BlockStat{},
+
+	Run: func(req cmds.Request, res cmds.Response) {
+		url := req.Arguments()[0]
+		n, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+
+		if !filestore.IsURL(url) {
+			res.SetError(fmt.Errorf("unsupported url syntax: %s", url), cmdkit.ErrNormal)
+			return
+		}
+
+		cfg, err := n.Repo.Config()
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+
+		if !cfg.Experimental.UrlstoreEnabled {
+			res.SetError(filestore.ErrUrlstoreNotEnabled, cmdkit.ErrNormal)
+			return
+		}
+
+		hreq, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+
+		hres, err := http.DefaultClient.Do(hreq)
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+		if hres.StatusCode != http.StatusOK {
+			res.SetError(fmt.Errorf("expected code 200, got: %d", hres.StatusCode), cmdkit.ErrNormal)
+			return
+		}
+
+		chk := chunk.NewSizeSplitter(hres.Body, chunk.DefaultBlockSize)
+		prefix := cid.NewPrefixV1(cid.DagProtobuf, mh.SHA2_256)
+		dbp := &ihelper.DagBuilderParams{
+			Dagserv:   n.DAG,
+			RawLeaves: true,
+			Maxlinks:  ihelper.DefaultLinksPerBlock,
+			NoCopy:    true,
+			Prefix:    &prefix,
+			URL:       url,
+		}
+
+		blc, err := balanced.Layout(dbp.New(chk))
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+
+		res.SetOutput(BlockStat{
+			Key:  blc.Cid().String(),
+			Size: int(hres.ContentLength),
+		})
+	},
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+			ch := res.Output().(<-chan interface{})
+			bs0 := <-ch
+			bs := bs0.(*BlockStat)
+			return strings.NewReader(bs.Key + "\n"), nil
+		},
+	},
+}

docs/experimental-features.md

@@ -17,6 +17,7 @@ the above issue.
 - [go-multiplex stream muxer](#go-multiplex-stream-muxer)
 - [Raw leaves for unixfs files](#raw-leaves-for-unixfs-files)
 - [ipfs filestore](#ipfs-filestore)
+- [ipfs urlstore](#ipfs-urlstore)
 - [BadgerDB datastore](#badger-datastore)
 - [Private Networks](#private-networks)
 - [ipfs p2p](#ipfs-p2p)
@@ -164,6 +165,26 @@ And then pass the `--nocopy` flag when running `ipfs add`
 
 ---
 
+## ipfs urlstore
+Allows ipfs to retrieve blocks contents via a url instead of storing it in the datastore
+
+### State
+experimental.
+
+### In Version
+???.
+
+### How to enable
+Modify your ipfs config:
+```
+ipfs config --json Experimental.UrlstoreEnabled true
+```
+
+### Road to being a real feature
+???.
+
+---
+
 ## Private Networks
 
 Allows ipfs to only connect to other peers who have a shared secret key.

filestore/filestore.go

@@ -9,6 +9,7 @@ package filestore
 
 import (
 	"context"
+	"errors"
 
 	blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
 	posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
@@ -20,6 +21,9 @@ import (
 
 var log = logging.Logger("filestore")
 
+var ErrFilestoreNotEnabled = errors.New("filestore is not enabled, see https://git.io/vNItf")
+var ErrUrlstoreNotEnabled = errors.New("urlstore is not enabled")
+
 // Filestore implements a Blockstore by combining a standard Blockstore
 // to store regular blocks and a special Blockstore called
 // FileManager to store blocks which data exists in an external file.

filestore/filestore_test.go

@@ -23,6 +23,7 @@ func newTestFilestore(t *testing.T) (string, *Filestore) {
 		t.Fatal(err)
 	}
 	fm := NewFileManager(mds, testdir)
+	fm.AllowFiles = true
 
 	bs := blockstore.NewBlockstore(mds)
 	fstore := NewFilestore(bs, fm)
@@ -162,3 +163,15 @@ func TestDeletes(t *testing.T) {
 		}
 	}
 }
+
+func TestIsURL(t *testing.T) {
+	if !IsURL("http://www.example.com") {
+		t.Fatal("IsURL failed: http://www.example.com")
+	}
+	if !IsURL("https://www.example.com") {
+		t.Fatal("IsURL failed: https://www.example.com")
+	}
+	if IsURL("adir/afile") || IsURL("http:/ /afile") || IsURL("http:/a/file") {
+		t.Fatal("IsURL recognized non-url")
+	}
+}