From e7ca9554d06b6ddcee776359edee665f5188ab50 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Mon, 6 Aug 2018 22:12:10 -0400 Subject: [PATCH 1/2] Add ability to retrieve blocks even if given using a different CID version. This is done via a wrapper blockstore. License: MIT Signed-off-by: Kevin Atkinson --- core/builder.go | 3 + test/sharness/t0276-cidv0v1.sh | 129 +++++++++++++++++++++++++++++++ thirdparty/cidv0v1/blockstore.go | 71 +++++++++++++++++ 3 files changed, 203 insertions(+) create mode 100755 test/sharness/t0276-cidv0v1.sh create mode 100644 thirdparty/cidv0v1/blockstore.go diff --git a/core/builder.go b/core/builder.go index 13641040e95..ce8eaa4deb4 100644 --- a/core/builder.go +++ b/core/builder.go @@ -13,6 +13,7 @@ import ( pin "github.com/ipfs/go-ipfs/pin" repo "github.com/ipfs/go-ipfs/repo" cfg "github.com/ipfs/go-ipfs/repo/config" + cidv0v1 "github.com/ipfs/go-ipfs/thirdparty/cidv0v1" "github.com/ipfs/go-ipfs/thirdparty/verifbs" uio "gx/ipfs/QmWdTRLi3H7ZJQ8s7NYo8oitz5JHEEPKLn1QPMsJVWg2Ew/go-unixfs/io" dag "gx/ipfs/Qma2BR57Wqp8w9vPreK4dEzoXXk8DFFRL3LresMZg4QpzN/go-merkledag" @@ -211,6 +212,8 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { wbs = bstore.NewIdStore(wbs) + wbs = cidv0v1.NewBlockstore(wbs) + n.BaseBlocks = wbs n.GCLocker = bstore.NewGCLocker() n.Blockstore = bstore.NewGCBlockstore(wbs, n.GCLocker) diff --git a/test/sharness/t0276-cidv0v1.sh b/test/sharness/t0276-cidv0v1.sh new file mode 100755 index 00000000000..d0b33f077d1 --- /dev/null +++ b/test/sharness/t0276-cidv0v1.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2017 Jakub Sztandera +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="CID Version 0/1 Duality" + +. lib/test-lib.sh + +test_init_ipfs + +# +# +# + +test_expect_success "create two small files" ' + random 1000 7 > afile + random 1000 9 > bfile +' + +test_expect_success "add file using CIDv1 but don't pin" ' + AHASHv1=$(ipfs add -q --cid-version=1 --raw-leaves=false --pin=false afile) +' + +test_expect_success "add file using CIDv0" ' + AHASHv0=$(ipfs add -q --cid-version=0 afile) +' + +test_expect_success "check hashes" ' + test "$(cid-fmt %v-%c $AHASHv0)" = "cidv0-protobuf" && + test "$(cid-fmt %v-%c $AHASHv1)" = "cidv1-protobuf" && + test "$(cid-fmt -v 0 %s $AHASHv1)" = "$AHASHv0" +' + +test_expect_success "make sure CIDv1 hash really is in the repo" ' + ipfs refs local | grep -q $AHASHv1 +' + +test_expect_success "make sure CIDv0 hash really is in the repo" ' + ipfs refs local | grep -q $AHASHv0 +' + +test_expect_success "run gc" ' + ipfs repo gc +' + +test_expect_success "make sure the CIDv0 hash is in the repo" ' + ipfs refs local | grep -q $AHASHv0 +' + +test_expect_success "make sure we can get CIDv0 added file" ' + ipfs cat $AHASHv0 > thefile && + test_cmp afile thefile +' + +test_expect_success "make sure the CIDv1 hash is not in the repo" ' + ! ipfs refs local | grep -q $AHASHv1 +' + +test_expect_success "clean up" ' + ipfs pin rm $AHASHv0 && + ipfs repo gc && + ! ipfs refs local | grep -q $AHASHv0 +' + +# +# +# + +test_expect_success "add file using CIDv1 but don't pin" ' + ipfs add -q --cid-version=1 --raw-leaves=false --pin=false afile +' + +test_expect_success "check that we can access the file when converted to CIDv0" ' + ipfs cat $AHASHv0 > thefile && + test_cmp afile thefile +' + +test_expect_success "clean up" ' + ipfs repo gc +' + +test_expect_success "add file using CIDv0 but don't pin" ' + ipfs add -q --cid-version=0 --raw-leaves=false --pin=false afile +' + +test_expect_success "check that we can access the file when converted to CIDv1" ' + ipfs cat $AHASHv1 > thefile && + test_cmp afile thefile +' + +# +# +# + +test_expect_success "set up iptb testbed" ' + iptb init -n 2 -p 0 -f --bootstrap=none +' + +test_expect_success "start nodes" ' + iptb start && + iptb connect 0 1 +' + +test_expect_success "add afile using CIDv0 to node 0" ' + iptb run 0 ipfs add -q --cid-version=0 afile +' + +test_expect_success "get afile using CIDv1 via node 1" ' + iptb run 1 ipfs --timeout=2s cat $AHASHv1 > thefile && + test_cmp afile thefile +' + +test_expect_success "add bfile using CIDv1 to node 0" ' + BHASHv1=$(iptb run 0 ipfs add -q --cid-version=1 --raw-leaves=false bfile) +' + +test_expect_success "get bfile using CIDv0 via node 1" ' + BHASHv0=$(cid-fmt -v 0 %s $BHASHv1) + iptb run 1 ipfs --timeout=2s cat $BHASHv0 > thefile && + test_cmp bfile thefile +' + +test_expect_success "stop testbed" ' + iptb stop +' + +test_done diff --git a/thirdparty/cidv0v1/blockstore.go b/thirdparty/cidv0v1/blockstore.go new file mode 100644 index 00000000000..6b745e79512 --- /dev/null +++ b/thirdparty/cidv0v1/blockstore.go @@ -0,0 +1,71 @@ +package cidv0v1 + +import ( + blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format" + cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" + bs "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore" +) + +type blockstore struct { + bs.Blockstore +} + +func NewBlockstore(b bs.Blockstore) bs.Blockstore { + return &blockstore{b} +} + +func (b *blockstore) Has(c *cid.Cid) (bool, error) { + have, err := b.Blockstore.Has(c) + if have || err != nil { + return have, err + } + c1 := tryOtherCidVersion(c) + if c1 == nil { + return false, nil + } + return b.Blockstore.Has(c1) +} + +func (b *blockstore) Get(c *cid.Cid) (blocks.Block, error) { + block, err := b.Blockstore.Get(c) + if err == nil { + return block, nil + } + if err != bs.ErrNotFound { + return nil, err + } + c1 := tryOtherCidVersion(c) + if c1 == nil { + return nil, bs.ErrNotFound + } + block, err = b.Blockstore.Get(c1) + if err != nil { + return nil, err + } + // modify block so it has the original CID + block, err = blocks.NewBlockWithCid(block.RawData(), c) + if err != nil { + return nil, err + } + // insert the block with the original CID to avoid problems + // with pinning + err = b.Blockstore.Put(block) + if err != nil { + return nil, err + } + return block, nil +} + +func tryOtherCidVersion(c *cid.Cid) *cid.Cid { + prefix := c.Prefix() + if prefix.Codec != cid.DagProtobuf { + return nil + } + var c1 *cid.Cid + if prefix.Version == 0 { + c1 = cid.NewCidV1(cid.DagProtobuf, c.Hash()) + } else { + c1 = cid.NewCidV0(c.Hash()) + } + return c1 +} From 877fa90f8cd930d9646b125317b031dc40311400 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Tue, 7 Aug 2018 19:03:07 -0400 Subject: [PATCH 2/2] Also check hash type in 'tryOtherCidVersion'. License: MIT Signed-off-by: Kevin Atkinson --- thirdparty/cidv0v1/blockstore.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/thirdparty/cidv0v1/blockstore.go b/thirdparty/cidv0v1/blockstore.go index 6b745e79512..7b9faf5216e 100644 --- a/thirdparty/cidv0v1/blockstore.go +++ b/thirdparty/cidv0v1/blockstore.go @@ -1,6 +1,7 @@ package cidv0v1 import ( + mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash" blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format" cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid" bs "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore" @@ -58,7 +59,7 @@ func (b *blockstore) Get(c *cid.Cid) (blocks.Block, error) { func tryOtherCidVersion(c *cid.Cid) *cid.Cid { prefix := c.Prefix() - if prefix.Codec != cid.DagProtobuf { + if prefix.Codec != cid.DagProtobuf || prefix.MhType != mh.SHA2_256 || prefix.MhLength != 32 { return nil } var c1 *cid.Cid