Merge pull request #68 from ipfs/feat/new-interfaces

Define GCDatastore/CheckedDatastore interfaces

.gx/lastpubver

@@ -1 +1 @@
-1.2.2: QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG
+1.3.0: QmTWw8RaXEE155hrm8GiMYAYoUrAMwi6qt72CBpzdeGq1s

basic_ds.go

@@ -226,3 +226,24 @@ func (d *LogDatastore) Close() error {
 	}
 	return nil
 }
+
+func (d *LogDatastore) Check() error {
+	if c, ok := d.child.(CheckedDatastore); ok {
+		return c.Check()
+	}
+	return nil
+}
+
+func (d *LogDatastore) Scrub() error {
+	if c, ok := d.child.(ScrubbedDatastore); ok {
+		return c.Scrub()
+	}
+	return nil
+}
+
+func (d *LogDatastore) CollectGarbage() error {
+	if c, ok := d.child.(GCDatastore); ok {
+		return c.CollectGarbage()
+	}
+	return nil
+}

datastore.go

@@ -81,9 +81,35 @@ var ErrBatchUnsupported = errors.New("this datastore does not support batching")
 // implement to leverage type safety checks.
 type ThreadSafeDatastore interface {
 	Datastore
+
 	IsThreadSafe()
 }
 
+// CheckedDatastore is an interface that should be implemented by datastores
+// which may need checking on-disk data integrity.
+type CheckedDatastore interface {
+	Datastore
+
+	Check() error
+}
+
+// CheckedDatastore is an interface that should be implemented by datastores
+// which want to provide a mechanism to check data integrity and/or
+// error correction
+type ScrubbedDatastore interface {
+	Datastore
+
+	Scrub() error
+}
+
+// GCDatastore is an interface that should be implemented by datastores which
+// don't free disk space by just removing data from them
+type GCDatastore interface {
+	Datastore
+
+	CollectGarbage() error
+}
+
 // Errors
 
 // ErrNotFound is returned by Get, Has, and Delete when a datastore does not

keytransform/keytransform.go

@@ -117,3 +117,24 @@ func (t *transformBatch) Delete(key ds.Key) error {
 func (t *transformBatch) Commit() error {
 	return t.dst.Commit()
 }
+
+func (d *ktds) Check() error {
+	if c, ok := d.child.(ds.CheckedDatastore); ok {
+		return c.Check()
+	}
+	return nil
+}
+
+func (d *ktds) Scrub() error {
+	if c, ok := d.child.(ds.ScrubbedDatastore); ok {
+		return c.Scrub()
+	}
+	return nil
+}
+
+func (d *ktds) CollectGarbage() error {
+	if c, ok := d.child.(ds.GCDatastore); ok {
+		return c.CollectGarbage()
+	}
+	return nil
+}

keytransform/keytransform_test.go

@@ -10,6 +10,7 @@ import (
 	ds "github.com/ipfs/go-datastore"
 	kt "github.com/ipfs/go-datastore/keytransform"
 	dsq "github.com/ipfs/go-datastore/query"
+	dstest "github.com/ipfs/go-datastore/test"
 )
 
 // Hook up gocheck into the "go test" runner.
@@ -35,7 +36,7 @@ func (ks *DSSuite) TestBasic(c *C) {
 		},
 	}
 
-	mpds := ds.NewMapDatastore()
+	mpds := dstest.NewTestDatastore(true)
 	ktds := kt.Wrap(mpds, pair)
 
 	keys := strsToKeys([]string{
@@ -88,6 +89,18 @@ func (ks *DSSuite) TestBasic(c *C) {
 
 	c.Log("listA: ", listA)
 	c.Log("listB: ", listB)
+
+	if err := ktds.Check(); err != dstest.TestError {
+		c.Errorf("Unexpected Check() error: %s", err)
+	}
+
+	if err := ktds.CollectGarbage(); err != dstest.TestError {
+		c.Errorf("Unexpected CollectGarbage() error: %s", err)
+	}
+
+	if err := ktds.Scrub(); err != dstest.TestError {
+		c.Errorf("Unexpected Scrub() error: %s", err)
+	}
 }
 
 func strsToKeys(strs []string) []ds.Key {

mount/mount.go

@@ -4,6 +4,7 @@ package mount
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"sort"
 	"strings"
@@ -251,3 +252,36 @@ func (mt *mountBatch) Commit() error {
 	}
 	return nil
 }
+
+func (d *Datastore) Check() error {
+	for _, m := range d.mounts {
+		if c, ok := m.Datastore.(datastore.CheckedDatastore); ok {
+			if err := c.Check(); err != nil {
+				return fmt.Errorf("checking datastore at %s: %s", m.Prefix.String(), err.Error())
+			}
+		}
+	}
+	return nil
+}
+
+func (d *Datastore) Scrub() error {
+	for _, m := range d.mounts {
+		if c, ok := m.Datastore.(datastore.ScrubbedDatastore); ok {
+			if err := c.Scrub(); err != nil {
+				return fmt.Errorf("scrubbing datastore at %s: %s", m.Prefix.String(), err.Error())
+			}
+		}
+	}
+	return nil
+}
+
+func (d *Datastore) CollectGarbage() error {
+	for _, m := range d.mounts {
+		if c, ok := m.Datastore.(datastore.GCDatastore); ok {
+			if err := c.CollectGarbage(); err != nil {
+				return fmt.Errorf("gc on datastore at %s: %s", m.Prefix.String(), err.Error())
+			}
+		}
+	}
+	return nil
+}

mount/mount_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/mount"
 	"github.com/ipfs/go-datastore/query"
+	dstest "github.com/ipfs/go-datastore/test"
 )
 
 func TestPutBadNothing(t *testing.T) {
@@ -371,3 +372,22 @@ func TestErrQueryClose(t *testing.T) {
 		t.Errorf("Query was ok or q.Error was nil")
 	}
 }
+
+func TestMaintenanceFunctions(t *testing.T) {
+	mapds := dstest.NewTestDatastore(true)
+	m := mount.New([]mount.Mount{
+		{Prefix: datastore.NewKey("/"), Datastore: mapds},
+	})
+
+	if err:= m.Check(); err.Error() != "checking datastore at /: test error" {
+		t.Errorf("Unexpected Check() error: %s", err)
+	}
+
+	if err:= m.CollectGarbage(); err.Error() != "gc on datastore at /: test error" {
+		t.Errorf("Unexpected CollectGarbage() error: %s", err)
+	}
+
+	if err:= m.Scrub(); err.Error() != "scrubbing datastore at /: test error" {
+		t.Errorf("Unexpected Scrub() error: %s", err)
+	}
+}

namespace/namespace_test.go

@@ -10,6 +10,7 @@ import (
 	ds "github.com/ipfs/go-datastore"
 	ns "github.com/ipfs/go-datastore/namespace"
 	dsq "github.com/ipfs/go-datastore/query"
+	dstest "github.com/ipfs/go-datastore/test"
 )
 
 // Hook up gocheck into the "go test" runner.
@@ -79,7 +80,7 @@ func (ks *DSSuite) testBasic(c *C, prefix string) {
 }
 
 func (ks *DSSuite) TestQuery(c *C) {
-	mpds := ds.NewMapDatastore()
+	mpds := dstest.NewTestDatastore(true)
 	nsds := ns.Wrap(mpds, ds.NewKey("/foo"))
 
 	keys := strsToKeys([]string{
@@ -137,6 +138,18 @@ func (ks *DSSuite) TestQuery(c *C) {
 		expval, _ := expect[i].Value.([]byte)
 		c.Check(string(entval), Equals, string(expval))
 	}
+
+	if err := nsds.Datastore.(ds.CheckedDatastore).Check(); err != dstest.TestError {
+		c.Errorf("Unexpected Check() error: %s", err)
+	}
+
+	if err := nsds.Datastore.(ds.GCDatastore).CollectGarbage(); err != dstest.TestError {
+		c.Errorf("Unexpected CollectGarbage() error: %s", err)
+	}
+
+	if err := nsds.Datastore.(ds.ScrubbedDatastore).Scrub(); err != dstest.TestError {
+		c.Errorf("Unexpected Scrub() error: %s", err)
+	}
 }
 
 func strsToKeys(strs []string) []ds.Key {

package.json

@@ -25,6 +25,6 @@
   "license": "MIT",
   "name": "go-datastore",
   "releaseCmd": "git commit -a -m \"gx publish $VERSION\"",
-  "version": "1.2.2"
+  "version": "1.3.0"
 }
 

sync/sync.go

@@ -114,3 +114,24 @@ func (b *syncBatch) Commit() error {
 	defer b.mds.Unlock()
 	return b.batch.Commit()
 }
+
+func (d *MutexDatastore) Check() error {
+	if c, ok := d.child.(ds.CheckedDatastore); ok {
+		return c.Check()
+	}
+	return nil
+}
+
+func (d *MutexDatastore) Scrub() error {
+	if c, ok := d.child.(ds.ScrubbedDatastore); ok {
+		return c.Scrub()
+	}
+	return nil
+}
+
+func (d *MutexDatastore) CollectGarbage() error {
+	if c, ok := d.child.(ds.GCDatastore); ok {
+		return c.CollectGarbage()
+	}
+	return nil
+}

syncmount/mount.go

@@ -4,6 +4,7 @@ package syncmount
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"sort"
 	"strings"
@@ -264,3 +265,36 @@ func (mt *mountBatch) Commit() error {
 	}
 	return nil
 }
+
+func (d *Datastore) Check() error {
+	for _, m := range d.mounts {
+		if c, ok := m.Datastore.(ds.CheckedDatastore); ok {
+			if err := c.Check(); err != nil {
+				return fmt.Errorf("checking datastore at %s: %s", m.Prefix.String(), err.Error())
+			}
+		}
+	}
+	return nil
+}
+
+func (d *Datastore) Scrub() error {
+	for _, m := range d.mounts {
+		if c, ok := m.Datastore.(ds.ScrubbedDatastore); ok {
+			if err := c.Scrub(); err != nil {
+				return fmt.Errorf("scrubbing datastore at %s: %s", m.Prefix.String(), err.Error())
+			}
+		}
+	}
+	return nil
+}
+
+func (d *Datastore) CollectGarbage() error {
+	for _, m := range d.mounts {
+		if c, ok := m.Datastore.(ds.GCDatastore); ok {
+			if err := c.CollectGarbage(); err != nil {
+				return fmt.Errorf("gc on datastore at %s: %s", m.Prefix.String(), err.Error())
+			}
+		}
+	}
+	return nil
+}

syncmount/mount_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/query"
 	mount "github.com/ipfs/go-datastore/syncmount"
+	dstest "github.com/ipfs/go-datastore/test"
 )
 
 func TestPutBadNothing(t *testing.T) {
@@ -371,3 +372,22 @@ func TestErrQueryClose(t *testing.T) {
 		t.Errorf("Query was ok or q.Error was nil")
 	}
 }
+
+func TestMaintenanceFunctions(t *testing.T) {
+	mapds := dstest.NewTestDatastore(true)
+	m := mount.New([]mount.Mount{
+		{Prefix: datastore.NewKey("/"), Datastore: mapds},
+	})
+
+	if err:= m.Check(); err.Error() != "checking datastore at /: test error" {
+		t.Errorf("Unexpected Check() error: %s", err)
+	}
+
+	if err:= m.CollectGarbage(); err.Error() != "gc on datastore at /: test error" {
+		t.Errorf("Unexpected CollectGarbage() error: %s", err)
+	}
+
+	if err:= m.Scrub(); err.Error() != "scrubbing datastore at /: test error" {
+		t.Errorf("Unexpected Scrub() error: %s", err)
+	}
+}