feat(mount): don't give up on error

Instead of giving up when a single datastore fails, operate on all datastores
first. That way:

* If one datastore doesn't support disk usage, we can at least return our best
  estimate for the ones that do.
* If one datastore fails to sync, we won't fail to sync the others.
* If one datastore refuses to close, we'll close the others.

Etc.

Also:

* Drop xerrors. We already need go 1.13 elsewhere.
* Use uber's multierr. It's much simpler than hashicorps (and is already a
  dependency through zap, fx, etc.).

go.mod

@@ -4,9 +4,9 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8
 	github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8
-	github.com/kr/pretty v0.1.0 // indirect
+	go.uber.org/multierr v1.4.0
 	golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
 )
 
-go 1.12
+go 1.13

go.sum

@@ -1,15 +1,55 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8 h1:NAviDvJ0WXgD+yiL2Rj35AmnfgI11+pHXbdciD917U0=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw=
 github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E=
+go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=

mount/mount.go

@@ -5,13 +5,15 @@ package mount
 import (
 	"container/heap"
 	"errors"
+	"fmt"
 	"sort"
 	"strings"
 	"sync"
 
 	ds "github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/query"
-	xerrors "golang.org/x/xerrors"
+
+	"go.uber.org/multierr"
 )
 
 var (
@@ -240,16 +242,22 @@ func (d *Datastore) Put(key ds.Key, value []byte) error {
 
 // Sync implements Datastore.Sync
 func (d *Datastore) Sync(prefix ds.Key) error {
+	var merr error
+
 	// Sync all mount points below the prefix
 	// Sync the mount point right at (or above) the prefix
-	dstores, _, rest := d.lookupAll(prefix)
+	dstores, prefixes, rest := d.lookupAll(prefix)
 	for i, suffix := range rest {
 		if err := dstores[i].Sync(suffix); err != nil {
-			return err
+			merr = multierr.Append(merr, fmt.Errorf(
+				"syncing datastore at %s: %w",
+				prefixes[i].String(),
+				err,
+			))
 		}
 	}
 
-	return nil
+	return merr
 }
 
 // Get returns the value associated with the key from the appropriate datastore.
@@ -355,27 +363,39 @@ func (d *Datastore) Query(master query.Query) (query.Results, error) {
 
 // Close closes all mounted datastores.
 func (d *Datastore) Close() error {
+	var merr error
 	for _, d := range d.mounts {
 		err := d.Datastore.Close()
 		if err != nil {
-			return err
+			merr = multierr.Append(merr, fmt.Errorf(
+				"closing datastore at %s: %w",
+				d.Prefix.String(),
+				err,
+			))
 		}
 	}
-	return nil
+	return merr
 }
 
 // DiskUsage returns the sum of DiskUsages for the mounted datastores.
 // Non PersistentDatastores will not be accounted.
 func (d *Datastore) DiskUsage() (uint64, error) {
-	var duTotal uint64 = 0
+	var (
+		merr    error
+		duTotal uint64 = 0
+	)
 	for _, d := range d.mounts {
 		du, err := ds.DiskUsage(d.Datastore)
 		duTotal += du
 		if err != nil {
-			return duTotal, err
+			merr = multierr.Append(merr, fmt.Errorf(
+				"getting disk usage at %s: %w",
+				d.Prefix.String(),
+				err,
+			))
 		}
 	}
-	return duTotal, nil
+	return duTotal, merr
 }
 
 type mountBatch struct {
@@ -436,44 +456,62 @@ func (mt *mountBatch) Commit() error {
 	mt.lk.Lock()
 	defer mt.lk.Unlock()
 
-	for _, t := range mt.mounts {
-		err := t.Commit()
-		if err != nil {
-			return err
+	var merr error
+	for p, t := range mt.mounts {
+		if err := t.Commit(); err != nil {
+			merr = multierr.Append(merr, fmt.Errorf(
+				"committing batch to datastore at %s: %w",
+				p, err,
+			))
 		}
 	}
-	return nil
+	return merr
 }
 
 func (d *Datastore) Check() error {
+	var merr error
 	for _, m := range d.mounts {
 		if c, ok := m.Datastore.(ds.CheckedDatastore); ok {
 			if err := c.Check(); err != nil {
-				return xerrors.Errorf("checking datastore at %s: %w", m.Prefix.String(), err)
+				merr = multierr.Append(merr, fmt.Errorf(
+					"checking datastore at %s: %w",
+					m.Prefix.String(),
+					err,
+				))
 			}
 		}
 	}
-	return nil
+	return merr
 }
 
 func (d *Datastore) Scrub() error {
+	var merr error
 	for _, m := range d.mounts {
 		if c, ok := m.Datastore.(ds.ScrubbedDatastore); ok {
 			if err := c.Scrub(); err != nil {
-				return xerrors.Errorf("scrubbing datastore at %s: %w", m.Prefix.String(), err)
+				merr = multierr.Append(merr, fmt.Errorf(
+					"scrubbing datastore at %s: %w",
+					m.Prefix.String(),
+					err,
+				))
 			}
 		}
 	}
-	return nil
+	return merr
 }
 
 func (d *Datastore) CollectGarbage() error {
+	var merr error
 	for _, m := range d.mounts {
 		if c, ok := m.Datastore.(ds.GCDatastore); ok {
 			if err := c.CollectGarbage(); err != nil {
-				return xerrors.Errorf("gc on datastore at %s: %w", m.Prefix.String(), err)
+				merr = multierr.Append(merr, fmt.Errorf(
+					"gc on datastore at %s: %w",
+					m.Prefix.String(),
+					err,
+				))
 			}
 		}
 	}
-	return nil
+	return merr
 }