Storage Market Extraction

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/google/go-cmp v0.3.1 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
-	github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099
+	github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/ipfs/go-bitswap v0.1.8 // indirect
 	github.com/ipfs/go-block-format v0.0.2
@@ -45,6 +45,7 @@ require (
 	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a
+	go.uber.org/multierr v1.1.0
 	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
 	golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 // indirect
 	golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898

go.sum

@@ -84,8 +84,8 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
 github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE=
-github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099 h1:vQqOW42RRM5LoM/1K5dK940VipLqpH8lEVGrMz+mNjU=
-github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k=
+github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c h1:+MSf4NEnLCYZoAgK6fqwc7NH88nM8haFSxKGUGIG3vA=
+github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=

pieceio/padreader/padreader.go

@@ -1,9 +1,11 @@
 package padreader
 
 import (
+	"io"
+	"math/bits"
+
 	ffi "github.com/filecoin-project/filecoin-ffi"
 	"github.com/filecoin-project/go-fil-components/pieceio"
-	"math/bits"
 )
 
 type padReader struct {
@@ -25,3 +27,21 @@ func (p padReader) PaddedSize(size uint64) uint64 {
 
 	return ffi.GetMaxUserBytesPerStagedSector(1 << (logv + 1))
 }
+
+type nullReader struct{}
+
+func (nr nullReader) Read(b []byte) (int, error) {
+	for i := range b {
+		b[i] = 0
+	}
+	return len(b), nil
+}
+
+func NewPaddedReader(r io.Reader, size uint64) (io.Reader, uint64) {
+	padSize := NewPadReader().PaddedSize(size)
+
+	return io.MultiReader(
+		io.LimitReader(r, int64(size)),
+		io.LimitReader(nullReader{}, int64(padSize-size)),
+	), padSize
+}

shared/statestore/store.go

@@ -0,0 +1,165 @@
+package statestore
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/query"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"go.uber.org/multierr"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-fil-components/shared/cborutil"
+)
+
+type StateStore struct {
+	ds datastore.Datastore
+}
+
+func New(ds datastore.Datastore) *StateStore {
+	return &StateStore{ds: ds}
+}
+
+func toKey(k interface{}) datastore.Key {
+	switch t := k.(type) {
+	case uint64:
+		return datastore.NewKey(fmt.Sprint(t))
+	case fmt.Stringer:
+		return datastore.NewKey(t.String())
+	default:
+		panic("unexpected key type")
+	}
+}
+
+func (st *StateStore) Begin(i interface{}, state interface{}) error {
+	k := toKey(i)
+	has, err := st.ds.Has(k)
+	if err != nil {
+		return err
+	}
+	if has {
+		return xerrors.Errorf("already tracking state for %v", i)
+	}
+
+	b, err := cborutil.Dump(state)
+	if err != nil {
+		return err
+	}
+
+	return st.ds.Put(k, b)
+}
+
+func (st *StateStore) End(i interface{}) error {
+	k := toKey(i)
+	has, err := st.ds.Has(k)
+	if err != nil {
+		return err
+	}
+	if !has {
+		return xerrors.Errorf("No state for %s", i)
+	}
+	return st.ds.Delete(k)
+}
+
+func cborMutator(mutator interface{}) func([]byte) ([]byte, error) {
+	rmut := reflect.ValueOf(mutator)
+
+	return func(in []byte) ([]byte, error) {
+		state := reflect.New(rmut.Type().In(0).Elem())
+
+		err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface())
+		if err != nil {
+			return nil, err
+		}
+
+		out := rmut.Call([]reflect.Value{state})
+
+		if err := out[0].Interface(); err != nil {
+			return nil, err.(error)
+		}
+
+		return cborutil.Dump(state.Interface())
+	}
+}
+
+// mutator func(*T) error
+func (st *StateStore) Mutate(i interface{}, mutator interface{}) error {
+	return st.mutate(i, cborMutator(mutator))
+}
+
+func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)) error {
+	k := toKey(i)
+	has, err := st.ds.Has(k)
+	if err != nil {
+		return err
+	}
+	if !has {
+		return xerrors.Errorf("No state for %s", i)
+	}
+
+	cur, err := st.ds.Get(k)
+	if err != nil {
+		return err
+	}
+
+	mutated, err := mutator(cur)
+	if err != nil {
+		return err
+	}
+
+	return st.ds.Put(k, mutated)
+}
+
+func (st *StateStore) Has(i interface{}) (bool, error) {
+	return st.ds.Has(toKey(i))
+}
+
+func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error {
+	k := toKey(i)
+	val, err := st.ds.Get(k)
+	if err != nil {
+		if xerrors.Is(err, datastore.ErrNotFound) {
+			return xerrors.Errorf("No state for %s: %w", i, err)
+		}
+		return err
+	}
+
+	return out.UnmarshalCBOR(bytes.NewReader(val))
+}
+
+// out: *[]T
+func (st *StateStore) List(out interface{}) error {
+	res, err := st.ds.Query(query.Query{})
+	if err != nil {
+		return err
+	}
+	defer res.Close()
+
+	outT := reflect.TypeOf(out).Elem().Elem()
+	rout := reflect.ValueOf(out)
+
+	var errs error
+
+	for {
+		res, ok := res.NextSync()
+		if !ok {
+			break
+		}
+		if res.Error != nil {
+			return res.Error
+		}
+
+		elem := reflect.New(outT)
+		err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface())
+		if err != nil {
+			errs = multierr.Append(errs, xerrors.Errorf("decoding state for key '%s': %w", res.Key, err))
+			continue
+		}
+
+		rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem()))
+	}
+
+	return nil
+}

shared/statestore/store_test.go

@@ -0,0 +1,38 @@
+package statestore
+
+import (
+	"testing"
+
+	"github.com/ipfs/go-datastore"
+
+	"github.com/filecoin-project/go-fil-components/shared/cborutil"
+	"github.com/filecoin-project/go-fil-components/shared/tokenamount"
+)
+
+func TestList(t *testing.T) {
+	ds := datastore.NewMapDatastore()
+
+	e, err := cborutil.Dump(tokenamount.FromInt(7))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ds.Put(datastore.NewKey("/2"), e); err != nil {
+		t.Fatal(err)
+	}
+
+	st := &StateStore{ds: ds}
+
+	var out []tokenamount.TokenAmount
+	if err := st.List(&out); err != nil {
+		t.Fatal(err)
+	}
+
+	if len(out) != 1 {
+		t.Fatal("wrong len")
+	}
+
+	if out[0].Int64() != 7 {
+		t.Fatal("wrong data")
+	}
+}

shared/types/ask.go

@@ -0,0 +1,30 @@
+package types
+
+import (
+	"github.com/filecoin-project/go-fil-components/shared/address"
+	"github.com/filecoin-project/go-fil-components/shared/tokenamount"
+	cbor "github.com/ipfs/go-ipld-cbor"
+)
+
+func init() {
+	cbor.RegisterCborType(SignedStorageAsk{})
+	cbor.RegisterCborType(StorageAsk{})
+}
+
+//go:generate cbor-gen-for SignedStorageAsk StorageAsk
+
+type SignedStorageAsk struct {
+	Ask       *StorageAsk
+	Signature *Signature
+}
+
+type StorageAsk struct {
+	// Price per GiB / Epoch
+	Price tokenamount.TokenAmount
+
+	MinPieceSize uint64
+	Miner        address.Address
+	Timestamp    uint64
+	Expiry       uint64
+	SeqNo        uint64
+}