Skip to content

Commit

Permalink
Storage Market Extraction (#15)
Browse files Browse the repository at this point in the history
* feat(storagemarket): extract storage market

Extract all components of storage market

* fix(storagemarket): fix linting error

* fix(storagemarket): store payload CID

* Tidy up go.mod

Co-authored-by: Ingar Shu <ingar@users.noreply.github.com>
  • Loading branch information
hannahhoward and ingar committed Dec 19, 2019
1 parent 920f643 commit cab0e34
Show file tree
Hide file tree
Showing 23 changed files with 4,157 additions and 4 deletions.
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/gogo/protobuf v1.3.1 // indirect
github.com/google/go-cmp v0.3.1 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099
github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/ipfs/go-bitswap v0.1.8 // indirect
github.com/ipfs/go-block-format v0.0.2
Expand Down Expand Up @@ -45,6 +45,7 @@ require (
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
github.com/stretchr/testify v1.4.0
github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a
go.uber.org/multierr v1.1.0
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 // indirect
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE=
github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099 h1:vQqOW42RRM5LoM/1K5dK940VipLqpH8lEVGrMz+mNjU=
github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k=
github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c h1:+MSf4NEnLCYZoAgK6fqwc7NH88nM8haFSxKGUGIG3vA=
github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
Expand Down
22 changes: 21 additions & 1 deletion pieceio/padreader/padreader.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package padreader

import (
"io"
"math/bits"

ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-fil-components/pieceio"
"math/bits"
)

type padReader struct {
Expand All @@ -25,3 +27,21 @@ func (p padReader) PaddedSize(size uint64) uint64 {

return ffi.GetMaxUserBytesPerStagedSector(1 << (logv + 1))
}

type nullReader struct{}

func (nr nullReader) Read(b []byte) (int, error) {
for i := range b {
b[i] = 0
}
return len(b), nil
}

func NewPaddedReader(r io.Reader, size uint64) (io.Reader, uint64) {
padSize := NewPadReader().PaddedSize(size)

return io.MultiReader(
io.LimitReader(r, int64(size)),
io.LimitReader(nullReader{}, int64(padSize-size)),
), padSize
}
165 changes: 165 additions & 0 deletions shared/statestore/store.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
package statestore

import (
"bytes"
"fmt"
"reflect"

"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
cbg "github.com/whyrusleeping/cbor-gen"
"go.uber.org/multierr"
"golang.org/x/xerrors"

"github.com/filecoin-project/go-fil-components/shared/cborutil"
)

type StateStore struct {
ds datastore.Datastore
}

func New(ds datastore.Datastore) *StateStore {
return &StateStore{ds: ds}
}

func toKey(k interface{}) datastore.Key {
switch t := k.(type) {
case uint64:
return datastore.NewKey(fmt.Sprint(t))
case fmt.Stringer:
return datastore.NewKey(t.String())
default:
panic("unexpected key type")
}
}

func (st *StateStore) Begin(i interface{}, state interface{}) error {
k := toKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if has {
return xerrors.Errorf("already tracking state for %v", i)
}

b, err := cborutil.Dump(state)
if err != nil {
return err
}

return st.ds.Put(k, b)
}

func (st *StateStore) End(i interface{}) error {
k := toKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", i)
}
return st.ds.Delete(k)
}

func cborMutator(mutator interface{}) func([]byte) ([]byte, error) {
rmut := reflect.ValueOf(mutator)

return func(in []byte) ([]byte, error) {
state := reflect.New(rmut.Type().In(0).Elem())

err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface())
if err != nil {
return nil, err
}

out := rmut.Call([]reflect.Value{state})

if err := out[0].Interface(); err != nil {
return nil, err.(error)
}

return cborutil.Dump(state.Interface())
}
}

// mutator func(*T) error
func (st *StateStore) Mutate(i interface{}, mutator interface{}) error {
return st.mutate(i, cborMutator(mutator))
}

func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)) error {
k := toKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", i)
}

cur, err := st.ds.Get(k)
if err != nil {
return err
}

mutated, err := mutator(cur)
if err != nil {
return err
}

return st.ds.Put(k, mutated)
}

func (st *StateStore) Has(i interface{}) (bool, error) {
return st.ds.Has(toKey(i))
}

func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error {
k := toKey(i)
val, err := st.ds.Get(k)
if err != nil {
if xerrors.Is(err, datastore.ErrNotFound) {
return xerrors.Errorf("No state for %s: %w", i, err)
}
return err
}

return out.UnmarshalCBOR(bytes.NewReader(val))
}

// out: *[]T
func (st *StateStore) List(out interface{}) error {
res, err := st.ds.Query(query.Query{})
if err != nil {
return err
}
defer res.Close()

outT := reflect.TypeOf(out).Elem().Elem()
rout := reflect.ValueOf(out)

var errs error

for {
res, ok := res.NextSync()
if !ok {
break
}
if res.Error != nil {
return res.Error
}

elem := reflect.New(outT)
err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface())
if err != nil {
errs = multierr.Append(errs, xerrors.Errorf("decoding state for key '%s': %w", res.Key, err))
continue
}

rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem()))
}

return nil
}
38 changes: 38 additions & 0 deletions shared/statestore/store_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package statestore

import (
"testing"

"github.com/ipfs/go-datastore"

"github.com/filecoin-project/go-fil-components/shared/cborutil"
"github.com/filecoin-project/go-fil-components/shared/tokenamount"
)

func TestList(t *testing.T) {
ds := datastore.NewMapDatastore()

e, err := cborutil.Dump(tokenamount.FromInt(7))
if err != nil {
t.Fatal(err)
}

if err := ds.Put(datastore.NewKey("/2"), e); err != nil {
t.Fatal(err)
}

st := &StateStore{ds: ds}

var out []tokenamount.TokenAmount
if err := st.List(&out); err != nil {
t.Fatal(err)
}

if len(out) != 1 {
t.Fatal("wrong len")
}

if out[0].Int64() != 7 {
t.Fatal("wrong data")
}
}
30 changes: 30 additions & 0 deletions shared/types/ask.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package types

import (
"github.com/filecoin-project/go-fil-components/shared/address"
"github.com/filecoin-project/go-fil-components/shared/tokenamount"
cbor "github.com/ipfs/go-ipld-cbor"
)

func init() {
cbor.RegisterCborType(SignedStorageAsk{})
cbor.RegisterCborType(StorageAsk{})
}

//go:generate cbor-gen-for SignedStorageAsk StorageAsk

type SignedStorageAsk struct {
Ask *StorageAsk
Signature *Signature
}

type StorageAsk struct {
// Price per GiB / Epoch
Price tokenamount.TokenAmount

MinPieceSize uint64
Miner address.Address
Timestamp uint64
Expiry uint64
SeqNo uint64
}
Loading

0 comments on commit cab0e34

Please sign in to comment.