diff --git a/Makefile b/Makefile index 6757bfa62..6e8061b6c 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,20 @@ test-go-parallel: go test ./... -race -timeout 30s +.PHONY: test-key-value-stores +test-key-value-stores: test-key-value-stores-go test-key-value-stores-wasm + + +.PHONY: test-key-value-stores-go +test-key-value-stores-go: + ENABLE_KEY_VALUE_TESTS=true go test ./db + + +.PHONY: test-key-value-stores-wasm +test-key-value-stores-wasm: + WASM_INIT_FILE="$$(pwd)/packages/mesh-browser-shim/dist/browser_shim.js" GOOS=js GOARCH=wasm ENABLE_KEY_VALUE_TESTS=true go test ./db -timeout 30m -tags=browser -exec="$$GOPATH/bin/wasmbrowsertest" + + .PHONY: test-go-serial test-go-serial: go test ./zeroex/ordervalidator ./zeroex/orderwatch ./core -race -timeout 90s -p=1 --serial diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index adb394d6f..fffe209a2 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -15,7 +15,6 @@ import ( "github.com/0xProject/0x-mesh/loghooks" "github.com/0xProject/0x-mesh/p2p" "github.com/0xProject/0x-mesh/p2p/banner" - sqlds "github.com/0xProject/sql-datastore" "github.com/ipfs/go-datastore" leveldbStore "github.com/ipfs/go-ds-leveldb" libp2p "github.com/libp2p/go-libp2p" @@ -29,7 +28,6 @@ import ( "github.com/libp2p/go-libp2p-core/routing" dht "github.com/libp2p/go-libp2p-kad-dht" dhtopts "github.com/libp2p/go-libp2p-kad-dht/opts" - peerstore "github.com/libp2p/go-libp2p-peerstore" "github.com/libp2p/go-libp2p-peerstore/pstoreds" "github.com/libp2p/go-libp2p/p2p/host/relay" filter "github.com/libp2p/go-maddr-filter" @@ -77,29 +75,6 @@ type Config struct { DataStoreType string `envvar:"DATA_STORE_TYPE" default:"leveldb"` // LevelDBDataDir is the directory used for storing data when using leveldb as data store type. LevelDBDataDir string `envvar:"LEVELDB_DATA_DIR" default:"0x_mesh"` - // SQLDBConnectionString is the connection URL used to connect to the - // database. - // NOTE: When set it has precedence over SQL_DB_HOST, SQL_DB_PORT etc. - SQLDBConnectionString string `envvar:"SQL_DB_CONNECTION_STRING" default:"" json:"-"` - // SQLDBHost is the database host used to connect to the database when - // using postgres as data store type. - SQLDBHost string `envvar:"SQL_DB_HOST" default:"localhost" json:"-"` - // SQLDBPort is the database port used to connect to the database when - // using postgres as data store type. - SQLDBPort string `envvar:"SQL_DB_PORT" default:"5432" json:"-"` - // SQLDBUser is the database user used to connect to the database when - // using postgres as data store type. - SQLDBUser string `envvar:"SQL_DB_USER" default:"postgres" json:"-"` - // SQLDBPassword is the database password used to connect to the database when - // using postgres as data store type. - SQLDBPassword string `envvar:"SQL_DB_PASSWORD" default:"" json:"-"` - // SQLDBName is the database name to connect to when using - // postgres as data store type. - SQLDBName string `envvar:"SQL_DB_NAME" default:"datastore" json:"-"` - // SQLDBEngine is the underyling database engine to use as the - // database driver. - // NOTE: Currently only `postgres` driver is supported. - SQLDBEngine string `envvar:"SQL_DB_ENGINE" default:"postgres"` // UseBootstrapList determines whether or not to use the list of hard-coded // peers to bootstrap the DHT for peer discovery. UseBootstrapList bool `envvar:"USE_BOOTSTRAP_LIST" default:"true"` @@ -162,66 +137,30 @@ func main() { var kadDHT *dht.IpfsDHT var newDHT func(h host.Host) (routing.PeerRouting, error) - var peerStore peerstore.Peerstore - - // TODO(oskar) - Figure out why returning an anonymous function from - // getNewDHT() is making kadDHT.runBootstrap panicing. - // When solved this big switch case can be removed from main() - switch config.DataStoreType { - case leveldbDataStore: - newDHT = func(h host.Host) (routing.PeerRouting, error) { - var err error - dhtDir := getDHTDir(config) - kadDHT, err = p2p.NewDHT(ctx, dhtDir, h) - if err != nil { - log.WithField("error", err).Fatal("could not create DHT") - } - return kadDHT, err - } - - // Set up the peerstore to use LevelDB. - store, err := leveldbStore.NewDatastore(getPeerstoreDir(config), nil) + newDHT = func(h host.Host) (routing.PeerRouting, error) { + var err error + dhtDir := getDHTDir(config) + // Set up the DHT to use LevelDB. + store, err := leveldbStore.NewDatastore(dhtDir, nil) if err != nil { - log.Fatal(err) - } - - peerStore, err = pstoreds.NewPeerstore(ctx, store, pstoreds.DefaultOpts()) - if err != nil { - log.Fatal(err) - } - - case sqlDataStore: - db, err := getSQLDatabase(config) - if err != nil { - log.WithField("error", err).Fatal("could not create SQL database") - } - - err = prepareSQLDatabase(db) - if err != nil { - log.WithField("error", err).Fatal("failed to repare SQL tables for datastores") - } - - newDHT = func(h host.Host) (routing.PeerRouting, error) { - var err error - dstore := sqlds.NewDatastore(db, sqlds.NewQueriesForTable(dhtTableName)) - - kadDHT, err = NewDHTWithDatastore(ctx, dstore, h) - if err != nil { - log.WithField("error", err).Fatal("could not create DHT") - } - - return kadDHT, err + return nil, err } - - pstore := sqlds.NewDatastore(db, sqlds.NewQueriesForTable(peerStoreTableName)) - peerStore, err = pstoreds.NewPeerstore(ctx, pstore, pstoreds.DefaultOpts()) + kadDHT, err = dht.New(ctx, h, dhtopts.Datastore(store), dhtopts.Protocols(p2p.DHTProtocolID)) if err != nil { - log.WithField("error", err).Fatal("could not create peerStore") + log.WithField("error", err).Fatal("could not create DHT") } + return kadDHT, err + } - default: - log.Fatalf("invalid datastore configured: %s. Expected either %s or %s", config.DataStoreType, leveldbDataStore, sqlDataStore) + // Set up the peerstore to use LevelDB. + store, err := leveldbStore.NewDatastore(getPeerstoreDir(config), nil) + if err != nil { + log.Fatal(err) + } + peerStore, err := pstoreds.NewPeerstore(ctx, store, pstoreds.DefaultOpts()) + if err != nil { + log.Fatal(err) } // Parse multiaddresses given in the config diff --git a/cmd/mesh-bootstrap/storage.go b/cmd/mesh-bootstrap/storage.go index cc45ee712..29f04571a 100644 --- a/cmd/mesh-bootstrap/storage.go +++ b/cmd/mesh-bootstrap/storage.go @@ -3,9 +3,6 @@ package main import ( - "database/sql" - "errors" - "fmt" "os" "path/filepath" @@ -33,40 +30,6 @@ func getPeerstoreDir(config Config) string { return filepath.Join(config.LevelDBDataDir, "p2p", "peerstore") } -func getSQLDatabase(config Config) (*sql.DB, error) { - // Currently we only support the postgres driver. - if config.SQLDBEngine != "postgres" { - return nil, errors.New("sqld currently only supports postgres driver") - } - - if config.SQLDBConnectionString != "" { - return sql.Open(config.SQLDBEngine, config.SQLDBConnectionString) - } - - fmtStr := "postgresql:///%s?host=%s&port=%s&user=%s&password=%s&sslmode=disable" - connstr := fmt.Sprintf(fmtStr, config.SQLDBName, config.SQLDBHost, config.SQLDBPort, config.SQLDBUser, config.SQLDBPassword) - - return sql.Open(config.SQLDBEngine, connstr) -} - -func prepareSQLDatabase(db *sql.DB) error { - createTableString := "CREATE TABLE IF NOT EXISTS %s (key TEXT NOT NULL UNIQUE, data BYTEA NOT NULL)" - createDHTTable := fmt.Sprintf(createTableString, dhtTableName) - createPeerStoreTable := fmt.Sprintf(createTableString, peerStoreTableName) - - _, err := db.Exec(createDHTTable) - if err != nil { - return err - } - - _, err = db.Exec(createPeerStoreTable) - if err != nil { - return err - } - - return nil -} - func initPrivateKey(path string) (p2pcrypto.PrivKey, error) { privKey, err := keys.GetPrivateKeyFromPath(path) if err == nil { diff --git a/core/core.go b/core/core.go index d242c2093..8d0c0e299 100644 --- a/core/core.go +++ b/core/core.go @@ -627,7 +627,7 @@ func (app *App) Start() error { RendezvousPoints: rendezvousPoints, UseBootstrapList: app.config.UseBootstrapList, BootstrapList: bootstrapList, - DataDir: filepath.Join(app.config.DataDir, "p2p"), + DB: app.db, CustomMessageValidator: app.orderFilter.ValidatePubSubMessage, MaxBytesPerSecond: app.config.MaxBytesPerSecond, } diff --git a/core/core_test.go b/core/core_test.go index 7e2e2b735..45edb92b8 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -246,7 +246,7 @@ func TestOrderSync(t *testing.T) { } for i, testCase := range testCases { testCaseName := fmt.Sprintf("%s (test case %d)", testCase.name, i) - t.Run(testCaseName, runOrdersyncTestCase(t, testCase)) + t.Run(testCaseName, runOrdersyncTestCase(testCase)) } } @@ -259,7 +259,7 @@ type ordersyncTestCase struct { const defaultOrderFilter = "{}" -func runOrdersyncTestCase(t *testing.T, testCase ordersyncTestCase) func(t *testing.T) { +func runOrdersyncTestCase(testCase ordersyncTestCase) func(t *testing.T) { return func(t *testing.T) { teardownSubTest := setupSubTest(t) defer teardownSubTest(t) diff --git a/core/ordersync/ordersync_test.go b/core/ordersync/ordersync_test.go index 613c4dcfb..c091c853e 100644 --- a/core/ordersync/ordersync_test.go +++ b/core/ordersync/ordersync_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/0xProject/0x-mesh/constants" + "github.com/0xProject/0x-mesh/db" "github.com/0xProject/0x-mesh/ethereum" "github.com/0xProject/0x-mesh/p2p" "github.com/0xProject/0x-mesh/scenario" @@ -30,12 +31,14 @@ func TestCalculateDelayWithJitters(t *testing.T) { } func TestHandleRawRequest(t *testing.T) { + db, err := db.New(context.Background(), db.TestOptions()) + require.NoError(t, err) n, err := p2p.New( context.Background(), p2p.Config{ MessageHandler: &noopMessageHandler{}, RendezvousPoints: []string{"/test-rendezvous-point"}, - DataDir: "/tmp", + DB: db, }, ) require.NoError(t, err) diff --git a/db/common.go b/db/common.go index 47e855af8..dbdfd24a1 100644 --- a/db/common.go +++ b/db/common.go @@ -12,6 +12,7 @@ import ( "github.com/0xProject/0x-mesh/zeroex" "github.com/ethereum/go-ethereum/common" "github.com/gibson042/canonicaljson-go" + ds "github.com/ipfs/go-datastore" ) const ( @@ -48,6 +49,8 @@ type Database interface { GetMetadata() (*types.Metadata, error) SaveMetadata(metadata *types.Metadata) error UpdateMetadata(updateFunc func(oldmetadata *types.Metadata) (newMetadata *types.Metadata)) error + PeerStore() ds.Batching + DHTStore() ds.Batching } type Options struct { diff --git a/db/db_test.go b/db/db_test.go index e09affdb7..15ec94d5a 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -19,12 +19,32 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" ethtypes "github.com/ethereum/go-ethereum/core/types" + dstest "github.com/ipfs/go-datastore/test" + "github.com/plaid/go-envvar/envvar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) var contractAddresses = ethereum.GanacheAddresses +// Since the key value store tests take a very long time to run, we disable them +// by default. These tests can be enabled with the `ENABLE_KEY_VALUE_TESTS` +// environment variable. +var keyValueTestsEnabled bool + +type TestingFlags struct { + EnableKeyValueTests bool `envvar:"ENABLE_KEY_VALUE_TESTS" default:"false"` +} + +func init() { + var flags TestingFlags + if err := envvar.Parse(&flags); err != nil { + panic(fmt.Sprintf("could not parse environment variables: %s", err.Error())) + } + keyValueTestsEnabled = flags.EnableKeyValueTests + testing.Init() +} + func TestAddOrders(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1818,6 +1838,70 @@ func makeMiniHeaderFilterTestCases(t *testing.T, db *DB) ([]*types.MiniHeader, [ return storedMiniHeaders, testCases } +func TestPeerStoreBasic(t *testing.T) { + if !keyValueTestsEnabled { + t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable") + } + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db := newTestDB(t, ctx) + peerstore := db.PeerStore() + + for _, test := range dstest.BasicSubtests { + test(t, peerstore) + } +} + +func TestPeerStoreBatch(t *testing.T) { + if !keyValueTestsEnabled { + t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable") + } + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db := newTestDB(t, ctx) + peerstore := db.PeerStore() + + for _, test := range dstest.BatchSubtests { + test(t, peerstore) + } +} + +func TestDHTStoreBasic(t *testing.T) { + if !keyValueTestsEnabled { + t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable") + } + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db := newTestDB(t, ctx) + dhtstore := db.DHTStore() + + for _, test := range dstest.BasicSubtests { + test(t, dhtstore) + } +} + +func TestDHTStoreBatch(t *testing.T) { + if !keyValueTestsEnabled { + t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable") + } + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db := newTestDB(t, ctx) + dhtstore := db.DHTStore() + + for _, test := range dstest.BatchSubtests { + test(t, dhtstore) + } +} + // safeSubsliceOrders returns a (shallow) subslice of orders without modifying // the original slice. Uses the same semantics as slice expressions: low is // inclusive, hi is exclusive. The returned slice still contains pointers, it diff --git a/db/dexie_datastore.go b/db/dexie_datastore.go new file mode 100644 index 000000000..1c85ec916 --- /dev/null +++ b/db/dexie_datastore.go @@ -0,0 +1,237 @@ +// +build js, wasm + +package db + +import ( + "context" + "syscall/js" + + "github.com/0xProject/0x-mesh/packages/mesh-browser/go/jsutil" + ds "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" +) + +// Ensure that we are implementing the ds.Batching interface. +var _ ds.Batching = &Datastore{} + +// NOTE(jalextowle): Close is a noop in this implementation. We do not want a close +// operation to shut down the database connection. +func (d *Datastore) Close() error { + return nil +} + +// NOTE(jalextowle): Sync is a noop in this implementation. Operations +// such as Put and Delete are completed before a result is returned. +func (d *Datastore) Sync(ds.Key) error { + return nil +} + +// Datastore provides a Dexie implementation of the ds.Batching interface. The +// corresponding Javascript bindings can be found in +// packages/mesh-browser-lite/src/datastore.ts. +type Datastore struct { + db *DB + ctx context.Context + dexieStore js.Value +} + +type OperationType byte + +const ( + PUT OperationType = iota + DELETE +) + +// Operation contains all of the data needed to communicate with the Javascript +// bindings that control access to the Dexie datastore. The Javascript bindings +// need to know what the operation should do (put or delete) and the data that +// should be used in the operation. +type Operation struct { + operationType OperationType + key ds.Key + value []byte +} + +func (o *Operation) JSValue() js.Value { + jsBytes, _ := jsutil.CopyBytesToJS(o.value) + return js.ValueOf(map[string]interface{}{ + "operationType": int(o.operationType), + "key": o.key.String(), + "value": jsBytes, + }) +} + +// Batch implements the ds.Batch interface, which allows Put and Delete operations +// to be queued and then committed all at once. +type Batch struct { + ctx context.Context + dexieStore js.Value + operations []*Operation +} + +func (d *Datastore) Batch() (ds.Batch, error) { + return &Batch{ + ctx: d.ctx, + dexieStore: d.dexieStore, + }, nil +} + +func (b *Batch) Put(key ds.Key, value []byte) error { + b.operations = append(b.operations, &Operation{ + operationType: PUT, + key: key, + value: value, + }) + return nil +} + +func (b *Batch) Delete(key ds.Key) error { + b.operations = append(b.operations, &Operation{ + operationType: DELETE, + key: key, + }) + return nil +} + +// Commit performs a batch of operations on the Dexie datastore. In this implementation, +// all of these operations occur in the same transactional context. +func (b *Batch) Commit() (err error) { + defer func() { + if r := recover(); r != nil { + err = jsutil.RecoverError(r) + } + }() + + convertibleOperations := make([]interface{}, len(b.operations)) + for i, operation := range b.operations { + convertibleOperations[i] = interface{}(operation) + } + _, err = jsutil.AwaitPromiseContext(b.ctx, b.dexieStore.Call("commitAsync", convertibleOperations)) + if err != nil { + return convertJSError(err) + } + return nil +} + +func (d *Datastore) Get(key ds.Key) (_ []byte, err error) { + defer func() { + if r := recover(); r != nil { + err = jsutil.RecoverError(r) + } + }() + + jsResult, err := jsutil.AwaitPromiseContext(d.ctx, d.dexieStore.Call("getAsync", key.String())) + if err != nil { + return nil, convertJSError(err) + } + return jsutil.CopyBytesToGo(jsResult) +} + +func (d *Datastore) Has(key ds.Key) (_ bool, err error) { + defer func() { + if r := recover(); r != nil { + err = jsutil.RecoverError(r) + } + }() + + jsResult, err := jsutil.AwaitPromiseContext(d.ctx, d.dexieStore.Call("hasAsync", key.String())) + if err != nil { + return false, convertJSError(err) + } + return jsResult.Bool(), nil +} + +func (d *Datastore) GetSize(key ds.Key) (_ int, err error) { + defer func() { + if r := recover(); r != nil { + err = jsutil.RecoverError(r) + } + }() + + jsResult, err := jsutil.AwaitPromiseContext(d.ctx, d.dexieStore.Call("getSizeAsync", key.String())) + if err != nil { + return -1, convertJSError(err) + } + return jsResult.Int(), nil +} + +func (d *Datastore) Query(q dsq.Query) (_ dsq.Results, err error) { + defer func() { + if r := recover(); r != nil { + err = jsutil.RecoverError(r) + } + }() + + jsResults, err := jsutil.AwaitPromiseContext(d.ctx, d.dexieStore.Call("queryAsync", q.Prefix)) + if err != nil { + return nil, convertJSError(err) + } + entries := make([]dsq.Entry, jsResults.Get("length").Int()) + for i := 0; i < jsResults.Get("length").Int(); i++ { + jsResult := jsResults.Index(i) + jsBytes, err := jsutil.CopyBytesToGo(jsResult.Get("value")) + if err != nil { + return nil, err + } + entries[i] = dsq.Entry{ + Key: jsResult.Get("key").String(), + Value: jsBytes, + Size: jsResult.Get("size").Int(), + } + } + filteredEntries := []dsq.Entry{} + for _, entry := range entries { + passes := true + for _, filter := range q.Filters { + if !filter.Filter(entry) { + passes = false + break + } + } + if passes { + filteredEntries = append(filteredEntries, entry) + } + } + dsq.Sort(q.Orders, filteredEntries) + if q.Offset > 0 && q.Offset <= len(filteredEntries) { + filteredEntries = filteredEntries[q.Offset:] + } else if q.Offset > len(filteredEntries) { + filteredEntries = []dsq.Entry{} + } + if q.Limit > 0 && q.Limit <= len(filteredEntries) { + filteredEntries = filteredEntries[:q.Limit] + } + return dsq.ResultsWithEntries(q, filteredEntries), nil +} + +func (d *Datastore) Put(key ds.Key, value []byte) (err error) { + defer func() { + if r := recover(); r != nil { + err = jsutil.RecoverError(r) + } + }() + + jsBytes, err := jsutil.CopyBytesToJS(value) + if err != nil { + return err + } + _, err = jsutil.AwaitPromiseContext(d.ctx, d.dexieStore.Call("putAsync", key.String(), jsBytes)) + if err != nil { + return convertJSError(err) + } + return nil +} + +func (d *Datastore) Delete(key ds.Key) (err error) { + defer func() { + if r := recover(); r != nil { + err = jsutil.RecoverError(r) + } + }() + + _, err = jsutil.AwaitPromiseContext(d.ctx, d.dexieStore.Call("deleteAsync", key.String())) + if err != nil { + return convertJSError(err) + } + return nil +} diff --git a/db/dexie_implementation.go b/db/dexie_implementation.go index 5bd199103..f251a4b7d 100644 --- a/db/dexie_implementation.go +++ b/db/dexie_implementation.go @@ -8,7 +8,6 @@ import ( "fmt" "math/big" "path/filepath" - "runtime/debug" "syscall/js" "time" @@ -19,6 +18,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/gibson042/canonicaljson-go" "github.com/google/uuid" + ds "github.com/ipfs/go-datastore" "github.com/sirupsen/logrus" ) @@ -65,7 +65,7 @@ func New(ctx context.Context, opts *Options) (database *DB, err error) { } defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() newDexieDatabase := js.Global().Get("__mesh_dexie_newDatabase__") @@ -90,10 +90,26 @@ func New(ctx context.Context, opts *Options) (database *DB, err error) { }, nil } +func (db *DB) PeerStore() ds.Batching { + dexieStore := db.dexie.Call("peerStore") + return &Datastore{ + ctx: db.ctx, + dexieStore: dexieStore, + } +} + +func (db *DB) DHTStore() ds.Batching { + dexieStore := db.dexie.Call("dhtStore") + return &Datastore{ + ctx: db.ctx, + dexieStore: dexieStore, + } +} + func (db *DB) AddOrders(orders []*types.OrderWithMetadata) (alreadyStored []common.Hash, added []*types.OrderWithMetadata, removed []*types.OrderWithMetadata, err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -129,7 +145,7 @@ func (db *DB) AddOrders(orders []*types.OrderWithMetadata) (alreadyStored []comm func (db *DB) GetOrder(hash common.Hash) (order *types.OrderWithMetadata, err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -148,7 +164,7 @@ func (db *DB) GetOrder(hash common.Hash) (order *types.OrderWithMetadata, err er func (db *DB) GetOrderStatuses(hashes []common.Hash) (statuses []*StoredOrderStatus, err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -184,7 +200,7 @@ func (db *DB) FindOrders(query *OrderQuery) (orders []*types.OrderWithMetadata, } defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -207,7 +223,7 @@ func (db *DB) CountOrders(query *OrderQuery) (count int, err error) { } defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -223,7 +239,7 @@ func (db *DB) CountOrders(query *OrderQuery) (count int, err error) { func (db *DB) DeleteOrder(hash common.Hash) (err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -241,7 +257,7 @@ func (db *DB) DeleteOrders(query *OrderQuery) (deletedOrders []*types.OrderWithM } defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -261,7 +277,7 @@ func (db *DB) DeleteOrders(query *OrderQuery) (deletedOrders []*types.OrderWithM func (db *DB) UpdateOrder(hash common.Hash, updateFunc func(existingOrder *types.OrderWithMetadata) (updatedOrder *types.OrderWithMetadata, err error)) (err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -294,7 +310,7 @@ func (db *DB) UpdateOrder(hash common.Hash, updateFunc func(existingOrder *types func (db *DB) AddMiniHeaders(miniHeaders []*types.MiniHeader) (added []*types.MiniHeader, removed []*types.MiniHeader, err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -314,7 +330,7 @@ func (db *DB) AddMiniHeaders(miniHeaders []*types.MiniHeader) (added []*types.Mi func (db *DB) ResetMiniHeaders(newMiniHeaders []*types.MiniHeader) (err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -330,7 +346,7 @@ func (db *DB) ResetMiniHeaders(newMiniHeaders []*types.MiniHeader) (err error) { func (db *DB) GetMiniHeader(hash common.Hash) (miniHeader *types.MiniHeader, err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -348,7 +364,7 @@ func (db *DB) FindMiniHeaders(query *MiniHeaderQuery) (miniHeaders []*types.Mini } defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -364,7 +380,7 @@ func (db *DB) FindMiniHeaders(query *MiniHeaderQuery) (miniHeaders []*types.Mini func (db *DB) DeleteMiniHeader(hash common.Hash) (err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -382,7 +398,7 @@ func (db *DB) DeleteMiniHeaders(query *MiniHeaderQuery) (deleted []*types.MiniHe } defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -398,7 +414,7 @@ func (db *DB) DeleteMiniHeaders(query *MiniHeaderQuery) (deleted []*types.MiniHe func (db *DB) GetMetadata() (metadata *types.Metadata, err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -417,7 +433,7 @@ func (db *DB) GetMetadata() (metadata *types.Metadata, err error) { func (db *DB) SaveMetadata(metadata *types.Metadata) (err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -437,7 +453,7 @@ func (db *DB) SaveMetadata(metadata *types.Metadata) (err error) { func (db *DB) UpdateMetadata(updateFunc func(oldmetadata *types.Metadata) (newMetadata *types.Metadata)) (err error) { defer func() { if r := recover(); r != nil { - err = recoverError(r) + err = jsutil.RecoverError(r) } }() start := time.Now() @@ -464,20 +480,6 @@ func (db *DB) UpdateMetadata(updateFunc func(oldmetadata *types.Metadata) (newMe return nil } -func recoverError(e interface{}) error { - if e != nil { - debug.PrintStack() - } - switch e := e.(type) { - case error: - return e - case string: - return errors.New(e) - default: - return fmt.Errorf("unexpected JavaScript error: (%T) %v", e, e) - } -} - func convertJSError(e error) error { switch e := e.(type) { case js.Error: @@ -495,6 +497,8 @@ func convertJSError(e error) error { return ErrMetadataAlreadyExists case ErrDBFilledWithPinnedOrders.Error(): return ErrDBFilledWithPinnedOrders + case ds.ErrNotFound.Error(): + return ds.ErrNotFound } } return e diff --git a/db/kv_store_queries.go b/db/kv_store_queries.go new file mode 100644 index 000000000..431bdf1b4 --- /dev/null +++ b/db/kv_store_queries.go @@ -0,0 +1,154 @@ +// +build !js + +package db + +import ( + "database/sql" + "fmt" + + "github.com/ipfs/go-ds-sql" + _ "github.com/lib/pq" //postgres driver +) + +/// PostgreSQL + +// Options are the postgres datastore options, reexported here for convenience. +type PostgreSQLOptions struct { + Host string + Port string + User string + Password string + Database string + Table string +} + +type postgreSQLQueries struct { + tableName string +} + +func NewPostgreSQLQueriesForTable(tableName string) *postgreSQLQueries { + return &postgreSQLQueries{tableName} +} + +func (q postgreSQLQueries) Delete() string { + return `DELETE FROM ` + q.tableName + ` WHERE key = $1` +} + +func (q postgreSQLQueries) Exists() string { + return `SELECT exists(SELECT 1 FROM ` + q.tableName + ` WHERE key=$1)` +} + +func (q postgreSQLQueries) Get() string { + return `SELECT data FROM ` + q.tableName + ` WHERE key = $1` +} + +func (q postgreSQLQueries) Put() string { + return `INSERT INTO ` + q.tableName + ` (key, data) SELECT $1, $2 ON CONFLICT(key) DO UPDATE SET data = $2 WHERE key = $1` +} + +func (q postgreSQLQueries) Query() string { + return `SELECT key, data FROM ` + q.tableName +} + +func (q postgreSQLQueries) Prefix() string { + return ` WHERE key LIKE '%s%%' ORDER BY key` +} + +func (q postgreSQLQueries) Limit() string { + return ` LIMIT %d` +} + +func (q postgreSQLQueries) Offset() string { + return ` OFFSET %d` +} + +func (q postgreSQLQueries) GetSize() string { + return `SELECT octet_length(data) FROM ` + q.tableName + ` WHERE key = $1` +} + +// Create returns a datastore connected to postgres initialized with a table +func (opts *PostgreSQLOptions) CreatePostgres() (*sqlds.Datastore, error) { + opts.setDefaults() + fmtstr := "postgresql:///%s?host=%s&port=%s&user=%s&password=%s&sslmode=disable" + constr := fmt.Sprintf(fmtstr, opts.Database, opts.Host, opts.Port, opts.User, opts.Password) + db, err := sql.Open("postgres", constr) + if err != nil { + return nil, err + } + + createTable := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (key TEXT NOT NULL UNIQUE, data BYTEA NOT NULL)", opts.Table) + _, err = db.Exec(createTable) + + if err != nil { + return nil, err + } + + return sqlds.NewDatastore(db, NewPostgreSQLQueriesForTable(opts.Table)), nil +} + +func (opts *PostgreSQLOptions) setDefaults() { + if opts.Table == "" { + opts.Table = "kv" + } + if opts.Host == "" { + opts.Host = "postgres" + } + + if opts.Port == "" { + opts.Port = "5432" + } + + if opts.User == "" { + opts.User = "postgres" + } + + if opts.Database == "" { + opts.Database = "datastore" + } +} + +/// Sqlite + +type sqliteQueries struct { + tableName string +} + +func NewSqliteQueriesForTable(tableName string) *sqliteQueries { + return &sqliteQueries{tableName} +} + +func (q sqliteQueries) Delete() string { + return `DELETE FROM ` + q.tableName + ` WHERE key = $1` +} + +func (q sqliteQueries) Exists() string { + return `SELECT exists(SELECT 1 FROM ` + q.tableName + ` WHERE key=$1)` +} + +func (q sqliteQueries) Get() string { + return `SELECT data FROM ` + q.tableName + ` WHERE key = $1` +} + +func (q sqliteQueries) Put() string { + return `INSERT INTO ` + q.tableName + ` (key, data) SELECT $1, $2 ON CONFLICT(key) DO UPDATE SET data = $2 WHERE key = $1` +} + +func (q sqliteQueries) Query() string { + return `SELECT key, data FROM ` + q.tableName +} + +func (q sqliteQueries) Prefix() string { + return ` WHERE key LIKE '%s%%' ORDER BY key` +} + +func (q sqliteQueries) Limit() string { + return ` LIMIT %d` +} + +func (q sqliteQueries) Offset() string { + return ` OFFSET %d` +} + +func (q sqliteQueries) GetSize() string { + return `SELECT length(data) FROM ` + q.tableName + ` WHERE key = $1` +} diff --git a/db/sql_implementation.go b/db/sql_implementation.go index a356c8367..a9d223b8f 100644 --- a/db/sql_implementation.go +++ b/db/sql_implementation.go @@ -19,6 +19,8 @@ import ( "github.com/gibson042/canonicaljson-go" "github.com/google/uuid" "github.com/ido50/sqlz" + ds "github.com/ipfs/go-datastore" + "github.com/ipfs/go-ds-sql" "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" ) @@ -96,6 +98,14 @@ func New(ctx context.Context, opts *Options) (*DB, error) { return db, nil } +func (db *DB) DHTStore() ds.Batching { + return sqlds.NewDatastore(db.sqldb.DB.DB, NewSqliteQueriesForTable("dhtstore")) +} + +func (db *DB) PeerStore() ds.Batching { + return sqlds.NewDatastore(db.sqldb.DB.DB, NewSqliteQueriesForTable("peerstore")) +} + // TODO(albrow): Use a proper migration tool. We don't technically need this // now but it will be necessary if we ever change the database schema. // Note(albrow): If needed, we can optimize this by adding indexes to the @@ -143,6 +153,16 @@ CREATE TABLE IF NOT EXISTS metadata ( ethRPCRequestsSentInCurrentUTCDay BIGINT NOT NULL, startOfCurrentUTCDay DATETIME NOT NULL ); + +CREATE TABLE IF NOT EXISTS peerstore ( + key TEXT NOT NULL UNIQUE, + data BYTEA NOT NULL +); + +CREATE TABLE IF NOT EXISTS dhtstore ( + key TEXT NOT NULL UNIQUE, + data BYTEA NOT NULL +); ` // Note(albrow): If needed, we can optimize this by using prepared diff --git a/go.mod b/go.mod index 231c52567..f9484cb4e 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ replace ( ) require ( - github.com/0xProject/sql-datastore v0.0.0-20200129193319-32397013f115 github.com/99designs/gqlgen v0.11.3 github.com/agnivade/levenshtein v1.1.0 // indirect github.com/albrow/stringset v2.1.0+incompatible @@ -37,14 +36,15 @@ require ( github.com/hashicorp/go-multierror v1.1.0 // indirect github.com/hashicorp/golang-lru v0.5.4 github.com/ido50/sqlz v0.0.0-20200308174337-487b8faf612c - github.com/ipfs/go-datastore v0.3.1 + github.com/ipfs/go-datastore v0.4.4 github.com/ipfs/go-ds-leveldb v0.4.0 + github.com/ipfs/go-ds-sql v0.2.0 github.com/jmoiron/sqlx v1.2.0 github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 // indirect github.com/karlseguin/ccache v2.0.3+incompatible github.com/karlseguin/expect v1.0.1 // indirect - github.com/lib/pq v1.2.0 + github.com/lib/pq v1.3.0 github.com/libp2p/go-libp2p v0.5.1 github.com/libp2p/go-libp2p-autonat-svc v0.1.0 github.com/libp2p/go-libp2p-circuit v0.1.4 diff --git a/go.sum b/go.sum index e0da55f23..1c2e2805d 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,6 @@ github.com/0xProject/go-ws-transport v0.1.1-0.20200602173532-300f0ff55a11/go.mod github.com/0xProject/goleveldb v1.0.1-0.20200602173211-6ee893c9b83a h1:7V/u2CwtbkJCQ4ctMzIMtxF8I1W1obt+y9zxw71Chg4= github.com/0xProject/goleveldb v1.0.1-0.20200602173211-6ee893c9b83a/go.mod h1:1Jo/RG6qMRJ3cgqzUqRWw592pkpuDJDKCIhMNXK5ar4= github.com/0xProject/qunit v0.0.0-20190730000255-81c18fdf7752/go.mod h1:Onz5mS+Tpffz0tyRWdHDrqKcQ1ZFNeRhYHrNAkaMgeQ= -github.com/0xProject/sql-datastore v0.0.0-20200129193319-32397013f115 h1:OHq6PP4Y8Pjmhm9UB3RtYWQSgxJBQplojLCDZZLrVDg= -github.com/0xProject/sql-datastore v0.0.0-20200129193319-32397013f115/go.mod h1:7icquWqYm+GkgQsUyBs0C0N1SyCHXQYBnoHaazVYDQ0= github.com/99designs/gqlgen v0.11.3 h1:oFSxl1DFS9X///uHV3y6CEfpcXWrDUxVblR4Xib2bs4= github.com/99designs/gqlgen v0.11.3/go.mod h1:RgX5GRRdDWNkh4pBrdzNpNPFVsdoUFY2+adM6nb1N+4= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= @@ -185,6 +183,7 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -234,6 +233,8 @@ github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRV github.com/ipfs/go-datastore v0.3.0/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= +github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -245,6 +246,8 @@ github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIyk github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.4.0 h1:l5RuxaIXDbCFqBonzGXGfMpCMBwrbJVrmqGTtqQKHaI= github.com/ipfs/go-ds-leveldb v0.4.0/go.mod h1:NX127692WYenEjmlzyZunFv+f8nrKbH1s9eIaRNnhzs= +github.com/ipfs/go-ds-sql v0.2.0 h1:ZUHUbU5IydNuBWzcRMOZYkBUwTg+L56o23fEVcbWC7o= +github.com/ipfs/go-ds-sql v0.2.0/go.mod h1:/c47NpRiHobwn+8F8EpW0yBy8d3Mx/j/tIlrVN1e1Ec= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= @@ -266,6 +269,8 @@ github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -303,6 +308,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -310,6 +316,8 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/libp2p/go-addr-util v0.0.1 h1:TpTQm9cXVRVSKsYbgQ7GKc3KbbHVTnbostgGaDEP+88= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= @@ -585,6 +593,7 @@ github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSg github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8= github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= @@ -676,6 +685,9 @@ go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -683,6 +695,7 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -696,6 +709,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -757,6 +772,9 @@ golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589 h1:rjUrONFu4kLchcZTfp3/96bR8bW8dIa8uz3cR5n0cgM= golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -782,6 +800,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/karlseguin/expect.v1 v1.0.1 h1:9u0iUltnhFbJTHaSIH0EP+cuTU5rafIgmcsEsg2JQFw= @@ -806,5 +825,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= sourcegraph.com/sourcegraph/appdash v0.0.0-20180110180208-2cc67fd64755/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= diff --git a/p2p/node.go b/p2p/node.go index d881346e5..4016ba6b8 100644 --- a/p2p/node.go +++ b/p2p/node.go @@ -14,6 +14,7 @@ import ( "time" "github.com/0xProject/0x-mesh/constants" + "github.com/0xProject/0x-mesh/db" "github.com/0xProject/0x-mesh/p2p/banner" "github.com/0xProject/0x-mesh/p2p/ratevalidator" "github.com/0xProject/0x-mesh/p2p/validatorset" @@ -136,8 +137,9 @@ type Config struct { // BootstrapList is a list of multiaddress strings to use for bootstrapping // the DHT. If empty, the default list will be used. BootstrapList []string - // DataDir is the directory to use for storing data. - DataDir string + // DB is a database instance that provides access to key value stores for + // the peerstore and the dht store. + DB *db.DB // GlobalPubSubMessageLimit is the maximum number of messages per second that // will be forwarded through GossipSub on behalf of other peers. It is an // important mechanism for limiting our own upload bandwidth. Without a global @@ -205,8 +207,7 @@ func New(ctx context.Context, config Config) (*Node, error) { var kadDHT *dht.IpfsDHT newDHT := func(h host.Host) (routing.PeerRouting, error) { var err error - dhtDir := getDHTDir(config.DataDir) - kadDHT, err = NewDHT(ctx, dhtDir, h) + kadDHT, err = NewDHT(ctx, config.DB, h) if err != nil { log.WithField("error", err).Error("could not create DHT") } diff --git a/p2p/node_test.go b/p2p/node_test.go index 5d4b5132b..72f7e5b3d 100644 --- a/p2p/node_test.go +++ b/p2p/node_test.go @@ -11,8 +11,8 @@ import ( "testing" "time" + "github.com/0xProject/0x-mesh/db" "github.com/0xProject/0x-mesh/p2p/banner" - "github.com/google/uuid" p2pcrypto "github.com/libp2p/go-libp2p-core/crypto" p2pnet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -81,6 +81,8 @@ func (n *testNotifee) OpenedStream(network p2pnet.Network, stream p2pnet.Stream) func newTestNode(t *testing.T, ctx context.Context, notifee p2pnet.Notifiee) *Node { privKey, _, err := p2pcrypto.GenerateSecp256k1Key(rand.Reader) require.NoError(t, err) + db, err := db.New(ctx, db.TestOptions()) + require.NoError(t, err) config := Config{ SubscribeTopic: testTopic, PublishTopics: []string{testTopic}, @@ -88,7 +90,7 @@ func newTestNode(t *testing.T, ctx context.Context, notifee p2pnet.Notifiee) *No MessageHandler: &dummyMessageHandler{}, RendezvousPoints: testRendezvousPoints, UseBootstrapList: false, - DataDir: "/tmp/0x-mesh/p2p-testing/" + uuid.New().String(), + DB: db, } return newTestNodeWithConfig(t, ctx, notifee, config) @@ -400,6 +402,8 @@ func TestRateValidatorGlobal(t *testing.T) { streams: make(chan p2pnet.Stream), } + db0, err := db.New(ctx, db.TestOptions()) + require.NoError(t, err) node0Config := Config{ SubscribeTopic: testTopic, PublishTopics: []string{testTopic}, @@ -408,10 +412,12 @@ func TestRateValidatorGlobal(t *testing.T) { }), RendezvousPoints: testRendezvousPoints, UseBootstrapList: false, - DataDir: "/tmp/0x-mesh/p2p-testing/" + uuid.New().String(), + DB: db0, GlobalPubSubMessageLimit: 1, GlobalPubSubMessageBurst: 5, } + db1, err := db.New(ctx, db.TestOptions()) + require.NoError(t, err) node1Config := Config{ SubscribeTopic: testTopic, PublishTopics: []string{testTopic}, @@ -420,10 +426,12 @@ func TestRateValidatorGlobal(t *testing.T) { }), RendezvousPoints: testRendezvousPoints, UseBootstrapList: false, - DataDir: "/tmp/0x-mesh/p2p-testing/" + uuid.New().String(), + DB: db1, GlobalPubSubMessageLimit: 1, GlobalPubSubMessageBurst: 5, } + db2, err := db.New(ctx, db.TestOptions()) + require.NoError(t, err) node2Config := Config{ SubscribeTopic: testTopic, PublishTopics: []string{testTopic}, @@ -432,7 +440,7 @@ func TestRateValidatorGlobal(t *testing.T) { }), RendezvousPoints: testRendezvousPoints, UseBootstrapList: false, - DataDir: "/tmp/0x-mesh/p2p-testing/" + uuid.New().String(), + DB: db2, GlobalPubSubMessageLimit: 1, GlobalPubSubMessageBurst: 5, } @@ -487,6 +495,8 @@ func TestRateValidatorPerPeer(t *testing.T) { streams: make(chan p2pnet.Stream), } + db0, err := db.New(ctx, db.TestOptions()) + require.NoError(t, err) node0Config := Config{ SubscribeTopic: testTopic, PublishTopics: []string{testTopic}, @@ -495,10 +505,12 @@ func TestRateValidatorPerPeer(t *testing.T) { }), RendezvousPoints: testRendezvousPoints, UseBootstrapList: false, - DataDir: "/tmp/0x-mesh/p2p-testing/" + uuid.New().String(), + DB: db0, PerPeerPubSubMessageLimit: 1, PerPeerPubSubMessageBurst: 5, } + db1, err := db.New(ctx, db.TestOptions()) + require.NoError(t, err) node1Config := Config{ SubscribeTopic: testTopic, PublishTopics: []string{testTopic}, @@ -507,10 +519,12 @@ func TestRateValidatorPerPeer(t *testing.T) { }), RendezvousPoints: testRendezvousPoints, UseBootstrapList: false, - DataDir: "/tmp/0x-mesh/p2p-testing/" + uuid.New().String(), + DB: db1, PerPeerPubSubMessageLimit: 1, PerPeerPubSubMessageBurst: 5, } + db2, err := db.New(ctx, db.TestOptions()) + require.NoError(t, err) node2Config := Config{ SubscribeTopic: testTopic, PublishTopics: []string{testTopic}, @@ -519,7 +533,7 @@ func TestRateValidatorPerPeer(t *testing.T) { }), RendezvousPoints: testRendezvousPoints, UseBootstrapList: false, - DataDir: "/tmp/0x-mesh/p2p-testing/" + uuid.New().String(), + DB: db2, PerPeerPubSubMessageLimit: 1, PerPeerPubSubMessageBurst: 5, } diff --git a/p2p/opts.go b/p2p/opts.go index 5a274f997..6031a7b4e 100644 --- a/p2p/opts.go +++ b/p2p/opts.go @@ -9,7 +9,7 @@ import ( "io/ioutil" "net/http" - leveldbStore "github.com/ipfs/go-ds-leveldb" + "github.com/0xProject/0x-mesh/db" libp2p "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p-core/host" dht "github.com/libp2p/go-libp2p-kad-dht" @@ -63,12 +63,7 @@ func getHostOptions(ctx context.Context, config Config) ([]libp2p.Option, error) } advertiseAddrs := []ma.Multiaddr{tcpAdvertiseAddr, wsAdvertiseAddr} - // Set up the peerstore to use LevelDB. - store, err := leveldbStore.NewDatastore(getPeerstoreDir(config.DataDir), nil) - if err != nil { - return nil, err - } - pstore, err := pstoreds.NewPeerstore(ctx, store, pstoreds.DefaultOpts()) + pstore, err := pstoreds.NewPeerstore(ctx, config.DB.PeerStore(), pstoreds.DefaultOpts()) if err != nil { return nil, err } @@ -117,14 +112,8 @@ func getPublicIP() (string, error) { } // NewDHT returns a new Kademlia DHT instance configured to work with 0x Mesh -// in native (pure Go) environments. storageDir is the directory to use for -// persisting the data with LevelDB. -func NewDHT(ctx context.Context, storageDir string, host host.Host) (*dht.IpfsDHT, error) { - // Set up the DHT to use LevelDB. - store, err := leveldbStore.NewDatastore(storageDir, nil) - if err != nil { - return nil, err - } - - return dht.New(ctx, host, dhtopts.Datastore(store), dhtopts.Protocols(DHTProtocolID)) +// in native (pure Go) environments. Standalone nodes use a SQL key value store +// to persist data and browser nodes use a Dexie key value store. +func NewDHT(ctx context.Context, db *db.DB, host host.Host) (*dht.IpfsDHT, error) { + return dht.New(ctx, host, dhtopts.Datastore(db.DHTStore()), dhtopts.Protocols(DHTProtocolID)) } diff --git a/p2p/opts_js.go b/p2p/opts_js.go index 5e2463f28..9ca945fb1 100644 --- a/p2p/opts_js.go +++ b/p2p/opts_js.go @@ -5,6 +5,7 @@ package p2p import ( "context" + "github.com/0xProject/0x-mesh/db" libp2p "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p-core/host" dht "github.com/libp2p/go-libp2p-kad-dht" @@ -44,6 +45,6 @@ func getPubSubOptions() []pubsub.Option { // NewDHT returns a new Kademlia DHT instance configured to work with 0x Mesh // in browser environments. -func NewDHT(ctx context.Context, storageDir string, host host.Host) (*dht.IpfsDHT, error) { - return dht.New(ctx, host, dhtopts.Client(true), dhtopts.Protocols(DHTProtocolID)) +func NewDHT(ctx context.Context, db *db.DB, host host.Host) (*dht.IpfsDHT, error) { + return dht.New(ctx, host, dhtopts.Client(true), dhtopts.Datastore(db.DHTStore()), dhtopts.Protocols(DHTProtocolID)) } diff --git a/packages/mesh-browser-lite/src/database.ts b/packages/mesh-browser-lite/src/database.ts index 2b2d3934d..20508fa01 100644 --- a/packages/mesh-browser-lite/src/database.ts +++ b/packages/mesh-browser-lite/src/database.ts @@ -11,6 +11,8 @@ import Dexie from 'dexie'; +import { BatchingDatastore } from './datastore'; + export type Record = Order | MiniHeader | Metadata; export interface Options { @@ -153,6 +155,8 @@ export class Database { private readonly _orders: Dexie.Table; private readonly _miniHeaders: Dexie.Table; private readonly _metadata: Dexie.Table; + private readonly _dhtstore: BatchingDatastore; + private readonly _peerstore: BatchingDatastore; constructor(opts: Options) { this._db = new Dexie(opts.dataSourceName); @@ -164,11 +168,23 @@ export class Database { '&hash,chainId,makerAddress,makerAssetData,makerAssetAmount,makerFee,makerFeeAssetData,takerAddress,takerAssetData,takerFeeAssetData,takerAssetAmount,takerFee,senderAddress,feeRecipientAddress,expirationTimeSeconds,salt,signature,exchangeAddress,fillableTakerAssetAmount,lastUpdated,isRemoved,isPinned,parsedMakerAssetData,parsedMakerFeeAssetData,lastValidatedBlockNumber,lastValidatedBlockHash,[isNotPinned+expirationTimeSeconds]', miniHeaders: '&hash,parent,number,timestamp', metadata: 'ðereumChainID', + dhtstore: '&key,data', + peerstore: '&key,data', }); this._orders = this._db.table('orders'); this._miniHeaders = this._db.table('miniHeaders'); this._metadata = this._db.table('metadata'); + this._dhtstore = new BatchingDatastore(this._db, 'dhtstore'); + this._peerstore = new BatchingDatastore(this._db, 'peerstore'); + } + + public dhtStore(): BatchingDatastore { + return this._dhtstore; + } + + public peerStore(): BatchingDatastore { + return this._peerstore; } public close(): void { diff --git a/packages/mesh-browser-lite/src/datastore.ts b/packages/mesh-browser-lite/src/datastore.ts new file mode 100644 index 000000000..1a484329a --- /dev/null +++ b/packages/mesh-browser-lite/src/datastore.ts @@ -0,0 +1,111 @@ +// tslint:disable:max-file-line-count + +/** + * @hidden + */ + +/** + * NOTE(jalextowle): This comment must be here so that typedoc knows that the above + * comment is a module comment + */ +import Dexie from 'dexie'; + +interface Entry { + key: string; + value: Uint8Array; + size: number; +} + +enum OperationType { + Addition, + Deletion, +} + +interface Operation { + operationType: OperationType; + key: string; + value?: Uint8Array; +} + +// This implements the subset of the ds.Batching interface that should be implemented +// on the Dexie side. The Go bindings for this system can be found in db/dexie_datastore.go. +// Some aspects of the ds.Batching interface make more sense to implement in Go +// for performance or dependency reasons. The most important example of this is +// that query filtering and ordering is performed on the Go side to avoid converting +// Go functions into Javascript functions. +export class BatchingDatastore { + private readonly _db: Dexie; + private readonly _table: Dexie.Table<{ key: string; value: Uint8Array }, string>; + + constructor(db: Dexie, tableName: string) { + this._db = db; + this._table = this._db._allTables[tableName] as Dexie.Table<{ key: string; value: Uint8Array }, string>; + if (this._table === undefined) { + throw new Error('BatchingDatastore: Attempting to use undefined table'); + } + } + + public async commitAsync(operations: Operation[]): Promise { + await this._db.transaction('rw!', this._table, async () => { + for (const operation of operations) { + if (operation.operationType === OperationType.Addition) { + if (!operation.value) { + throw new Error('commitAsync: no value for key'); + } + await this._table.put({ key: operation.key, value: operation.value }); + } else { + await this._table.delete(operation.key); + } + } + }); + } + + public async putAsync(key: string, value: Uint8Array): Promise { + await this._table.put({ key, value }); + } + + public async deleteAsync(key: string): Promise { + await this._table.delete(key); + } + + public async getAsync(key: string): Promise { + const result = await this._table.get(key); + if (result === undefined) { + throw new Error('datastore: key not found'); + } + return result.value; + } + + public async getSizeAsync(key: string): Promise { + const value = await this.getAsync(key); + if (value === undefined) { + throw new Error('datastore: key not found'); + } + return value.length; + } + + public async hasAsync(key: string): Promise { + const result = await this._table.get(key); + return result !== undefined; + } + + // NOTE(jalextowle): This function only filters the database based on prefix + // and generates entries for each row of the database. The other query + // operations (filtering, sorting, etc.) are implemented in + // db/dexie_datastore.go for performance reasons. The prefixes are + // interpreted as regular expressions to satisfy the ds.Datastore interface. + public async queryAsync(prefix: string): Promise { + return this._db.transaction('rw!', this._table, async () => { + const prefixRegExp = new RegExp(prefix); + const col = + prefix !== '' ? this._table.filter(entry => prefixRegExp.test(entry.key)) : this._table.toCollection(); + return (await col.toArray()).map(entry => { + return { + key: entry.key, + value: entry.value, + size: entry.value.length, + }; + }); + }); + } +} diff --git a/packages/mesh-browser/go/jsutil/jsutil.go b/packages/mesh-browser/go/jsutil/jsutil.go index d8681986a..3127c470f 100644 --- a/packages/mesh-browser/go/jsutil/jsutil.go +++ b/packages/mesh-browser/go/jsutil/jsutil.go @@ -7,10 +7,43 @@ package jsutil import ( "context" "encoding/json" + "errors" "fmt" + "runtime/debug" "syscall/js" ) +func CopyBytesToJS(bytes []byte) (jsBytes js.Value, err error) { + defer func() { + if r := recover(); r != nil { + err = RecoverError(r) + } + }() + + jsBytes = js.Global().Get("Uint8Array").New(len(bytes)) + copied := js.CopyBytesToJS(jsBytes, bytes) + if copied != len(bytes) { + return js.Undefined(), fmt.Errorf("should have copied %d bytes to JS but only copied %d", len(bytes), copied) + } + return jsBytes, nil +} + +func CopyBytesToGo(jsBytes js.Value) (bytes []byte, err error) { + defer func() { + if r := recover(); r != nil { + err = RecoverError(r) + } + }() + + jsBytesLength := jsBytes.Get("length").Int() + bytes = make([]byte, jsBytesLength) + copied := js.CopyBytesToGo(bytes, jsBytes) + if copied != jsBytesLength { + return nil, fmt.Errorf("should have copied %d bytes to Go but only copied %d", jsBytesLength, copied) + } + return bytes, nil +} + // ErrorToJS converts a Go error to a JavaScript Error. func ErrorToJS(err error) js.Value { return js.Global().Get("Error").New(err.Error()) @@ -123,3 +156,19 @@ func InefficientlyConvertFromJS(jsValue js.Value, value interface{}) (err error) jsonString := js.Global().Get("JSON").Call("stringify", jsValue) return json.Unmarshal([]byte(jsonString.String()), value) } + +// RecoverError allows a function to recover from a thrown Javascript error if +// called inside of a deferred function with a recover statement. +func RecoverError(e interface{}) error { + if e != nil { + debug.PrintStack() + } + switch e := e.(type) { + case error: + return e + case string: + return errors.New(e) + default: + return fmt.Errorf("unexpected JavaScript error: (%T) %v", e, e) + } +} diff --git a/yarn.lock b/yarn.lock index 813223bc6..0bd2d0604 100644 --- a/yarn.lock +++ b/yarn.lock @@ -468,15 +468,6 @@ bignumber.js "~9.0.0" ethereum-types "^3.1.0" -"@0x/types@^3.1.3": - version "3.1.3" - resolved "https://registry.yarnpkg.com/@0x/types/-/types-3.1.3.tgz#f5edbcb0fe38f6c2979cd6cedacf7c132a72aa23" - integrity sha512-6lHKOlr90zN5P/Rrg/SfdHXUASU4ZDBr5Y4IBwwKrrPo/XetxNFxdZQcDxmVJT8aG13f7r6xnDwwlEyFtvnWEQ== - dependencies: - "@types/node" "*" - bignumber.js "~9.0.0" - ethereum-types "^3.1.1" - "@0x/types@^3.2.0": version "3.2.0" resolved "https://registry.yarnpkg.com/@0x/types/-/types-3.2.0.tgz#490627e9440c3ff4e4660b0c10960ccd02d7105f" @@ -508,17 +499,6 @@ ethereum-types "^3.1.0" popper.js "1.14.3" -"@0x/typescript-typings@^5.1.0": - version "5.1.0" - resolved "https://registry.yarnpkg.com/@0x/typescript-typings/-/typescript-typings-5.1.0.tgz#8e81a8605e3ef02bea16b73e4bd229ce010912c6" - integrity sha512-djQWgwabVgQ5jH3KFlrzOdLVZhYRpOIwlZtvkeznjToi8Xw2YXBoX0OL6ZJ/PhyNCEgtopO11HsHTR2mcyKyIg== - dependencies: - "@types/bn.js" "^4.11.0" - "@types/react" "*" - bignumber.js "~9.0.0" - ethereum-types "^3.1.1" - popper.js "1.14.3" - "@0x/typescript-typings@^5.1.1": version "5.1.1" resolved "https://registry.yarnpkg.com/@0x/typescript-typings/-/typescript-typings-5.1.1.tgz#e5216f33e5859d48cd441a059da9c56c2b6df36a" @@ -568,25 +548,6 @@ js-sha3 "^0.7.0" lodash "^4.17.11" -"@0x/utils@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@0x/utils/-/utils-5.5.0.tgz#b167a09e3a950035e45e05050f95be709b4d2b79" - integrity sha512-2rDKKzdbPEjKXv5HSrkB6VzukZrd1EJGXFhJIlLEN/c2Z9svFnWhQiSlPQX0o1bK435gJQbd1I3B6O3I2TkLrQ== - dependencies: - "@0x/types" "^3.1.3" - "@0x/typescript-typings" "^5.1.0" - "@types/node" "*" - abortcontroller-polyfill "^1.1.9" - bignumber.js "~9.0.0" - chalk "^2.3.0" - detect-node "2.0.3" - ethereum-types "^3.1.1" - ethereumjs-util "^5.1.1" - ethers "~4.0.4" - isomorphic-fetch "2.2.1" - js-sha3 "^0.7.0" - lodash "^4.17.11" - "@0x/utils@^5.5.1": version "5.5.1" resolved "https://registry.yarnpkg.com/@0x/utils/-/utils-5.5.1.tgz#5f131755e1692e7c7b74d03d224d94808d475dd7" @@ -3750,14 +3711,6 @@ ethereum-types@^3.0.0, ethereum-types@^3.1.0: "@types/node" "*" bignumber.js "~9.0.0" -ethereum-types@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/ethereum-types/-/ethereum-types-3.1.1.tgz#fc9ea72f0d4b93cd651e31ed79b9af4126e15f5a" - integrity sha512-4PRpHfzN4v+IhgrEOS4KYugtKliuDESGtWjmhpPmOC2RUilo6wQDYKK2PKaq4rYG+dzHxIfXrPh/AnQpRgSNhw== - dependencies: - "@types/node" "*" - bignumber.js "~9.0.0" - ethereum-types@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/ethereum-types/-/ethereum-types-3.2.0.tgz#5bd27cadc3c1b3c2e2bf94654fa2bc3618a4520f"