Skip to content
This repository has been archived by the owner on Oct 11, 2024. It is now read-only.

Commit

Permalink
Finalized new key store implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
jalextowle committed Aug 19, 2020
1 parent acb6809 commit 499b272
Show file tree
Hide file tree
Showing 7 changed files with 247 additions and 45 deletions.
14 changes: 14 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,20 @@ test-go-parallel:
go test ./... -race -timeout 30s


.PHONY: test-key-value-stores
test-key-value-stores: test-key-value-stores-go test-key-value-stores-wasm


.PHONY: test-key-value-stores-go
test-key-value-stores-go:
ENABLE_KEY_VALUE_TESTS=true go test ./db


.PHONY: test-key-value-stores-wasm
test-key-value-stores-wasm:
WASM_INIT_FILE="$$(pwd)/packages/mesh-browser-shim/dist/browser_shim.js" GOOS=js GOARCH=wasm ENABLE_KEY_VALUE_TESTS=true go test ./db -tags=browser -exec="$$GOPATH/bin/wasmbrowsertest"


.PHONY: test-go-serial
test-go-serial:
go test ./zeroex/ordervalidator ./zeroex/orderwatch ./core -race -timeout 90s -p=1 --serial
Expand Down
7 changes: 4 additions & 3 deletions cmd/mesh-bootstrap/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@ import (
"strings"
"time"

meshdb "github.com/0xProject/0x-mesh/db"
"github.com/0xProject/0x-mesh/loghooks"
"github.com/0xProject/0x-mesh/p2p"
"github.com/0xProject/0x-mesh/p2p/banner"
sqlds "github.com/0xProject/sql-datastore"
"github.com/ipfs/go-datastore"
leveldbStore "github.com/ipfs/go-ds-leveldb"
sqlds "github.com/ipfs/go-ds-sql"
libp2p "github.com/libp2p/go-libp2p"
autonat "github.com/libp2p/go-libp2p-autonat-svc"
circuit "github.com/libp2p/go-libp2p-circuit"
Expand Down Expand Up @@ -208,7 +209,7 @@ func main() {

newDHT = func(h host.Host) (routing.PeerRouting, error) {
var err error
dstore := sqlds.NewDatastore(db, sqlds.NewPostgreSQLQueriesForTable(dhtTableName))
dstore := sqlds.NewDatastore(db, meshdb.NewPostgreSQLQueriesForTable(dhtTableName))

kadDHT, err = NewDHTWithDatastore(ctx, dstore, h)
if err != nil {
Expand All @@ -218,7 +219,7 @@ func main() {
return kadDHT, err
}

pstore := sqlds.NewDatastore(db, sqlds.NewPostgreSQLQueriesForTable(peerStoreTableName))
pstore := sqlds.NewDatastore(db, meshdb.NewPostgreSQLQueriesForTable(peerStoreTableName))
peerStore, err = pstoreds.NewPeerstore(ctx, pstore, pstoreds.DefaultOpts())
if err != nil {
log.WithField("error", err).Fatal("could not create peerStore")
Expand Down
94 changes: 63 additions & 31 deletions db/db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,31 @@ import (
"github.com/ethereum/go-ethereum/common/math"
ethtypes "github.com/ethereum/go-ethereum/core/types"
dstest "github.com/ipfs/go-datastore/test"
"github.com/plaid/go-envvar/envvar"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

var contractAddresses = ethereum.GanacheAddresses

// Since the key value store tests take a very long time to run, we disable them
// by default. These tests can be enabled with the `ENABLE_KEY_VALUE_TESTS`
// environment variable.
var keyValueTestsEnabled bool

type TestingFlags struct {
EnableKeyValueTests bool `envvar:"ENABLE_KEY_VALUE_TESTS" default:"false"`
}

func init() {
var flags TestingFlags
if err := envvar.Parse(&flags); err != nil {
panic(fmt.Sprintf("could not parse environment variables: %s", err.Error()))
}
keyValueTestsEnabled = flags.EnableKeyValueTests
testing.Init()
}

func TestAddOrders(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand Down Expand Up @@ -1819,14 +1838,11 @@ func makeMiniHeaderFilterTestCases(t *testing.T, db *DB) ([]*types.MiniHeader, [
return storedMiniHeaders, testCases
}

// FIXME - Remove this
func randValue() []byte {
value := make([]byte, 64)
rand.Read(value)
return value
}
func TestPeerStoreBasic(t *testing.T) {
if !keyValueTestsEnabled {
t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable")
}

func TestPeerStore(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db := newTestDB(t, ctx)
Expand All @@ -1835,36 +1851,52 @@ func TestPeerStore(t *testing.T) {
for _, test := range dstest.BasicSubtests {
test(t, peerstore)
}
}

func TestPeerStoreBatch(t *testing.T) {
if !keyValueTestsEnabled {
t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable")
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db := newTestDB(t, ctx)
peerstore := db.PeerStore()

for _, test := range dstest.BatchSubtests {
test(t, peerstore)
}
}

// func TestDHTStore(t *testing.T) {
// ctx, cancel := context.WithCancel(context.Background())
// defer cancel()
// db := newTestDB(t, ctx)
// dhtstore := db.DHTStore()
//
// bytes := make([]byte, 64)
//
// dhtstore.Put()
//
// // for _, test := range dstest.BasicSubtests {
// // test(t, dhtstore)
// // if t.Failed() {
// // t.FailNow()
// // }
// // }
//
// // for _, test := range dstest.BatchSubtests {
// // test(t, dhtstore)
// // if t.Failed() {
// // t.FailNow()
// // }
// // }
// }
func TestDHTStoreBasic(t *testing.T) {
if !keyValueTestsEnabled {
t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable")
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db := newTestDB(t, ctx)
dhtstore := db.DHTStore()

for _, test := range dstest.BasicSubtests {
test(t, dhtstore)
}
}

func TestDHTStoreBatch(t *testing.T) {
if !keyValueTestsEnabled {
t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable")
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db := newTestDB(t, ctx)
dhtstore := db.DHTStore()

for _, test := range dstest.BatchSubtests {
test(t, dhtstore)
}
}

// safeSubsliceOrders returns a (shallow) subslice of orders without modifying
// the original slice. Uses the same semantics as slice expressions: low is
Expand Down
154 changes: 154 additions & 0 deletions db/kv_store_queries.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
// +build !js

package db

import (
"database/sql"
"fmt"

"github.com/ipfs/go-ds-sql"
_ "github.com/lib/pq" //postgres driver
)

/// PostgreSQL

// Options are the postgres datastore options, reexported here for convenience.
type PostgreSQLOptions struct {
Host string
Port string
User string
Password string
Database string
Table string
}

type postgreSQLQueries struct {
tableName string
}

func NewPostgreSQLQueriesForTable(tableName string) *postgreSQLQueries {
return &postgreSQLQueries{tableName}
}

func (q postgreSQLQueries) Delete() string {
return `DELETE FROM ` + q.tableName + ` WHERE key = $1`
}

func (q postgreSQLQueries) Exists() string {
return `SELECT exists(SELECT 1 FROM ` + q.tableName + ` WHERE key=$1)`
}

func (q postgreSQLQueries) Get() string {
return `SELECT data FROM ` + q.tableName + ` WHERE key = $1`
}

func (q postgreSQLQueries) Put() string {
return `INSERT INTO ` + q.tableName + ` (key, data) SELECT $1, $2 ON CONFLICT(key) DO UPDATE SET data = $2 WHERE key = $1`
}

func (q postgreSQLQueries) Query() string {
return `SELECT key, data FROM ` + q.tableName
}

func (q postgreSQLQueries) Prefix() string {
return ` WHERE key LIKE '%s%%' ORDER BY key`
}

func (q postgreSQLQueries) Limit() string {
return ` LIMIT %d`
}

func (q postgreSQLQueries) Offset() string {
return ` OFFSET %d`
}

func (q postgreSQLQueries) GetSize() string {
return `SELECT octet_length(data) FROM ` + q.tableName + ` WHERE key = $1`
}

// Create returns a datastore connected to postgres initialized with a table
func (opts *PostgreSQLOptions) CreatePostgres() (*sqlds.Datastore, error) {
opts.setDefaults()
fmtstr := "postgresql:///%s?host=%s&port=%s&user=%s&password=%s&sslmode=disable"
constr := fmt.Sprintf(fmtstr, opts.Database, opts.Host, opts.Port, opts.User, opts.Password)
db, err := sql.Open("postgres", constr)
if err != nil {
return nil, err
}

createTable := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (key TEXT NOT NULL UNIQUE, data BYTEA NOT NULL)", opts.Table)
_, err = db.Exec(createTable)

if err != nil {
return nil, err
}

return sqlds.NewDatastore(db, NewPostgreSQLQueriesForTable(opts.Table)), nil
}

func (opts *PostgreSQLOptions) setDefaults() {
if opts.Table == "" {
opts.Table = "kv"
}
if opts.Host == "" {
opts.Host = "postgres"
}

if opts.Port == "" {
opts.Port = "5432"
}

if opts.User == "" {
opts.User = "postgres"
}

if opts.Database == "" {
opts.Database = "datastore"
}
}

/// Sqlite

type sqliteQueries struct {
tableName string
}

func NewSqliteQueriesForTable(tableName string) *sqliteQueries {
return &sqliteQueries{tableName}
}

func (q sqliteQueries) Delete() string {
return `DELETE FROM ` + q.tableName + ` WHERE key = $1`
}

func (q sqliteQueries) Exists() string {
return `SELECT exists(SELECT 1 FROM ` + q.tableName + ` WHERE key=$1)`
}

func (q sqliteQueries) Get() string {
return `SELECT data FROM ` + q.tableName + ` WHERE key = $1`
}

func (q sqliteQueries) Put() string {
return `INSERT INTO ` + q.tableName + ` (key, data) SELECT $1, $2 ON CONFLICT(key) DO UPDATE SET data = $2 WHERE key = $1`
}

func (q sqliteQueries) Query() string {
return `SELECT key, data FROM ` + q.tableName
}

func (q sqliteQueries) Prefix() string {
return ` WHERE key LIKE '%s%%' ORDER BY key`
}

func (q sqliteQueries) Limit() string {
return ` LIMIT %d`
}

func (q sqliteQueries) Offset() string {
return ` OFFSET %d`
}

func (q sqliteQueries) GetSize() string {
return `SELECT length(data) FROM ` + q.tableName + ` WHERE key = $1`
}
6 changes: 3 additions & 3 deletions db/sql_implementation.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@ import (

"github.com/0xProject/0x-mesh/common/types"
"github.com/0xProject/0x-mesh/db/sqltypes"
"github.com/0xProject/sql-datastore"
"github.com/ethereum/go-ethereum/common"
"github.com/gibson042/canonicaljson-go"
"github.com/google/uuid"
"github.com/ido50/sqlz"
ds "github.com/ipfs/go-datastore"
"github.com/ipfs/go-ds-sql"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
)
Expand Down Expand Up @@ -99,11 +99,11 @@ func New(ctx context.Context, opts *Options) (*DB, error) {
}

func (db *DB) DHTStore() ds.Batching {
return sqlds.NewDatastore(db.sqldb.DB.DB, sqlds.NewSqliteQueriesForTable("dhtstore"))
return sqlds.NewDatastore(db.sqldb.DB.DB, NewSqliteQueriesForTable("dhtstore"))
}

func (db *DB) PeerStore() ds.Batching {
return sqlds.NewDatastore(db.sqldb.DB.DB, sqlds.NewSqliteQueriesForTable("peerstore"))
return sqlds.NewDatastore(db.sqldb.DB.DB, NewSqliteQueriesForTable("peerstore"))
}

// TODO(albrow): Use a proper migration tool. We don't technically need this
Expand Down
7 changes: 3 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ module github.com/0xProject/0x-mesh
go 1.14

replace (
github.com/0xProject/sql-datastore => /Users/alextowle/sql-datastore
// github.com/ethereum/go-ethereum => github.com/0xProject/go-ethereum wasm-support
github.com/ethereum/go-ethereum => github.com/0xProject/go-ethereum v1.8.8-0.20200603225022-cb1f52043425
github.com/libp2p/go-flow-metrics => github.com/libp2p/go-flow-metrics v0.0.3
Expand All @@ -16,7 +15,6 @@ replace (
)

require (
github.com/0xProject/sql-datastore v0.0.0-20200812212451-239b36c67c16
github.com/99designs/gqlgen v0.11.3
github.com/agnivade/levenshtein v1.1.0 // indirect
github.com/albrow/stringset v2.1.0+incompatible
Expand All @@ -38,14 +36,15 @@ require (
github.com/hashicorp/go-multierror v1.1.0 // indirect
github.com/hashicorp/golang-lru v0.5.4
github.com/ido50/sqlz v0.0.0-20200308174337-487b8faf612c
github.com/ipfs/go-datastore v0.3.1
github.com/ipfs/go-datastore v0.4.4
github.com/ipfs/go-ds-leveldb v0.4.0
github.com/ipfs/go-ds-sql v0.2.0
github.com/jmoiron/sqlx v1.2.0
github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d
github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9 // indirect
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/karlseguin/expect v1.0.1 // indirect
github.com/lib/pq v1.2.0
github.com/lib/pq v1.3.0
github.com/libp2p/go-libp2p v0.5.1
github.com/libp2p/go-libp2p-autonat-svc v0.1.0
github.com/libp2p/go-libp2p-circuit v0.1.4
Expand Down
Loading

0 comments on commit 499b272

Please sign in to comment.