Skip to content
This repository has been archived by the owner on Oct 11, 2024. It is now read-only.

Commit

Permalink
Addressed review feedback from @recmo
Browse files Browse the repository at this point in the history
  • Loading branch information
@jalextowle
jalextowle committed Aug 25, 2020
1 parent 1df11d5 commit e9d602a
Show file tree
Hide file tree
Showing 5 changed files with 34 additions and 130 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ test-key-value-stores-go:

.PHONY: test-key-value-stores-wasm
test-key-value-stores-wasm:
WASM_INIT_FILE="$$(pwd)/packages/mesh-browser-shim/dist/browser_shim.js" GOOS=js GOARCH=wasm ENABLE_KEY_VALUE_TESTS=true go test ./db -timeout 20m -tags=browser -exec="$$GOPATH/bin/wasmbrowsertest"
WASM_INIT_FILE="$$(pwd)/packages/mesh-browser-shim/dist/browser_shim.js" GOOS=js GOARCH=wasm ENABLE_KEY_VALUE_TESTS=true go test ./db -timeout 10h -tags=browser -exec="$$GOPATH/bin/wasmbrowsertest"


.PHONY: test-go-serial
Expand Down
103 changes: 18 additions & 85 deletions cmd/mesh-bootstrap/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,11 @@ import (
"strings"
"time"

meshdb "github.com/0xProject/0x-mesh/db"
"github.com/0xProject/0x-mesh/loghooks"
"github.com/0xProject/0x-mesh/p2p"
"github.com/0xProject/0x-mesh/p2p/banner"
"github.com/ipfs/go-datastore"
leveldbStore "github.com/ipfs/go-ds-leveldb"
sqlds "github.com/ipfs/go-ds-sql"
libp2p "github.com/libp2p/go-libp2p"
autonat "github.com/libp2p/go-libp2p-autonat-svc"
circuit "github.com/libp2p/go-libp2p-circuit"
Expand All @@ -30,7 +28,6 @@ import (
"github.com/libp2p/go-libp2p-core/routing"
dht "github.com/libp2p/go-libp2p-kad-dht"
dhtopts "github.com/libp2p/go-libp2p-kad-dht/opts"
peerstore "github.com/libp2p/go-libp2p-peerstore"
"github.com/libp2p/go-libp2p-peerstore/pstoreds"
"github.com/libp2p/go-libp2p/p2p/host/relay"
filter "github.com/libp2p/go-maddr-filter"
Expand Down Expand Up @@ -78,29 +75,6 @@ type Config struct {
DataStoreType string `envvar:"DATA_STORE_TYPE" default:"leveldb"`
// LevelDBDataDir is the directory used for storing data when using leveldb as data store type.
LevelDBDataDir string `envvar:"LEVELDB_DATA_DIR" default:"0x_mesh"`
// SQLDBConnectionString is the connection URL used to connect to the
// database.
// NOTE: When set it has precedence over SQL_DB_HOST, SQL_DB_PORT etc.
SQLDBConnectionString string `envvar:"SQL_DB_CONNECTION_STRING" default:"" json:"-"`
// SQLDBHost is the database host used to connect to the database when
// using postgres as data store type.
SQLDBHost string `envvar:"SQL_DB_HOST" default:"localhost" json:"-"`
// SQLDBPort is the database port used to connect to the database when
// using postgres as data store type.
SQLDBPort string `envvar:"SQL_DB_PORT" default:"5432" json:"-"`
// SQLDBUser is the database user used to connect to the database when
// using postgres as data store type.
SQLDBUser string `envvar:"SQL_DB_USER" default:"postgres" json:"-"`
// SQLDBPassword is the database password used to connect to the database when
// using postgres as data store type.
SQLDBPassword string `envvar:"SQL_DB_PASSWORD" default:"" json:"-"`
// SQLDBName is the database name to connect to when using
// postgres as data store type.
SQLDBName string `envvar:"SQL_DB_NAME" default:"datastore" json:"-"`
// SQLDBEngine is the underyling database engine to use as the
// database driver.
// NOTE: Currently only `postgres` driver is supported.
SQLDBEngine string `envvar:"SQL_DB_ENGINE" default:"postgres"`
// UseBootstrapList determines whether or not to use the list of hard-coded
// peers to bootstrap the DHT for peer discovery.
UseBootstrapList bool `envvar:"USE_BOOTSTRAP_LIST" default:"true"`
Expand Down Expand Up @@ -163,71 +137,30 @@ func main() {
var kadDHT *dht.IpfsDHT
var newDHT func(h host.Host) (routing.PeerRouting, error)

var peerStore peerstore.Peerstore

// TODO(oskar) - Figure out why returning an anonymous function from
// getNewDHT() is making kadDHT.runBootstrap panicing.
// When solved this big switch case can be removed from main()
switch config.DataStoreType {
case leveldbDataStore:
newDHT = func(h host.Host) (routing.PeerRouting, error) {
var err error
dhtDir := getDHTDir(config)
// Set up the DHT to use LevelDB.
store, err := leveldbStore.NewDatastore(dhtDir, nil)
if err != nil {
return nil, err
}
kadDHT, err = dht.New(ctx, h, dhtopts.Datastore(store), dhtopts.Protocols(p2p.DHTProtocolID))
if err != nil {
log.WithField("error", err).Fatal("could not create DHT")
}
return kadDHT, err
}

// Set up the peerstore to use LevelDB.
store, err := leveldbStore.NewDatastore(getPeerstoreDir(config), nil)
if err != nil {
log.Fatal(err)
}

peerStore, err = pstoreds.NewPeerstore(ctx, store, pstoreds.DefaultOpts())
if err != nil {
log.Fatal(err)
}

case sqlDataStore:
db, err := getSQLDatabase(config)
if err != nil {
log.WithField("error", err).Fatal("could not create SQL database")
}

err = prepareSQLDatabase(db)
newDHT = func(h host.Host) (routing.PeerRouting, error) {
var err error
dhtDir := getDHTDir(config)
// Set up the DHT to use LevelDB.
store, err := leveldbStore.NewDatastore(dhtDir, nil)
if err != nil {
log.WithField("error", err).Fatal("failed to repare SQL tables for datastores")
}

newDHT = func(h host.Host) (routing.PeerRouting, error) {
var err error
dstore := sqlds.NewDatastore(db, meshdb.NewPostgreSQLQueriesForTable(dhtTableName))

kadDHT, err = NewDHTWithDatastore(ctx, dstore, h)
if err != nil {
log.WithField("error", err).Fatal("could not create DHT")
}

return kadDHT, err
return nil, err
}

pstore := sqlds.NewDatastore(db, meshdb.NewPostgreSQLQueriesForTable(peerStoreTableName))
peerStore, err = pstoreds.NewPeerstore(ctx, pstore, pstoreds.DefaultOpts())
kadDHT, err = dht.New(ctx, h, dhtopts.Datastore(store), dhtopts.Protocols(p2p.DHTProtocolID))
if err != nil {
log.WithField("error", err).Fatal("could not create peerStore")
log.WithField("error", err).Fatal("could not create DHT")
}
return kadDHT, err
}

default:
log.Fatalf("invalid datastore configured: %s. Expected either %s or %s", config.DataStoreType, leveldbDataStore, sqlDataStore)
// Set up the peerstore to use LevelDB.
store, err := leveldbStore.NewDatastore(getPeerstoreDir(config), nil)
if err != nil {
log.Fatal(err)
}

peerStore, err := pstoreds.NewPeerstore(ctx, store, pstoreds.DefaultOpts())
if err != nil {
log.Fatal(err)
}

// Parse multiaddresses given in the config
Expand Down
37 changes: 0 additions & 37 deletions cmd/mesh-bootstrap/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@
package main

import (
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"

Expand Down Expand Up @@ -33,40 +30,6 @@ func getPeerstoreDir(config Config) string {
return filepath.Join(config.LevelDBDataDir, "p2p", "peerstore")
}

func getSQLDatabase(config Config) (*sql.DB, error) {
// Currently we only support the postgres driver.
if config.SQLDBEngine != "postgres" {
return nil, errors.New("sqld currently only supports postgres driver")
}

if config.SQLDBConnectionString != "" {
return sql.Open(config.SQLDBEngine, config.SQLDBConnectionString)
}

fmtStr := "postgresql:///%s?host=%s&port=%s&user=%s&password=%s&sslmode=disable"
connstr := fmt.Sprintf(fmtStr, config.SQLDBName, config.SQLDBHost, config.SQLDBPort, config.SQLDBUser, config.SQLDBPassword)

return sql.Open(config.SQLDBEngine, connstr)
}

func prepareSQLDatabase(db *sql.DB) error {
createTableString := "CREATE TABLE IF NOT EXISTS %s (key TEXT NOT NULL UNIQUE, data BYTEA NOT NULL)"
createDHTTable := fmt.Sprintf(createTableString, dhtTableName)
createPeerStoreTable := fmt.Sprintf(createTableString, peerStoreTableName)

_, err := db.Exec(createDHTTable)
if err != nil {
return err
}

_, err = db.Exec(createPeerStoreTable)
if err != nil {
return err
}

return nil
}

func initPrivateKey(path string) (p2pcrypto.PrivKey, error) {
privKey, err := keys.GetPrivateKeyFromPath(path)
if err == nil {
Expand Down
4 changes: 4 additions & 0 deletions db/db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1842,6 +1842,7 @@ func TestPeerStoreBasic(t *testing.T) {
if !keyValueTestsEnabled {
t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable")
}
t.Parallel()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -1857,6 +1858,7 @@ func TestPeerStoreBatch(t *testing.T) {
if !keyValueTestsEnabled {
t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable")
}
t.Parallel()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -1872,6 +1874,7 @@ func TestDHTStoreBasic(t *testing.T) {
if !keyValueTestsEnabled {
t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable")
}
t.Parallel()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -1887,6 +1890,7 @@ func TestDHTStoreBatch(t *testing.T) {
if !keyValueTestsEnabled {
t.Skip("Key-value store tests are disabled. You can enable them with the ENABLE_KEY_VALUE_TESTS environment variable")
}
t.Parallel()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand Down
18 changes: 11 additions & 7 deletions packages/mesh-browser-lite/src/datastore.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ enum OperationType {

interface Operation {
operationType: OperationType;
key: Uint8Array;
value?: string;
key: string;
value?: Uint8Array;
}

// This implements the subset of the ds.Batching interface that should be implemented
Expand All @@ -35,11 +35,11 @@ interface Operation {
// Go functions into Javascript functions.
export class BatchingDatastore {
private readonly _db: Dexie;
private readonly _table: Dexie.Table;
private readonly _table: Dexie.Table<{ key: string; value: Uint8Array }, string>;

constructor(db: Dexie, tableName: string) {
this._db = db;
this._table = this._db._allTables[tableName];
this._table = this._db._allTables[tableName] as Dexie.Table<{ key: string; value: Uint8Array }, string>;
if (this._table === undefined) {
throw new Error('BatchingDatastore: Attempting to use undefined table');
}
Expand Down Expand Up @@ -90,11 +90,15 @@ export class BatchingDatastore {
}

// NOTE(jalextowle): This function only filters the database based on prefix
// and generates entries for each row of the database. All of the other query
// operations are implemented in db/dexie_datastore.go for performance reasons.
// and generates entries for each row of the database. All of the other
// query operations are implemented in db/dexie_datastore.go for performance
// reasons. The prefixes are interpreted as regular expressions to satisfy
// the ds.Datastore interface.
public async queryAsync(prefix: string): Promise<Entry[]> {
return this._db.transaction('rw!', this._table, async () => {
const col = prefix !== '' ? this._table.where('key').startsWith(prefix) : this._table.toCollection();
const prefixRegExp = new RegExp(prefix);
const col =
prefix !== '' ? this._table.filter(entry => prefixRegExp.test(entry.key)) : this._table.toCollection();
return (await col.toArray()).map(entry => {
return {
key: entry.key,
Expand Down

0 comments on commit e9d602a

Please sign in to comment.