Optimize FindMiniHeaders by using gob encoding for event logs

db/db_test.go

@@ -1128,7 +1128,7 @@ func TestParseContractAddressesAndTokenIdsFromAssetData(t *testing.T) {
 	}
 }
 
-func newTestDB(t *testing.T, ctx context.Context) *DB {
+func newTestDB(t testing.TB, ctx context.Context) *DB {
 	db, err := New(ctx, TestOptions())
 	require.NoError(t, err)
 	count, err := db.CountOrders(nil)
@@ -1722,33 +1722,9 @@ func makeMiniHeaderFilterTestCases(t *testing.T, db *DB) ([]*types.MiniHeader, [
 			expectedMatchingMiniHeaders: storedMiniHeaders[5:],
 		},
 
-		// Filter on Logs (type ParsedAssetData/TEXT)
-		{
-			name: "Logs CONTAINS query that matches all",
-			filters: []MiniHeaderFilter{
-				{
-					Field: MFLogs,
-					Kind:  Contains,
-					Value: `"address":"0x21ab6c9fac80c59d401b37cb43f81ea9dde7fe34"`,
-				},
-			},
-			expectedMatchingMiniHeaders: storedMiniHeaders,
-		},
-		{
-			name: "Logs CONTAINS query that matches one",
-			filters: []MiniHeaderFilter{
-				{
-					Field: MFLogs,
-					Kind:  Contains,
-					Value: `"blockNumber":"0x5"`,
-				},
-			},
-			expectedMatchingMiniHeaders: storedMiniHeaders[5:6],
-		},
-
 		// Combining two or more filters
 		{
-			name: "Number >= 3 AND Timestamp < h",
+			name: "Number >= 3 AND Timestamp < 700",
 			filters: []MiniHeaderFilter{
 				{
 					Field: MFNumber,
@@ -1963,7 +1939,7 @@ func lessByTimestampDescAndNumberDesc(miniHeaders []*types.MiniHeader) func(i, j
 	}
 }
 
-func assertMiniHeaderSlicesAreEqual(t *testing.T, expected, actual []*types.MiniHeader) {
+func assertMiniHeaderSlicesAreEqual(t testing.TB, expected, actual []*types.MiniHeader) {
 	assert.Len(t, actual, len(expected), "wrong number of miniheaders")
 	for i, expectedMiniHeader := range expected {
 		if i >= len(actual) {
@@ -1983,7 +1959,7 @@ func assertMiniHeaderSlicesAreEqual(t *testing.T, expected, actual []*types.Mini
 	}
 }
 
-func assertMiniHeaderSlicesAreUnsortedEqual(t *testing.T, expected, actual []*types.MiniHeader) {
+func assertMiniHeaderSlicesAreUnsortedEqual(t testing.TB, expected, actual []*types.MiniHeader) {
 	// Make a copy of the given mini headers so we don't mess up the original when sorting them.
 	expectedCopy := make([]*types.MiniHeader, len(expected))
 	copy(expectedCopy, expected)
@@ -1994,7 +1970,7 @@ func assertMiniHeaderSlicesAreUnsortedEqual(t *testing.T, expected, actual []*ty
 	assertMiniHeaderSlicesAreEqual(t, expected, actual)
 }
 
-func assertMiniHeadersAreEqual(t *testing.T, expected, actual *types.MiniHeader) {
+func assertMiniHeadersAreEqual(t testing.TB, expected, actual *types.MiniHeader) {
 	if expected.Timestamp.Equal(actual.Timestamp) {
 		// HACK(albrow): In this case, the two values represent the same time.
 		// This is what we care about, but the assert package might consider

db/dexie_implementation.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"math/big"
 	"path/filepath"
+	"runtime/debug"
 	"syscall/js"
 
 	"github.com/0xProject/0x-mesh/common/types"
@@ -230,25 +231,14 @@ func (db *DB) AddMiniHeaders(miniHeaders []*types.MiniHeader) (added []*types.Mi
 			err = recoverError(r)
 		}
 	}()
-	jsMiniHeaders, err := jsutil.InefficientlyConvertToJS(dexietypes.MiniHeadersFromCommonType(miniHeaders))
-	if err != nil {
-		return nil, nil, err
-	}
+	jsMiniHeaders := dexietypes.MiniHeadersFromCommonType(miniHeaders)
 	jsResult, err := jsutil.AwaitPromiseContext(db.ctx, db.dexie.Call("addMiniHeadersAsync", jsMiniHeaders))
 	if err != nil {
 		return nil, nil, convertJSError(err)
 	}
 	jsAdded := jsResult.Get("added")
-	var dexieAdded []*dexietypes.MiniHeader
-	if err := jsutil.InefficientlyConvertFromJS(jsAdded, &dexieAdded); err != nil {
-		return nil, nil, err
-	}
 	jsRemoved := jsResult.Get("removed")
-	var dexieRemoved []*dexietypes.MiniHeader
-	if err := jsutil.InefficientlyConvertFromJS(jsRemoved, &dexieRemoved); err != nil {
-		return nil, nil, err
-	}
-	return dexietypes.MiniHeadersToCommonType(dexieAdded), dexietypes.MiniHeadersToCommonType(dexieRemoved), nil
+	return dexietypes.MiniHeadersToCommonType(jsAdded), dexietypes.MiniHeadersToCommonType(jsRemoved), nil
 }
 
 func (db *DB) GetMiniHeader(hash common.Hash) (miniHeader *types.MiniHeader, err error) {
@@ -261,11 +251,7 @@ func (db *DB) GetMiniHeader(hash common.Hash) (miniHeader *types.MiniHeader, err
 	if err != nil {
 		return nil, convertJSError(err)
 	}
-	var dexieMiniHeader dexietypes.MiniHeader
-	if err := jsutil.InefficientlyConvertFromJS(jsMiniHeader, &dexieMiniHeader); err != nil {
-		return nil, err
-	}
-	return dexietypes.MiniHeaderToCommonType(&dexieMiniHeader), nil
+	return dexietypes.MiniHeaderToCommonType(jsMiniHeader), nil
 }
 
 func (db *DB) FindMiniHeaders(query *MiniHeaderQuery) (miniHeaders []*types.MiniHeader, err error) {
@@ -282,11 +268,7 @@ func (db *DB) FindMiniHeaders(query *MiniHeaderQuery) (miniHeaders []*types.Mini
 	if err != nil {
 		return nil, convertJSError(err)
 	}
-	var dexieMiniHeaders []*dexietypes.MiniHeader
-	if err := jsutil.InefficientlyConvertFromJS(jsMiniHeaders, &dexieMiniHeaders); err != nil {
-		return nil, err
-	}
-	return dexietypes.MiniHeadersToCommonType(dexieMiniHeaders), nil
+	return dexietypes.MiniHeadersToCommonType(jsMiniHeaders), nil
 }
 
 func (db *DB) DeleteMiniHeader(hash common.Hash) (err error) {
@@ -316,11 +298,7 @@ func (db *DB) DeleteMiniHeaders(query *MiniHeaderQuery) (deleted []*types.MiniHe
 	if err != nil {
 		return nil, convertJSError(err)
 	}
-	var dexieMiniHeaders []*dexietypes.MiniHeader
-	if err := jsutil.InefficientlyConvertFromJS(jsMiniHeaders, &dexieMiniHeaders); err != nil {
-		return nil, err
-	}
-	return dexietypes.MiniHeadersToCommonType(dexieMiniHeaders), nil
+	return dexietypes.MiniHeadersToCommonType(jsMiniHeaders), nil
 }
 
 func (db *DB) GetMetadata() (metadata *types.Metadata, err error) {
@@ -387,6 +365,9 @@ func (db *DB) UpdateMetadata(updateFunc func(oldmetadata *types.Metadata) (newMe
 }
 
 func recoverError(e interface{}) error {
+	if e != nil {
+		debug.PrintStack()
+	}
 	switch e := e.(type) {
 	case error:
 		return e

db/dexietypes/dexietypes.go

@@ -1,16 +1,24 @@
+// +build js,wasm
+
 package dexietypes
 
 // Note(albrow): Could be optimized if needed by more directly converting between
 // Go types and JavaScript types instead of using jsutil.IneffecientlyConvertX.
+// The technique we used for MiniHeaders could be used in more places if needed.
 
 import (
+	"bytes"
+	"encoding/gob"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"math/big"
 	"strconv"
+	"syscall/js"
 	"time"
 
 	"github.com/0xProject/0x-mesh/common/types"
+	"github.com/0xProject/0x-mesh/packages/browser/go/jsutil"
 	"github.com/ethereum/go-ethereum/common"
 	ethmath "github.com/ethereum/go-ethereum/common/math"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
@@ -87,15 +95,19 @@ func SortedBigIntFromInt64(v int64) *SortedBigInt {
 	return NewSortedBigInt(big.NewInt(v))
 }
 
-func (i *SortedBigInt) MarshalJSON() ([]byte, error) {
-	if i == nil || i.Int == nil {
-		return json.Marshal(nil)
-	}
+func (i *SortedBigInt) FixedLengthString() string {
 	// Note(albrow), strings in Dexie.js are sorted in alphanumerical order, not
 	// numerical order. In order to sort by numerical order, we need to pad with
 	// zeroes. The maximum length of an unsigned 256 bit integer is 80, so we
 	// pad with zeroes such that the length of the number is always 80.
-	return json.Marshal(fmt.Sprintf("%080s", i.Int.String()))
+	return fmt.Sprintf("%080s", i.Int.String())
+}
+
+func (i *SortedBigInt) MarshalJSON() ([]byte, error) {
+	if i == nil || i.Int == nil {
+		return json.Marshal(nil)
+	}
+	return json.Marshal(i.FixedLengthString())
 }
 
 func (i *SortedBigInt) UnmarshalJSON(data []byte) error {
@@ -149,14 +161,6 @@ type Order struct {
 	ParsedMakerFeeAssetData  string         `json:"parsedMakerFeeAssetData"`
 }
 
-type MiniHeader struct {
-	Hash      common.Hash   `json:"hash"`
-	Parent    common.Hash   `json:"parent"`
-	Number    *SortedBigInt `json:"number"`
-	Timestamp time.Time     `json:"timestamp"`
-	Logs      string        `json:"logs"`
-}
-
 type Metadata struct {
 	EthereumChainID                   int       `json:"ethereumChainID"`
 	EthRPCRequestsSentInCurrentUTCDay int       `json:"ethRPCRequestsSentInCurrentUTCDay"`
@@ -294,57 +298,76 @@ func SingleAssetDataFromCommonType(singleAssetData *types.SingleAssetData) *Sing
 	}
 }
 
-func MiniHeaderToCommonType(miniHeader *MiniHeader) *types.MiniHeader {
-	if miniHeader == nil {
+func MiniHeaderToCommonType(miniHeader js.Value) *types.MiniHeader {
+	if jsutil.IsNullOrUndefined(miniHeader) {
 		return nil
 	}
+	number, ok := ethmath.ParseBig256(miniHeader.Get("number").String())
+	if !ok {
+		panic(errors.New("could not convert number to uint64"))
+	}
+	timestamp, err := time.Parse(time.RFC3339Nano, miniHeader.Get("timestamp").String())
+	if err != nil {
+		panic(errors.New("could not convert timestamp: " + err.Error()))
+	}
 	return &types.MiniHeader{
-		Hash:      miniHeader.Hash,
-		Parent:    miniHeader.Parent,
-		Number:    miniHeader.Number.Int,
-		Timestamp: miniHeader.Timestamp,
-		Logs:      EventLogsToCommonType(miniHeader.Logs),
+		Hash:      common.HexToHash(miniHeader.Get("hash").String()),
+		Parent:    common.HexToHash(miniHeader.Get("parent").String()),
+		Number:    number,
+		Timestamp: timestamp,
+		Logs:      EventLogsToCommonType(miniHeader.Get("logs")),
 	}
 }
 
-func MiniHeaderFromCommonType(miniHeader *types.MiniHeader) *MiniHeader {
+func MiniHeaderFromCommonType(miniHeader *types.MiniHeader) js.Value {
 	if miniHeader == nil {
-		return nil
-	}
-	return &MiniHeader{
-		Hash:      miniHeader.Hash,
-		Parent:    miniHeader.Parent,
-		Number:    NewSortedBigInt(miniHeader.Number),
-		Timestamp: miniHeader.Timestamp,
-		Logs:      EventLogsFromCommonType(miniHeader.Logs),
+		return js.Null()
 	}
+	return js.ValueOf(
+		map[string]interface{}{
+			"hash":      miniHeader.Hash.Hex(),
+			"parent":    miniHeader.Parent.Hex(),
+			"number":    NewSortedBigInt(miniHeader.Number).FixedLengthString(),
+			"timestamp": miniHeader.Timestamp.Format(time.RFC3339Nano),
+			"logs":      EventLogsFromCommonType(miniHeader.Logs),
+		},
+	)
 }
 
-func MiniHeadersToCommonType(miniHeaders []*MiniHeader) []*types.MiniHeader {
-	result := make([]*types.MiniHeader, len(miniHeaders))
-	for i, miniHeader := range miniHeaders {
-		result[i] = MiniHeaderToCommonType(miniHeader)
+func MiniHeadersToCommonType(miniHeaders js.Value) []*types.MiniHeader {
+	result := make([]*types.MiniHeader, miniHeaders.Length())
+	for i := range result {
+		result[i] = MiniHeaderToCommonType(miniHeaders.Index(i))
 	}
 	return result
 }
 
-func MiniHeadersFromCommonType(miniHeaders []*types.MiniHeader) []*MiniHeader {
-	result := make([]*MiniHeader, len(miniHeaders))
+func MiniHeadersFromCommonType(miniHeaders []*types.MiniHeader) js.Value {
+	result := make([]interface{}, len(miniHeaders))
 	for i, miniHeader := range miniHeaders {
 		result[i] = MiniHeaderFromCommonType(miniHeader)
 	}
-	return result
+	return js.ValueOf(result)
 }
 
-func EventLogsToCommonType(eventLogs string) []ethtypes.Log {
+func EventLogsToCommonType(eventLogs js.Value) []ethtypes.Log {
 	var result []ethtypes.Log
-	_ = json.Unmarshal([]byte(eventLogs), &result)
+	buf := make([]byte, eventLogs.Get("length").Int())
+	js.CopyBytesToGo(buf, eventLogs)
+	if err := gob.NewDecoder(bytes.NewBuffer(buf)).Decode(&result); err != nil {
+		panic(err)
+	}
 	return result
 }
 
-func EventLogsFromCommonType(eventLogs []ethtypes.Log) string {
-	result, _ := json.Marshal(eventLogs)
-	return string(result)
+func EventLogsFromCommonType(eventLogs []ethtypes.Log) js.Value {
+	buf := &bytes.Buffer{}
+	if err := gob.NewEncoder(buf).Encode(eventLogs); err != nil {
+		panic(err)
+	}
+	result := js.Global().Get("Uint8Array").New(len(buf.Bytes()))
+	js.CopyBytesToJS(result, buf.Bytes())
+	return result
 }
 
 func MetadataToCommonType(metadata *Metadata) *types.Metadata {

db/dexietypes/dexietypes_test.go

@@ -0,0 +1,59 @@
+// +build js,wasm
+
+package dexietypes
+
+import (
+	"testing"
+	"time"
+
+	"github.com/0xProject/0x-mesh/common/types"
+	"github.com/ethereum/go-ethereum/common"
+	ethmath "github.com/ethereum/go-ethereum/common/math"
+	ethtypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMiniHeadersConversion(t *testing.T) {
+	originalMiniHeader := &types.MiniHeader{
+		Hash:      common.HexToHash("0x00a3ce0e9cbcb5c4d79c1c19df276a0db954a487b895dca1d4deb35e39859eb8"),
+		Parent:    common.HexToHash("0x302febf685d86eaa2339e6f9b226e36d69ebf48b1bfd10b44fc51fcaaefbf148"),
+		Number:    ethmath.MaxBig256,
+		Timestamp: time.Date(1992, time.September, 29, 8, 45, 15, 1230, time.UTC),
+		Logs: []ethtypes.Log{
+			{
+				Address: common.HexToAddress("0x21ab6c9fac80c59d401b37cb43f81ea9dde7fe34"),
+				Topics: []common.Hash{
+					common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"),
+					common.HexToHash("0x0000000000000000000000004d8a4aa1f304f9632cf3877473445d85c577fe5d"),
+					common.HexToHash("0x0000000000000000000000004bdd0d16cfa18e33860470fc4d65c6f5cee60959"),
+				},
+				Data:        common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000337ad34c0"),
+				BlockNumber: 30,
+				TxHash:      common.HexToHash("0xd9bb5f9e888ee6f74bedcda811c2461230f247c205849d6f83cb6c3925e54586"),
+				TxIndex:     0,
+				BlockHash:   common.HexToHash("0x6bbf9b6e836207ab25379c20e517a89090cbbaf8877746f6ed7fb6820770816b"),
+				Index:       0,
+				Removed:     false,
+			},
+			{
+				Address: common.HexToAddress("0x21ab6c9fac80c59d401b37cb43f81ea9dde7fe34"),
+				Topics: []common.Hash{
+					common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"),
+					common.HexToHash("0x0000000000000000000000004d8a4aa1f304f9632cf3877473445d85c577fe5d"),
+					common.HexToHash("0x0000000000000000000000004bdd0d16cfa18e33860470fc4d65c6f5cee60959"),
+				},
+				Data:        common.Hex2Bytes("00000000000000000000000000000000000000000000000000000000deadbeef"),
+				BlockNumber: 31,
+				TxHash:      common.HexToHash("0xd9bb5f9e888ee6f74bedcda811c2461230f247c205849d6f83cb6c3925e54586"),
+				TxIndex:     1,
+				BlockHash:   common.HexToHash("0x6bbf9b6e836207ab25379c20e517a89090cbbaf8877746f6ed7fb6820770816b"),
+				Index:       2,
+				Removed:     true,
+			},
+		},
+	}
+
+	// Convert to JS/Dexie type and back. Make sure we get back the same values that we started with.
+	convertedMiniHeader := MiniHeaderToCommonType(MiniHeaderFromCommonType(originalMiniHeader))
+	assert.Equal(t, originalMiniHeader, convertedMiniHeader)
+}