Big database upgrade (#793)

* Strip out all existing database code

* rename meshdb package to db

* Implement first db tests for adding/finding orders

* Add basic support for Orders and MiniHeaders

* Implement and test UpdateOrder

* Implement sorting orders

* Imlpement and test limit and offset

* Implement and test filters for finding orders

* Add fields for parsed MakerAssetData and MakerFeeAssetData

* Add test cases for filtering based on parsed maker asset data

* Implement and test helper methods for filtering based on parsed asset data

* Rename some FilterOpts fields

* Remove unneeded quotes

* Add OF prefix to OrderField constants

* Implement and test MiniHeaders query options

* Implement and test deleting orders with a filter

* Implement and test DeleteOrder

* Refactor db package to use common/types

* Implement and test deleting miniHeaders. Adjust implementation of DeleteOrders

* Update all types. Fix build

* Automatically remove miniHeaders above retention limit

* Add new options for database to configure max mini headers

* Get all blockwatch tests passing 😄

* Add new db options

* Implement and test CountOrders

* Begin fixing some bugs in order watcher tests

* Uncomment all remaining orderwatcher tests (except max expiration)

* Implement and test metadata methods

* Fix bug in sqltypes.BigInt. Get ratelimit tests passing

* Fix some additional tests and unimplemented methods in core

* Implement core.GetStats. Move data dir for most integration tests to /tmp

* Implement new getOrders RPC endpoint

* Re-enable ordersync tests

* Remove expirationwatch package

* Return db.ErrNotFound where appropriate. Remove old TODOs

* Set up basic skeleton for Dexie.js implementation

* Implement most order methods in Dexie.js

Not implemented:

- Query filters
- Multiple sort parameters

* Fix bugs and add support for order queries with one filter

* Add support for multiple filters

* Use checkOrderQuery function for both Dexie and SQL

* Fix bug in SQL implementation involving big numbers

* Add support for multiple sort values in Dexie

* Implement AddMiniHeaders in Dexie

* Implement remaining MiniHeaders methods in Dexie

* Implement Metadata methods in Dexie

* Fix failing TypeScript and Wasm tests

* Fix some build issues

* Fix bug in OrderWatcher

* Fix failing conversion tests

* Fix any remaining failing tests

* Fix linter errors

* Fix build errors

* Update yarn.lock

* Add backwards compatibility layer for FilteredPaginationSubprotocol

* Re-use existing asset data decoder from orderwatcher

* Remove remaining uses of goleveldb

* Improve handling of db closed error

* Rename some identifiers

* Address some low hanging TODOs

* De-dupe and refactor DB test cases

* Change how context is passed in to core.App

* Add missing exit 0 to some clean commands

* Address remaining TODOs in sql_implementation

* Address remaining TODOs in database.ts

* Address remaining TODOs in order_watcher.go

* Minor changes and cleanup

* Fix yarn.lock

* Respond to PR feedback

* Fix bug in database.ts

* Address remaining PR feedback

* Address third round PR feedback

* Address review feedback part 4

* Fix linter

cmd/mesh/main.go

@@ -38,13 +38,13 @@ func main() {
 		log.WithField("error", err.Error()).Fatal("could not parse environment variables")
 	}
 
-	// Start core.App.
-	app, err := core.New(coreConfig)
+	// Initialize core.App.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	app, err := core.New(ctx, coreConfig)
 	if err != nil {
 		log.WithField("error", err.Error()).Fatal("could not initialize app")
 	}
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
 
 	// Below, we will start several independent goroutines. We use separate
 	// channels to communicate errors and a waitgroup to wait for all goroutines
@@ -55,7 +55,7 @@ func main() {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		if err := app.Start(ctx); err != nil {
+		if err := app.Start(); err != nil {
 			coreErrChan <- err
 		}
 	}()

cmd/mesh/rpc_handler.go

@@ -18,6 +18,7 @@ import (
 	"github.com/0xProject/0x-mesh/rpc"
 	"github.com/0xProject/0x-mesh/zeroex"
 	"github.com/0xProject/0x-mesh/zeroex/ordervalidator"
+	"github.com/ethereum/go-ethereum/common"
 	ethrpc "github.com/ethereum/go-ethereum/rpc"
 	peerstore "github.com/libp2p/go-libp2p-peerstore"
 	log "github.com/sirupsen/logrus"
@@ -60,11 +61,10 @@ func instantiateServer(ctx context.Context, app *core.App, rpcAddr string) *rpc.
 }
 
 // GetOrders is called when an RPC client calls GetOrders.
-func (handler *rpcHandler) GetOrders(page, perPage int, snapshotID string) (result *types.GetOrdersResponse, err error) {
+func (handler *rpcHandler) GetOrders(perPage int, minOrderHashHex string) (result *types.GetOrdersResponse, err error) {
 	log.WithFields(map[string]interface{}{
-		"page":       page,
-		"perPage":    perPage,
-		"snapshotID": snapshotID,
+		"perPage":         perPage,
+		"minOrderHashHex": minOrderHashHex,
 	}).Debug("received GetOrders request via RPC")
 	// Catch panics, log stack trace and return RPC error message
 	defer func() {
@@ -82,7 +82,8 @@ func (handler *rpcHandler) GetOrders(page, perPage int, snapshotID string) (resu
 			err = errors.New("method handler crashed in GetOrders RPC call (check logs for stack trace)")
 		}
 	}()
-	getOrdersResponse, err := handler.app.GetOrders(page, perPage, snapshotID)
+	var minOrderHash = common.HexToHash(minOrderHashHex)
+	getOrdersResponse, err := handler.app.GetOrders(perPage, minOrderHash)
 	if err != nil {
 		if _, ok := err.(core.ErrSnapshotNotFound); ok {
 			return nil, err

common/types/types.go

@@ -11,6 +11,7 @@ import (
 	"github.com/0xProject/0x-mesh/zeroex"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 // Stats is the return value for core.GetStats. Also used in the browser and RPC
@@ -42,9 +43,8 @@ type LatestBlock struct {
 // GetOrdersResponse is the return value for core.GetOrders. Also used in the
 // browser and RPC interface.
 type GetOrdersResponse struct {
-	SnapshotID        string       `json:"snapshotID"`
-	SnapshotTimestamp time.Time    `json:"snapshotTimestamp"`
-	OrdersInfos       []*OrderInfo `json:"ordersInfos"`
+	Timestamp   time.Time    `json:"timestamp"`
+	OrdersInfos []*OrderInfo `json:"ordersInfos"`
 }
 
 // AddOrdersOpts is a set of options for core.AddOrders. Also used in the
@@ -96,3 +96,84 @@ func (o *OrderInfo) UnmarshalJSON(data []byte) error {
 	}
 	return nil
 }
+
+type OrderWithMetadata struct {
+	Hash                     common.Hash    `json:"hash"`
+	ChainID                  *big.Int       `json:"chainID"`
+	ExchangeAddress          common.Address `json:"exchangeAddress"`
+	MakerAddress             common.Address `json:"makerAddress"`
+	MakerAssetData           []byte         `json:"makerAssetData"`
+	MakerFeeAssetData        []byte         `json:"makerFeeAssetData"`
+	MakerAssetAmount         *big.Int       `json:"makerAssetAmount"`
+	MakerFee                 *big.Int       `json:"makerFee"`
+	TakerAddress             common.Address `json:"takerAddress"`
+	TakerAssetData           []byte         `json:"takerAssetData"`
+	TakerFeeAssetData        []byte         `json:"takerFeeAssetData"`
+	TakerAssetAmount         *big.Int       `json:"takerAssetAmount"`
+	TakerFee                 *big.Int       `json:"takerFee"`
+	SenderAddress            common.Address `json:"senderAddress"`
+	FeeRecipientAddress      common.Address `json:"feeRecipientAddress"`
+	ExpirationTimeSeconds    *big.Int       `json:"expirationTimeSeconds"`
+	Salt                     *big.Int       `json:"salt"`
+	Signature                []byte         `json:"signature"`
+	FillableTakerAssetAmount *big.Int       `json:"fillableTakerAssetAmount"`
+	LastUpdated              time.Time      `json:"lastUpdated"`
+	// Was this order flagged for removal? Due to the possibility of block-reorgs, instead
+	// of immediately removing an order when FillableTakerAssetAmount becomes 0, we instead
+	// flag it for removal. After this order isn't updated for X time and has IsRemoved = true,
+	// the order can be permanently deleted.
+	IsRemoved bool `json:"isRemoved"`
+	// IsPinned indicates whether or not the order is pinned. Pinned orders are
+	// not removed from the database unless they become unfillable.
+	IsPinned bool `json:"isPinned"`
+	// JSON-encoded list of assetdatas contained in MakerAssetData. For non-MAP
+	// orders, the list contains only one element which is equal to MakerAssetData.
+	// For MAP orders, it contains each component assetdata.
+	ParsedMakerAssetData []*SingleAssetData `json:"parsedMakerAssetData"`
+	// Same as ParsedMakerAssetData but for MakerFeeAssetData instead of MakerAssetData.
+	ParsedMakerFeeAssetData []*SingleAssetData `json:"parsedMakerFeeAssetData"`
+}
+
+func (order OrderWithMetadata) SignedOrder() *zeroex.SignedOrder {
+	return &zeroex.SignedOrder{
+		Order: zeroex.Order{
+			ChainID:               order.ChainID,
+			ExchangeAddress:       order.ExchangeAddress,
+			MakerAddress:          order.MakerAddress,
+			MakerAssetData:        order.MakerAssetData,
+			MakerFeeAssetData:     order.MakerFeeAssetData,
+			MakerAssetAmount:      order.MakerAssetAmount,
+			MakerFee:              order.MakerFee,
+			TakerAddress:          order.TakerAddress,
+			TakerAssetData:        order.TakerAssetData,
+			TakerFeeAssetData:     order.TakerFeeAssetData,
+			TakerAssetAmount:      order.TakerAssetAmount,
+			TakerFee:              order.TakerFee,
+			SenderAddress:         order.SenderAddress,
+			FeeRecipientAddress:   order.FeeRecipientAddress,
+			ExpirationTimeSeconds: order.ExpirationTimeSeconds,
+			Salt:                  order.Salt,
+		},
+		Signature: order.Signature,
+	}
+}
+
+type SingleAssetData struct {
+	Address common.Address `json:"address"`
+	TokenID *big.Int       `json:"tokenID"`
+}
+
+type MiniHeader struct {
+	Hash      common.Hash `json:"hash"`
+	Parent    common.Hash `json:"parent"`
+	Number    *big.Int    `json:"number"`
+	Timestamp time.Time   `json:"timestamp"`
+	Logs      []types.Log `json:"logs"`
+}
+
+type Metadata struct {
+	EthereumChainID                   int
+	MaxExpirationTime                 *big.Int
+	EthRPCRequestsSentInCurrentUTCDay int
+	StartOfCurrentUTCDay              time.Time
+}

common/types/types_js.go

@@ -5,6 +5,8 @@ package types
 import (
 	"encoding/json"
 	"syscall/js"
+
+	"github.com/0xProject/0x-mesh/packages/browser/go/jsutil"
 )
 
 func (r GetOrdersResponse) JSValue() js.Value {
@@ -48,3 +50,8 @@ func (s Stats) JSValue() js.Value {
 		"ethRPCRateLimitExpiredRequests":    s.EthRPCRateLimitExpiredRequests,
 	})
 }
+
+func (o OrderWithMetadata) JSValue() js.Value {
+	value, _ := jsutil.InefficientlyConvertToJS(o)
+	return value
+}

constants/constants.go

@@ -105,3 +105,8 @@ var (
 const ParityFilterUnknownBlock = "One of the blocks specified in filter (fromBlock, toBlock or blockHash) cannot be found"
 
 const GethFilterUnknownBlock = "unknown block"
+
+var (
+	ZRXAssetData  = common.Hex2Bytes("f47261b0000000000000000000000000871dd7c2b4b25e1aa18728e9d5f2af4c4e431f5c")
+	WETHAssetData = common.Hex2Bytes("f47261b00000000000000000000000000b1ba0af832d7c05fd64161e0db78e85978e8082")
+)