From 483de81a5e97ca09b5b70511be11753562b0db88 Mon Sep 17 00:00:00 2001 From: JoeGruff Date: Tue, 21 Jan 2025 16:48:52 +0900 Subject: [PATCH] tatanka: Add orderbook db. --- tatanka/db/db.go | 53 +++++++ tatanka/db/orderbook.go | 216 ++++++++++++++++++++++++++++ tatanka/db/orderbook_test.go | 270 +++++++++++++++++++++++++++++++++++ tatanka/tanka/swaps.go | 17 ++- 4 files changed, 551 insertions(+), 5 deletions(-) create mode 100644 tatanka/db/orderbook.go create mode 100644 tatanka/db/orderbook_test.go diff --git a/tatanka/db/db.go b/tatanka/db/db.go index 9a62ad1542..c75a5690b6 100644 --- a/tatanka/db/db.go +++ b/tatanka/db/db.go @@ -7,6 +7,7 @@ import ( "context" "encoding" "encoding/binary" + "errors" "fmt" "os" "path/filepath" @@ -99,6 +100,58 @@ func New(dir string, log dex.Logger) (*DB, error) { }, nil } +func (db *DB) NewOrderBook(baseID, quoteID uint32) (*OrderBook, error) { + bSym := dex.BipIDSymbol(baseID) + qSym := dex.BipIDSymbol(quoteID) + if bSym == "" || qSym == "" { + return nil, errors.New("could not find base or quote symbol") + } + prefix := fmt.Sprintf("%s-%s", bSym, qSym) + orderBookTable, err := db.Table(fmt.Sprintf("%s-orderbook", prefix)) + if err != nil { + return nil, fmt.Errorf("error constructing orderbook table: %w", err) + } + + orderBookOrderIDIdx, err := orderBookTable.AddIndex(fmt.Sprintf("%s-orderbook-orderid", prefix), func(_, v encoding.BinaryMarshaler) ([]byte, error) { + o, is := v.(*OrderUpdate) + if !is { + return nil, fmt.Errorf("wrong type %T", v) + } + oID := o.ID() + return oID[:], nil + }) + + orderBookStampIdx, err := orderBookTable.AddIndex(fmt.Sprintf("%s-orderbook-stamp", prefix), func(_, v encoding.BinaryMarshaler) ([]byte, error) { + o, is := v.(*OrderUpdate) + if !is { + return nil, fmt.Errorf("wrong type %T", v) + } + tB := make([]byte, 8) + binary.BigEndian.PutUint64(tB, uint64(o.Stamp.UnixMilli())) + return tB, nil + }) + + orderBookSellRateIdx, err := orderBookTable.AddIndex(fmt.Sprintf("%s-orderbook-sellrate", prefix), func(_, v encoding.BinaryMarshaler) ([]byte, error) { + o, is := v.(*OrderUpdate) + if !is { + return nil, fmt.Errorf("wrong type %T", v) + } + srB := make([]byte, 1+8) + if o.Sell { + srB[0] = 1 + } + binary.BigEndian.PutUint64(srB[1:], o.Rate) + return srB, nil + }) + + return &OrderBook{ + orderBook: orderBookTable, + orderBookOrderIDIdx: orderBookOrderIDIdx, + orderBookStampIdx: orderBookStampIdx, + orderBookSellRateIdx: orderBookSellRateIdx, + }, nil +} + func (db *DB) Connect(ctx context.Context) (*sync.WaitGroup, error) { var wg sync.WaitGroup go func() { diff --git a/tatanka/db/orderbook.go b/tatanka/db/orderbook.go new file mode 100644 index 0000000000..22719020ae --- /dev/null +++ b/tatanka/db/orderbook.go @@ -0,0 +1,216 @@ +// This code is available on the terms of the project LICENSE.md file, +// also available online at https://blueoakcouncil.org/license/1.0.0. + +package db + +import ( + "errors" + "fmt" + "time" + + "decred.org/dcrdex/dex/encode" + "decred.org/dcrdex/dex/lexi" + "decred.org/dcrdex/tatanka/tanka" +) + +type OrderUpdate struct { + Sig []byte `json:"sig"` + *tanka.Order +} + +func (o *OrderUpdate) MarshalBinary() ([]byte, error) { + const orderVer = 0 + var b encode.BuildyBytes = make([]byte, 1, 32+4+4+1+8+8+8+8+8+8+4+8+len(o.Sig)) + b[0] = orderVer + sell := encode.ByteFalse + if o.Sell { + sell = encode.ByteTrue + } + b = b.AddData(o.From[:]). + AddData(encode.Uint32Bytes(o.BaseID)). + AddData(encode.Uint32Bytes(o.QuoteID)). + AddData(sell). + AddData(encode.Uint64Bytes(o.Qty)). + AddData(encode.Uint64Bytes(o.Rate)). + AddData(encode.Uint64Bytes(o.LotSize)). + AddData(encode.Uint64Bytes(o.MinFeeRate)). + AddData(encode.Uint64Bytes(uint64(o.Stamp.UnixMilli()))). + AddData(encode.Uint32Bytes(o.Nonce)). + AddData(encode.Uint64Bytes(uint64(o.Expiration.UnixMilli()))). + AddData(encode.Uint64Bytes(o.Settled)). + AddData(o.Sig) + + return b, nil +} + +func (o *OrderUpdate) UnmarshalBinary(b []byte) error { + if o == nil { + return errors.New("nil order update") + } + const orderVer = 0 + o.Order = new(tanka.Order) + ver, pushes, err := encode.DecodeBlob(b, 13) + if err != nil { + return fmt.Errorf("error decoding order update blob: %w", err) + } + if ver != orderVer { + return fmt.Errorf("unknown order update version %d", ver) + } + if len(pushes) != 13 { + return fmt.Errorf("unknown number of order update blob pushes %d", len(pushes)) + } + copy(o.From[:], pushes[0]) + o.BaseID = encode.BytesToUint32(pushes[1]) + o.QuoteID = encode.BytesToUint32(pushes[2]) + o.Sell = pushes[3][0] == encode.ByteTrue[0] + o.Qty = encode.BytesToUint64(pushes[4]) + o.Rate = encode.BytesToUint64(pushes[5]) + o.LotSize = encode.BytesToUint64(pushes[6]) + o.MinFeeRate = encode.BytesToUint64(pushes[7]) + o.Stamp = time.UnixMilli(int64(encode.BytesToUint64(pushes[8]))) + o.Nonce = encode.BytesToUint32(pushes[9]) + o.Expiration = time.UnixMilli(int64(encode.BytesToUint64(pushes[10]))) + o.Settled = encode.BytesToUint64(pushes[11]) + o.Sig = pushes[12] + + return nil +} + +type OrderBooker interface { + OrderIDs(fromIdx, nOrders uint64) ([]tanka.ID32, bool, error) + Order(id tanka.ID32) (*OrderUpdate, error) + Orders(id []tanka.ID32) ([]*OrderUpdate, error) + FindOrders(filter *OrderFilter) ([]*OrderUpdate, error) + Add(*OrderUpdate) error + Update(*OrderUpdate) error + Delete(id tanka.ID32) error +} + +type OrderFilter struct { + IsSell *bool + Check func(*OrderUpdate) (ok, done bool) +} + +var _ OrderBooker = (*OrderBook)(nil) + +type OrderBook struct { + orderBook *lexi.Table + orderBookOrderIDIdx *lexi.Index + orderBookStampIdx *lexi.Index + orderBookSellRateIdx *lexi.Index +} + +func (ob *OrderBook) OrderIDs(fromIdx, nOrders uint64) (oids []tanka.ID32, all bool, err error) { + var ( + i uint64 + enoughErr = errors.New("enough") + ) + all = true + err = ob.orderBookOrderIDIdx.Iterate([]byte{}, func(it *lexi.Iter) error { + if i < fromIdx { + i++ + return nil + } + if i-fromIdx >= nOrders { + all = false + return enoughErr + } + k, err := it.K() + if err != nil { + return fmt.Errorf("error getting order key: %w", err) + } + var oid tanka.ID32 + copy(oid[:], k) + oids = append(oids, oid) + i++ + return nil + }) + if err != nil && !errors.Is(err, enoughErr) { + return nil, false, err + } + return oids, all, nil +} + +func (ob *OrderBook) Order(id tanka.ID32) (ou *OrderUpdate, err error) { + return ou, ob.orderBook.Get(id, ou) +} + +func (ob *OrderBook) Orders(ids []tanka.ID32) ([]*OrderUpdate, error) { + ous := make([]*OrderUpdate, 0, len(ids)) + for _, id := range ids { + ou := new(OrderUpdate) + if err := ob.orderBook.Get(id, ou); err != nil { + return nil, err + } + ous = append(ous, ou) + } + return ous, nil +} + +func (ob *OrderBook) FindOrders(filter *OrderFilter) (ous []*OrderUpdate, err error) { + if filter == nil { + return nil, errors.New("filter is nil") + } + var enoughErr = errors.New("enough") + find := func(isSell bool) error { + var i uint64 + prefix := [1]byte{} + // Find best orders first. + opts := lexi.WithReverse() + if isSell { + prefix[0] = 1 + opts = lexi.WithForward() + } + err = ob.orderBookSellRateIdx.Iterate(prefix[:], func(it *lexi.Iter) error { + ou := new(OrderUpdate) + if err := it.V(func(vB []byte) error { + return ou.UnmarshalBinary(vB) + }); err != nil { + return err + } + if filter.Check != nil { + ok, done := filter.Check(ou) + if !ok { + return nil + } + if done { + return enoughErr + } + } + ous = append(ous, ou) + i++ + return nil + }, opts) + if err != nil && !errors.Is(err, enoughErr) { + return err + } + return nil + } + if filter.IsSell != nil { + if err := find(*filter.IsSell); err != nil { + return nil, err + } + } else { + if err := find(false); err != nil { + return nil, err + } + if err := find(true); err != nil { + return nil, err + } + } + return ous, nil +} + +func (ob *OrderBook) Add(ou *OrderUpdate) error { + id := ou.ID() + return ob.orderBook.Set(id, ou, lexi.WithReplace()) +} + +func (ob *OrderBook) Update(ou *OrderUpdate) error { + id := ou.ID() + return ob.orderBook.Set(id, ou, lexi.WithReplace()) +} + +func (ob *OrderBook) Delete(id tanka.ID32) error { + return ob.orderBook.Delete(id[:]) +} diff --git a/tatanka/db/orderbook_test.go b/tatanka/db/orderbook_test.go new file mode 100644 index 0000000000..08a008771c --- /dev/null +++ b/tatanka/db/orderbook_test.go @@ -0,0 +1,270 @@ +package db + +import ( + "testing" + "time" + + "decred.org/dcrdex/dex/encode" + "decred.org/dcrdex/tatanka/tanka" +) + +func testOrders() []*OrderUpdate { + mustParse := func(s string) time.Time { + t, err := time.Parse(time.RFC1123, s) + if err != nil { + panic(err) + } + return t + } + + var peer tanka.PeerID + copy(peer[:], encode.RandomBytes(32)) + baseID, quoteID := uint32(0), uint32(42) + lowbuy := &OrderUpdate{ + Sig: encode.RandomBytes(32), + Order: &tanka.Order{ + From: peer, + BaseID: baseID, + QuoteID: quoteID, + Sell: false, + Qty: 1000, + Settled: 500, + Rate: 123, + LotSize: 3, + MinFeeRate: 10000, + Nonce: 0, + Stamp: mustParse("Sun, 12 Dec 2024 12:23:00 UTC"), + Expiration: mustParse("Sun, 12 Jan 2025 12:23:00 UTC"), + }, + } + highbuy := &OrderUpdate{ + Sig: encode.RandomBytes(32), + Order: &tanka.Order{ + From: peer, + BaseID: baseID, + QuoteID: quoteID, + Sell: false, + Qty: 1000, + Settled: 500, + Rate: 1234, + LotSize: 2, + MinFeeRate: 10000, + Nonce: 1, + Stamp: mustParse("Sun, 12 Dec 2024 12:23:00 UTC"), + Expiration: mustParse("Sun, 12 Jan 2025 12:23:00 UTC"), + }, + } + lowsell := &OrderUpdate{ + Sig: encode.RandomBytes(32), + Order: &tanka.Order{ + From: peer, + BaseID: baseID, + QuoteID: quoteID, + Sell: true, + Qty: 1000, + Settled: 500, + Rate: 12345, + LotSize: 2, + MinFeeRate: 10000, + Nonce: 2, + Stamp: mustParse("Sun, 12 Dec 2024 12:23:00 UTC"), + Expiration: mustParse("Sun, 12 Jan 2025 12:23:00 UTC"), + }, + } + highsell := &OrderUpdate{ + Sig: encode.RandomBytes(32), + Order: &tanka.Order{ + From: peer, + BaseID: baseID, + QuoteID: quoteID, + Sell: true, + Qty: 1000, + Settled: 500, + Rate: 123456, + LotSize: 4, + MinFeeRate: 10000, + Nonce: 3, + Stamp: mustParse("Sun, 12 Dec 2024 12:23:00 UTC"), + Expiration: mustParse("Sun, 12 Jan 2025 12:23:00 UTC"), + }, + } + return []*OrderUpdate{lowbuy, highbuy, lowsell, highsell} +} + +func TestOrderIDs(t *testing.T) { + db, shutdown := tNewDB() + defer shutdown() + ob, err := db.NewOrderBook(0, 42) + if err != nil { + t.Fatal(err) + } + for _, o := range testOrders() { + if err := ob.Add(o); err != nil { + t.Fatal(err) + } + } + + tests := []struct { + name string + fromIdx, nOrders uint64 + wantOrderLen int + wantAll bool + }{{ + name: "all orders", + nOrders: 100, + wantOrderLen: 4, + wantAll: true, + }, { + name: "from 2", + fromIdx: 2, + nOrders: 100, + wantOrderLen: 2, + wantAll: true, + }, { + name: "3 orders", + nOrders: 3, + wantOrderLen: 3, + }, { + name: "from 1 until 2 orders", + fromIdx: 1, + nOrders: 2, + wantOrderLen: 2, + }} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + oids, all, err := ob.OrderIDs(test.fromIdx, test.nOrders) + if err != nil { + t.Fatal(err) + } + if len(oids) != test.wantOrderLen { + t.Fatalf("wanted %d but got %d orders", test.wantOrderLen, len(oids)) + } + if all != test.wantAll { + t.Fatalf("wanted all to be %v but got %v", test.wantAll, all) + } + }) + } +} + +func TestOrders(t *testing.T) { + db, shutdown := tNewDB() + defer shutdown() + ob, err := db.NewOrderBook(0, 42) + if err != nil { + t.Fatal(err) + } + var oids []tanka.ID32 + for _, o := range testOrders() { + if err := ob.Add(o); err != nil { + t.Fatal(err) + } + oids = append(oids, o.ID()) + } + ous, err := ob.Orders(oids) + if err != nil { + t.Fatal(err) + } + if len(ous) != 4 { + t.Fatalf("wanted 4 but got %d orders", len(oids)) + } +} + +func TestFindOrders(t *testing.T) { + db, shutdown := tNewDB() + defer shutdown() + ob, err := db.NewOrderBook(0, 42) + if err != nil { + t.Fatal(err) + } + tOrds := testOrders() + for _, o := range tOrds { + if err := ob.Add(o); err != nil { + t.Fatal(err) + } + } + yes, no := true, false + + tests := []struct { + name string + filter *OrderFilter + wantOrderLen int + wantErr bool + wantOrderIdx []int + }{{ + name: "all orders", + filter: new(OrderFilter), + wantOrderLen: 4, + wantOrderIdx: []int{1, 0, 2, 3}, + }, { + name: "sells", + filter: &OrderFilter{ + IsSell: &yes, + }, + wantOrderLen: 2, + wantOrderIdx: []int{2, 3}, + }, { + name: "buys", + filter: &OrderFilter{ + IsSell: &no, + }, + wantOrderLen: 2, + wantOrderIdx: []int{1, 0}, + }, { + name: "lot size over 2", + filter: &OrderFilter{ + Check: func(ou *OrderUpdate) (ok, done bool) { + return ou.LotSize > 2, false + }, + }, + wantOrderLen: 2, + wantOrderIdx: []int{0, 3}, + }, { + name: "lot size over 2 and sell", + filter: &OrderFilter{ + IsSell: &yes, + Check: func(ou *OrderUpdate) (ok, done bool) { + return ou.LotSize > 2, false + }, + }, + wantOrderLen: 1, + wantOrderIdx: []int{3}, + }, { + name: "buy done after one", + filter: &OrderFilter{ + IsSell: &no, + Check: func() func(*OrderUpdate) (ok, done bool) { + var i int + return func(ou *OrderUpdate) (ok, done bool) { + defer func() { i++ }() + return true, i == 1 + } + }(), + }, + wantOrderLen: 1, + wantOrderIdx: []int{1}, + }, { + name: "nil filter", + wantErr: true, + }} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ous, err := ob.FindOrders(test.filter) + if test.wantErr { + if err == nil { + t.Fatal("expected error") + } + return + } else if err != nil { + t.Fatal(err) + } + if len(ous) != test.wantOrderLen { + t.Fatalf("wanted %d but got %d orders", test.wantOrderLen, len(ous)) + } + for i, idx := range test.wantOrderIdx { + if tOrds[idx].ID() != ous[i].ID() { + t.Fatalf("returned order at idx %d not equal to test order at %d", idx, i) + } + } + }) + } +} diff --git a/tatanka/tanka/swaps.go b/tatanka/tanka/swaps.go index fe1244a113..3832a7e3e2 100644 --- a/tatanka/tanka/swaps.go +++ b/tatanka/tanka/swaps.go @@ -17,6 +17,7 @@ type Order struct { QuoteID uint32 `json:"quoteID"` Sell bool `json:"sell"` Qty uint64 `json:"qty"` + Settled uint64 `json:"settled"` Rate uint64 `json:"rate"` // LotSize: Tatankanet does not prescribe a lot size. Instead, users must // select their own minimum minimum lot size. The user's UI should ignore @@ -28,13 +29,15 @@ type Order struct { // does supply a suggested fee rate that is updated periodically. The user's // UI should ignore an order from the order book if its MinFeeRate falls // below the Tatnkanet suggested rate. - MinFeeRate uint64 `json:"minFeeRate"` + MinFeeRate uint64 `json:"minFeeRate"` + // Nonce can be used to force unique ids while other values are the same. + Nonce uint32 `json:"nonce"` Stamp time.Time `json:"stamp"` Expiration time.Time `json:"expiration"` } -func (ord *Order) ID() [32]byte { - const msgLen = 32 + 4 + 4 + 1 + 8 + 8 + 8 + 8 + 8 + 8 +func (ord *Order) ID() ID32 { + const msgLen = 32 + 4 + 4 + 1 + 8 + 8 + 8 + 8 + 8 + 8 + 4 b := make([]byte, msgLen) copy(b[:32], ord.From[:]) binary.BigEndian.PutUint32(b[32:36], ord.BaseID) @@ -47,9 +50,9 @@ func (ord *Order) ID() [32]byte { binary.BigEndian.PutUint64(b[57:65], ord.LotSize) binary.BigEndian.PutUint64(b[65:73], ord.MinFeeRate) binary.BigEndian.PutUint64(b[73:81], uint64(ord.Stamp.UnixMilli())) - binary.BigEndian.PutUint64(b[81:89], uint64(ord.Expiration.UnixMilli())) + binary.BigEndian.PutUint32(b[81:89], ord.Nonce) + // Settled and Expiration can be updated while keeping the same ID. return blake256.Sum256(b) - } type ID32 [32]byte @@ -58,6 +61,10 @@ func (i ID32) String() string { return hex.EncodeToString(i[:]) } +func (i ID32) MarshalBinary() ([]byte, error) { + return i[:], nil +} + type Match struct { From PeerID `json:"from"` OrderID ID32 `json:"orderID"`