server/{matcher,market}: base start and end rate calcs on actual matc…

…h rates

We have always used the mid-gap at the beginning and end of the epoch
for our candle's start rate and end rate. In a traditional market, you would
use the first and last match price. Use the expected numbers instead.
The candle should also not go to zero on an empty book anymore.

server/db/driver/pg/epochs.go

@@ -5,7 +5,9 @@ package pg
 
 import (
 	"context"
+	"database/sql"
 	"database/sql/driver"
+	"errors"
 	"fmt"
 	"time"
 
@@ -82,6 +84,22 @@ func (a *Archiver) InsertEpoch(ed *db.EpochResults) error {
 	return err
 }
 
+// LastEpochRate gets the EndRate of the last EpochResults inserted for the
+// market. If the database is empty, no error and a rate of zero are returned.
+func (a *Archiver) LastEpochRate(base, quote uint32) (rate uint64, err error) {
+	marketSchema, err := a.marketSchema(base, quote)
+	if err != nil {
+		return 0, err
+	}
+
+	epochReportsTableName := fullEpochReportsTableName(a.dbName, marketSchema)
+	stmt := fmt.Sprintf(internal.SelectLastEpochRate, epochReportsTableName)
+	if err = a.db.QueryRowContext(a.ctx, stmt).Scan(&rate); err != nil && !errors.Is(sql.ErrNoRows, err) {
+		return 0, err
+	}
+	return rate, nil
+}
+
 // LoadEpochStats reads all market epoch history from the database, updating the
 // provided caches along the way.
 func (a *Archiver) LoadEpochStats(base, quote uint32, caches []*candles.Cache) error {

server/db/driver/pg/internal/epochs.go

@@ -18,6 +18,11 @@ const (
 	InsertEpoch = `INSERT INTO %s (epoch_idx, epoch_dur, match_time, csum, seed, revealed, missed)
 		VALUES ($1, $2, $3, $4, $5, $6, $7);`
 
+	SelectLastEpochRate = `SELECT end_rate
+		FROM %s
+		ORDER BY epoch_end DESC
+		LIMIT 1;`
+
 	// CreateEpochReportTable creates an epoch_reports table that holds
 	// epoch-end reports that can be used to construct market history data sets.
 	CreateEpochReportTable = `CREATE TABLE IF NOT EXISTS %s (
@@ -33,8 +38,8 @@ const (
 		book_sells_25 INT8,         -- booked sell volume within 25 pct of market
 		high_rate INT8,             -- the highest rate matched
 		low_rate INT8,              -- the lowest rate matched
-		start_rate INT8,            -- the mid-gap rate at the beginning of the match cycle
-		end_rate INT8               -- the mid-gap rate at the end of the match cycle
+		start_rate INT8,            -- the rate of the first match in the epoch
+		end_rate INT8               -- the rate of the last match in the epoch
 	);`
 
 	// InsertEpochReport inserts a row into the epoch_reports table.

server/db/driver/pg/matches_online_test.go

@@ -1197,8 +1197,16 @@ func TestEpochReport(t *testing.T) {
 		t.Fatalf("cleanTables: %v", err)
 	}
 
+	lastRate, err := archie.LastEpochRate(42, 0)
+	if err != nil {
+		t.Fatalf("error getting last epoch rate from empty table (should be err = nil, rate = 0): %v", err)
+	}
+	if lastRate != 0 {
+		t.Fatalf("wrong initial last rate. expected 0, got %d", lastRate)
+	}
+
 	var epochIdx, epochDur int64 = 13245678, 6000
-	err := archie.InsertEpoch(&db.EpochResults{
+	err = archie.InsertEpoch(&db.EpochResults{
 		MktBase:     42,
 		MktQuote:    0,
 		Idx:         epochIdx,
@@ -1215,6 +1223,14 @@ func TestEpochReport(t *testing.T) {
 		t.Fatalf("error inserting first epoch: %v", err)
 	}
 
+	lastRate, err = archie.LastEpochRate(42, 0)
+	if err != nil {
+		t.Fatalf("error getting last epoch rate from after first epoch: %v", err)
+	}
+	if lastRate != 5 {
+		t.Fatalf("wrong first epoch last rate. expected 5, got %d", lastRate)
+	}
+
 	// Trying for the same epoch should violate a primary key constraint.
 	err = archie.InsertEpoch(&db.EpochResults{
 		MktBase:  42,
@@ -1242,6 +1258,14 @@ func TestEpochReport(t *testing.T) {
 		t.Fatalf("error inserting second epoch: %v", err)
 	}
 
+	lastRate, err = archie.LastEpochRate(42, 0)
+	if err != nil {
+		t.Fatalf("error getting last epoch rate from after second-to-last epoch: %v", err)
+	}
+	if lastRate != 15 {
+		t.Fatalf("wrong second-to-last epoch last rate. expected 15, got %d", lastRate)
+	}
+
 	archie.InsertEpoch(&db.EpochResults{
 		MktBase:     42,
 		MktQuote:    0,

server/db/interface.go

@@ -78,6 +78,11 @@ type DEXArchivist interface {
 	// InsertEpoch stores the results of a newly-processed epoch.
 	InsertEpoch(ed *EpochResults) error
 
+	// LastEpochRate gets the EndRate of the last EpochResults inserted for the
+	// market. If the database is empty, no error and a rate of zero are
+	// returned.
+	LastEpochRate(b, q uint32) (uint64, error)
+
 	// LoadEpochStats reads all market epoch history from the database.
 	LoadEpochStats(uint32, uint32, []*candles.Cache) error
 

server/market/integ/booker_matcher_test.go

@@ -597,14 +597,16 @@ func TestMatchWithBook_limitsOnly_multipleQueued(t *testing.T) {
 	_, matches, passed, failed, doneOK, partial, booked, nomatched, unbooked, _, stats := me.Match(b, epochQueue)
 	//t.Log(matches, passed, failed, doneOK, partial, booked, unbooked)
 
+	lastMatch := matches[len(matches)-1]
+
 	compareMatchStats(t, &matcher.MatchCycleStats{
 		BookBuys:    (bookBuyLots - 4) * LotSize,
 		BookSells:   (bookSellLots + 39) * LotSize,
 		MatchVolume: 6 * LotSize,
 		HighRate:    4550000,
 		LowRate:     4300000,
-		StartRate:   (4550000 + 4500000) / 2,
-		EndRate:     (4600000 + 4300000) / 2,
+		StartRate:   matches[0].Makers[0].Rate,
+		EndRate:     lastMatch.Makers[len(lastMatch.Makers)-1].Rate,
 	}, stats)
 
 	// PASSED orders
@@ -928,10 +930,10 @@ func TestMatch_marketSellsOnly(t *testing.T) {
 				BookBuys:    (bookBuyLots - 1) * LotSize,
 				BookSells:   bookSellLots * LotSize,
 				MatchVolume: LotSize,
-				HighRate:    initialMidGap,
-				LowRate:     4500000,
-				StartRate:   initialMidGap,
-				EndRate:     (4300000 + 4550000) / 2,
+				HighRate:    bookBuyOrders[nBuy-1].Rate,
+				LowRate:     bookBuyOrders[nBuy-1].Rate,
+				StartRate:   bookBuyOrders[nBuy-1].Rate,
+				EndRate:     bookBuyOrders[nBuy-1].Rate,
 			},
 		},
 		{
@@ -955,10 +957,10 @@ func TestMatch_marketSellsOnly(t *testing.T) {
 				BookBuys:    (bookBuyLots - 3) * LotSize,
 				BookSells:   bookSellLots * LotSize,
 				MatchVolume: 3 * LotSize,
-				HighRate:    initialMidGap,
-				LowRate:     4300000,
-				StartRate:   initialMidGap,
-				EndRate:     (4550000 + 4300000) / 2,
+				HighRate:    bookBuyOrders[nBuy-1].Rate,
+				LowRate:     bookBuyOrders[nBuy-2].Rate,
+				StartRate:   bookBuyOrders[nBuy-1].Rate,
+				EndRate:     bookBuyOrders[nBuy-2].Rate,
 			},
 			// newLimitOrder(false, 4300000, 2, order.StandingTiF, 0), // older
 			// newLimitOrder(false, 4500000, 1, order.StandingTiF, 0),
@@ -987,10 +989,10 @@ func TestMatch_marketSellsOnly(t *testing.T) {
 				BookSells:   bookSellLots * LotSize,
 				MatchVolume: 5 * LotSize,
 
-				HighRate:  initialMidGap,
-				LowRate:   4300000,
-				StartRate: initialMidGap,
-				EndRate:   (4550000 + 4300000) / 2,
+				HighRate:  bookBuyOrders[nBuy-1].Rate,
+				LowRate:   bookBuyOrders[nBuy-3].Rate,
+				StartRate: bookBuyOrders[nBuy-1].Rate,
+				EndRate:   bookBuyOrders[nBuy-3].Rate,
 			},
 		},
 		{

server/market/market.go

@@ -167,6 +167,7 @@ type Market struct {
 
 	// Data API
 	dataCollector DataCollector
+	lastRate      uint64
 }
 
 // Storage is the DB interface required by Market.
@@ -176,6 +177,7 @@ type Storage interface {
 	Fatal() <-chan struct{}
 	Close() error
 	InsertEpoch(ed *db.EpochResults) error
+	LastEpochRate(base, quote uint32) (uint64, error)
 	MarketMatches(base, quote uint32) ([]*db.MatchDataWithCoins, error)
 	InsertMatch(match *order.Match) error
 }
@@ -475,6 +477,11 @@ ordersLoop:
 	}
 	log.Infof("Tracking %d orders with %d active matches.", len(settling), len(activeMatches))
 
+	lastEpochEndRate, err := storage.LastEpochRate(base, quote)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load last epoch end rate: %w", err)
+	}
+
 	return &Market{
 		running:          make(chan struct{}), // closed on market start
 		marketInfo:       mktInfo,
@@ -492,6 +499,7 @@ ordersLoop:
 		baseFeeFetcher:   cfg.FeeFetcherBase,
 		quoteFeeFetcher:  cfg.FeeFetcherQuote,
 		dataCollector:    cfg.DataCollector,
+		lastRate:         lastEpochEndRate,
 	}, nil
 }
 
@@ -2375,6 +2383,17 @@ func (m *Market) processReadyEpoch(epoch *readyEpoch, notifyChan chan<- *updateS
 		oidsMissed = append(oidsMissed, om.ID())
 	}
 
+	// If there were no matches, we need to persist that last rate from the last
+	// match recorded.
+	if stats.EndRate == 0 {
+		stats.EndRate = m.lastRate
+		stats.StartRate = m.lastRate
+		stats.HighRate = m.lastRate
+		stats.LowRate = m.lastRate
+	} else {
+		m.lastRate = stats.EndRate
+	}
+
 	err := m.storage.InsertEpoch(&db.EpochResults{
 		MktBase:        m.marketInfo.Base,
 		MktQuote:       m.marketInfo.Quote,

server/market/market_test.go

@@ -135,6 +135,9 @@ func (ta *TArchivist) InsertEpoch(ed *db.EpochResults) error {
 	}
 	return nil
 }
+func (ta *TArchivist) LastEpochRate(base, quote uint32) (rate uint64, err error) {
+	return 1, nil
+}
 func (ta *TArchivist) BookOrder(lo *order.LimitOrder) error {
 	ta.mtx.Lock()
 	defer ta.mtx.Unlock()

server/matcher/match.go

@@ -152,10 +152,6 @@ func (m *Matcher) Match(book Booker, queue []*OrderRevealed) (seed []byte, match
 
 	updates = new(OrdersUpdated)
 	stats = new(MatchCycleStats)
-	startRate := midGap(book)
-	stats.StartRate = startRate
-	stats.LowRate = startRate
-	stats.HighRate = startRate
 
 	appendTradeSet := func(matchSet *order.MatchSet) {
 		matches = append(matches, matchSet)
@@ -315,9 +311,23 @@ func (m *Matcher) Match(book Booker, queue []*OrderRevealed) (seed []byte, match
 		nomatched = append(nomatched, q)
 	}
 
-	midGap := midGap(book)
-	stats.EndRate = midGap
-	bookVolumes(book, midGap, stats)
+	for _, matchSet := range matches {
+		if matchSet.Total > 0 { // cancel filter
+			stats.StartRate = matchSet.Makers[0].Rate
+			break
+		}
+	}
+	if stats.StartRate > 0 { // If we didn't find anything going forward, no need to check going backwards.
+		for i := len(matches) - 1; i >= 0; i-- {
+			matchSet := matches[i]
+			if matchSet.Total > 0 { // cancel filter
+				stats.EndRate = matchSet.Makers[len(matchSet.Makers)-1].Rate
+				break
+			}
+		}
+	}
+
+	bookVolumes(book, stats)
 
 	return
 }
@@ -658,7 +668,8 @@ func sideVolume(ords []*order.LimitOrder) (q uint64) {
 	return
 }
 
-func bookVolumes(book Booker, midGap uint64, stats *MatchCycleStats) {
+func bookVolumes(book Booker, stats *MatchCycleStats) {
+	midGap := midGap(book)
 	cutoff5 := midGap - midGap/20 // 5%
 	cutoff25 := midGap - midGap/4 // 25%
 	for _, ord := range book.BuyOrders() {

server/matcher/match_test.go

@@ -775,10 +775,10 @@ func TestMatch_limitsOnly(t *testing.T) {
 				BookBuys:    bookBuyLots * LotSize,
 				BookSells:   (bookSellLots - 1) * LotSize,
 				MatchVolume: LotSize,
-				HighRate:    4550000,
-				LowRate:     initialMidGap,
-				StartRate:   initialMidGap,
-				EndRate:     (4500000 + 4600000) / 2,
+				HighRate:    bookSellOrders[nSell-1].Rate,
+				LowRate:     bookSellOrders[nSell-1].Rate,
+				StartRate:   bookSellOrders[nSell-1].Rate,
+				EndRate:     bookSellOrders[nSell-1].Rate,
 			},
 		},
 		{
@@ -810,10 +810,10 @@ func TestMatch_limitsOnly(t *testing.T) {
 				BookSells:   (bookSellLots - 1) * LotSize,
 				BookBuys:    (bookBuyLots + 1) * LotSize,
 				MatchVolume: LotSize,
-				HighRate:    4550000,
-				LowRate:     initialMidGap,
-				StartRate:   initialMidGap,
-				EndRate:     (4550000 + 4600000) / 2,
+				HighRate:    bookSellOrders[nSell-1].Rate,
+				LowRate:     bookSellOrders[nSell-1].Rate,
+				StartRate:   bookSellOrders[nSell-1].Rate,
+				EndRate:     bookSellOrders[nSell-1].Rate,
 			},
 		},
 		{
@@ -844,10 +844,10 @@ func TestMatch_limitsOnly(t *testing.T) {
 				BookSells:   (bookSellLots - 1) * LotSize,
 				BookBuys:    bookBuyLots * LotSize,
 				MatchVolume: LotSize,
-				HighRate:    4550000,
-				LowRate:     initialMidGap,
-				StartRate:   initialMidGap,
-				EndRate:     (4500000 + 4600000) / 2,
+				HighRate:    bookSellOrders[nSell-1].Rate,
+				LowRate:     bookSellOrders[nSell-1].Rate,
+				StartRate:   bookSellOrders[nSell-1].Rate,
+				EndRate:     bookSellOrders[nSell-1].Rate,
 			},
 		},
 		{
@@ -874,10 +874,10 @@ func TestMatch_limitsOnly(t *testing.T) {
 			matchStats: &MatchCycleStats{
 				BookSells: bookSellLots * LotSize,
 				BookBuys:  bookBuyLots * LotSize,
-				HighRate:  initialMidGap,
-				LowRate:   initialMidGap,
-				StartRate: initialMidGap,
-				EndRate:   initialMidGap,
+				HighRate:  0,
+				LowRate:   0,
+				StartRate: 0,
+				EndRate:   0,
 			},
 		},
 		{
@@ -1393,10 +1393,10 @@ func TestMatch_marketBuysOnly(t *testing.T) {
 				BookSells:   (bookSellLots - 1) * LotSize,
 				BookBuys:    bookBuyLots * LotSize,
 				MatchVolume: LotSize,
-				HighRate:    4550000,
-				LowRate:     initialMidGap,
-				StartRate:   initialMidGap,
-				EndRate:     (4500000 + 4600000) / 2,
+				HighRate:    bookSellOrders[nSell-1].Rate,
+				LowRate:     bookSellOrders[nSell-1].Rate,
+				StartRate:   bookSellOrders[nSell-1].Rate,
+				EndRate:     bookSellOrders[nSell-1].Rate,
 			},
 		},
 		{
@@ -1428,10 +1428,10 @@ func TestMatch_marketBuysOnly(t *testing.T) {
 				BookSells:   (bookSellLots - 2) * LotSize,
 				BookBuys:    bookBuyLots * LotSize,
 				MatchVolume: 2 * LotSize,
-				HighRate:    4600000,
-				LowRate:     initialMidGap,
-				StartRate:   initialMidGap,
-				EndRate:     (4500000 + 4600000) / 2,
+				HighRate:    bookSellOrders[nSell-2].Rate,
+				LowRate:     bookSellOrders[nSell-1].Rate,
+				StartRate:   bookSellOrders[nSell-1].Rate,
+				EndRate:     bookSellOrders[nSell-2].Rate,
 			},
 		},
 		{
@@ -1462,10 +1462,10 @@ func TestMatch_marketBuysOnly(t *testing.T) {
 				BookSells:   (bookSellLots - 3) * LotSize,
 				BookBuys:    bookBuyLots * LotSize,
 				MatchVolume: 3 * LotSize,
-				HighRate:    4600000,
-				LowRate:     initialMidGap,
-				StartRate:   initialMidGap,
-				EndRate:     (4500000 + 4700000) / 2,
+				HighRate:    bookSellOrders[nSell-2].Rate,
+				LowRate:     bookSellOrders[nSell-1].Rate,
+				StartRate:   bookSellOrders[nSell-1].Rate,
+				EndRate:     bookSellOrders[nSell-2].Rate,
 			},
 		},
 		{