diff --git a/packages/eventindexer/.l1.env b/packages/eventindexer/.l1.env index e137fc178ce..1a0131b7af8 100644 --- a/packages/eventindexer/.l1.env +++ b/packages/eventindexer/.l1.env @@ -13,4 +13,5 @@ RPC_URL=wss://l1ws.internal.taiko.xyz CORS_ORIGINS=* BLOCK_BATCH_SIZE=10 CACHE_INTERVAL_IN_SECONDS=60 -LAYER=l1 \ No newline at end of file +LAYER=l1 +GENESIS_DATE=2023-09-08 \ No newline at end of file diff --git a/packages/eventindexer/cmd/flags/generator.go b/packages/eventindexer/cmd/flags/generator.go index b9388df43ac..9c53b437f50 100644 --- a/packages/eventindexer/cmd/flags/generator.go +++ b/packages/eventindexer/cmd/flags/generator.go @@ -10,7 +10,15 @@ var ( Category: generatorCategory, EnvVars: []string{"GENESIS_DATE"}, } + Regenerate = &cli.StringFlag{ + Name: "regenerate", + Usage: "True to delete all existing data and regenerate from genesis, false to not", + Required: false, + Category: generatorCategory, + EnvVars: []string{"REGENERATE"}, + } ) var GeneratorFlags = MergeFlags(CommonFlags, []cli.Flag{ GenesisDate, + Regenerate, }) diff --git a/packages/eventindexer/db/db.go b/packages/eventindexer/db/db.go index 9223263b3cb..3b865643fb7 100644 --- a/packages/eventindexer/db/db.go +++ b/packages/eventindexer/db/db.go @@ -46,14 +46,14 @@ func OpenDBConnection(opts DBConnectionOpts) (*DB, error) { dsn := "" if opts.Password == "" { dsn = fmt.Sprintf( - "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", + "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True", opts.Name, opts.Host, opts.Database, ) } else { dsn = fmt.Sprintf( - "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", + "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True", opts.Name, opts.Password, opts.Host, diff --git a/packages/eventindexer/event.go b/packages/eventindexer/event.go index 288801324ac..8fd589ae4b0 100644 --- a/packages/eventindexer/event.go +++ b/packages/eventindexer/event.go @@ -34,6 +34,8 @@ type Event struct { Address string `json:"address"` BlockID sql.NullInt64 `json:"blockID"` Amount decimal.NullDecimal `json:"amount"` + ProofReward decimal.NullDecimal `json:"proofReward"` + ProposerReward decimal.NullDecimal `json:"proposerReward"` AssignedProver string `json:"assignedProver"` To string `json:"to"` TokenID sql.NullInt64 `json:"tokenID"` @@ -50,6 +52,8 @@ type SaveEventOpts struct { Address string BlockID *int64 Amount *big.Int + ProposerReward *big.Int + ProofReward *big.Int AssignedProver *string To *string TokenID *int64 diff --git a/packages/eventindexer/generator/config.go b/packages/eventindexer/generator/config.go index 1cf9ec83675..378b03c24c0 100644 --- a/packages/eventindexer/generator/config.go +++ b/packages/eventindexer/generator/config.go @@ -28,6 +28,7 @@ type Config struct { DatabaseMaxConnLifetime uint64 MetricsHTTPPort uint64 GenesisDate time.Time + Regenerate bool OpenDBFunc func() (DB, error) } @@ -48,6 +49,7 @@ func NewConfigFromCliContext(c *cli.Context) (*Config, error) { DatabaseMaxConnLifetime: c.Uint64(flags.DatabaseConnMaxLifetime.Name), MetricsHTTPPort: c.Uint64(flags.MetricsHTTPPort.Name), GenesisDate: date, + Regenerate: c.Bool(flags.Regenerate.Name), OpenDBFunc: func() (DB, error) { return db.OpenDBConnection(db.DBConnectionOpts{ Name: c.String(flags.DatabaseUsername.Name), diff --git a/packages/eventindexer/generator/config_test.go b/packages/eventindexer/generator/config_test.go index 2731d49fb0f..f38f1ed1205 100644 --- a/packages/eventindexer/generator/config_test.go +++ b/packages/eventindexer/generator/config_test.go @@ -32,6 +32,7 @@ func TestNewConfigFromCliContext(t *testing.T) { assert.Equal(t, "dbpass", c.DatabasePassword) assert.Equal(t, "dbname", c.DatabaseName) assert.Equal(t, "dbhost", c.DatabaseHost) + assert.Equal(t, true, c.Regenerate) wantTime, _ := time.Parse("2006-01-02", "2023-07-07") assert.Equal(t, wantTime, c.GenesisDate) @@ -52,5 +53,6 @@ func TestNewConfigFromCliContext(t *testing.T) { "-" + flags.DatabaseHost.Name, "dbhost", "-" + flags.DatabaseName.Name, "dbname", "-" + flags.GenesisDate.Name, "2023-07-07", + "-" + flags.Regenerate.Name, "true", })) } diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index 0b933732148..b60228f9b4c 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -4,11 +4,10 @@ import ( "context" "errors" "log/slog" - "strconv" - "syscall" "time" "github.com/ethereum/go-ethereum/common" + "github.com/shopspring/decimal" "github.com/taikoxyz/taiko-mono/packages/eventindexer" "github.com/taikoxyz/taiko-mono/packages/eventindexer/tasks" "github.com/urfave/cli/v2" @@ -24,6 +23,7 @@ var ( type Generator struct { db DB genesisDate time.Time + regenerate bool } func (g *Generator) InitFromCli(ctx context.Context, c *cli.Context) error { @@ -43,6 +43,7 @@ func InitFromConfig(ctx context.Context, g *Generator, cfg *Config) error { g.db = db g.genesisDate = cfg.GenesisDate + g.regenerate = cfg.Regenerate return nil } @@ -52,13 +53,17 @@ func (g *Generator) Name() string { } func (g *Generator) Start() error { - slog.Info("generating time series data") + if g.regenerate { + slog.Info("regenerating, deleting existing data") - if err := g.generateTimeSeriesData(context.Background()); err != nil { - return err + if err := g.deleteTimeSeriesData(context.Background()); err != nil { + return err + } } - if err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM); err != nil { + slog.Info("generating time series data") + + if err := g.generateTimeSeriesData(context.Background()); err != nil { return err } @@ -76,6 +81,15 @@ func (g *Generator) Close(ctx context.Context) { } } +func (g *Generator) deleteTimeSeriesData(ctx context.Context) error { + deleteStmt := "DELETE FROM time_series_data;" + if err := g.db.GormDB().Exec(deleteStmt).Error; err != nil { + return err + } + + return nil +} + // generateTimeSeriesData iterates over each task and generates time series data. func (g *Generator) generateTimeSeriesData(ctx context.Context) error { for _, task := range tasks.Tasks { @@ -93,19 +107,25 @@ func (g *Generator) generateTimeSeriesData(ctx context.Context) error { func (g *Generator) generateByTask(ctx context.Context, task string) error { slog.Info("generating for task", "task", task) - latestDate, err := g.getLatestDateByTask(ctx, task) + startingDate, err := g.getStartingDateByTask(ctx, task) if err != nil { return err } currentDate := g.getCurrentDate() - if latestDate.AddDate(0, 0, 1).Compare(currentDate) == 0 { - slog.Info("data already generated up-to-date for task", "task", task, "date", latestDate.Format("2006-01-02")) + if startingDate.Compare(currentDate) == 0 { + slog.Info( + "data already generated up-to-date for task", + "task", task, + "date", startingDate.Format("2006-01-02"), + "currentDate", currentDate.Format("2006-01-02"), + ) + return nil } // Loop through each date from latestDate to currentDate - for d := latestDate; d.Before(currentDate); d = d.AddDate(0, 0, 1) { + for d := startingDate; d.Before(currentDate); d = d.AddDate(0, 0, 1) { slog.Info("Processing", "task", task, "date", d.Format("2006-01-02"), "currentDate", currentDate.Format("2006-01-02")) result, err := g.queryByTask(task, d) @@ -114,7 +134,7 @@ func (g *Generator) generateByTask(ctx context.Context, task string) error { return err } - slog.Info("Query successful", "task", task, "date", d.Format("2006-01-02"), "result", result) + slog.Info("Query successful", "task", task, "date", d.Format("2006-01-02"), "result", result.String()) insertStmt := ` INSERT INTO time_series_data(task, value, date) @@ -132,12 +152,12 @@ func (g *Generator) generateByTask(ctx context.Context, task string) error { return nil } -// getLatestDateByTask returns the last time time series data has been generated -// for the given task. -func (g *Generator) getLatestDateByTask(ctx context.Context, task string) (time.Time, error) { +// getStartingDateByTask returns first required time series data, one after the latest date entry, +// or the genesis date. +func (g *Generator) getStartingDateByTask(ctx context.Context, task string) (time.Time, error) { var latestDateString string - var latestDate time.Time + var nextRequiredDate time.Time q := `SELECT date FROM time_series_data WHERE task = ? ORDER BY date DESC LIMIT 1;` @@ -146,178 +166,207 @@ func (g *Generator) getLatestDateByTask(ctx context.Context, task string) (time. slog.Info("latestDateString", "task", task, "date", latestDateString) if err != nil || latestDateString == "" { - latestDate = g.genesisDate + nextRequiredDate = g.genesisDate } else { - latestDate, err = time.Parse("2006-01-02", latestDateString) - } + latestDate, err := time.Parse("2006-01-02", latestDateString) + if err != nil { + return time.Time{}, err + } - if err != nil { - return time.Time{}, err + nextRequiredDate = latestDate.AddDate(0, 0, 1) } - slog.Info("latest date for task", "task", task, "latestDate", latestDate.Format("2006-01-02")) + slog.Info("next required date for task", "task", task, "nextRequiredDate", nextRequiredDate.Format("2006-01-02")) - return latestDate, nil + return nextRequiredDate, nil } // getCurrentDate returns the current date in YYYY-MM-DD format func (g *Generator) getCurrentDate() time.Time { // Get current date - currentTime := time.Now() + currentTime := time.Now().UTC() currentDate := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(), 0, 0, 0, 0, time.UTC) return currentDate } -// nolint: funlen +// nolint: funlen, gocognit // queryByTask runs a database query which should return result data based on the // task -func (g *Generator) queryByTask(task string, date time.Time) (string, error) { +func (g *Generator) queryByTask(task string, date time.Time) (decimal.Decimal, error) { dateString := date.Format("2006-01-02") - var result string + var result decimal.Decimal var err error switch task { + case tasks.ProposerRewardsPerDay: + query := "SELECT COALESCE(SUM(proposer_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + err = g.db.GormDB(). + Raw(query, eventindexer.EventNameBlockProposed, dateString). + Scan(&result).Error + + case tasks.TotalProposerRewards: + var dailyProposerRewards decimal.NullDecimal + + query := "SELECT COALESCE(SUM(proposer_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + err = g.db.GormDB(). + Raw(query, eventindexer.EventNameBlockProposed, dateString). + Scan(&dailyProposerRewards).Error + + tsdResult, err := g.previousDayTsdResultByTask(task, date) + if err != nil { + return result, err + } + + result = tsdResult.Decimal.Add(dailyProposerRewards.Decimal) + + case tasks.TotalProofRewards: + var dailyProofRewards decimal.NullDecimal + + query := "SELECT COALESCE(SUM(proof_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + err = g.db.GormDB(). + Raw(query, eventindexer.EventNameBlockProposed, dateString). + Scan(&dailyProofRewards).Error + + tsdResult, err := g.previousDayTsdResultByTask(task, date) + if err != nil { + return result, err + } + + result = tsdResult.Decimal.Add(dailyProofRewards.Decimal) + case tasks.ProofRewardsPerDay: + query := "SELECT COALESCE(SUM(proof_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" + err = g.db.GormDB(). + Raw(query, eventindexer.EventNameBlockProposed, dateString). + Scan(&result).Error case tasks.BridgeMessagesSentPerDay: err = g.eventCount(task, date, eventindexer.EventNameMessageSent, &result) case tasks.TotalBridgeMessagesSent: - var dailyMsgSentCount int + var dailyMsgSentCount decimal.NullDecimal err = g.eventCount(task, date, eventindexer.EventNameMessageSent, &dailyMsgSentCount) if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyMsgSentCount + tsdResult) + result = tsdResult.Decimal.Add(dailyMsgSentCount.Decimal) case tasks.ProposeBlockTxPerDay: err = g.eventCount(task, date, eventindexer.EventNameBlockProposed, &result) case tasks.TotalProposeBlockTx: - var dailyProposerCount int + var dailyProposerCount decimal.NullDecimal err = g.eventCount(task, date, eventindexer.EventNameBlockProposed, &dailyProposerCount) if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyProposerCount + tsdResult) + result = tsdResult.Decimal.Add(dailyProposerCount.Decimal) case tasks.UniqueProposersPerDay: query := "SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, date). + Raw(query, eventindexer.EventNameBlockProposed, dateString). Scan(&result).Error case tasks.TotalUniqueProposers: - var dailyProposerCount int - - query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?` + query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ?` - err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProposed, dateString).Scan(&dailyProposerCount).Error + err = g.db.GormDB().Raw( + query, + eventindexer.EventNameBlockProposed, + ).Scan(&result).Error if err != nil { - return "", err + return result, err } - - tsdResult, err := g.previousDayTsdResultByTask(task, date) - if err != nil { - return "", err - } - - result = strconv.Itoa(dailyProposerCount + tsdResult) case tasks.UniqueProversPerDay: query := "SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProven, date). + Raw(query, eventindexer.EventNameBlockProven, dateString). Scan(&result).Error case tasks.TotalUniqueProvers: - var dailyProposerCount int - - query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?` + query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ?` - err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProposerCount).Error + err = g.db.GormDB().Raw( + query, + eventindexer.EventNameBlockProven, + ).Scan(&result).Error if err != nil { - return "", err + return result, err } - - tsdResult, err := g.previousDayTsdResultByTask(task, date) - if err != nil { - return "", err - } - - result = strconv.Itoa(dailyProposerCount + tsdResult) case tasks.ProveBlockTxPerDay: query := "SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProven, date). + Raw(query, eventindexer.EventNameBlockProven, dateString). Scan(&result).Error case tasks.TotalProveBlockTx: - var dailyProposerCount int + var dailyProveBlockCount decimal.NullDecimal query := `SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?` - err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProposerCount).Error + err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProveBlockCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyProposerCount + tsdResult) + result = tsdResult.Decimal.Add(dailyProveBlockCount.Decimal) case tasks.AccountsPerDay: query := `SELECT COUNT(*) FROM accounts WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&result).Error case tasks.TotalAccounts: - var dailyAccountsCount int + var dailyAccountsCount decimal.NullDecimal query := `SELECT COUNT(*) FROM accounts WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&dailyAccountsCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyAccountsCount + tsdResult) + result = tsdResult.Decimal.Add(dailyAccountsCount.Decimal) case tasks.BlocksPerDay: query := `SELECT COUNT(*) FROM blocks WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&result).Error case tasks.TotalBlocks: - var dailyBlockCount int + var dailyBlockCount decimal.NullDecimal query := `SELECT COUNT(*) FROM blocks WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&dailyBlockCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyBlockCount + tsdResult) + result = tsdResult.Decimal.Add(dailyBlockCount.Decimal) case tasks.TransactionsPerDay: query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&result).Error case tasks.TotalTransactions: - var dailyTxCount int + var dailyTxCount decimal.NullDecimal // get current days txs, get previous entry for the time series data, add them together. @@ -325,41 +374,41 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { err = g.db.GormDB().Raw(query, dateString).Scan(&dailyTxCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyTxCount + tsdResult) + result = tsdResult.Decimal.Add(dailyTxCount.Decimal) case tasks.ContractDeploymentsPerDay: query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ? AND contract_address != ?` - err = g.db.GormDB().Raw(query, dateString, ZeroAddress).Scan(&result).Error + err = g.db.GormDB().Raw(query, dateString, ZeroAddress.Hex()).Scan(&result).Error case tasks.TotalContractDeployments: - var dailyContractCount int + var dailyContractCount decimal.NullDecimal // get current days txs, get previous entry for the time series data, add them together. query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ? AND contract_address != ?` - err = g.db.GormDB().Raw(query, dateString, ZeroAddress).Scan(&dailyContractCount).Error + err = g.db.GormDB().Raw(query, dateString, ZeroAddress.Hex()).Scan(&dailyContractCount).Error if err != nil { - return "", err + return result, err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return "", err + return result, err } - result = strconv.Itoa(dailyContractCount + tsdResult) + result = tsdResult.Decimal.Add(dailyContractCount.Decimal) default: - return "", errors.New("task not supported") + return result, errors.New("task not supported") } if err != nil { - return "", err + return result, err } return result, nil @@ -367,14 +416,21 @@ func (g *Generator) queryByTask(task string, date time.Time) (string, error) { // previousDayTsdResultByTask returns the previous day's time series data, based on // task and time passed in. -func (g *Generator) previousDayTsdResultByTask(task string, date time.Time) (int, error) { - var tsdResult int +func (g *Generator) previousDayTsdResultByTask(task string, date time.Time) (decimal.NullDecimal, error) { + var tsdResult decimal.NullDecimal tsdQuery := `SELECT value FROM time_series_data WHERE task = ? AND date = ?` err := g.db.GormDB().Raw(tsdQuery, task, date.AddDate(0, 0, -1).Format("2006-01-02")).Scan(&tsdResult).Error if err != nil { - return 0, err + return tsdResult, err + } + + if !tsdResult.Valid { + return decimal.NullDecimal{ + Valid: true, + Decimal: decimal.Zero, + }, nil } return tsdResult, nil @@ -386,6 +442,6 @@ func (g *Generator) eventCount(task string, date time.Time, event string, result query := "SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?" return g.db.GormDB(). - Raw(query, event, date). + Raw(query, event, date.Format("2006-01-02")). Scan(result).Error } diff --git a/packages/eventindexer/http/get_stats.go b/packages/eventindexer/http/get_stats.go index cbb68858c6d..c66e26a4109 100644 --- a/packages/eventindexer/http/get_stats.go +++ b/packages/eventindexer/http/get_stats.go @@ -2,10 +2,10 @@ package http import ( "net/http" + "time" "github.com/cyberhorsey/webutils" "github.com/labstack/echo/v4" - "github.com/patrickmn/go-cache" "github.com/taikoxyz/taiko-mono/packages/eventindexer" ) @@ -26,7 +26,7 @@ func (srv *Server) GetStats(c echo.Context) error { return webutils.LogAndRenderErrors(c, http.StatusUnprocessableEntity, err) } - srv.cache.Set(CacheKeyStats, stats, cache.DefaultExpiration) + srv.cache.Set(CacheKeyStats, stats, 1*time.Minute) } return c.JSON(http.StatusOK, stats) diff --git a/packages/eventindexer/http/get_stats_test.go b/packages/eventindexer/http/get_stats_test.go index a4c5c6000fb..9926d1f9a25 100644 --- a/packages/eventindexer/http/get_stats_test.go +++ b/packages/eventindexer/http/get_stats_test.go @@ -37,7 +37,8 @@ func Test_GetStats(t *testing.T) { "success", "0x123", http.StatusOK, - []string{`{"id":1,"averageProofTime":"5","averageProofReward":"7","numProofs":1,"numVerifiedBlocks":1}`}, + // nolint: lll + []string{`{"id":1,"averageProofTime":"5","averageProofReward":"7","averageProposerReward":"","numProposerRewards":0,"numProofs":1,"numVerifiedBlocks":1}`}, }, } diff --git a/packages/eventindexer/indexer/save_block_proposed_event.go b/packages/eventindexer/indexer/save_block_proposed_event.go index 6d2c2b027fd..6564efcef1f 100644 --- a/packages/eventindexer/indexer/save_block_proposed_event.go +++ b/packages/eventindexer/indexer/save_block_proposed_event.go @@ -75,6 +75,16 @@ func (indxr *Indexer) saveBlockProposedEvent( return errors.Wrap(err, "indxr.ethClient.BlockByNumber") } + proposerReward, err := indxr.updateAverageProposerReward(ctx, event) + if err != nil { + return errors.Wrap(err, "indxr.updateAverageProposerReward") + } + + proverReward, err := indxr.updateAverageProverReward(ctx, event) + if err != nil { + return errors.Wrap(err, "indxr.updateAverageProposerReward") + } + _, err = indxr.eventRepo.Save(ctx, eventindexer.SaveEventOpts{ Name: eventindexer.EventNameBlockProposed, Data: string(marshaled), @@ -83,7 +93,10 @@ func (indxr *Indexer) saveBlockProposedEvent( Address: sender.Hex(), BlockID: &blockID, AssignedProver: &assignedProver, - TransactedAt: time.Unix(int64(block.Time()), 0), + TransactedAt: time.Unix(int64(block.Time()), 0).UTC(), + Amount: event.Reward, + ProposerReward: proposerReward, + ProofReward: proverReward, }) if err != nil { return errors.Wrap(err, "indxr.eventRepo.Save") @@ -93,3 +106,94 @@ func (indxr *Indexer) saveBlockProposedEvent( return nil } + +func (indxr *Indexer) updateAverageProposerReward( + ctx context.Context, + event *taikol1.TaikoL1BlockProposed, +) (*big.Int, error) { + stat, err := indxr.statRepo.Find(ctx) + if err != nil { + return nil, errors.Wrap(err, "indxr.statRepo.Find") + } + + reward := event.Reward + + avg, ok := new(big.Int).SetString(stat.AverageProposerReward, 10) + if !ok { + return nil, errors.New("unable to convert average proposer to string") + } + + newAverageProposerReward := calcNewAverage( + avg, + new(big.Int).SetUint64(stat.NumProposerRewards), + reward, + ) + + slog.Info("newAverageProposerReward update", + "id", + event.BlockId.Int64(), + "prover", + event.Prover.Hex(), + "avg", + avg.String(), + "newAvg", + newAverageProposerReward.String(), + ) + + _, err = indxr.statRepo.Save(ctx, eventindexer.SaveStatOpts{ + ProposerReward: newAverageProposerReward, + }) + if err != nil { + return nil, errors.Wrap(err, "indxr.statRepo.Save") + } + + return reward, err +} + +func (indxr *Indexer) updateAverageProverReward( + ctx context.Context, + event *taikol1.TaikoL1BlockProposed, +) (*big.Int, error) { + stat, err := indxr.statRepo.Find(ctx) + if err != nil { + return nil, errors.Wrap(err, "indxr.statRepo.Find") + } + + tx, _, err := indxr.ethClient.TransactionByHash(ctx, event.Raw.TxHash) + if err != nil { + return nil, errors.Wrap(err, "indxr.ethClient.TransactionByHash") + } + + reward := tx.Value() + + avg, ok := new(big.Int).SetString(stat.AverageProofReward, 10) + if !ok { + return nil, errors.New("unable to convert average proof time to string") + } + + newAverageProofReward := calcNewAverage( + avg, + new(big.Int).SetUint64(stat.NumProofs), + reward, + ) + + slog.Info("newAverageProofReward update", + "id", + event.BlockId.Int64(), + "prover", + event.Prover.Hex(), + "avg", + avg.String(), + "newAvg", + newAverageProofReward.String(), + ) + + _, err = indxr.statRepo.Save(ctx, eventindexer.SaveStatOpts{ + ProofReward: newAverageProofReward, + }) + if err != nil { + return nil, errors.Wrap(err, "indxr.statRepo.Save") + } + + return reward, nil +} diff --git a/packages/eventindexer/migrations/1666650599_create_events_table.sql b/packages/eventindexer/migrations/1666650599_create_events_table.sql index e5b027fa73c..40fe273e1f1 100644 --- a/packages/eventindexer/migrations/1666650599_create_events_table.sql +++ b/packages/eventindexer/migrations/1666650599_create_events_table.sql @@ -9,6 +9,8 @@ CREATE TABLE IF NOT EXISTS events ( address VARCHAR(42) NOT NULL DEFAULT "", block_id int DEFAULT NULL, amount DECIMAL(65, 0) DEFAULT NULL, + proof_reward VARCHAR(255) DEFAULT NULL, + proposer_reward VARCHAR(255) DEFAULT NULL, assigned_prover VARCHAR(42) NOT NULL DEFAULT "", transacted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP , diff --git a/packages/eventindexer/migrations/1666650701_create_stats_table.sql b/packages/eventindexer/migrations/1666650701_create_stats_table.sql index 9a8ea8a6e93..60bbbeea6fc 100644 --- a/packages/eventindexer/migrations/1666650701_create_stats_table.sql +++ b/packages/eventindexer/migrations/1666650701_create_stats_table.sql @@ -4,6 +4,8 @@ CREATE TABLE IF NOT EXISTS stats ( id int NOT NULL PRIMARY KEY AUTO_INCREMENT, average_proof_time VARCHAR(255) NOT NULL DEFAULT "0", average_proof_reward VARCHAR(255) NOT NULL DEFAULT "0", + average_proposer_reward VARCHAR(255) NOT NULL DEFAULT "0", + num_proposer_rewards int NOT NULL default 0, num_proofs int NOT NULL default 0, num_verified_blocks int NOT NULL default 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP , diff --git a/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql b/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql index 64bd23e013e..b41b7cf0bcc 100644 --- a/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql +++ b/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS time_series_data ( id int NOT NULL PRIMARY KEY AUTO_INCREMENT, task VARCHAR(40) NOT NULL, - value VARCHAR(100) NOT NULL, + value DECIMAL(65, 0) NOT NULL, date VARCHAR(20) NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP , updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, diff --git a/packages/eventindexer/repo/chart_repo.go b/packages/eventindexer/repo/chart.go similarity index 96% rename from packages/eventindexer/repo/chart_repo.go rename to packages/eventindexer/repo/chart.go index e1326c4a35c..590ed4e92f5 100644 --- a/packages/eventindexer/repo/chart_repo.go +++ b/packages/eventindexer/repo/chart.go @@ -48,7 +48,7 @@ func (r *ChartRepository) Find( for _, d := range tsd { chart.Chart = append(chart.Chart, eventindexer.ChartItem{ Date: d.Date, - Value: d.Value, + Value: d.Value.Decimal.String(), }) } diff --git a/packages/eventindexer/repo/event.go b/packages/eventindexer/repo/event.go index 4b940615a6d..cb28860426a 100644 --- a/packages/eventindexer/repo/event.go +++ b/packages/eventindexer/repo/event.go @@ -56,6 +56,30 @@ func (r *EventRepository) Save(ctx context.Context, opts eventindexer.SaveEventO } } + if opts.ProposerReward != nil { + amt, err := decimal.NewFromString(opts.ProposerReward.String()) + if err != nil { + return nil, errors.Wrap(err, "decimal.NewFromString") + } + + e.ProposerReward = decimal.NullDecimal{ + Valid: true, + Decimal: amt, + } + } + + if opts.ProofReward != nil { + amt, err := decimal.NewFromString(opts.ProofReward.String()) + if err != nil { + return nil, errors.Wrap(err, "decimal.NewFromString") + } + + e.ProofReward = decimal.NullDecimal{ + Valid: true, + Decimal: amt, + } + } + if opts.AssignedProver != nil { e.AssignedProver = *opts.AssignedProver } diff --git a/packages/eventindexer/repo/stat.go b/packages/eventindexer/repo/stat.go index 65174abb912..82a5efd31b4 100644 --- a/packages/eventindexer/repo/stat.go +++ b/packages/eventindexer/repo/stat.go @@ -41,6 +41,11 @@ func (r *StatRepository) Save(ctx context.Context, opts eventindexer.SaveStatOpt s.AverageProofTime = opts.ProofTime.String() } + if opts.ProposerReward != nil { + s.NumProposerRewards++ + s.AverageProposerReward = opts.ProposerReward.String() + } + if err := r.db.GormDB().Save(s).Error; err != nil { return nil, errors.Wrap(err, "r.db.Save") } @@ -66,5 +71,9 @@ func (r *StatRepository) Find(ctx context.Context) (*eventindexer.Stat, error) { s.AverageProofTime = "0" } + if s.AverageProposerReward == "" { + s.AverageProposerReward = "0" + } + return s, nil } diff --git a/packages/eventindexer/repo/stat_test.go b/packages/eventindexer/repo/stat_test.go index 1c7d9128c3a..37abb5b81cf 100644 --- a/packages/eventindexer/repo/stat_test.go +++ b/packages/eventindexer/repo/stat_test.go @@ -5,7 +5,6 @@ import ( "math/big" "testing" - "github.com/davecgh/go-spew/spew" "github.com/taikoxyz/taiko-mono/packages/eventindexer" "gotest.tools/assert" ) @@ -54,9 +53,14 @@ func TestIntegration_Stat_Find(t *testing.T) { var proofReward = big.NewInt(4) - _, err = statRepo.Save(context.Background(), eventindexer.SaveStatOpts{ - ProofReward: proofReward, - }) + var proposerReward = big.NewInt(7) + + for i := 0; i < 3; i++ { + _, err = statRepo.Save(context.Background(), eventindexer.SaveStatOpts{ + ProofReward: proofReward, + ProposerReward: proposerReward, + }) + } assert.Equal(t, nil, err) @@ -68,11 +72,13 @@ func TestIntegration_Stat_Find(t *testing.T) { { "success", &eventindexer.Stat{ - ID: 1, - AverageProofReward: proofReward.String(), - AverageProofTime: "0", - NumProofs: 0, - NumVerifiedBlocks: 1, + ID: 1, + AverageProofReward: proofReward.String(), + AverageProofTime: "0", + AverageProposerReward: proposerReward.String(), + NumProposerRewards: 3, + NumProofs: 0, + NumVerifiedBlocks: 3, }, nil, }, @@ -81,7 +87,7 @@ func TestIntegration_Stat_Find(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { resp, err := statRepo.Find(context.Background()) - spew.Dump(resp) + assert.Equal(t, tt.wantErr, err) assert.Equal(t, *tt.wantResp, *resp) }) diff --git a/packages/eventindexer/stat.go b/packages/eventindexer/stat.go index 58752298b66..0734ca64def 100644 --- a/packages/eventindexer/stat.go +++ b/packages/eventindexer/stat.go @@ -9,17 +9,20 @@ import ( // into the Data field to be unmarshalled into a concrete struct // dependant on the name of the event type Stat struct { - ID int `json:"id"` - AverageProofTime string `json:"averageProofTime"` - AverageProofReward string `json:"averageProofReward"` - NumProofs uint64 `json:"numProofs"` - NumVerifiedBlocks uint64 `json:"numVerifiedBlocks"` + ID int `json:"id"` + AverageProofTime string `json:"averageProofTime"` + AverageProofReward string `json:"averageProofReward"` + AverageProposerReward string `json:"averageProposerReward"` + NumProposerRewards uint64 `json:"numProposerRewards"` + NumProofs uint64 `json:"numProofs"` + NumVerifiedBlocks uint64 `json:"numVerifiedBlocks"` } // SaveStatOpts type SaveStatOpts struct { - ProofTime *big.Int - ProofReward *big.Int + ProofTime *big.Int + ProofReward *big.Int + ProposerReward *big.Int } // StatRepository is used to interact with stats in the store diff --git a/packages/eventindexer/tasks/tasks.go b/packages/eventindexer/tasks/tasks.go index 971a8ffb717..62e4bc5cab5 100644 --- a/packages/eventindexer/tasks/tasks.go +++ b/packages/eventindexer/tasks/tasks.go @@ -19,6 +19,10 @@ var ( TotalProposeBlockTx = "total-propose-block-tx" BridgeMessagesSentPerDay = "bridge-messages-sent-per-day" TotalBridgeMessagesSent = "total-bridge-messages-sent" + TotalProofRewards = "total-proof-rewards" + ProofRewardsPerDay = "proof-rewards-per-day" + TotalProposerRewards = "total-proposer-rewards" + ProposerRewardsPerDay = "proposer-rewards-per-day" ) var Tasks = []string{ @@ -40,4 +44,8 @@ var Tasks = []string{ TotalProposeBlockTx, BridgeMessagesSentPerDay, TotalBridgeMessagesSent, + TotalProofRewards, + ProofRewardsPerDay, + TotalProposerRewards, + ProposerRewardsPerDay, } diff --git a/packages/eventindexer/time_series_data.go b/packages/eventindexer/time_series_data.go index 4d593b901b0..35c7dd8ec84 100644 --- a/packages/eventindexer/time_series_data.go +++ b/packages/eventindexer/time_series_data.go @@ -1,11 +1,15 @@ package eventindexer -import "time" +import ( + "time" + + "github.com/shopspring/decimal" +) type TimeSeriesData struct { ID int Task string - Value string + Value decimal.NullDecimal Date string CreatedAt time.Time UpdatedAt time.Time diff --git a/packages/relayer/db/db.go b/packages/relayer/db/db.go index 9223263b3cb..e2d19c61ca0 100644 --- a/packages/relayer/db/db.go +++ b/packages/relayer/db/db.go @@ -46,18 +46,20 @@ func OpenDBConnection(opts DBConnectionOpts) (*DB, error) { dsn := "" if opts.Password == "" { dsn = fmt.Sprintf( - "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", + "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=%v", opts.Name, opts.Host, opts.Database, + "UTC", ) } else { dsn = fmt.Sprintf( - "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", + "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=%v", opts.Name, opts.Password, opts.Host, opts.Database, + "UTC", ) }