Skip to content

Commit

Permalink
fix: use local db when this should be
Browse files Browse the repository at this point in the history
  • Loading branch information
@Majorfi
Majorfi committed Jun 23, 2023
1 parent 77edbbd commit 765312d
Show file tree
Hide file tree
Showing 11 changed files with 193 additions and 227 deletions.
1 change: 0 additions & 1 deletion cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ func SummonDaemonsw(chainID uint64, parentWg *sync.WaitGroup) {

func main() {
initFlags()
logs.Info(`Loading initial state in memory`)
loadDaemonsForAllChains(trace)
summonDaemonsForAllChains(trace)
var wg sync.WaitGroup
Expand Down
11 changes: 6 additions & 5 deletions common/store/db.badger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,11 @@ func _assertMap() {
}
if badgerDBMutex[chainID] == nil {
badgerDBMutex[chainID] = make(map[string]*sync.Mutex)
badgerDBMutex[chainID][TABLES.BLOCK_TIME] = &sync.Mutex{}
badgerDBMutex[chainID][TABLES.PRICES] = &sync.Mutex{}
badgerDBMutex[chainID][TABLES.HISTORICAL_PRICES] = &sync.Mutex{}
badgerDBMutex[chainID][TABLES.STRATEGIES] = &sync.Mutex{}
badgerDBMutex[chainID][TABLES.TOKENS] = &sync.Mutex{}
badgerDBMutex[chainID][TABLES.VAULTS] = &sync.Mutex{}
badgerDBMutex[chainID][TABLES.VAULTS_LEGACY] = &sync.Mutex{}
badgerDBMutex[chainID][TABLES.VAULTS_FROM_REGISTRY_SYNC] = &sync.Mutex{}
badgerDBMutex[chainID][TABLES.STRATEGIES_FROM_VAULT_SYNC] = &sync.Mutex{}
badgerDBMutex[chainID][TABLES.REGISTRY_SYNC] = &sync.Mutex{}
}
}
isBadgerDBLoaded = true
Expand Down Expand Up @@ -143,6 +141,7 @@ func ListFromBadgerDB(chainID uint64, dbKey string, dest interface{}) error {
k := item.Key()
v, err := item.ValueCopy(nil)
if err != nil {
logs.Error(err)
traces.
Capture(`error`, `impossible to get value for key: `+string(k)).
SetEntity(`database`).
Expand Down Expand Up @@ -205,6 +204,8 @@ func ListFromBadgerDB(chainID uint64, dbKey string, dest interface{}) error {
** Assign our reflected value in our dest pointer
**************************************************************************************/
reflect.ValueOf(dest).Elem().Set(destMap)
} else {
logs.Warning(`Unsupported type: `, elem.Kind())
}
return nil
})
Expand Down
3 changes: 1 addition & 2 deletions common/store/initializer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"sync"

"github.com/yearn/ydaemon/common/env"
"github.com/yearn/ydaemon/common/logs"
)

type TDBType string
Expand All @@ -17,6 +16,7 @@ const (
var _dbType TDBType
var _blockTimeSyncMap = make(map[uint64]*sync.Map)
var _historicalPriceSyncMap = make(map[uint64]*sync.Map)

var _newVaultsFromRegistrySyncMap = make(map[uint64]*sync.Map)
var _vaultsSyncMap = make(map[uint64]*sync.Map)
var _erc20SyncMap = make(map[uint64]*sync.Map)
Expand Down Expand Up @@ -44,7 +44,6 @@ func init() {
}

wg := &sync.WaitGroup{}
logs.Info(`Loading DB`)
for _, chainID := range env.SUPPORTED_CHAIN_IDS {
wg.Add(3)
go LoadBlockTime(chainID, nil)
Expand Down
24 changes: 10 additions & 14 deletions common/store/keys.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,17 @@ package store

// TTables lists the keys used in the database
type TTables = struct {
BLOCK_TIME string
PRICES string
HISTORICAL_PRICES string
STRATEGIES string
TOKENS string
VAULTS string
VAULTS_LEGACY string
PRICES string
VAULTS_LEGACY string
VAULTS_FROM_REGISTRY_SYNC string // For a given registry, sync the last block checked for a given vault
STRATEGIES_FROM_VAULT_SYNC string // For a given vault, current last block checked for the strategies added
REGISTRY_SYNC string
}

var TABLES = TTables{
BLOCK_TIME: "blocktime",
PRICES: "prices",
HISTORICAL_PRICES: "historical_prices",
STRATEGIES: "strategies",
TOKENS: "tokens",
VAULTS: "vaults",
VAULTS_LEGACY: "legacy_vaults",
PRICES: `prices`,
VAULTS_LEGACY: `legacy_vaults`,
VAULTS_FROM_REGISTRY_SYNC: `db_new_vaults_from_registries`,
STRATEGIES_FROM_VAULT_SYNC: `db_new_strategies_from_vaults`,
REGISTRY_SYNC: `db_registry_sync`,
}
2 changes: 1 addition & 1 deletion common/store/store.getter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ func ListAllVaults(chainID uint64) (asMap map[common.Address]models.TVault, asSl
** given chainID. Both a map and a slice are returned.
** Note: It's for the TStrategyAdded structure.
**************************************************************************************************/
func ListAllStrategiess(chainID uint64) (
func ListAllStrategies(chainID uint64) (
asMap map[common.Address]models.TStrategyAdded,
asSlice []models.TStrategyAdded,
) {
Expand Down
122 changes: 62 additions & 60 deletions common/store/store.loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,8 @@ func LoadBlockTime(chainID uint64, wg *sync.WaitGroup) {

switch _dbType {
case DBBadger:
temp := make(map[uint64]uint64)
ListFromBadgerDB(chainID, TABLES.BLOCK_TIME, &temp)
if temp != nil && (len(temp) > 0) {
for k, v := range temp {
syncMap.Store(k, v)
}
}

// LEGACY: Deprecated
logs.Warning(`BadgerDB is deprecated for LoadBlockTime`)
case DBSql:
var temp []DBBlockTime
DATABASE.Table(`db_block_times`).
Expand All @@ -59,13 +53,8 @@ func LoadHistoricalPrice(chainID uint64, wg *sync.WaitGroup) {

switch _dbType {
case DBBadger:
temp := make(map[string]string)
ListFromBadgerDB(chainID, TABLES.HISTORICAL_PRICES, &temp)
if temp != nil && (len(temp) > 0) {
for k, v := range temp {
syncMap.Store(k, bigNumber.NewInt(0).SetString(v))
}
}
// LEGACY: Deprecated
logs.Warning(`BadgerDB is deprecated for LoadHistoricalPrice`)
case DBSql:
var temp []DBHistoricalPrice
DATABASE.Table(`db_historical_prices`).
Expand All @@ -82,44 +71,38 @@ func LoadHistoricalPrice(chainID uint64, wg *sync.WaitGroup) {

/**************************************************************************************************
** LoadNewVaultsFromRegistry will retrieve the all the vaults added to the registries from the
** configured DB and store them in the _newVaultsFromRegistrySyncMap for fast access during that
** local DB and store them in the _newVaultsFromRegistrySyncMap for fast access during that same
** execution.
** Use local DB to not screw up the DB with the same data over and over again.
**************************************************************************************************/
func LoadNewVaultsFromRegistry(chainID uint64, wg *sync.WaitGroup) {
if wg != nil {
defer wg.Done()
}
syncMap := _newVaultsFromRegistrySyncMap[chainID]

switch _dbType {
case DBBadger:
// Not implemented
logs.Warning(`LoadNewVaultsFromRegistry not implemented for DBBadger. Skipping...`)
case DBSql:
var temp []DBNewVaultsFromRegistry
temp := make(map[string]DBNewVaultsFromRegistry)
ListFromBadgerDB(chainID, TABLES.VAULTS_FROM_REGISTRY_SYNC, &temp)

DATABASE.Table(`db_new_vaults_from_registries`).
Where("chain_id = ?", chainID).
FindInBatches(&temp, 10_000, func(tx *gorm.DB, batch int) error {
for _, vaultFromDB := range temp {
key := strconv.FormatUint(vaultFromDB.Block, 10) + "_" + addresses.ToAddress(vaultFromDB.RegistryAddress).Hex() + "_" + addresses.ToAddress(vaultFromDB.VaultsAddress).Hex() + "_" + addresses.ToAddress(vaultFromDB.TokenAddress).Hex() + "_" + vaultFromDB.APIVersion
vaultFromRegistry := models.TVaultsFromRegistry{
Address: addresses.ToAddress(vaultFromDB.VaultsAddress),
RegistryAddress: addresses.ToAddress(vaultFromDB.RegistryAddress),
TokenAddress: addresses.ToAddress(vaultFromDB.TokenAddress),
BlockHash: common.HexToHash(vaultFromDB.BlockHash),
Type: vaultFromDB.Type,
APIVersion: vaultFromDB.APIVersion,
ChainID: vaultFromDB.ChainID,
BlockNumber: vaultFromDB.Block,
Activation: vaultFromDB.Activation,
ManagementFee: vaultFromDB.ManagementFee,
TxIndex: vaultFromDB.TxIndex,
LogIndex: vaultFromDB.LogIndex,
}
syncMap.Store(key, vaultFromRegistry)
}
return nil
})
if temp != nil && (len(temp) > 0) {
for _, vaultFromDB := range temp {
key := strconv.FormatUint(vaultFromDB.Block, 10) + "_" + addresses.ToAddress(vaultFromDB.RegistryAddress).Hex() + "_" + addresses.ToAddress(vaultFromDB.VaultsAddress).Hex() + "_" + addresses.ToAddress(vaultFromDB.TokenAddress).Hex() + "_" + vaultFromDB.APIVersion
vaultFromRegistry := models.TVaultsFromRegistry{
Address: addresses.ToAddress(vaultFromDB.VaultsAddress),
RegistryAddress: addresses.ToAddress(vaultFromDB.RegistryAddress),
TokenAddress: addresses.ToAddress(vaultFromDB.TokenAddress),
BlockHash: common.HexToHash(vaultFromDB.BlockHash),
Type: vaultFromDB.Type,
APIVersion: vaultFromDB.APIVersion,
ChainID: vaultFromDB.ChainID,
BlockNumber: vaultFromDB.Block,
Activation: vaultFromDB.Activation,
ManagementFee: vaultFromDB.ManagementFee,
TxIndex: vaultFromDB.TxIndex,
LogIndex: vaultFromDB.LogIndex,
}
syncMap.Store(key, vaultFromRegistry)
}
}
}

Expand All @@ -135,13 +118,8 @@ func LoadERC20(chainID uint64, wg *sync.WaitGroup) {

switch _dbType {
case DBBadger:
temp := make(map[common.Address]models.TERC20Token)
ListFromBadgerDB(chainID, TABLES.TOKENS, &temp)
if temp != nil && (len(temp) > 0) {
for _, v := range temp {
syncMap.Store(v.Address, v)
}
}
// LEGACY: Deprecated
logs.Warning(`BadgerDB is deprecated for LoadERC20`)
case DBSql:
var temp []DBERC20
DATABASE.Table(`db_erc20`).
Expand Down Expand Up @@ -183,13 +161,8 @@ func LoadVaults(chainID uint64, wg *sync.WaitGroup) {

switch _dbType {
case DBBadger:
temp := make(map[common.Address]models.TVault)
ListFromBadgerDB(chainID, TABLES.VAULTS, &temp)
if temp != nil && (len(temp) > 0) {
for _, v := range temp {
syncMap.Store(v.Address, v)
}
}
// LEGACY: Deprecated
logs.Warning(`BadgerDB is deprecated for LoadVaults`)
case DBSql:
var temp []DBVault
DATABASE.Table(`db_vaults`).
Expand Down Expand Up @@ -269,7 +242,8 @@ func LoadStrategies(chainID uint64, wg *sync.WaitGroup) {

switch _dbType {
case DBBadger:
// not implemented
// LEGACY: Deprecated
logs.Warning(`BadgerDB is deprecated for LoadStrategies`)
case DBSql:
var temp []DBStrategy
DATABASE.Table(`db_strategies`).
Expand Down Expand Up @@ -298,3 +272,31 @@ func LoadStrategies(chainID uint64, wg *sync.WaitGroup) {
})
}
}

/**************************************************************************************************
** LoadSyncStrategiesAdded will try to retrieve all the sync for vaults/strategies for a given
** chain on the local DB.
**************************************************************************************************/
func LoadSyncStrategiesAdded(chainID uint64) []DBStrategyAddedSync {
syncMap := make(map[string]DBStrategyAddedSync)
ListFromBadgerDB(chainID, TABLES.STRATEGIES_FROM_VAULT_SYNC, &syncMap)
arr := []DBStrategyAddedSync{}
for _, v := range syncMap {
arr = append(arr, v)
}
return arr
}

/**************************************************************************************************
** LoadSyncRegistry will try to retrieve all the sync for registry/vault for a given chain on local
** DB.
**************************************************************************************************/
func LoadSyncRegistry(chainID uint64) []DBRegistrySync {
syncMap := make(map[string]DBRegistrySync)
ListFromBadgerDB(chainID, TABLES.REGISTRY_SYNC, &syncMap)
arr := []DBRegistrySync{}
for _, v := range syncMap {
arr = append(arr, v)
}
return arr
}
Loading

1 comment on commit 765312d

@vercel
Copy link

@vercel vercel bot commented on 765312d Jun 23, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

ydaemon – ./docs

ydaemon-git-main.yearn.farm
ydaemon.yearn.farm

Please sign in to comment.