Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat!: complete the PSS reward distribution #1709

Merged
merged 8 commits into from
Mar 19, 2024
Merged

feat!: complete the PSS reward distribution #1709

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
51a70bb
update compute consumer total power for reward distribution
sainoe Mar 14, 2024
76bcf98
update distribution logic to work with epochcs
sainoe Mar 14, 2024
ab0906a
Adapt reward distribution mem test to epochs
sainoe Mar 15, 2024
dcb569c
doc
sainoe Mar 15, 2024
4d51ae6
nits
sainoe Mar 15, 2024
3252dd7
other nits
sainoe Mar 15, 2024
d06c17c
nits
sainoe Mar 18, 2024
e949f66
Update tests/integration/distribution.go
sainoe Mar 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 81 additions & 54 deletions tests/integration/distribution.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
distrtypes "github.com/cosmos/cosmos-sdk/x/distribution/types"

abci "github.com/cometbft/cometbft/abci/types"
"github.com/cometbft/cometbft/libs/bytes"

icstestingutils "github.com/cosmos/interchain-security/v4/testutil/integration"
consumerkeeper "github.com/cosmos/interchain-security/v4/x/ccv/consumer/keeper"
Expand Down Expand Up @@ -51,6 +50,8 @@ func (s *CCVTestSuite) TestRewardsDistribution() {
providerAccountKeeper := s.providerApp.GetTestAccountKeeper()
consumerBankKeeper := s.consumerApp.GetTestBankKeeper()
providerBankKeeper := s.providerApp.GetTestBankKeeper()
providerKeeper := s.providerApp.GetProviderKeeper()
providerDistributionKeeper := s.providerApp.GetTestDistributionKeeper()

// send coins to the fee pool which is used for reward distribution
consumerFeePoolAddr := consumerAccountKeeper.GetModuleAccount(s.consumerCtx(), authtypes.FeeCollectorName).GetAddress()
Expand Down Expand Up @@ -79,7 +80,6 @@ func (s *CCVTestSuite) TestRewardsDistribution() {
s.Require().Equal(providerExpectedRewards.AmountOf(sdk.DefaultBondDenom), providerTokens.AmountOf(sdk.DefaultBondDenom))

// send the reward to provider chain after 2 blocks

s.consumerChain.NextBlock()
providerTokens = consumerBankKeeper.GetAllBalances(s.consumerCtx(), providerRedistributeAddr)
s.Require().Equal(0, len(providerTokens))
Expand All @@ -91,52 +91,85 @@ func (s *CCVTestSuite) TestRewardsDistribution() {
rewardPool := providerAccountKeeper.GetModuleAccount(s.providerCtx(), providertypes.ConsumerRewardsPool).GetAddress()
rewardCoins := providerBankKeeper.GetAllBalances(s.providerCtx(), rewardPool)

ibcCoinIndex := -1
for i, coin := range rewardCoins {
// Check that the reward pool contains a coin with an IBC denom
rewardsIBCdenom := ""
for _, coin := range rewardCoins {
if strings.HasPrefix(coin.Denom, "ibc") {
ibcCoinIndex = i
rewardsIBCdenom = coin.Denom
}
}

// Check that we found an ibc denom in the reward pool
s.Require().Greater(ibcCoinIndex, -1)
s.Require().NotZero(rewardsIBCdenom)

// Check that the coins got into the ConsumerRewardsPool
s.Require().Equal(rewardCoins[ibcCoinIndex].Amount, (providerExpectedRewards[0].Amount))
providerExpRewardsAmount := providerExpectedRewards.AmountOf(sdk.DefaultBondDenom)
s.Require().Equal(rewardCoins.AmountOf(rewardsIBCdenom), providerExpRewardsAmount)

// Advance a block and check that the coins are still in the ConsumerRewardsPool
s.providerChain.NextBlock()
rewardCoins = providerBankKeeper.GetAllBalances(s.providerCtx(), rewardPool)
s.Require().Equal(rewardCoins[ibcCoinIndex].Amount, (providerExpectedRewards[0].Amount))
s.Require().Equal(rewardCoins.AmountOf(rewardsIBCdenom), providerExpRewardsAmount)

// Set the consumer reward denom. This would be done by a governance proposal in prod
s.providerApp.GetProviderKeeper().SetConsumerRewardDenom(s.providerCtx(), rewardCoins[ibcCoinIndex].Denom)
// Set the consumer reward denom. This would be done by a governance proposal in prod.
providerKeeper.SetConsumerRewardDenom(s.providerCtx(), rewardsIBCdenom)

// Refill the consumer fee pool
err = consumerBankKeeper.SendCoinsFromAccountToModule(s.consumerCtx(), s.consumerChain.SenderAccount.GetAddress(), authtypes.FeeCollectorName, fees)
err = consumerBankKeeper.SendCoinsFromAccountToModule(
s.consumerCtx(),
s.consumerChain.SenderAccount.GetAddress(),
authtypes.FeeCollectorName,
fees,
)
s.Require().NoError(err)

// pass two blocks
// Pass two blocks
s.consumerChain.NextBlock()
s.consumerChain.NextBlock()

// transfer rewards from consumer to provider
relayAllCommittedPackets(s, s.consumerChain, s.transferPath, transfertypes.PortID, s.transferPath.EndpointA.ChannelID, 1)
// Save the consumer validators total outstanding rewards on the provider
consumerValsOutstandingRewardsFunc := func(ctx sdk.Context) sdk.DecCoins {
totalRewards := sdk.DecCoins{}
for _, v := range providerKeeper.GetConsumerValSet(ctx, s.consumerChain.ChainID) {
val, ok := s.providerApp.GetTestStakingKeeper().GetValidatorByConsAddr(ctx, sdk.ConsAddress(v.ProviderConsAddr))
s.Require().True(ok)
valReward := providerDistributionKeeper.GetValidatorOutstandingRewards(ctx, val.GetOperator())
totalRewards = totalRewards.Add(valReward.Rewards...)
}
return totalRewards
}
consuValsRewards := consumerValsOutstandingRewardsFunc(s.providerCtx())

// Save community pool balance
communityPool := providerDistributionKeeper.GetFeePoolCommunityCoins(s.providerCtx())

// Transfer rewards from consumer to provider
relayAllCommittedPackets(
s,
s.consumerChain,
s.transferPath,
transfertypes.PortID,
s.transferPath.EndpointA.ChannelID,
1,
)

// check that the consumer rewards allocation are empty since relayAllCommittedPackets call BeginBlock
rewardsAlloc := s.providerApp.GetProviderKeeper().GetConsumerRewardsAllocation(s.providerCtx(), s.consumerChain.ChainID)
// Check that the consumer rewards allocation are empty since relayAllCommittedPackets calls BeginBlockRD,
// which in turns calls AllocateTokens.
rewardsAlloc := providerKeeper.GetConsumerRewardsAllocation(s.providerCtx(), s.consumerChain.ChainID)
s.Require().Empty(rewardsAlloc.Rewards)

// Check that the reward pool still has the first coins transferred that were never allocated
// Check that the reward pool still holds the coins from the first transfer,
// which were never allocated since they were not whitelisted
rewardCoins = providerBankKeeper.GetAllBalances(s.providerCtx(), rewardPool)
s.Require().Equal(rewardCoins[ibcCoinIndex].Amount, (providerExpectedRewards[0].Amount))

// check that the fee pool has the expected amount of coins
// Note that all rewards are allocated to the community pool since
// BeginBlock is called without the validators' votes in ibctesting.
// See NextBlock() in https://github.com/cosmos/ibc-go/blob/release/v7.3.x/testing/chain.go#L281
communityCoins := s.providerApp.GetTestDistributionKeeper().GetFeePoolCommunityCoins(s.providerCtx())
s.Require().Equal(communityCoins[ibcCoinIndex].Amount, (sdk.NewDecCoinFromCoin(providerExpectedRewards[0]).Amount))
s.Require().Equal(rewardCoins.AmountOf(rewardsIBCdenom), providerExpRewardsAmount)

// Check that summing the rewards received by the consumer validators and the community pool
// is equal to the expected provider rewards
consuValsRewardsReceived := consumerValsOutstandingRewardsFunc(s.providerCtx()).Sub(consuValsRewards)
communityPoolDelta := providerDistributionKeeper.GetFeePoolCommunityCoins(s.providerCtx()).Sub(communityPool)

s.Require().Equal(
sdk.NewDecFromInt(providerExpRewardsAmount),
consuValsRewardsReceived.AmountOf(rewardsIBCdenom).Add(communityPoolDelta.AmountOf(rewardsIBCdenom)),
)
}

// TestSendRewardsRetries tests that failed reward transmissions are retried every BlocksPerDistributionTransmission blocks
Expand Down Expand Up @@ -906,19 +939,11 @@ func (s *CCVTestSuite) TestAllocateTokensToValidator() {
distributionKeeper := s.providerApp.GetTestDistributionKeeper()
bankKeeper := s.providerApp.GetTestBankKeeper()

chainID := "consumer"
validators := []bytes.HexBytes{
s.providerChain.Vals.Validators[0].Address,
s.providerChain.Vals.Validators[1].Address,
}
votes := []abci.VoteInfo{
{Validator: abci.Validator{Address: validators[0], Power: 1}},
{Validator: abci.Validator{Address: validators[1], Power: 1}},
}
chainID := s.consumerChain.ChainID

testCases := []struct {
name string
votes []abci.VoteInfo
consuValLen int
tokens sdk.DecCoins
rate sdk.Dec
expAllocated sdk.DecCoins
Expand All @@ -930,21 +955,21 @@ func (s *CCVTestSuite) TestAllocateTokensToValidator() {
expAllocated: nil,
},
{
name: "total voting power is zero",
name: "consumer valset is empty - total voting power is zero",
tokens: sdk.DecCoins{sdk.NewDecCoin(sdk.DefaultBondDenom, math.NewInt(100_000))},
rate: sdk.ZeroDec(),
expAllocated: nil,
},
{
name: "expect all tokens to be allocated to a single validator",
votes: []abci.VoteInfo{votes[0]},
consuValLen: 1,
tokens: sdk.DecCoins{sdk.NewDecCoin(sdk.DefaultBondDenom, math.NewInt(999))},
rate: sdk.NewDecWithPrec(5, 1),
expAllocated: sdk.DecCoins{sdk.NewDecCoin(sdk.DefaultBondDenom, math.NewInt(999))},
},
{
name: "expect tokens to be allocated evenly between validators",
votes: []abci.VoteInfo{votes[0], votes[1]},
consuValLen: 2,
tokens: sdk.DecCoins{sdk.NewDecCoinFromDec(sdk.DefaultBondDenom, math.LegacyNewDecFromIntWithPrec(math.NewInt(999), 2))},
rate: sdk.OneDec(),
expAllocated: sdk.DecCoins{sdk.NewDecCoinFromDec(sdk.DefaultBondDenom, math.LegacyNewDecFromIntWithPrec(math.NewInt(999), 2))},
Expand All @@ -953,27 +978,29 @@ func (s *CCVTestSuite) TestAllocateTokensToValidator() {

for _, tc := range testCases {
s.Run(tc.name, func() {
// set the same consumer commission rate for all validators
for _, v := range s.providerChain.Vals.Validators {
provAddr := providertypes.NewProviderConsAddress(sdk.ConsAddress(v.Address))
ctx, _ := s.providerCtx().CacheContext()

// change the consumer valset
consuVals := providerKeeper.GetConsumerValSet(ctx, chainID)
providerKeeper.DeleteConsumerValSet(ctx, chainID)
providerKeeper.SetConsumerValSet(ctx, chainID, consuVals[0:tc.consuValLen])
consuVals = providerKeeper.GetConsumerValSet(ctx, chainID)

// set the same consumer commission rate for all consumer validators
for _, v := range consuVals {
provAddr := providertypes.NewProviderConsAddress(sdk.ConsAddress(v.ProviderConsAddr))
providerKeeper.SetConsumerCommissionRate(
s.providerCtx(),
ctx,
chainID,
provAddr,
tc.rate,
)
}

// TODO: opt validators in and verify
// that rewards are only allocated to them
ctx, _ := s.providerCtx().CacheContext()

// allocate tokens
res := providerKeeper.AllocateTokensToConsumerValidators(
ctx,
chainID,
tc.votes,
tc.tokens,
)

Expand All @@ -982,11 +1009,11 @@ func (s *CCVTestSuite) TestAllocateTokensToValidator() {

if !tc.expAllocated.Empty() {
// rewards are expected to be allocated evenly between validators
rewardsPerVal := tc.expAllocated.QuoDec(sdk.NewDec(int64(len(tc.votes))))
rewardsPerVal := tc.expAllocated.QuoDec(sdk.NewDec(int64(len(consuVals))))

// check that the rewards are allocated to validators
for _, v := range tc.votes {
valAddr := sdk.ValAddress(v.Validator.Address)
for _, v := range consuVals {
valAddr := sdk.ValAddress(v.ProviderConsAddr)
rewards := s.providerApp.GetTestDistributionKeeper().GetValidatorOutstandingRewards(
ctx,
valAddr,
Expand Down Expand Up @@ -1019,8 +1046,8 @@ func (s *CCVTestSuite) TestAllocateTokensToValidator() {
s.Require().Equal(withdrawnCoins, bankKeeper.GetAllBalances(ctx, sdk.AccAddress(valAddr)))
}
} else {
for _, v := range tc.votes {
valAddr := sdk.ValAddress(v.Validator.Address)
for _, v := range consuVals {
valAddr := sdk.ValAddress(v.ProviderConsAddr)
rewards := s.providerApp.GetTestDistributionKeeper().GetValidatorOutstandingRewards(
ctx,
valAddr,
Expand Down
56 changes: 22 additions & 34 deletions x/ccv/provider/keeper/distribution.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,11 @@ import (

// BeginBlockRD executes BeginBlock logic for the Reward Distribution sub-protocol.
func (k Keeper) BeginBlockRD(ctx sdk.Context, req abci.RequestBeginBlock) {
// determine the total power signing the block
var previousTotalPower int64
for _, voteInfo := range req.LastCommitInfo.GetVotes() {
previousTotalPower += voteInfo.Validator.Power
}

// TODO this is Tendermint-dependent
// ref https://github.com/cosmos/cosmos-sdk/issues/3095
if ctx.BlockHeight() > 1 {
k.AllocateTokens(ctx, previousTotalPower, req.LastCommitInfo.GetVotes())
k.AllocateTokens(ctx)
}
}

Expand Down Expand Up @@ -75,7 +70,7 @@ func (k Keeper) GetAllConsumerRewardDenoms(ctx sdk.Context) (consumerRewardDenom

// AllocateTokens performs rewards distribution to the community pool and validators
// based on the Partial Set Security distribution specification.
func (k Keeper) AllocateTokens(ctx sdk.Context, totalPreviousPower int64, bondedVotes []abci.VoteInfo) {
func (k Keeper) AllocateTokens(ctx sdk.Context) {
// return if there is no coins in the consumer rewards pool
if k.GetConsumerRewardsPool(ctx).IsZero() {
return
Expand All @@ -95,6 +90,9 @@ func (k Keeper) AllocateTokens(ctx sdk.Context, totalPreviousPower int64, bonded
continue
}

// note that it's possible that no rewards are collected even though the
// reward pool isn't empty. This can happen if the reward pool holds some tokens
// of non-whitelisted denominations.
if rewardsCollected.IsZero() {
continue
}
Expand All @@ -104,13 +102,13 @@ func (k Keeper) AllocateTokens(ctx sdk.Context, totalPreviousPower int64, bonded
// temporary workaround to keep CanWithdrawInvariant happy
// general discussions here: https://github.com/cosmos/cosmos-sdk/issues/2906#issuecomment-441867634
feePool := k.distributionKeeper.GetFeePool(ctx)
if k.ComputeConsumerTotalVotingPower(ctx, consumer.ChainId, bondedVotes) == 0 {
if k.ComputeConsumerTotalVotingPower(ctx, consumer.ChainId) == 0 {
feePool.CommunityPool = feePool.CommunityPool.Add(rewardsCollectedDec...)
k.distributionKeeper.SetFeePool(ctx, feePool)
return
}

// Calculate the reward allocations
// calculate the reward allocations
remaining := rewardsCollectedDec
communityTax := k.distributionKeeper.GetCommunityTax(ctx)
voteMultiplier := math.LegacyOneDec().Sub(communityTax)
Expand All @@ -120,7 +118,6 @@ func (k Keeper) AllocateTokens(ctx sdk.Context, totalPreviousPower int64, bonded
feeAllocated := k.AllocateTokensToConsumerValidators(
ctx,
consumer.ChainId,
bondedVotes,
feeMultiplier,
)
remaining = remaining.Sub(feeAllocated)
Expand All @@ -131,33 +128,30 @@ func (k Keeper) AllocateTokens(ctx sdk.Context, totalPreviousPower int64, bonded
}
}

// TODO: allocate tokens to validators that opted-in and for long enough e.g. 1000 blocks
// once the opt-in logic is integrated QueueVSCPackets()
//
// AllocateTokensToConsumerValidators allocates the given tokens from the
// from consumer rewards pool to validator according to their voting power
// AllocateTokensToConsumerValidators allocates tokens
// to the given consumer chain's validator set
func (k Keeper) AllocateTokensToConsumerValidators(
ctx sdk.Context,
chainID string,
bondedVotes []abci.VoteInfo,
tokens sdk.DecCoins,
) (allocated sdk.DecCoins) {
// return early if the tokens are empty
if tokens.Empty() {
return allocated
}

// get the consumer total voting power from the votes
totalPower := k.ComputeConsumerTotalVotingPower(ctx, chainID, bondedVotes)
// get the total voting power of the consumer valset
totalPower := k.ComputeConsumerTotalVotingPower(ctx, chainID)
if totalPower == 0 {
return allocated
}

for _, vote := range bondedVotes {
// TODO: should check if validator IsOptIn or continue here
consAddr := sdk.ConsAddress(vote.Validator.Address)
// Allocate tokens by iterating over the consumer validators
for _, consumerVal := range k.GetConsumerValSet(ctx, chainID) {
consAddr := sdk.ConsAddress(consumerVal.ProviderConsAddr)

powerFraction := math.LegacyNewDec(vote.Validator.Power).QuoTruncate(math.LegacyNewDec(totalPower))
// get the validator tokens fraction using its voting power
powerFraction := math.LegacyNewDec(consumerVal.Power).QuoTruncate(math.LegacyNewDec(totalPower))
tokensFraction := tokens.MulDecTruncate(powerFraction)

// get the validator type struct for the consensus address
Expand Down Expand Up @@ -242,21 +236,15 @@ func (k Keeper) GetConsumerRewardsPool(ctx sdk.Context) sdk.Coins {
)
}

// ComputeConsumerTotalVotingPower returns the total voting power for a given consumer chain
// by summing its opted-in validators votes
func (k Keeper) ComputeConsumerTotalVotingPower(ctx sdk.Context, chainID string, votes []abci.VoteInfo) int64 {
// TODO: create a optedIn set from the OptedIn validators
// and sum their validator power
var totalPower int64

// ComputeConsumerTotalVotingPower returns the validator set total voting power
// for the given consumer chain
func (k Keeper) ComputeConsumerTotalVotingPower(ctx sdk.Context, chainID string) (totalPower int64) {
// sum the opted-in validators set voting powers
for _, vote := range votes {
// TODO: check that val is in the optedIn set

totalPower += vote.Validator.Power
for _, v := range k.GetConsumerValSet(ctx, chainID) {
totalPower += v.Power
}

return totalPower
return
}

// IdentifyConsumerChainIDFromIBCPacket checks if the packet destination matches a registered consumer chain.
Expand Down
Loading
Loading