Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add concurrency to liquidated vault validation #1801

Closed
wants to merge 17 commits into from
Closed

Add concurrency to liquidated vault validation #1801

Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
40eafe9
Make CLockFreeGuard safer
prasannavl Mar 3, 2023
86f3db3
Fix getburninfo consortium and result allocations
prasannavl Mar 3, 2023
b9dcc1e
Fix CLockFreeGuard, introduce AtomicMutex
prasannavl Mar 6, 2023
5de5a53
Fix missing expected
prasannavl Mar 6, 2023
2ebd23d
Use larger thread quantums
prasannavl Mar 6, 2023
d84f789
Increase default yields since sleep quantums are larger
prasannavl Mar 6, 2023
bf61ef3
Fix syntax
prasannavl Mar 6, 2023
6c00d71
Add more comments
prasannavl Mar 6, 2023
2e09252
Introduce BufferPool
prasannavl Mar 6, 2023
8283344
Merge branch 'pvl/fix-get-burninfo' into pvl/burn-info-switch-to-pool
prasannavl Mar 6, 2023
9f4f983
Switch getburninfo to use BufferPool
prasannavl Mar 6, 2023
162694b
Eliminate CLockFreeGuard, use AtomicMutex
prasannavl Mar 6, 2023
b051b9d
Add concurrency to liquidated vault validation
prasannavl Mar 6, 2023
6093051
Merge branch 'master' into pvl/parallel-vault-validation
prasannavl Mar 6, 2023
625e8f0
Merge branch 'master' into pvl/parallel-vault-validation
prasannavl Mar 6, 2023
2310808
Add missing include. Fix compiler warning.
Bushstar Mar 6, 2023
519279f
Merge branch 'master' into pvl/parallel-vault-validation
prasannavl Mar 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
275 changes: 173 additions & 102 deletions src/masternodes/validation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <masternodes/masternodes.h>
#include <masternodes/mn_checks.h>
#include <masternodes/mn_rpc.h>
#include <masternodes/threadpool.h>
#include <masternodes/validation.h>
#include <masternodes/vaulthistory.h>
#include <validation.h>
Expand Down Expand Up @@ -490,130 +491,200 @@ static void ProcessLoanEvents(const CBlockIndex* pindex, CCustomCSView& cache, c
if (pindex->nHeight % chainparams.GetConsensus().blocksCollateralizationRatioCalculation() == 0) {
bool useNextPrice = false, requireLivePrice = true;

cache.ForEachVaultCollateral([&](const CVaultId& vaultId, const CBalances& collaterals) {
auto collateral = cache.GetLoanCollaterals(vaultId, collaterals, pindex->nHeight, pindex->nTime, useNextPrice, requireLivePrice);
if (!collateral) {
return true;
}
auto &pool = DfTxTaskPool->pool;
auto nWorkers = DfTxTaskPool->GetAvailableThreads();

struct VaultWithCollateralInfo {
CVaultId vaultId;
CBalances collaterals;
CCollateralLoans vaultAssets;
CVaultData vault;
CLoanSchemeData scheme;
};

BufferPool<std::vector<VaultWithCollateralInfo>> resultsPool{nWorkers};
TaskGroup g;

const auto markCompleted = [&g] { g.RemoveTask(); };
const auto errorAndShutdown = [&g](const std::string &msg) {
g.RemoveTask();
g.MarkCancellation();
LogPrintf("ERROR: %s. Shutting down\n", msg);
StartShutdown();
};

cache.ForEachVaultCollateral(
[&](const CVaultId &vaultId, const CBalances &collaterals) {
g.AddTask();

CVaultId vaultIdCopy = vaultId;
CBalances assets = collaterals;
boost::asio::post(pool, [vaultId=vaultIdCopy, assets, &cache, pindex, useNextPrice,
requireLivePrice, &g, &resultsPool, &markCompleted, &errorAndShutdown] {
if (g.IsCancelled()) {
return;
}

auto vault = cache.GetVault(vaultId);
assert(vault);
auto scheme = cache.GetLoanScheme(vault->schemeId);
assert(scheme);
if (scheme->ratio <= collateral.val->ratio()) {
// All good, within ratio, nothing more to do.
return true;
}
auto collateral = cache.GetLoanCollaterals(
vaultId, assets, pindex->nHeight, pindex->nTime, useNextPrice, requireLivePrice);

// Time to liquidate vault.
vault->isUnderLiquidation = true;
cache.StoreVault(vaultId, *vault);
auto loanTokens = cache.GetLoanTokens(vaultId);
assert(loanTokens);

// Get the interest rate for each loan token in the vault, find
// the interest value and move it to the totals, removing it from the
// vault, while also stopping the vault from accumulating interest
// further. Note, however, it's added back so that it's accurate
// for auction calculations.
CBalances totalInterest;
for (auto it = loanTokens->balances.begin(); it != loanTokens->balances.end();) {
const auto &[tokenId, tokenValue] = *it;

auto rate = cache.GetInterestRate(vaultId, tokenId, pindex->nHeight);
assert(rate);

auto subInterest = TotalInterest(*rate, pindex->nHeight);
if (subInterest > 0) {
totalInterest.Add({tokenId, subInterest});
}
if (!collateral) {
errorAndShutdown("unexpected collateral");
return;
}

// Remove loan from the vault
cache.SubLoanToken(vaultId, {tokenId, tokenValue});
auto vault = cache.GetVault(vaultId);
if (!vault) {
errorAndShutdown("unexpected vault");
return;
}
auto scheme = cache.GetLoanScheme(vault->schemeId);
if (!scheme) {
errorAndShutdown("unexpected scheme");
return;
}

if (const auto token = cache.GetToken("DUSD"); token && token->first == tokenId) {
TrackDUSDSub(cache, {tokenId, tokenValue});
}
if (scheme->ratio <= collateral.val->ratio()) {
// All good, within ratio, nothing more to do.
markCompleted();
return;
}

// Remove interest from the vault
cache.DecreaseInterest(pindex->nHeight, vaultId, vault->schemeId, tokenId, tokenValue,
subInterest < 0 || (!subInterest && rate->interestPerBlock.negative) ? std::numeric_limits<CAmount>::max() : subInterest);
auto result = resultsPool.Acquire();
result->push_back(VaultWithCollateralInfo{vaultId, assets, collateral, *vault, *scheme});
resultsPool.Release(result);
markCompleted();
});
return true;
});

// Putting this back in now for auction calculations.
it->second += subInterest;
if (g.IsCancelled()) {
StartShutdown();
return;
}

g.WaitForCompletion(true);


for (auto &result: resultsPool.GetBuffer()) {
for (auto &v: *result) {
auto collaterals = v.collaterals;
auto collateral = v.vaultAssets;
auto vault = v.vault;
auto vaultId = v.vaultId;

// Time to liquidate vault.
vault.isUnderLiquidation = true;
cache.StoreVault(vaultId, vault);
auto loanTokens = cache.GetLoanTokens(vaultId);
assert(loanTokens);

// Get the interest rate for each loan token in the vault, find
// the interest value and move it to the totals, removing it from the
// vault, while also stopping the vault from accumulating interest
// further. Note, however, it's added back so that it's accurate
// for auction calculations.
CBalances totalInterest;
for (auto it = loanTokens->balances.begin(); it != loanTokens->balances.end();) {
const auto &[tokenId, tokenValue] = *it;

auto rate = cache.GetInterestRate(vaultId, tokenId, pindex->nHeight);
assert(rate);

auto subInterest = TotalInterest(*rate, pindex->nHeight);
if (subInterest > 0) {
totalInterest.Add({tokenId, subInterest});
}

// If loan amount fully negated then remove it
if (it->second < 0) {
// Remove loan from the vault
cache.SubLoanToken(vaultId, {tokenId, tokenValue});

TrackNegativeInterest(cache, {tokenId, tokenValue});
if (const auto token = cache.GetToken("DUSD"); token && token->first == tokenId) {
TrackDUSDSub(cache, {tokenId, tokenValue});
}

it = loanTokens->balances.erase(it);
} else {
// Remove interest from the vault
cache.DecreaseInterest(pindex->nHeight,
vaultId,
vault.schemeId,
tokenId,
tokenValue,
subInterest < 0 || (!subInterest && rate->interestPerBlock.negative)
? std::numeric_limits<CAmount>::max()
: subInterest);

// Putting this back in now for auction calculations.
it->second += subInterest;

// If loan amount fully negated then remove it
if (it->second < 0) {
TrackNegativeInterest(cache, {tokenId, tokenValue});

it = loanTokens->balances.erase(it);
} else {
if (subInterest < 0) {
TrackNegativeInterest(cache, {tokenId, std::abs(subInterest)});
}

if (subInterest < 0) {
TrackNegativeInterest(cache, {tokenId, std::abs(subInterest)});
++it;
}

++it;
}
}

// Remove the collaterals out of the vault.
// (Prep to get the auction batches instead)
for (const auto& col : collaterals.balances) {
auto tokenId = col.first;
auto tokenValue = col.second;
cache.SubVaultCollateral(vaultId, {tokenId, tokenValue});
}
// Remove the collaterals out of the vault.
// (Prep to get the auction batches instead)
for (const auto &col : collaterals.balances) {
auto tokenId = col.first;
auto tokenValue = col.second;
cache.SubVaultCollateral(vaultId, {tokenId, tokenValue});
}

auto batches = CollectAuctionBatches(*collateral.val, collaterals.balances, loanTokens->balances);

// Now, let's add the remaining amounts and store the batch.
CBalances totalLoanInBatches{};
for (auto i = 0u; i < batches.size(); i++) {
auto& batch = batches[i];
totalLoanInBatches.Add(batch.loanAmount);
auto tokenId = batch.loanAmount.nTokenId;
auto interest = totalInterest.balances[tokenId];
if (interest > 0) {
auto balance = loanTokens->balances[tokenId];
auto interestPart = DivideAmounts(batch.loanAmount.nValue, balance);
batch.loanInterest = MultiplyAmounts(interestPart, interest);
totalLoanInBatches.Sub({tokenId, batch.loanInterest});
auto batches = CollectAuctionBatches(collateral, collaterals.balances, loanTokens->balances);

// Now, let's add the remaining amounts and store the batch.
CBalances totalLoanInBatches{};
for (auto i = 0u; i < batches.size(); i++) {
auto &batch = batches[i];
totalLoanInBatches.Add(batch.loanAmount);
auto tokenId = batch.loanAmount.nTokenId;
auto interest = totalInterest.balances[tokenId];
if (interest > 0) {
auto balance = loanTokens->balances[tokenId];
auto interestPart = DivideAmounts(batch.loanAmount.nValue, balance);
batch.loanInterest = MultiplyAmounts(interestPart, interest);
totalLoanInBatches.Sub({tokenId, batch.loanInterest});
}
cache.StoreAuctionBatch({vaultId, i}, batch);
}
cache.StoreAuctionBatch({vaultId, i}, batch);
}

// Check if more than loan amount was generated.
CBalances balances;
for (const auto& [tokenId, amount] : loanTokens->balances) {
if (totalLoanInBatches.balances.count(tokenId)) {
const auto interest = totalInterest.balances.count(tokenId) ? totalInterest.balances[tokenId] : 0;
if (totalLoanInBatches.balances[tokenId] > amount - interest) {
balances.Add({tokenId, totalLoanInBatches.balances[tokenId] - (amount - interest)});
// Check if more than loan amount was generated.
CBalances balances;
for (const auto &[tokenId, amount] : loanTokens->balances) {
if (totalLoanInBatches.balances.count(tokenId)) {
const auto interest =
totalInterest.balances.count(tokenId) ? totalInterest.balances[tokenId] : 0;
if (totalLoanInBatches.balances[tokenId] > amount - interest) {
balances.Add({tokenId, totalLoanInBatches.balances[tokenId] - (amount - interest)});
}
}
}
}

// Only store to attributes if there has been a rounding error.
if (!balances.balances.empty()) {
TrackLiveBalances(cache, balances, EconomyKeys::BatchRoundingExcess);
}
// Only store to attributes if there has been a rounding error.
if (!balances.balances.empty()) {
TrackLiveBalances(cache, balances, EconomyKeys::BatchRoundingExcess);
}

// All done. Ready to save the overall auction.
cache.StoreAuction(vaultId, CAuctionData{
uint32_t(batches.size()),
pindex->nHeight + chainparams.GetConsensus().blocksCollateralAuction(),
cache.GetLoanLiquidationPenalty()
});
// All done. Ready to save the overall auction.
cache.StoreAuction(vaultId,
CAuctionData{uint32_t(batches.size()),
pindex->nHeight + chainparams.GetConsensus().blocksCollateralAuction(),
cache.GetLoanLiquidationPenalty()});

// Store state in vault DB
if (pvaultHistoryDB) {
pvaultHistoryDB->WriteVaultState(cache, *pindex, vaultId, collateral.val->ratio());
// Store state in vault DB
if (pvaultHistoryDB) {
pvaultHistoryDB->WriteVaultState(cache, *pindex, vaultId, collateral.ratio());
}
}

return true;
});
}
}

CAccountsHistoryWriter view(cache, pindex->nHeight, ~0u, pindex->GetBlockHash(), uint8_t(CustomTxType::AuctionBid));
Expand Down