From 87b1dfc6ae398222f55d0a73f4fc4666685b3f4a Mon Sep 17 00:00:00 2001 From: Pawel Jakubas Date: Mon, 17 Aug 2020 10:41:02 +0200 Subject: [PATCH 1/5] use hardfork infrastructure in latency --- lib/shelley/bench/Latency.hs | 79 ++++++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/lib/shelley/bench/Latency.hs b/lib/shelley/bench/Latency.hs index 8880d050531..4aa067f89dc 100644 --- a/lib/shelley/bench/Latency.hs +++ b/lib/shelley/bench/Latency.hs @@ -1,6 +1,7 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE LambdaCase #-} +{-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedLabels #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE ScopedTypeVariables #-} @@ -36,10 +37,13 @@ import Cardano.Wallet.Api.Types , ApiTransaction , ApiUtxoStatistics , ApiWallet + , EncodeAddress (..) , WalletStyle (..) ) import Cardano.Wallet.LatencyBenchShared ( LogCaptureFunc, fmtResult, fmtTitle, measureApiLogs, withLatencyLogging ) +import Cardano.Wallet.Logging + ( stdoutTextTracer ) import Cardano.Wallet.Logging ( trMessage ) import Cardano.Wallet.Network.Ports @@ -62,7 +66,17 @@ import Cardano.Wallet.Shelley.Compatibility import Cardano.Wallet.Shelley.Faucet ( initFaucet ) import Cardano.Wallet.Shelley.Launch - ( singleNodeParams, withBFTNode, withSystemTempDir ) + ( PoolConfig (..) + , RunningNode (..) + , moveInstantaneousRewardsTo + , oneMillionAda + , sendFaucetFundsTo + , withCluster + , withSystemTempDir + , withTempDir + ) +import Control.Arrow + ( first ) import Control.Concurrent.Async ( race_ ) import Control.Concurrent.MVar @@ -70,7 +84,7 @@ import Control.Concurrent.MVar import Control.Concurrent.STM.TVar ( TVar ) import Control.Monad - ( mapM_, replicateM, replicateM_, void ) + ( mapM_, replicateM, replicateM_ ) import Data.Generics.Internal.VL.Lens ( (^.) ) import Data.Proxy @@ -87,10 +101,14 @@ import Numeric.Natural ( Natural ) import System.Directory ( createDirectory ) +import System.Environment + ( lookupEnv ) import System.FilePath ( () ) import Test.Hspec ( shouldBe ) +import Test.Integration.Faucet + ( genRewardAccounts, mirMnemonics, shelleyIntegrationTestFunds ) import Test.Integration.Framework.DSL ( Context (..) , Headers (..) @@ -135,16 +153,16 @@ walletApiBench capture benchWithServer = do fmtTitle "Latencies for 2 fixture wallets scenario" runScenario (nFixtureWallet 2) - +{-- fmtTitle "Latencies for 10 fixture wallets scenario" runScenario (nFixtureWallet 10) fmtTitle "Latencies for 100 fixture wallets scenario" runScenario (nFixtureWallet 100) - +--} fmtTitle "Latencies for 2 fixture wallets with 10 txs scenario" runScenario (nFixtureWalletWithTxs 2 10) - +{-- fmtTitle "Latencies for 2 fixture wallets with 20 txs scenario" runScenario (nFixtureWalletWithTxs 2 20) @@ -159,10 +177,10 @@ walletApiBench capture benchWithServer = do fmtTitle "Latencies for 10 fixture wallets with 100 txs scenario" runScenario (nFixtureWalletWithTxs 10 100) - +--} fmtTitle "Latencies for 2 fixture wallets with 100 utxos scenario" runScenario (nFixtureWalletWithUTxOs 2 100) - +{-- fmtTitle "Latencies for 2 fixture wallets with 200 utxos scenario" runScenario (nFixtureWalletWithUTxOs 2 200) @@ -171,6 +189,7 @@ walletApiBench capture benchWithServer = do fmtTitle "Latencies for 2 fixture wallets with 1000 utxos scenario" runScenario (nFixtureWalletWithUTxOs 2 1000) +--} where -- Creates n fixture wallets and return two of them @@ -345,11 +364,31 @@ benchWithShelleyServer tracers action = do where withServer act = withSystemTempDir nullTracer "latency" $ \dir -> do - params <- singleNodeParams Error - let db = dir "wallets" - createDirectory db - withBFTNode nullTracer dir params $ \socketPath block0 (gp, vData) -> - void $ serveWallet + testPoolConfigs' <- poolConfigsFromEnv + let db = dir "wallets" + createDirectory db + withCluster + nullTracer + Error + testPoolConfigs' + dir + onByron + (afterFork dir) + (onClusterStart act dir) + onByron _ = pure () + afterFork dir _ = do + let encodeAddr = T.unpack . encodeAddress @'Mainnet + let addresses = map (first encodeAddr) shelleyIntegrationTestFunds + sendFaucetFundsTo stdoutTextTracer dir addresses + let rewards = (,Coin $ fromIntegral oneMillionAda) <$> + concatMap genRewardAccounts mirMnemonics + moveInstantaneousRewardsTo stdoutTextTracer dir rewards + + onClusterStart act dir (RunningNode socketPath block0 (gp, vData)) = do + -- NOTE: We may want to keep a wallet running across the fork, but + -- having three callbacks like this might not work well for that. + withTempDir nullTracer dir "wallets" $ \db -> do + serveWallet @(IO Shelley) (SomeNetworkDiscriminant $ Proxy @'Mainnet) tracers (SyncTolerance 10) @@ -362,3 +401,19 @@ benchWithShelleyServer tracers action = do block0 (gp, vData) (act gp) + + poolConfigsFromEnv = lookupEnv "NO_POOLS" >>= \case + Nothing -> pure testPoolConfigs + Just "" -> pure testPoolConfigs + Just _ -> pure [] + + testPoolConfigs = + [ -- This pool should never retire: + PoolConfig {retirementEpoch = Nothing} + -- This pool should retire almost immediately: + , PoolConfig {retirementEpoch = Just 3} + -- This pool should retire, but not within the duration of a test run: + , PoolConfig {retirementEpoch = Just 100_000} + -- This pool should retire, but not within the duration of a test run: + , PoolConfig {retirementEpoch = Just 1_000_000} + ] From 648ca13877b5b5cfc42eaa38b485abcdc460c32b Mon Sep 17 00:00:00 2001 From: Pawel Jakubas Date: Mon, 17 Aug 2020 11:15:31 +0200 Subject: [PATCH 2/5] slim cluster warm up --- lib/shelley/bench/Latency.hs | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/lib/shelley/bench/Latency.hs b/lib/shelley/bench/Latency.hs index 4aa067f89dc..7d17f6d983b 100644 --- a/lib/shelley/bench/Latency.hs +++ b/lib/shelley/bench/Latency.hs @@ -66,10 +66,7 @@ import Cardano.Wallet.Shelley.Compatibility import Cardano.Wallet.Shelley.Faucet ( initFaucet ) import Cardano.Wallet.Shelley.Launch - ( PoolConfig (..) - , RunningNode (..) - , moveInstantaneousRewardsTo - , oneMillionAda + ( RunningNode (..) , sendFaucetFundsTo , withCluster , withSystemTempDir @@ -101,14 +98,12 @@ import Numeric.Natural ( Natural ) import System.Directory ( createDirectory ) -import System.Environment - ( lookupEnv ) import System.FilePath ( () ) import Test.Hspec ( shouldBe ) import Test.Integration.Faucet - ( genRewardAccounts, mirMnemonics, shelleyIntegrationTestFunds ) + ( shelleyIntegrationTestFunds ) import Test.Integration.Framework.DSL ( Context (..) , Headers (..) @@ -364,13 +359,12 @@ benchWithShelleyServer tracers action = do where withServer act = withSystemTempDir nullTracer "latency" $ \dir -> do - testPoolConfigs' <- poolConfigsFromEnv let db = dir "wallets" createDirectory db withCluster nullTracer Error - testPoolConfigs' + [] dir onByron (afterFork dir) @@ -380,9 +374,6 @@ benchWithShelleyServer tracers action = do let encodeAddr = T.unpack . encodeAddress @'Mainnet let addresses = map (first encodeAddr) shelleyIntegrationTestFunds sendFaucetFundsTo stdoutTextTracer dir addresses - let rewards = (,Coin $ fromIntegral oneMillionAda) <$> - concatMap genRewardAccounts mirMnemonics - moveInstantaneousRewardsTo stdoutTextTracer dir rewards onClusterStart act dir (RunningNode socketPath block0 (gp, vData)) = do -- NOTE: We may want to keep a wallet running across the fork, but @@ -401,19 +392,3 @@ benchWithShelleyServer tracers action = do block0 (gp, vData) (act gp) - - poolConfigsFromEnv = lookupEnv "NO_POOLS" >>= \case - Nothing -> pure testPoolConfigs - Just "" -> pure testPoolConfigs - Just _ -> pure [] - - testPoolConfigs = - [ -- This pool should never retire: - PoolConfig {retirementEpoch = Nothing} - -- This pool should retire almost immediately: - , PoolConfig {retirementEpoch = Just 3} - -- This pool should retire, but not within the duration of a test run: - , PoolConfig {retirementEpoch = Just 100_000} - -- This pool should retire, but not within the duration of a test run: - , PoolConfig {retirementEpoch = Just 1_000_000} - ] From 9ad87639475f879efebe84ce13c521f87fe466d1 Mon Sep 17 00:00:00 2001 From: KtorZ Date: Mon, 24 Aug 2020 14:30:47 +0200 Subject: [PATCH 3/5] fix HLint suggestion about redundant imports --- lib/shelley/bench/Latency.hs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/shelley/bench/Latency.hs b/lib/shelley/bench/Latency.hs index 7d17f6d983b..61b32d0b410 100644 --- a/lib/shelley/bench/Latency.hs +++ b/lib/shelley/bench/Latency.hs @@ -43,9 +43,7 @@ import Cardano.Wallet.Api.Types import Cardano.Wallet.LatencyBenchShared ( LogCaptureFunc, fmtResult, fmtTitle, measureApiLogs, withLatencyLogging ) import Cardano.Wallet.Logging - ( stdoutTextTracer ) -import Cardano.Wallet.Logging - ( trMessage ) + ( stdoutTextTracer, trMessage ) import Cardano.Wallet.Network.Ports ( unsafePortNumber ) import Cardano.Wallet.Primitive.AddressDerivation From b62cbe32fce1996c79986e1109e65c3afd2cc14e Mon Sep 17 00:00:00 2001 From: KtorZ Date: Mon, 24 Aug 2020 22:15:46 +0200 Subject: [PATCH 4/5] start local cluster only once in latency benchmarks Previously, was started before each bench although, some state was shared in-memory causing issues for the second bench to run. --- lib/shelley/bench/Latency.hs | 66 +++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/lib/shelley/bench/Latency.hs b/lib/shelley/bench/Latency.hs index 61b32d0b410..f6de69067c7 100644 --- a/lib/shelley/bench/Latency.hs +++ b/lib/shelley/bench/Latency.hs @@ -128,8 +128,8 @@ import qualified Network.HTTP.Types.Status as HTTP main :: forall t n. (t ~ Shelley, n ~ 'Mainnet) => IO () main = withUtf8Encoding $ withLatencyLogging setupTracers $ \tracers capture -> - walletApiBench @t @n capture (benchWithShelleyServer tracers) - + withShelleyServer tracers $ \ctx -> do + walletApiBench @t @n capture ctx where setupTracers :: TVar [LogObject ApiLog] -> Tracers IO setupTracers tvar = nullTracers @@ -138,24 +138,27 @@ main = withUtf8Encoding $ walletApiBench :: forall t (n :: NetworkDiscriminant). (t ~ Shelley, n ~ 'Mainnet) => LogCaptureFunc ApiLog () - -> ((Context t -> IO ()) -> IO ()) + -> Context t -> IO () -walletApiBench capture benchWithServer = do +walletApiBench capture ctx = do fmtTitle "Non-cached run" runWarmUpScenario fmtTitle "Latencies for 2 fixture wallets scenario" runScenario (nFixtureWallet 2) -{-- + fmtTitle "Latencies for 10 fixture wallets scenario" runScenario (nFixtureWallet 10) + {-- PENDING: We currently have a limited amount of available fixture + wallets, so we can't just run a benchmark with 100 wallets in parallel. fmtTitle "Latencies for 100 fixture wallets scenario" runScenario (nFixtureWallet 100) ---} + --} + fmtTitle "Latencies for 2 fixture wallets with 10 txs scenario" runScenario (nFixtureWalletWithTxs 2 10) -{-- + fmtTitle "Latencies for 2 fixture wallets with 20 txs scenario" runScenario (nFixtureWalletWithTxs 2 20) @@ -170,23 +173,30 @@ walletApiBench capture benchWithServer = do fmtTitle "Latencies for 10 fixture wallets with 100 txs scenario" runScenario (nFixtureWalletWithTxs 10 100) ---} + fmtTitle "Latencies for 2 fixture wallets with 100 utxos scenario" runScenario (nFixtureWalletWithUTxOs 2 100) -{-- + fmtTitle "Latencies for 2 fixture wallets with 200 utxos scenario" runScenario (nFixtureWalletWithUTxOs 2 200) fmtTitle "Latencies for 2 fixture wallets with 500 utxos scenario" runScenario (nFixtureWalletWithUTxOs 2 500) + {-- PENDING: Fee estimation is taking way longer than it should and this + scenario is not resolving in a timely manner. + + To be re-enabled once #2006 & #2051 are fixed. + fmtTitle "Latencies for 2 fixture wallets with 1000 utxos scenario" runScenario (nFixtureWalletWithUTxOs 2 1000) ---} + --} + fmtTitle "Latencies for 2 fixture wallets with 1000 utxos scenario" + fmtTitle "CURRENTLY DISABLED. SEE #2006 & #2051" where -- Creates n fixture wallets and return two of them - nFixtureWallet n ctx = do + nFixtureWallet n = do wal1 : wal2 : _ <- replicateM n (fixtureWallet ctx) pure (wal1, wal2) @@ -195,8 +205,8 @@ walletApiBench capture benchWithServer = do -- additionally created source fixture wallet. Then we wait for the money -- to be accommodated in recipient wallet. After that the source fixture -- wallet is removed. - nFixtureWalletWithTxs n m ctx = do - (wal1, wal2) <- nFixtureWallet n ctx + nFixtureWalletWithTxs n m = do + (wal1, wal2) <- nFixtureWallet n let amt = (1 :: Natural) let batchSize = 10 @@ -210,13 +220,13 @@ walletApiBench capture benchWithServer = do [lastBit] let expInflows' = filter (/=0) expInflows - mapM_ (repeatPostTx ctx wal1 amt batchSize . amtExp) expInflows' + mapM_ (repeatPostTx wal1 amt batchSize . amtExp) expInflows' pure (wal1, wal2) - nFixtureWalletWithUTxOs n utxoNumber ctx = do + nFixtureWalletWithUTxOs n utxoNumber = do let utxoExp = replicate utxoNumber 1 wal1 <- fixtureWalletWith @n ctx utxoExp - (_, wal2) <- nFixtureWallet n ctx + (_, wal2) <- nFixtureWallet n eventually "Wallet balance is as expected" $ do rWal1 <- request @ApiWallet ctx @@ -234,10 +244,10 @@ walletApiBench capture benchWithServer = do expectWalletUTxO (fromIntegral <$> utxoExp) (snd rStat) pure (wal1, wal2) - repeatPostTx ctx wDest amtToSend batchSize amtExp = do + repeatPostTx wDest amtToSend batchSize amtExp = do wSrc <- fixtureWallet ctx replicateM_ batchSize - (postTx ctx (wSrc, Link.createTransaction @'Shelley, fixturePassphrase) wDest amtToSend) + (postTx (wSrc, Link.createTransaction @'Shelley, fixturePassphrase) wDest amtToSend) eventually "repeatPostTx: wallet balance is as expected" $ do rWal1 <- request @ApiWallet ctx (Link.getWallet @'Shelley wDest) Default Empty verify rWal1 @@ -250,9 +260,9 @@ walletApiBench capture benchWithServer = do expectResponseCode @IO HTTP.status204 rDel pure () - postTx ctx (wSrc, postTxEndp, pass) wDest amt = do + postTx (wSrc, postTxEndp, pass) wDest amt = do (_, addrs) <- unsafeRequest @[ApiAddress n] ctx - (Link.listAddresses @'Shelley wDest) Empty + (Link.listAddresses @'Shelley wDest) Empty let destination = (addrs !! 1) ^. #id let payload = Json [json|{ "payments": [{ @@ -268,8 +278,8 @@ walletApiBench capture benchWithServer = do expectResponseCode HTTP.status202 r return r - runScenario scenario = benchWithServer $ \ctx -> do - (wal1, wal2) <- scenario ctx + runScenario scenario = do + (wal1, wal2) <- scenario t1 <- measureApiLogs capture (request @[ApiWallet] ctx (Link.listWallets @'Shelley) Default Empty) @@ -318,11 +328,11 @@ walletApiBench capture benchWithServer = do pure () where - arbitraryStake :: Maybe Coin - arbitraryStake = Just $ ada 10000 - where ada = Coin . (1000*1000*) + arbitraryStake :: Maybe Coin + arbitraryStake = Just $ ada 10000 + where ada = Coin . (1000*1000*) - runWarmUpScenario = benchWithServer $ \ctx -> do + runWarmUpScenario = do -- this one is to have comparable results from first to last measurement -- in runScenario t <- measureApiLogs capture $ request @ApiNetworkInformation ctx @@ -330,11 +340,11 @@ walletApiBench capture benchWithServer = do fmtResult "getNetworkInfo " t pure () -benchWithShelleyServer +withShelleyServer :: Tracers IO -> (Context Shelley -> IO ()) -> IO () -benchWithShelleyServer tracers action = do +withShelleyServer tracers action = do ctx <- newEmptyMVar let setupContext np wAddr = do let baseUrl = "http://" <> T.pack (show wAddr) <> "/" From 882a921201b940a75cc00837665166035e52a595 Mon Sep 17 00:00:00 2001 From: KtorZ Date: Tue, 25 Aug 2020 13:09:59 +0200 Subject: [PATCH 5/5] tentatively attempt to provide 'cardano-cli' in scope for the latency benchmarks --- nix/haskell.nix | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nix/haskell.nix b/nix/haskell.nix index 8d5975c831b..8688e762b44 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -111,7 +111,19 @@ let # Add jormungandr to the PATH of the latency benchmark packages.cardano-wallet-jormungandr.components.benchmarks.latency = wrapBench jmPkgs.jormungandr; - packages.cardano-wallet.components.benchmarks.latency = wrapBench pkgs.cardano-node; + packages.cardano-wallet.components.benchmarks.latency = + lib.optionalAttrs (!stdenv.hostPlatform.isWindows) { + build-tools = [ pkgs.makeWrapper ]; + postInstall = '' + wrapProgram $out/bin/* \ + --run "cd $src" \ + --prefix PATH : ${pkgs.cardano-node}/bin + + wrapProgram $out/bin/* \ + --run "cd $src" \ + --prefix PATH : ${pkgs.cardano-cli}/bin + ''; + }; # Add cardano-node to the PATH of the byroon restore benchmark. # cardano-node will want to write logs to a subdirectory of the working directory.