From c30527e4c23d1386d05ddd4dc95482480d3f76bc Mon Sep 17 00:00:00 2001 From: Hai Nguyen Quang Date: Mon, 15 Nov 2021 19:47:13 +0700 Subject: [PATCH] Chain Index: users can configure to only store txs from a block no onward (#72) * ChainIndex: users can configure to only store txs from a block no onward * Add BlockProcessOption to make appendBlock calls clearer. Also prep for future options. * plutus-chain-index-core: add BlockProcessOption tests. Tighten and start sharing code among existing tests as well. * updateMaterialized --- .../.plan.nix/plutus-chain-index-core.nix | 2 + .../.plan.nix/plutus-chain-index-core.nix | 2 + .../.plan.nix/plutus-chain-index-core.nix | 2 + .../plutus-chain-index-core.cabal | 3 +- .../src/Plutus/ChainIndex/Effects.hs | 4 +- .../Plutus/ChainIndex/Emulator/Handlers.hs | 8 ++- .../src/Plutus/ChainIndex/Handlers.hs | 8 ++- .../src/Plutus/ChainIndex/Types.hs | 17 +++++ .../ChainIndex/Emulator/HandlersSpec.hs | 62 ++++++++++-------- .../test/Plutus/ChainIndex/HandlersSpec.hs | 65 +++++++++++-------- plutus-chain-index-core/test/Util.hs | 27 ++++++++ .../src/Plutus/ChainIndex/App.hs | 30 +++++---- .../src/Plutus/ChainIndex/Config.hs | 15 +++-- .../src/Plutus/Trace/Emulator/System.hs | 3 +- .../test-node/testnet/chain-index-config.json | 3 + 15 files changed, 171 insertions(+), 80 deletions(-) create mode 100644 plutus-chain-index-core/test/Util.hs diff --git a/nix/pkgs/haskell/materialized-darwin/.plan.nix/plutus-chain-index-core.nix b/nix/pkgs/haskell/materialized-darwin/.plan.nix/plutus-chain-index-core.nix index b6e00c18f8..d6096b3327 100644 --- a/nix/pkgs/haskell/materialized-darwin/.plan.nix/plutus-chain-index-core.nix +++ b/nix/pkgs/haskell/materialized-darwin/.plan.nix/plutus-chain-index-core.nix @@ -123,6 +123,7 @@ (hsPkgs."bytestring" or (errorHandler.buildDepError "bytestring")) (hsPkgs."containers" or (errorHandler.buildDepError "containers")) (hsPkgs."contra-tracer" or (errorHandler.buildDepError "contra-tracer")) + (hsPkgs."data-default" or (errorHandler.buildDepError "data-default")) (hsPkgs."fingertree" or (errorHandler.buildDepError "fingertree")) (hsPkgs."freer-extras" or (errorHandler.buildDepError "freer-extras")) (hsPkgs."freer-simple" or (errorHandler.buildDepError "freer-simple")) @@ -140,6 +141,7 @@ "Plutus/ChainIndex/Emulator/DiskStateSpec" "Plutus/ChainIndex/Emulator/HandlersSpec" "Plutus/ChainIndex/HandlersSpec" + "Util" ]; hsSourceDirs = [ "test" ]; mainPath = [ "Spec.hs" ]; diff --git a/nix/pkgs/haskell/materialized-linux/.plan.nix/plutus-chain-index-core.nix b/nix/pkgs/haskell/materialized-linux/.plan.nix/plutus-chain-index-core.nix index b6e00c18f8..d6096b3327 100644 --- a/nix/pkgs/haskell/materialized-linux/.plan.nix/plutus-chain-index-core.nix +++ b/nix/pkgs/haskell/materialized-linux/.plan.nix/plutus-chain-index-core.nix @@ -123,6 +123,7 @@ (hsPkgs."bytestring" or (errorHandler.buildDepError "bytestring")) (hsPkgs."containers" or (errorHandler.buildDepError "containers")) (hsPkgs."contra-tracer" or (errorHandler.buildDepError "contra-tracer")) + (hsPkgs."data-default" or (errorHandler.buildDepError "data-default")) (hsPkgs."fingertree" or (errorHandler.buildDepError "fingertree")) (hsPkgs."freer-extras" or (errorHandler.buildDepError "freer-extras")) (hsPkgs."freer-simple" or (errorHandler.buildDepError "freer-simple")) @@ -140,6 +141,7 @@ "Plutus/ChainIndex/Emulator/DiskStateSpec" "Plutus/ChainIndex/Emulator/HandlersSpec" "Plutus/ChainIndex/HandlersSpec" + "Util" ]; hsSourceDirs = [ "test" ]; mainPath = [ "Spec.hs" ]; diff --git a/nix/pkgs/haskell/materialized-windows/.plan.nix/plutus-chain-index-core.nix b/nix/pkgs/haskell/materialized-windows/.plan.nix/plutus-chain-index-core.nix index b6e00c18f8..d6096b3327 100644 --- a/nix/pkgs/haskell/materialized-windows/.plan.nix/plutus-chain-index-core.nix +++ b/nix/pkgs/haskell/materialized-windows/.plan.nix/plutus-chain-index-core.nix @@ -123,6 +123,7 @@ (hsPkgs."bytestring" or (errorHandler.buildDepError "bytestring")) (hsPkgs."containers" or (errorHandler.buildDepError "containers")) (hsPkgs."contra-tracer" or (errorHandler.buildDepError "contra-tracer")) + (hsPkgs."data-default" or (errorHandler.buildDepError "data-default")) (hsPkgs."fingertree" or (errorHandler.buildDepError "fingertree")) (hsPkgs."freer-extras" or (errorHandler.buildDepError "freer-extras")) (hsPkgs."freer-simple" or (errorHandler.buildDepError "freer-simple")) @@ -140,6 +141,7 @@ "Plutus/ChainIndex/Emulator/DiskStateSpec" "Plutus/ChainIndex/Emulator/HandlersSpec" "Plutus/ChainIndex/HandlersSpec" + "Util" ]; hsSourceDirs = [ "test" ]; mainPath = [ "Spec.hs" ]; diff --git a/plutus-chain-index-core/plutus-chain-index-core.cabal b/plutus-chain-index-core/plutus-chain-index-core.cabal index 5e9612334b..fa5da361b2 100644 --- a/plutus-chain-index-core/plutus-chain-index-core.cabal +++ b/plutus-chain-index-core/plutus-chain-index-core.cabal @@ -94,7 +94,6 @@ library typed-protocols -any, unordered-containers -any, bytestring -any, - data-default -any, text -any, servant -any, servant-server -any, @@ -116,6 +115,7 @@ test-suite plutus-chain-index-test Plutus.ChainIndex.Emulator.DiskStateSpec Plutus.ChainIndex.Emulator.HandlersSpec Plutus.ChainIndex.HandlersSpec + Util build-depends: plutus-ledger -any, plutus-ledger-api -any, @@ -128,6 +128,7 @@ test-suite plutus-chain-index-test bytestring -any, containers -any, contra-tracer -any, + data-default -any, fingertree -any, freer-extras -any, freer-simple -any, diff --git a/plutus-chain-index-core/src/Plutus/ChainIndex/Effects.hs b/plutus-chain-index-core/src/Plutus/ChainIndex/Effects.hs index e525a0a7f8..e5b1f08043 100644 --- a/plutus-chain-index-core/src/Plutus/ChainIndex/Effects.hs +++ b/plutus-chain-index-core/src/Plutus/ChainIndex/Effects.hs @@ -33,7 +33,7 @@ import Ledger (AssetClass, Datum, DatumHash, MintingPolicy, MintingPolicyHash, R import Ledger.Credential (Credential) import Ledger.Tx (ChainIndexTxOut, TxOutRef) import Plutus.ChainIndex.Tx (ChainIndexTx) -import Plutus.ChainIndex.Types (Diagnostics, Point, Tip) +import Plutus.ChainIndex.Types (BlockProcessOption, Diagnostics, Point, Tip) data ChainIndexQueryEffect r where @@ -78,7 +78,7 @@ makeEffect ''ChainIndexQueryEffect data ChainIndexControlEffect r where -- | Add a new block to the chain index by giving a new tip and list of tx. - AppendBlock :: Tip -> [ChainIndexTx] -> ChainIndexControlEffect () + AppendBlock :: Tip -> [ChainIndexTx] -> BlockProcessOption -> ChainIndexControlEffect () -- | Roll back to a previous state (previous tip) Rollback :: Point -> ChainIndexControlEffect () diff --git a/plutus-chain-index-core/src/Plutus/ChainIndex/Emulator/Handlers.hs b/plutus-chain-index-core/src/Plutus/ChainIndex/Emulator/Handlers.hs index 9bdeee7c1c..998dbe2073 100644 --- a/plutus-chain-index-core/src/Plutus/ChainIndex/Emulator/Handlers.hs +++ b/plutus-chain-index-core/src/Plutus/ChainIndex/Emulator/Handlers.hs @@ -43,7 +43,8 @@ import Plutus.ChainIndex.Emulator.DiskState (DiskState, addressMap, assetClassMa import Plutus.ChainIndex.Emulator.DiskState qualified as DiskState import Plutus.ChainIndex.Tx (ChainIndexTx, _ValidTx, citxOutputs) import Plutus.ChainIndex.TxUtxoBalance qualified as TxUtxoBalance -import Plutus.ChainIndex.Types (Diagnostics (..), Point (PointAtGenesis), Tip (..), TxUtxoBalance (..)) +import Plutus.ChainIndex.Types (BlockProcessOption (..), Diagnostics (..), Point (PointAtGenesis), Tip (..), + TxUtxoBalance (..)) import Plutus.ChainIndex.UtxoState (InsertUtxoSuccess (..), RollbackResult (..), UtxoIndex, tip, utxoState) import Plutus.ChainIndex.UtxoState qualified as UtxoState import Plutus.V1.Ledger.Api (Credential (PubKeyCredential, ScriptCredential)) @@ -159,7 +160,7 @@ handleControl :: => ChainIndexControlEffect ~> Eff effs handleControl = \case - AppendBlock tip_ transactions -> do + AppendBlock tip_ transactions opts -> do oldState <- get @ChainIndexEmulatorState case UtxoState.insert (TxUtxoBalance.fromBlock tip_ transactions) (view utxoIndex oldState) of Left err -> do @@ -169,7 +170,8 @@ handleControl = \case Right InsertUtxoSuccess{newIndex, insertPosition} -> do put $ oldState & set utxoIndex newIndex - & over diskState (mappend $ foldMap DiskState.fromTx transactions) + & over diskState + (mappend $ foldMap DiskState.fromTx (if bpoStoreTxs opts then transactions else [])) logDebug $ InsertionSuccess tip_ insertPosition Rollback tip_ -> do oldState <- get @ChainIndexEmulatorState diff --git a/plutus-chain-index-core/src/Plutus/ChainIndex/Handlers.hs b/plutus-chain-index-core/src/Plutus/ChainIndex/Handlers.hs index 8f4eb44c5a..dd3780558b 100644 --- a/plutus-chain-index-core/src/Plutus/ChainIndex/Handlers.hs +++ b/plutus-chain-index-core/src/Plutus/ChainIndex/Handlers.hs @@ -21,6 +21,7 @@ module Plutus.ChainIndex.Handlers import Cardano.Api qualified as C import Control.Applicative (Const (..)) import Control.Lens (Lens', _Just, ix, view, (^?)) +import Control.Monad (when) import Control.Monad.Freer (Eff, Member, type (~>)) import Control.Monad.Freer.Error (Error, throwError) import Control.Monad.Freer.Extras.Beam (BeamEffect (..), BeamableSqlite, addRowsInBatches, combined, deleteRows, @@ -53,7 +54,8 @@ import Plutus.ChainIndex.DbSchema import Plutus.ChainIndex.Effects (ChainIndexControlEffect (..), ChainIndexQueryEffect (..)) import Plutus.ChainIndex.Tx import Plutus.ChainIndex.TxUtxoBalance qualified as TxUtxoBalance -import Plutus.ChainIndex.Types (Depth (..), Diagnostics (..), Point (..), Tip (..), TxUtxoBalance (..), tipAsPoint) +import Plutus.ChainIndex.Types (BlockProcessOption (..), Depth (..), Diagnostics (..), Point (..), Tip (..), + TxUtxoBalance (..), tipAsPoint) import Plutus.ChainIndex.UtxoState (InsertUtxoSuccess (..), RollbackResult (..), UtxoIndex) import Plutus.ChainIndex.UtxoState qualified as UtxoState import Plutus.V1.Ledger.Ada qualified as Ada @@ -239,7 +241,7 @@ handleControl :: => ChainIndexControlEffect ~> Eff effs handleControl = \case - AppendBlock tip_ transactions -> do + AppendBlock tip_ transactions opts -> do oldIndex <- get @ChainIndexState let newUtxoState = TxUtxoBalance.fromBlock tip_ transactions case UtxoState.insert newUtxoState oldIndex of @@ -254,7 +256,7 @@ handleControl = \case lbcResult -> do put $ UtxoState.reducedIndex lbcResult reduceOldUtxoDb $ UtxoState._usTip $ UtxoState.combinedState lbcResult - insert $ foldMap fromTx transactions + when (bpoStoreTxs opts) $ insert $ foldMap fromTx transactions insertUtxoDb newUtxoState logDebug $ InsertionSuccess tip_ insertPosition Rollback tip_ -> do diff --git a/plutus-chain-index-core/src/Plutus/ChainIndex/Types.hs b/plutus-chain-index-core/src/Plutus/ChainIndex/Types.hs index 84b83449f4..07bc1f1d4d 100644 --- a/plutus-chain-index-core/src/Plutus/ChainIndex/Types.hs +++ b/plutus-chain-index-core/src/Plutus/ChainIndex/Types.hs @@ -32,6 +32,7 @@ module Plutus.ChainIndex.Types( , TxOutBalance(..) , tobUnspentOutputs , tobSpentOutputs + , BlockProcessOption(..) ) where import Codec.Serialise (Serialise) @@ -42,6 +43,7 @@ import Crypto.Hash (SHA256, hash) import Data.Aeson (FromJSON, ToJSON) import Data.ByteArray qualified as BA import Data.ByteString.Lazy qualified as BSL +import Data.Default (Default (..)) import Data.Map (Map) import Data.Map qualified as Map import Data.Monoid (Last (..), Sum (..)) @@ -345,3 +347,18 @@ instance Semigroup TxUtxoBalance where instance Monoid TxUtxoBalance where mappend = (<>) mempty = TxUtxoBalance mempty mempty + +-- | User-customizable options to process a block. +-- See #73 for more motivations. +newtype BlockProcessOption = + BlockProcessOption + { bpoStoreTxs :: Bool + -- ^ Should the chain index store this batch of transactions or not. + -- If not, only handle the tip and UTXOs. + -- This, for example, allows applications to skip unwanted pre-Alonzo transactions. + } + +-- We should think twice when setting the default option. +-- For now, it should store all data to avoid weird non-backward-compatible bugs in the future. +instance Default BlockProcessOption where + def = BlockProcessOption True diff --git a/plutus-chain-index-core/test/Plutus/ChainIndex/Emulator/HandlersSpec.hs b/plutus-chain-index-core/test/Plutus/ChainIndex/Emulator/HandlersSpec.hs index b07e9f1473..0fe2876f26 100644 --- a/plutus-chain-index-core/test/Plutus/ChainIndex/Emulator/HandlersSpec.hs +++ b/plutus-chain-index-core/test/Plutus/ChainIndex/Emulator/HandlersSpec.hs @@ -8,28 +8,31 @@ module Plutus.ChainIndex.Emulator.HandlersSpec (tests) where import Control.Lens -import Control.Monad (forM, forM_) +import Control.Monad (forM) import Control.Monad.Freer (Eff, interpret, reinterpret, runM) import Control.Monad.Freer.Error (Error, runError) import Control.Monad.Freer.Extras (LogMessage, LogMsg (..), handleLogWriter) import Control.Monad.Freer.State (State, runState) import Control.Monad.Freer.Writer (runWriter) import Control.Monad.IO.Class (liftIO) +import Data.Default (def) import Data.Sequence (Seq) -import Data.Set (member) +import Data.Set qualified as S import Generators qualified as Gen -import Ledger (Address (Address, addressCredential), TxOut (TxOut, txOutAddress), outValue) +import Ledger (outValue) import Plutus.ChainIndex (ChainIndexLog, Page (pageItems), PageQuery (PageQuery), appendBlock, txFromTxId, - utxoSetAtAddress, utxoSetWithCurrency) + utxoSetMembership, utxoSetWithCurrency) import Plutus.ChainIndex.ChainIndexError (ChainIndexError) import Plutus.ChainIndex.Effects (ChainIndexControlEffect, ChainIndexQueryEffect) import Plutus.ChainIndex.Emulator.Handlers (ChainIndexEmulatorState, handleControl, handleQuery) import Plutus.ChainIndex.Tx (_ValidTx, citxOutputs, citxTxId) +import Plutus.ChainIndex.Types (BlockProcessOption (..)) import Plutus.V1.Ledger.Value (AssetClass (AssetClass), flattenValue) import Hedgehog (Property, assert, forAll, property, (===)) import Test.Tasty import Test.Tasty.Hedgehog (testProperty) +import Util (utxoSetFromBlockAddrs) tests :: TestTree tests = do @@ -44,6 +47,9 @@ tests = do [ testProperty "each txOutRef should be unspent" eachTxOutRefWithCurrencyShouldBeUnspentSpec , testProperty "should restrict to non-ADA currencies" cantRequestForTxOutRefsWithAdaSpec ] + , testGroup "BlockProcessOption" + [ testProperty "do not store txs" doNotStoreTxs + ] ] -- | Tests we can correctly query a tx in the database using a tx id. We also @@ -53,7 +59,7 @@ txFromTxIdSpec = property $ do (tip, block@(fstTx:_)) <- forAll $ Gen.evalTxGenState Gen.genNonEmptyBlock unknownTxId <- forAll Gen.genRandomTxId txs <- liftIO $ runEmulatedChainIndex mempty $ do - appendBlock tip block + appendBlock tip block def tx <- txFromTxId (view citxTxId fstTx) tx' <- txFromTxId unknownTxId pure (tx, tx') @@ -69,24 +75,14 @@ eachTxOutRefAtAddressShouldBeUnspentSpec :: Property eachTxOutRefAtAddressShouldBeUnspentSpec = property $ do ((tip, block), state) <- forAll $ Gen.runTxGenState Gen.genNonEmptyBlock - let addresses = - fmap (\TxOut { txOutAddress = Address { addressCredential }} -> addressCredential) - $ view (traverse . citxOutputs . _ValidTx) block - result <- liftIO $ runEmulatedChainIndex mempty $ do -- Append the generated block in the chain index - appendBlock tip block - - forM addresses $ \addr -> do - let pq = PageQuery 200 Nothing - (_, utxoRefs) <- utxoSetAtAddress pq addr - pure $ pageItems utxoRefs + appendBlock tip block def + utxoSetFromBlockAddrs block case result of - Left _ -> Hedgehog.assert False - Right utxoRefsGroups -> do - forM_ (concat utxoRefsGroups) $ \utxoRef -> do - Hedgehog.assert $ utxoRef `member` view Gen.txgsUtxoSet state + Left _ -> Hedgehog.assert False + Right utxoGroups -> S.fromList (concat utxoGroups) === view Gen.txgsUtxoSet state -- | After generating and appending a block in the chain index, verify that -- querying the chain index with each of the asset classes in the block returns @@ -102,7 +98,7 @@ eachTxOutRefWithCurrencyShouldBeUnspentSpec = property $ do result <- liftIO $ runEmulatedChainIndex mempty $ do -- Append the generated block in the chain index - appendBlock tip block + appendBlock tip block def forM assetClasses $ \ac -> do let pq = PageQuery 200 Nothing @@ -110,11 +106,8 @@ eachTxOutRefWithCurrencyShouldBeUnspentSpec = property $ do pure $ pageItems utxoRefs case result of - Left _ -> Hedgehog.assert False - Right utxoRefsGroups -> do - let utxoRefs = concat utxoRefsGroups - forM_ utxoRefs $ \utxoRef -> do - Hedgehog.assert $ utxoRef `member` view Gen.txgsUtxoSet state + Left _ -> Hedgehog.assert False + Right utxoGroups -> S.fromList (concat utxoGroups) === view Gen.txgsUtxoSet state -- | Requesting UTXOs containing Ada should not return anything because every -- transaction output must have a minimum of 1 ADA. So asking for UTXOs with ADA @@ -125,7 +118,7 @@ cantRequestForTxOutRefsWithAdaSpec = property $ do result <- liftIO $ runEmulatedChainIndex mempty $ do -- Append the generated block in the chain index - appendBlock tip block + appendBlock tip block def let pq = PageQuery 200 Nothing (_, utxoRefs) <- utxoSetWithCurrency pq (AssetClass ("", "")) @@ -135,6 +128,23 @@ cantRequestForTxOutRefsWithAdaSpec = property $ do Left _ -> Hedgehog.assert False Right utxoRefs -> Hedgehog.assert $ null utxoRefs +-- | Do not store txs through BlockProcessOption. +-- The UTxO set must still be stored. +-- But cannot be fetched through addresses as addresses are not stored. +doNotStoreTxs :: Property +doNotStoreTxs = property $ do + ((tip, block), state) <- forAll $ Gen.runTxGenState Gen.genNonEmptyBlock + result <- liftIO $ runEmulatedChainIndex mempty $ do + appendBlock tip block BlockProcessOption{bpoStoreTxs=False} + tx <- txFromTxId (view citxTxId (head block)) + utxosFromAddr <- utxoSetFromBlockAddrs block + utxosStored <- traverse utxoSetMembership (S.toList (view Gen.txgsUtxoSet state)) + pure (tx, concat utxosFromAddr, utxosStored) + case result of + Right (Nothing, [], utxosStored) -> Hedgehog.assert $ and (snd <$> utxosStored) + _ -> Hedgehog.assert False + +-- | Run an emulated chain index effect against a starting state runEmulatedChainIndex :: ChainIndexEmulatorState -> Eff '[ ChainIndexControlEffect diff --git a/plutus-chain-index-core/test/Plutus/ChainIndex/HandlersSpec.hs b/plutus-chain-index-core/test/Plutus/ChainIndex/HandlersSpec.hs index d79f610370..8034832998 100644 --- a/plutus-chain-index-core/test/Plutus/ChainIndex/HandlersSpec.hs +++ b/plutus-chain-index-core/test/Plutus/ChainIndex/HandlersSpec.hs @@ -8,30 +8,33 @@ module Plutus.ChainIndex.HandlersSpec (tests) where import Control.Concurrent.STM (newTVarIO) -import Control.Lens -import Control.Monad (forM, forM_) +import Control.Lens (view) +import Control.Monad (forM) import Control.Monad.Freer (Eff) import Control.Monad.Freer.Extras.Beam (BeamEffect) import Control.Monad.IO.Class (liftIO) import Control.Tracer (nullTracer) -import Data.Set (member) +import Data.Default (def) +import Data.Set qualified as S import Database.Beam.Migrate.Simple (autoMigrate) import Database.Beam.Sqlite qualified as Sqlite import Database.Beam.Sqlite.Migrate qualified as Sqlite import Database.SQLite.Simple qualified as Sqlite import Generators qualified as Gen import Hedgehog (Property, assert, forAll, property, (===)) -import Ledger (Address (Address, addressCredential), TxOut (TxOut, txOutAddress), outValue) +import Ledger (outValue) import Plutus.ChainIndex (Page (pageItems), PageQuery (PageQuery), RunRequirements (..), appendBlock, citxOutputs, - runChainIndexEffects, txFromTxId, utxoSetAtAddress, utxoSetWithCurrency) + runChainIndexEffects, txFromTxId, utxoSetMembership, utxoSetWithCurrency) import Plutus.ChainIndex.ChainIndexError (ChainIndexError (..)) import Plutus.ChainIndex.DbSchema (checkedSqliteDb) import Plutus.ChainIndex.Effects (ChainIndexControlEffect, ChainIndexQueryEffect) import Plutus.ChainIndex.Tx (_ValidTx, citxTxId) +import Plutus.ChainIndex.Types (BlockProcessOption (..)) import Plutus.V1.Ledger.Ada qualified as Ada import Plutus.V1.Ledger.Value (AssetClass (AssetClass), flattenValue) import Test.Tasty import Test.Tasty.Hedgehog (testProperty) +import Util (utxoSetFromBlockAddrs) tests :: TestTree tests = do @@ -46,6 +49,9 @@ tests = do [ testProperty "each txOutRef should be unspent" eachTxOutRefWithCurrencyShouldBeUnspentSpec , testProperty "should restrict to non-ADA currencies" cantRequestForTxOutRefsWithAdaSpec ] + , testGroup "BlockProcessOption" + [ testProperty "do not store txs" doNotStoreTxs + ] ] -- | Tests we can correctly query a tx in the database using a tx id. We also @@ -57,7 +63,7 @@ txFromTxIdSpec = property $ do txs <- liftIO $ Sqlite.withConnection ":memory:" $ \conn -> do Sqlite.runBeamSqlite conn $ autoMigrate Sqlite.migrationBackend checkedSqliteDb liftIO $ runChainIndex conn $ do - appendBlock tip block + appendBlock tip block def tx <- txFromTxId (view citxTxId fstTx) tx' <- txFromTxId unknownTxId pure (tx, tx') @@ -73,26 +79,16 @@ eachTxOutRefAtAddressShouldBeUnspentSpec :: Property eachTxOutRefAtAddressShouldBeUnspentSpec = property $ do ((tip, block), state) <- forAll $ Gen.runTxGenState Gen.genNonEmptyBlock - let addresses = - fmap (\TxOut { txOutAddress = Address { addressCredential }} -> addressCredential) - $ view (traverse . citxOutputs . _ValidTx) block - result <- liftIO $ Sqlite.withConnection ":memory:" $ \conn -> do Sqlite.runBeamSqlite conn $ autoMigrate Sqlite.migrationBackend checkedSqliteDb liftIO $ runChainIndex conn $ do -- Append the generated block in the chain index - appendBlock tip block - - forM addresses $ \addr -> do - let pq = PageQuery 200 Nothing - (_, utxoRefs) <- utxoSetAtAddress pq addr - pure $ pageItems utxoRefs + appendBlock tip block def + utxoSetFromBlockAddrs block case result of - Left _ -> Hedgehog.assert False - Right utxoRefsGroups -> do - forM_ (concat utxoRefsGroups) $ \utxoRef -> do - Hedgehog.assert $ utxoRef `member` view Gen.txgsUtxoSet state + Left _ -> Hedgehog.assert False + Right utxoGroups -> S.fromList (concat utxoGroups) === view Gen.txgsUtxoSet state -- | After generating and appending a block in the chain index, verify that -- querying the chain index with each of the addresses in the block returns @@ -111,7 +107,7 @@ eachTxOutRefWithCurrencyShouldBeUnspentSpec = property $ do Sqlite.runBeamSqlite conn $ autoMigrate Sqlite.migrationBackend checkedSqliteDb liftIO $ runChainIndex conn $ do -- Append the generated block in the chain index - appendBlock tip block + appendBlock tip block def forM assetClasses $ \ac -> do let pq = PageQuery 200 Nothing @@ -119,10 +115,8 @@ eachTxOutRefWithCurrencyShouldBeUnspentSpec = property $ do pure $ pageItems utxoRefs case result of - Left _ -> Hedgehog.assert False - Right utxoRefsGroups -> do - forM_ (concat utxoRefsGroups) $ \utxoRef -> - Hedgehog.assert $ utxoRef `member` view Gen.txgsUtxoSet state + Left _ -> Hedgehog.assert False + Right utxoGroups -> S.fromList (concat utxoGroups) === view Gen.txgsUtxoSet state -- | Requesting UTXOs containing Ada should not return anything because every -- transaction output must have a minimum of 1 ADA. So asking for UTXOs with ADA @@ -135,7 +129,7 @@ cantRequestForTxOutRefsWithAdaSpec = property $ do Sqlite.runBeamSqlite conn $ autoMigrate Sqlite.migrationBackend checkedSqliteDb liftIO $ runChainIndex conn $ do -- Append the generated block in the chain index - appendBlock tip block + appendBlock tip block def let pq = PageQuery 200 Nothing (_, utxoRefs) <- utxoSetWithCurrency pq (AssetClass (Ada.adaSymbol, Ada.adaToken)) @@ -145,6 +139,25 @@ cantRequestForTxOutRefsWithAdaSpec = property $ do Left _ -> Hedgehog.assert False Right utxoRefs -> Hedgehog.assert $ null utxoRefs +-- | Do not store txs through BlockProcessOption. +-- The UTxO set must still be stored. +-- But cannot be fetched through addresses as addresses are not stored. +doNotStoreTxs :: Property +doNotStoreTxs = property $ do + ((tip, block), state) <- forAll $ Gen.runTxGenState Gen.genNonEmptyBlock + result <- liftIO $ Sqlite.withConnection ":memory:" $ \conn -> do + Sqlite.runBeamSqlite conn $ autoMigrate Sqlite.migrationBackend checkedSqliteDb + liftIO $ runChainIndex conn $ do + appendBlock tip block BlockProcessOption{bpoStoreTxs=False} + tx <- txFromTxId (view citxTxId (head block)) + utxosFromAddr <- utxoSetFromBlockAddrs block + utxosStored <- traverse utxoSetMembership (S.toList (view Gen.txgsUtxoSet state)) + pure (tx, concat utxosFromAddr, utxosStored) + case result of + Right (Nothing, [], utxosStored) -> Hedgehog.assert $ and (snd <$> utxosStored) + _ -> Hedgehog.assert False + +-- | Run a chain index action against a SQLite connection. runChainIndex :: Sqlite.Connection -> Eff '[ ChainIndexQueryEffect diff --git a/plutus-chain-index-core/test/Util.hs b/plutus-chain-index-core/test/Util.hs new file mode 100644 index 0000000000..e2d8e2a90c --- /dev/null +++ b/plutus-chain-index-core/test/Util.hs @@ -0,0 +1,27 @@ +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE MonoLocalBinds #-} +{-# LANGUAGE NamedFieldPuns #-} + +module Util where + +import Control.Lens (view) +import Control.Monad (forM) +import Control.Monad.Freer (Eff, Member) +import Ledger (Address (Address, addressCredential), TxOut (TxOut, txOutAddress), TxOutRef) +import Ledger.Credential (Credential) +import Plutus.ChainIndex (Page (pageItems), PageQuery (PageQuery), citxOutputs, utxoSetAtAddress) +import Plutus.ChainIndex.Effects (ChainIndexQueryEffect) +import Plutus.ChainIndex.Tx (ChainIndexTx, _ValidTx) + +-- | Get all address credentials from a block. +addrCredsFromBlock :: [ChainIndexTx] -> [Credential] +addrCredsFromBlock = + fmap (\TxOut { txOutAddress = Address { addressCredential }} -> addressCredential) + . view (traverse . citxOutputs . _ValidTx) + +-- | Get the UTxO set from a block. +utxoSetFromBlockAddrs :: Member ChainIndexQueryEffect effs => [ChainIndexTx] -> Eff effs [[TxOutRef]] +utxoSetFromBlockAddrs block = forM (addrCredsFromBlock block) $ \addr -> do + let pq = PageQuery 200 Nothing + (_, utxoRefs) <- utxoSetAtAddress pq addr + pure $ pageItems utxoRefs diff --git a/plutus-chain-index/src/Plutus/ChainIndex/App.hs b/plutus-chain-index/src/Plutus/ChainIndex/App.hs index d06acd3ff5..aede59a492 100644 --- a/plutus-chain-index/src/Plutus/ChainIndex/App.hs +++ b/plutus-chain-index/src/Plutus/ChainIndex/App.hs @@ -2,6 +2,7 @@ {-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE GADTs #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} @@ -36,7 +37,7 @@ import Cardano.BM.Configuration.Model qualified as CM import Cardano.BM.Setup (setupTrace_) import Cardano.BM.Trace (Trace, logDebug, logError) -import Cardano.Api (ChainPoint, ChainTip (..), ConsensusModeParams (..), LocalNodeConnectInfo (..), getLocalChainTip) +import Cardano.Api qualified as C import Cardano.Protocol.Socket.Client (ChainSyncEvent (..), runChainSync) import Cardano.Protocol.Socket.Type (epochSlots) import Ledger (Slot (..)) @@ -50,7 +51,7 @@ import Plutus.ChainIndex.Effects (ChainIndexControlEffect (..), ChainIndexQueryE import Plutus.ChainIndex.Handlers (getResumePoints) import Plutus.ChainIndex.Logging qualified as Logging import Plutus.ChainIndex.Server qualified as Server -import Plutus.ChainIndex.Types (pointSlot) +import Plutus.ChainIndex.Types (BlockProcessOption (..), pointSlot) import Plutus.Monitoring.Util (runLogEffects) @@ -72,31 +73,32 @@ runChainIndex runReq effect = do chainSyncHandler :: RunRequirements + -> C.BlockNo -> ChainSyncEvent -> Slot -> IO () -chainSyncHandler runReq - (RollForward block _) _ = do +chainSyncHandler runReq storeFrom + (RollForward block@(C.BlockInMode (C.Block (C.BlockHeader _ _ blockNo) _) _) _) _ = do let ciBlock = fromCardanoBlock block case ciBlock of Left err -> logError (trace runReq) (ConversionFailed err) - Right txs -> - void $ runChainIndex runReq $ appendBlock (tipFromCardanoBlock block) txs -chainSyncHandler runReq + Right txs -> void $ runChainIndex runReq $ + appendBlock (tipFromCardanoBlock block) txs (BlockProcessOption (blockNo >= storeFrom)) +chainSyncHandler runReq _ (RollBackward point _) _ = do putStr "Rolling back to " print point -- Do we really want to pass the tip of the new blockchain to the -- rollback function (rather than the point where the chains diverge)? void $ runChainIndex runReq $ rollback (fromCardanoPoint point) -chainSyncHandler runReq +chainSyncHandler runReq _ (Resume point) _ = do putStr "Resuming from " print point void $ runChainIndex runReq $ resumeSync $ fromCardanoPoint point -showResumePoints :: [ChainPoint] -> String +showResumePoints :: [C.ChainPoint] -> String showResumePoints = \case [] -> "none" [x] -> showPoint x @@ -142,10 +144,10 @@ main = do -- The primary purpose of this query is to get the first response of the node for potential errors before opening the DB and starting the chain index. -- See #69. putStr "\nThe tip of the local node: " - ChainTip slotNo _ _ <- getLocalChainTip $ LocalNodeConnectInfo - { localConsensusModeParams = CardanoModeParams epochSlots - , localNodeNetworkId = Config.cicNetworkId config - , localNodeSocketPath = Config.cicSocketPath config + C.ChainTip slotNo _ _ <- C.getLocalChainTip $ C.LocalNodeConnectInfo + { C.localConsensusModeParams = C.CardanoModeParams epochSlots + , C.localNodeNetworkId = Config.cicNetworkId config + , C.localNodeSocketPath = Config.cicSocketPath config } print slotNo @@ -185,7 +187,7 @@ runMain trace config = do (Config.cicSlotConfig config) (Config.cicNetworkId config) resumePoints - (chainSyncHandler runReq) + (chainSyncHandler runReq (Config.cicStoreFrom config)) putStrLn $ "Starting webserver on port " <> show (Config.cicPort config) Server.serveChainIndexQueryServer (Config.cicPort config) runReq diff --git a/plutus-chain-index/src/Plutus/ChainIndex/Config.hs b/plutus-chain-index/src/Plutus/ChainIndex/Config.hs index 89a7831287..71c14b11c3 100644 --- a/plutus-chain-index/src/Plutus/ChainIndex/Config.hs +++ b/plutus-chain-index/src/Plutus/ChainIndex/Config.hs @@ -17,10 +17,11 @@ module Plutus.ChainIndex.Config( port, networkId, securityParam, - slotConfig + slotConfig, + storeFrom ) where -import Cardano.Api (NetworkId (..)) +import Cardano.Api (BlockNo (..), NetworkId (..)) import Control.Exception (Exception) import Control.Lens (makeLensesFor) import Data.Aeson (FromJSON, ToJSON) @@ -36,6 +37,7 @@ data ChainIndexConfig = ChainIndexConfig , cicNetworkId :: NetworkId , cicSecurityParam :: Int -- ^ The number of blocks after which a transaction cannot be rolled back anymore , cicSlotConfig :: SlotConfig + , cicStoreFrom :: BlockNo -- ^ Only store transactions from this block number onward } deriving stock (Show, Eq, Generic) deriving anyclass (FromJSON, ToJSON) @@ -47,6 +49,8 @@ deriving anyclass instance FromJSON NetworkId deriving anyclass instance ToJSON NetworkId deriving anyclass instance FromJSON NetworkMagic deriving anyclass instance ToJSON NetworkMagic +deriving anyclass instance FromJSON BlockNo +deriving anyclass instance ToJSON BlockNo -- | These settings work with the Alonzo Purple testnet defaultConfig :: ChainIndexConfig @@ -61,15 +65,17 @@ defaultConfig = ChainIndexConfig { scSlotZeroTime = 1596059091000 , scSlotLength = 1000 } + , cicStoreFrom = BlockNo 0 } instance Pretty ChainIndexConfig where - pretty ChainIndexConfig{cicSocketPath, cicDbPath, cicPort, cicNetworkId, cicSecurityParam} = + pretty ChainIndexConfig{cicSocketPath, cicDbPath, cicPort, cicNetworkId, cicSecurityParam, cicStoreFrom} = vsep [ "Socket:" <+> pretty cicSocketPath , "Db:" <+> pretty cicDbPath , "Port:" <+> pretty cicPort , "Network Id:" <+> viaShow cicNetworkId , "Security Param:" <+> pretty cicSecurityParam + , "Store from:" <+> viaShow cicStoreFrom ] makeLensesFor [ @@ -78,7 +84,8 @@ makeLensesFor [ ("cicPort", "port"), ("cicNetworkId", "networkId"), ("cicSecurityParam", "securityParam"), - ("cicSlotConfig", "slotConfig") + ("cicSlotConfig", "slotConfig"), + ("cicStoreFrom", "storeFrom") ] 'ChainIndexConfig newtype DecodeConfigException = DecodeConfigException String diff --git a/plutus-contract/src/Plutus/Trace/Emulator/System.hs b/plutus-contract/src/Plutus/Trace/Emulator/System.hs index def2bd55ae..3246ddb390 100644 --- a/plutus-contract/src/Plutus/Trace/Emulator/System.hs +++ b/plutus-contract/src/Plutus/Trace/Emulator/System.hs @@ -20,6 +20,7 @@ module Plutus.Trace.Emulator.System import Control.Monad (forM_, void) import Control.Monad.Freer import Control.Monad.Freer.Coroutine +import Data.Default (def) import Data.Foldable (traverse_) import Data.Maybe (maybeToList) import Data.Text qualified as Text @@ -148,4 +149,4 @@ appendNewTipBlock lastTip block newSlot = do $ (Text.encodeUtf8 . Text.pack . show . hash) $ foldMap (getTxId . eitherTx txId txId) block let newTip = Tip newSlot blockId nextBlockNo - appendBlock newTip (fmap fromOnChainTx block) + appendBlock newTip (fmap fromOnChainTx block) def diff --git a/plutus-pab/test-node/testnet/chain-index-config.json b/plutus-pab/test-node/testnet/chain-index-config.json index 4e4f44c503..9657202eb7 100644 --- a/plutus-pab/test-node/testnet/chain-index-config.json +++ b/plutus-pab/test-node/testnet/chain-index-config.json @@ -12,5 +12,8 @@ "unNetworkMagic": 1097911063 }, "tag": "Testnet" + }, + "cicStoreFrom": { + "unBlockNo": 0 } }