Chain Index: users can configure to only store txs from a block no on…

…ward (#72)

* ChainIndex: users can configure to only store txs from a block no onward

* Add BlockProcessOption to make appendBlock calls clearer. Also prep for future options.

* plutus-chain-index-core: add BlockProcessOption tests.
Tighten and start sharing code among existing tests as well.

* updateMaterialized

plutus-chain-index-core/plutus-chain-index-core.cabal

@@ -94,7 +94,6 @@ library
         typed-protocols -any,
         unordered-containers -any,
         bytestring -any,
-        data-default -any,
         text -any,
         servant -any,
         servant-server -any,
@@ -116,6 +115,7 @@ test-suite plutus-chain-index-test
         Plutus.ChainIndex.Emulator.DiskStateSpec
         Plutus.ChainIndex.Emulator.HandlersSpec
         Plutus.ChainIndex.HandlersSpec
+        Util
     build-depends:
         plutus-ledger -any,
         plutus-ledger-api -any,
@@ -128,6 +128,7 @@ test-suite plutus-chain-index-test
         bytestring -any,
         containers -any,
         contra-tracer -any,
+        data-default -any,
         fingertree -any,
         freer-extras -any,
         freer-simple -any,

plutus-chain-index-core/src/Plutus/ChainIndex/Effects.hs

@@ -33,7 +33,7 @@ import Ledger (AssetClass, Datum, DatumHash, MintingPolicy, MintingPolicyHash, R
 import Ledger.Credential (Credential)
 import Ledger.Tx (ChainIndexTxOut, TxOutRef)
 import Plutus.ChainIndex.Tx (ChainIndexTx)
-import Plutus.ChainIndex.Types (Diagnostics, Point, Tip)
+import Plutus.ChainIndex.Types (BlockProcessOption, Diagnostics, Point, Tip)
 
 data ChainIndexQueryEffect r where
 
@@ -78,7 +78,7 @@ makeEffect ''ChainIndexQueryEffect
 data ChainIndexControlEffect r where
 
     -- | Add a new block to the chain index by giving a new tip and list of tx.
-    AppendBlock :: Tip -> [ChainIndexTx] -> ChainIndexControlEffect ()
+    AppendBlock :: Tip -> [ChainIndexTx] -> BlockProcessOption -> ChainIndexControlEffect ()
 
     -- | Roll back to a previous state (previous tip)
     Rollback    :: Point -> ChainIndexControlEffect ()

plutus-chain-index-core/src/Plutus/ChainIndex/Emulator/Handlers.hs

@@ -43,7 +43,8 @@ import Plutus.ChainIndex.Emulator.DiskState (DiskState, addressMap, assetClassMa
 import Plutus.ChainIndex.Emulator.DiskState qualified as DiskState
 import Plutus.ChainIndex.Tx (ChainIndexTx, _ValidTx, citxOutputs)
 import Plutus.ChainIndex.TxUtxoBalance qualified as TxUtxoBalance
-import Plutus.ChainIndex.Types (Diagnostics (..), Point (PointAtGenesis), Tip (..), TxUtxoBalance (..))
+import Plutus.ChainIndex.Types (BlockProcessOption (..), Diagnostics (..), Point (PointAtGenesis), Tip (..),
+                                TxUtxoBalance (..))
 import Plutus.ChainIndex.UtxoState (InsertUtxoSuccess (..), RollbackResult (..), UtxoIndex, tip, utxoState)
 import Plutus.ChainIndex.UtxoState qualified as UtxoState
 import Plutus.V1.Ledger.Api (Credential (PubKeyCredential, ScriptCredential))
@@ -159,7 +160,7 @@ handleControl ::
     => ChainIndexControlEffect
     ~> Eff effs
 handleControl = \case
-    AppendBlock tip_ transactions -> do
+    AppendBlock tip_ transactions opts -> do
         oldState <- get @ChainIndexEmulatorState
         case UtxoState.insert (TxUtxoBalance.fromBlock tip_ transactions) (view utxoIndex oldState) of
             Left err -> do
@@ -169,7 +170,8 @@ handleControl = \case
             Right InsertUtxoSuccess{newIndex, insertPosition} -> do
                 put $ oldState
                         & set utxoIndex newIndex
-                        & over diskState (mappend $ foldMap DiskState.fromTx transactions)
+                        & over diskState
+                            (mappend $ foldMap DiskState.fromTx (if bpoStoreTxs opts then transactions else []))
                 logDebug $ InsertionSuccess tip_ insertPosition
     Rollback tip_ -> do
         oldState <- get @ChainIndexEmulatorState

plutus-chain-index-core/src/Plutus/ChainIndex/Handlers.hs

@@ -21,6 +21,7 @@ module Plutus.ChainIndex.Handlers
 import Cardano.Api qualified as C
 import Control.Applicative (Const (..))
 import Control.Lens (Lens', _Just, ix, view, (^?))
+import Control.Monad (when)
 import Control.Monad.Freer (Eff, Member, type (~>))
 import Control.Monad.Freer.Error (Error, throwError)
 import Control.Monad.Freer.Extras.Beam (BeamEffect (..), BeamableSqlite, addRowsInBatches, combined, deleteRows,
@@ -53,7 +54,8 @@ import Plutus.ChainIndex.DbSchema
 import Plutus.ChainIndex.Effects (ChainIndexControlEffect (..), ChainIndexQueryEffect (..))
 import Plutus.ChainIndex.Tx
 import Plutus.ChainIndex.TxUtxoBalance qualified as TxUtxoBalance
-import Plutus.ChainIndex.Types (Depth (..), Diagnostics (..), Point (..), Tip (..), TxUtxoBalance (..), tipAsPoint)
+import Plutus.ChainIndex.Types (BlockProcessOption (..), Depth (..), Diagnostics (..), Point (..), Tip (..),
+                                TxUtxoBalance (..), tipAsPoint)
 import Plutus.ChainIndex.UtxoState (InsertUtxoSuccess (..), RollbackResult (..), UtxoIndex)
 import Plutus.ChainIndex.UtxoState qualified as UtxoState
 import Plutus.V1.Ledger.Ada qualified as Ada
@@ -239,7 +241,7 @@ handleControl ::
     => ChainIndexControlEffect
     ~> Eff effs
 handleControl = \case
-    AppendBlock tip_ transactions -> do
+    AppendBlock tip_ transactions opts -> do
         oldIndex <- get @ChainIndexState
         let newUtxoState = TxUtxoBalance.fromBlock tip_ transactions
         case UtxoState.insert newUtxoState oldIndex of
@@ -254,7 +256,7 @@ handleControl = \case
                   lbcResult -> do
                     put $ UtxoState.reducedIndex lbcResult
                     reduceOldUtxoDb $ UtxoState._usTip $ UtxoState.combinedState lbcResult
-                insert $ foldMap fromTx transactions
+                when (bpoStoreTxs opts) $ insert $ foldMap fromTx transactions
                 insertUtxoDb newUtxoState
                 logDebug $ InsertionSuccess tip_ insertPosition
     Rollback tip_ -> do

plutus-chain-index-core/src/Plutus/ChainIndex/Types.hs

@@ -32,6 +32,7 @@ module Plutus.ChainIndex.Types(
     , TxOutBalance(..)
     , tobUnspentOutputs
     , tobSpentOutputs
+    , BlockProcessOption(..)
     ) where
 
 import Codec.Serialise (Serialise)
@@ -42,6 +43,7 @@ import Crypto.Hash (SHA256, hash)
 import Data.Aeson (FromJSON, ToJSON)
 import Data.ByteArray qualified as BA
 import Data.ByteString.Lazy qualified as BSL
+import Data.Default (Default (..))
 import Data.Map (Map)
 import Data.Map qualified as Map
 import Data.Monoid (Last (..), Sum (..))
@@ -345,3 +347,18 @@ instance Semigroup TxUtxoBalance where
 instance Monoid TxUtxoBalance where
     mappend = (<>)
     mempty = TxUtxoBalance mempty mempty
+
+-- | User-customizable options to process a block.
+-- See #73 for more motivations.
+newtype BlockProcessOption =
+  BlockProcessOption
+    { bpoStoreTxs :: Bool
+    -- ^ Should the chain index store this batch of transactions or not.
+    -- If not, only handle the tip and UTXOs.
+    -- This, for example, allows applications to skip unwanted pre-Alonzo transactions.
+    }
+
+-- We should think twice when setting the default option.
+-- For now, it should store all data to avoid weird non-backward-compatible bugs in the future.
+instance Default BlockProcessOption where
+  def = BlockProcessOption True

plutus-chain-index-core/test/Plutus/ChainIndex/Emulator/HandlersSpec.hs

@@ -8,28 +8,31 @@
 module Plutus.ChainIndex.Emulator.HandlersSpec (tests) where
 
 import Control.Lens
-import Control.Monad (forM, forM_)
+import Control.Monad (forM)
 import Control.Monad.Freer (Eff, interpret, reinterpret, runM)
 import Control.Monad.Freer.Error (Error, runError)
 import Control.Monad.Freer.Extras (LogMessage, LogMsg (..), handleLogWriter)
 import Control.Monad.Freer.State (State, runState)
 import Control.Monad.Freer.Writer (runWriter)
 import Control.Monad.IO.Class (liftIO)
+import Data.Default (def)
 import Data.Sequence (Seq)
-import Data.Set (member)
+import Data.Set qualified as S
 import Generators qualified as Gen
-import Ledger (Address (Address, addressCredential), TxOut (TxOut, txOutAddress), outValue)
+import Ledger (outValue)
 import Plutus.ChainIndex (ChainIndexLog, Page (pageItems), PageQuery (PageQuery), appendBlock, txFromTxId,
-                          utxoSetAtAddress, utxoSetWithCurrency)
+                          utxoSetMembership, utxoSetWithCurrency)
 import Plutus.ChainIndex.ChainIndexError (ChainIndexError)
 import Plutus.ChainIndex.Effects (ChainIndexControlEffect, ChainIndexQueryEffect)
 import Plutus.ChainIndex.Emulator.Handlers (ChainIndexEmulatorState, handleControl, handleQuery)
 import Plutus.ChainIndex.Tx (_ValidTx, citxOutputs, citxTxId)
+import Plutus.ChainIndex.Types (BlockProcessOption (..))
 import Plutus.V1.Ledger.Value (AssetClass (AssetClass), flattenValue)
 
 import Hedgehog (Property, assert, forAll, property, (===))
 import Test.Tasty
 import Test.Tasty.Hedgehog (testProperty)
+import Util (utxoSetFromBlockAddrs)
 
 tests :: TestTree
 tests = do
@@ -44,6 +47,9 @@ tests = do
       [ testProperty "each txOutRef should be unspent" eachTxOutRefWithCurrencyShouldBeUnspentSpec
       , testProperty "should restrict to non-ADA currencies" cantRequestForTxOutRefsWithAdaSpec
       ]
+    , testGroup "BlockProcessOption"
+      [ testProperty "do not store txs" doNotStoreTxs
+      ]
     ]
 
 -- | Tests we can correctly query a tx in the database using a tx id. We also
@@ -53,7 +59,7 @@ txFromTxIdSpec = property $ do
   (tip, block@(fstTx:_)) <- forAll $ Gen.evalTxGenState Gen.genNonEmptyBlock
   unknownTxId <- forAll Gen.genRandomTxId
   txs <- liftIO $ runEmulatedChainIndex mempty $ do
-    appendBlock tip block
+    appendBlock tip block def
     tx <- txFromTxId (view citxTxId fstTx)
     tx' <- txFromTxId unknownTxId
     pure (tx, tx')
@@ -69,24 +75,14 @@ eachTxOutRefAtAddressShouldBeUnspentSpec :: Property
 eachTxOutRefAtAddressShouldBeUnspentSpec = property $ do
   ((tip, block), state) <- forAll $ Gen.runTxGenState Gen.genNonEmptyBlock
 
-  let addresses =
-        fmap (\TxOut { txOutAddress = Address { addressCredential }} -> addressCredential)
-        $ view (traverse . citxOutputs . _ValidTx) block
-
   result <- liftIO $ runEmulatedChainIndex mempty $ do
     -- Append the generated block in the chain index
-    appendBlock tip block
-
-    forM addresses $ \addr -> do
-      let pq = PageQuery 200 Nothing
-      (_, utxoRefs) <- utxoSetAtAddress pq addr
-      pure $ pageItems utxoRefs
+    appendBlock tip block def
+    utxoSetFromBlockAddrs block
 
   case result of
-    Left _ -> Hedgehog.assert False
-    Right utxoRefsGroups -> do
-      forM_ (concat utxoRefsGroups) $ \utxoRef -> do
-        Hedgehog.assert $ utxoRef `member` view Gen.txgsUtxoSet state
+    Left _           -> Hedgehog.assert False
+    Right utxoGroups -> S.fromList (concat utxoGroups) === view Gen.txgsUtxoSet state
 
 -- | After generating and appending a block in the chain index, verify that
 -- querying the chain index with each of the asset classes in the block returns
@@ -102,19 +98,16 @@ eachTxOutRefWithCurrencyShouldBeUnspentSpec = property $ do
 
   result <- liftIO $ runEmulatedChainIndex mempty $ do
     -- Append the generated block in the chain index
-    appendBlock tip block
+    appendBlock tip block def
 
     forM assetClasses $ \ac -> do
       let pq = PageQuery 200 Nothing
       (_, utxoRefs) <- utxoSetWithCurrency pq ac
       pure $ pageItems utxoRefs
 
   case result of
-    Left _ -> Hedgehog.assert False
-    Right utxoRefsGroups -> do
-      let utxoRefs = concat utxoRefsGroups
-      forM_ utxoRefs $ \utxoRef -> do
-        Hedgehog.assert $ utxoRef `member` view Gen.txgsUtxoSet state
+    Left _           -> Hedgehog.assert False
+    Right utxoGroups -> S.fromList (concat utxoGroups) === view Gen.txgsUtxoSet state
 
 -- | Requesting UTXOs containing Ada should not return anything because every
 -- transaction output must have a minimum of 1 ADA. So asking for UTXOs with ADA
@@ -125,7 +118,7 @@ cantRequestForTxOutRefsWithAdaSpec = property $ do
 
   result <- liftIO $ runEmulatedChainIndex mempty $ do
     -- Append the generated block in the chain index
-    appendBlock tip block
+    appendBlock tip block def
 
     let pq = PageQuery 200 Nothing
     (_, utxoRefs) <- utxoSetWithCurrency pq (AssetClass ("", ""))
@@ -135,6 +128,23 @@ cantRequestForTxOutRefsWithAdaSpec = property $ do
     Left _         -> Hedgehog.assert False
     Right utxoRefs -> Hedgehog.assert $ null utxoRefs
 
+-- | Do not store txs through BlockProcessOption.
+-- The UTxO set must still be stored.
+-- But cannot be fetched through addresses as addresses are not stored.
+doNotStoreTxs :: Property
+doNotStoreTxs = property $ do
+  ((tip, block), state) <- forAll $ Gen.runTxGenState Gen.genNonEmptyBlock
+  result <- liftIO $ runEmulatedChainIndex mempty $ do
+    appendBlock tip block BlockProcessOption{bpoStoreTxs=False}
+    tx <- txFromTxId (view citxTxId (head block))
+    utxosFromAddr <- utxoSetFromBlockAddrs block
+    utxosStored <- traverse utxoSetMembership (S.toList (view Gen.txgsUtxoSet state))
+    pure (tx, concat utxosFromAddr, utxosStored)
+  case result of
+    Right (Nothing, [], utxosStored) -> Hedgehog.assert $ and (snd <$> utxosStored)
+    _                                -> Hedgehog.assert False
+
+-- | Run an emulated chain index effect against a starting state
 runEmulatedChainIndex
   :: ChainIndexEmulatorState
   -> Eff '[ ChainIndexControlEffect