diff --git a/cardano-chain-gen/cardano-chain-gen.cabal b/cardano-chain-gen/cardano-chain-gen.cabal index f7d91c1d7..9d9a441d0 100644 --- a/cardano-chain-gen/cardano-chain-gen.cabal +++ b/cardano-chain-gen/cardano-chain-gen.cabal @@ -61,6 +61,7 @@ library build-depends: base >= 4.14 && < 5 , async , aeson + , base16-bytestring , bytestring , cardano-binary , cardano-crypto-class @@ -166,6 +167,7 @@ test-suite cardano-chain-gen Test.Cardano.Db.Mock.Unit.Conway.Simple Test.Cardano.Db.Mock.Unit.Conway.Stake Test.Cardano.Db.Mock.Unit.Conway.Tx + Test.Cardano.Db.Mock.Unit.Conway.Whitelist Test.Cardano.Db.Mock.UnifiedApi Test.Cardano.Db.Mock.Validate diff --git a/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Alonzo/ScriptsExamples.hs b/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Alonzo/ScriptsExamples.hs index d69eccde0..f76befd8d 100644 --- a/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Alonzo/ScriptsExamples.hs +++ b/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Alonzo/ScriptsExamples.hs @@ -23,6 +23,7 @@ module Cardano.Mock.Forging.Tx.Alonzo.ScriptsExamples ( alwaysMintScriptHash, alwaysMintScriptAddr, alwaysMintScriptStake, + alwaysMintScriptHashRandomPolicyVal, scriptHash, assetNames, plutusData2, @@ -47,6 +48,7 @@ import Codec.Serialise import Codec.Serialise.Encoding import Data.ByteString.Short import Data.Maybe +import Numeric.Natural (Natural) import Ouroboros.Consensus.Cardano.Block (StandardAlonzo) import qualified PlutusCore.Data as Plutus import qualified PlutusLedgerApi.Test.Examples as Plutus @@ -103,6 +105,15 @@ alwaysMintScriptStake = ScriptHashObj alwaysMintScriptHash mkPlutusScriptEra :: AlonzoEraScript era => PlutusBinary -> AlonzoScript era mkPlutusScriptEra sh = PlutusScript $ fromJust $ mkBinaryPlutusScript PlutusV1 sh +alwaysMintScriptHashRandomPolicyVal :: Natural -> ScriptHash StandardCrypto +alwaysMintScriptHashRandomPolicyVal n = scriptHash @StandardAlonzo $ alwaysMintRandomScript n + +alwaysMintRandomScript :: AlonzoEraScript era => Natural -> AlonzoScript era +alwaysMintRandomScript n = mkPlutusScriptEra $ alwaysMintRandomPlutusBinary n + +alwaysMintRandomPlutusBinary :: Natural -> PlutusBinary +alwaysMintRandomPlutusBinary n = PlutusBinary $ Plutus.alwaysFailingNAryFunction n + scriptHash :: forall era. ( EraCrypto era ~ StandardCrypto diff --git a/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Conway.hs b/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Conway.hs index 589c1d170..bfc017f2e 100644 --- a/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Conway.hs +++ b/cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Conway.hs @@ -38,6 +38,7 @@ module Cardano.Mock.Forging.Tx.Conway ( mkNewConstitutionTx, mkDummyRegisterTx, mkDummyTxBody, + mkDummyTxBodyWithFee, mkTxDelegCert, mkRegTxCert, mkUnRegTxCert, @@ -589,6 +590,21 @@ mkDummyTxBody = mempty (Withdrawals mempty) +mkDummyTxBodyWithFee :: + Coin -> + ConwayTxBody StandardConway +mkDummyTxBodyWithFee coin' = + consTxBody + mempty + mempty + mempty + mempty + SNothing + coin' + mempty + mempty + (Withdrawals mempty) + mkFullTx :: Int -> Integer -> diff --git a/cardano-chain-gen/src/Cardano/Mock/Query.hs b/cardano-chain-gen/src/Cardano/Mock/Query.hs index 416ddcfe5..343202de6 100644 --- a/cardano-chain-gen/src/Cardano/Mock/Query.hs +++ b/cardano-chain-gen/src/Cardano/Mock/Query.hs @@ -11,10 +11,20 @@ module Cardano.Mock.Query ( queryGovActionCounts, queryConstitutionAnchor, queryRewardRests, + queryMultiAssetMetadataPolicy, + queryStakeAddressHashRaw, + queryStakeAddressCount, + queryCollateralTxOutCount, + queryPoolUpdateCount, + queryStakeDeRegCount, + queryStakeRegCount, + countTxOutNonNullStakeAddrIds, ) where import qualified Cardano.Db as Db -import Cardano.Prelude hiding (from, on) +import Cardano.Prelude hiding (from, isNothing, on) +import qualified Data.ByteString.Base16 as Base16 +import Data.ByteString.Short (ShortByteString, toShort) import Database.Esqueleto.Experimental import Prelude () @@ -150,3 +160,60 @@ queryRewardRests = do pure (reward ^. Db.RewardRestType, reward ^. Db.RewardRestAmount) pure $ map (bimap unValue (Db.unDbLovelace . unValue)) res + +queryMultiAssetMetadataPolicy :: MonadIO io => ReaderT SqlBackend io (Maybe ShortByteString) +queryMultiAssetMetadataPolicy = do + res <- selectOne $ do + metadataPolicy <- from $ table @Db.MultiAsset + pure $ metadataPolicy ^. Db.MultiAssetPolicy + pure $ toShort . Base16.encode . unValue <$> res + +queryStakeAddressHashRaw :: MonadIO io => ReaderT SqlBackend io (Maybe ShortByteString) +queryStakeAddressHashRaw = do + res <- selectOne $ do + stakeAddress <- from $ table @Db.StakeAddress + pure $ stakeAddress ^. Db.StakeAddressHashRaw + pure $ toShort . Base16.encode . unValue <$> res + +queryStakeAddressCount :: MonadIO io => ReaderT SqlBackend io Word +queryStakeAddressCount = do + res <- selectOne $ do + _ <- from (table @Db.StakeAddress) + pure countRows + pure $ maybe 0 unValue res + +queryCollateralTxOutCount :: MonadIO io => ReaderT SqlBackend io Word +queryCollateralTxOutCount = do + res <- selectOne $ do + _ <- from (table @Db.CollateralTxOut) + pure countRows + pure $ maybe 0 unValue res + +queryPoolUpdateCount :: MonadIO io => ReaderT SqlBackend io Word +queryPoolUpdateCount = do + res <- selectOne $ do + _ <- from (table @Db.PoolUpdate) + pure countRows + pure $ maybe 0 unValue res + +queryStakeDeRegCount :: MonadIO io => ReaderT SqlBackend io Word +queryStakeDeRegCount = do + res <- selectOne $ do + _ <- from (table @Db.StakeDeregistration) + pure countRows + pure $ maybe 0 unValue res + +queryStakeRegCount :: MonadIO io => ReaderT SqlBackend io Word +queryStakeRegCount = do + res <- selectOne $ do + _ <- from (table @Db.StakeRegistration) + pure countRows + pure $ maybe 0 unValue res + +countTxOutNonNullStakeAddrIds :: (MonadIO m) => SqlPersistT m Word +countTxOutNonNullStakeAddrIds = do + result <- selectOne $ do + txOut <- from $ table @Db.TxOut + where_ $ not_ (isNothing $ txOut ^. Db.TxOutStakeAddressId) + pure countRows + pure $ maybe 0 unValue result diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs index 39605a1da..8a44eff13 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs @@ -16,6 +16,7 @@ import qualified Test.Cardano.Db.Mock.Unit.Conway.Rollback as Rollback import qualified Test.Cardano.Db.Mock.Unit.Conway.Simple as Simple import qualified Test.Cardano.Db.Mock.Unit.Conway.Stake as Stake import qualified Test.Cardano.Db.Mock.Unit.Conway.Tx as Tx +import qualified Test.Cardano.Db.Mock.Unit.Conway.Whitelist as Whitelist import Test.Cardano.Db.Mock.Validate (expectFailSilent) import Test.Tasty (TestTree (), testGroup) import Test.Tasty.HUnit (Assertion (), testCase) @@ -42,6 +43,12 @@ unitTests iom knownMigrations = "remove jsonb from schema and add back" Config.configJsonbInSchemaShouldRemoveThenAdd ] + , testGroup + "invalid whitelist hashes" + [ testCase "Fail if Shelley stake address hash is invalid" Config.invalidShelleyStkAddrHash + , testCase "Fail if multi-asset policies hash is invalid" Config.invalidMultiAssetPoliciesHash + , testCase "Fail if Plutus script hash invalid" Config.invalidPlutusScriptHash + ] , testGroup "tx-out" [ test "consumed_by_tx_id column check" MigrateConsumedPruneTxOut.txConsumedColumnCheck @@ -112,7 +119,6 @@ unitTests iom knownMigrations = , test "consume utxo same block" Tx.consumeSameBlock , test "tx with metadata" Tx.addTxMetadata , test "tx with metadata disabled" Tx.addTxMetadataDisabled - , test "tx with metadata whitelist" Tx.addTxMetadataWhitelist ] , testGroup "stake addresses" @@ -174,6 +180,14 @@ unitTests iom knownMigrations = , test "swap many multi assets" Plutus.swapMultiAssets , test "swap with multi assets disabled" Plutus.swapMultiAssetsDisabled ] + , testGroup + "Whitelist" + [ test "add tx with whitelist" Whitelist.addTxMultiAssetsWhitelist + , test "tx with metadata whitelist" Whitelist.addTxMetadataWhitelist + , test "tx with metadata whitelist multiple" Whitelist.addTxMetadataWhitelistMultiple + , test "add simple tx, whitelist tx address" Whitelist.addSimpleTxStakeAddrsWhitelist + , test "add full tx, with stake address whitelist" Whitelist.fullTxStakeAddressWhitelist + ] , testGroup "Pools and smash" [ test "pool registration" Other.poolReg diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs index ee7e84de0..224eab6e1 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/Parse.hs @@ -8,7 +8,11 @@ module Test.Cardano.Db.Mock.Unit.Conway.Config.Parse ( wrongConwayGenesisHash, insertConfig, defaultInsertConfig, -) where + invalidShelleyStkAddrHash, + invalidMultiAssetPoliciesHash, + invalidPlutusScriptHash, +) +where import Cardano.DbSync.Config import Cardano.DbSync.Config.Types @@ -16,6 +20,8 @@ import Cardano.DbSync.Error import Cardano.Prelude hiding (from, isNothing) import qualified Data.Aeson as Aeson import Data.Default.Class (Default (..)) +import Data.String (String) +import Data.Text (pack) import Test.Cardano.Db.Mock.Config import Test.Tasty.HUnit (Assertion (), assertBool, (@?=)) import Prelude () @@ -23,7 +29,8 @@ import Prelude () conwayGenesis :: Assertion conwayGenesis = mkSyncNodeConfig configDir initCommandLineArgs - >>= void . mkConfig configDir mutableDir cmdLineArgs + >>= void + . mkConfig configDir mutableDir cmdLineArgs where configDir = "config-conway" mutableDir = mkMutableDir "conwayConfigSimple" @@ -108,3 +115,27 @@ insertConfig = do dncInsertOptions cfg @?= expected where configDir = "config-conway-insert-options" + +invalidShelleyStkAddrHash :: Assertion +invalidShelleyStkAddrHash = + let invalidJson = "{ \"enable\": true, \"stake_addresses\": " <> invalidHash <> " }" + decodedResult :: Either String ShelleyInsertConfig + decodedResult = Aeson.eitherDecodeStrict $ encodeUtf8 $ pack invalidJson + in assertBool "Decoding should fail for invalid Shelley stake address hash" (isLeft decodedResult) + +invalidMultiAssetPoliciesHash :: Assertion +invalidMultiAssetPoliciesHash = + let invalidJson = "{ \"enable\": true, \"policies\": " <> invalidHash <> " }" + decodedResult :: Either String MultiAssetConfig + decodedResult = Aeson.eitherDecodeStrict $ encodeUtf8 $ pack invalidJson + in assertBool "Decoding should fail for invalid MultiAsset policies hash" (isLeft decodedResult) + +invalidPlutusScriptHash :: Assertion +invalidPlutusScriptHash = + let invalidJson = "{ \"enable\": true, \"script_hashes\": " <> invalidHash <> " }" + decodedResult :: Either String PlutusConfig + decodedResult = Aeson.eitherDecodeStrict $ encodeUtf8 $ pack invalidJson + in assertBool "Decoding should fail for invalid Plutus script hash" (isLeft decodedResult) + +invalidHash :: String +invalidHash = "[\"\\xe0758b08dea05dabd1cd3510689ebd9efb6a49316acb30eead750e2e9e\"]" diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Plutus.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Plutus.hs index bb5f2c784..c78258b02 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Plutus.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Plutus.hs @@ -1,4 +1,5 @@ {-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} module Test.Cardano.Db.Mock.Unit.Conway.Plutus ( @@ -32,6 +33,8 @@ module Test.Cardano.Db.Mock.Unit.Conway.Plutus ( import Cardano.Crypto.Hash.Class (hashToBytes) import qualified Cardano.Db as DB +import Cardano.DbSync.Config (SyncNodeConfig (..)) +import Cardano.DbSync.Config.Types (MultiAssetConfig (..), SyncInsertOptions (..)) import Cardano.DbSync.Era.Shelley.Generic.Util (renderAddress) import Cardano.Ledger.Coin (Coin (..)) import Cardano.Ledger.Mary.Value (MaryValue (..), MultiAsset (..), PolicyID (..)) @@ -763,41 +766,49 @@ swapMultiAssets = testLabel = "conwaySwapMultiAssets" swapMultiAssetsDisabled :: IOManager -> [(Text, Text)] -> Assertion -swapMultiAssetsDisabled = - withCustomConfig args Nothing cfgDir testLabel $ \interpreter mockServer dbSync -> do - startDBSync dbSync +swapMultiAssetsDisabled ioManager metadata = do + syncNodeConfig <- mksNodeConfig + withCustomConfig cmdlArgs (Just syncNodeConfig) cfgDir testLabel action ioManager metadata + where + action = \interpreter mockServer dbSync -> do + startDBSync dbSync + + -- Forge a block with multiple multi-asset scripts + void $ Api.withConwayFindLeaderAndSubmit interpreter mockServer $ \state' -> do + let policy = PolicyID Examples.alwaysMintScriptHash + assets = Map.singleton (Prelude.head Examples.assetNames) 1 + mintedValue = MultiAsset $ Map.singleton policy assets + outValue = MaryValue (Coin 20) (MultiAsset $ Map.singleton policy assets) + + -- Forge a multi-asset script + tx0 <- + Conway.mkMultiAssetsScriptTx + [UTxOIndex 0] + (UTxOIndex 1) + [(UTxOAddress Examples.alwaysSucceedsScriptAddr, outValue)] + [] + mintedValue + True + 100 + state' - -- Forge a block with multiple multi-asset scripts - void $ Api.withConwayFindLeaderAndSubmit interpreter mockServer $ \state' -> do - let policy = PolicyID Examples.alwaysMintScriptHash - assets = Map.singleton (Prelude.head Examples.assetNames) 1 - mintedValue = MultiAsset $ Map.singleton policy assets - outValue = MaryValue (Coin 20) (MultiAsset $ Map.singleton policy assets) + pure [tx0] - -- Forge a multi-asset script - tx0 <- - Conway.mkMultiAssetsScriptTx - [UTxOIndex 0] - (UTxOIndex 1) - [(UTxOAddress Examples.alwaysSucceedsScriptAddr, outValue)] - [] - mintedValue - True - 100 - state' + -- Wait for it to sync + assertBlockNoBackoff dbSync 1 + -- Verify multi-assets + assertEqBackoff dbSync queryMultiAssetCount 0 [] "Unexpected multi-assets" - pure [tx0] + cmdlArgs = initCommandLineArgs {claFullMode = False} - -- Wait for it to sync - assertBlockNoBackoff dbSync 1 - -- Verify multi-assets - assertEqBackoff dbSync queryMultiAssetCount 0 [] "Unexpected multi-assets" - where - args = - initCommandLineArgs - { claConfigFilename = "test-db-sync-config-no-multi-assets.json" - , claFullMode = False - } + mksNodeConfig :: IO SyncNodeConfig + mksNodeConfig = do + initConfigFile <- mkSyncNodeConfig cfgDir cmdlArgs + let dncInsertOptions' = dncInsertOptions initConfigFile + pure $ + initConfigFile + { dncInsertOptions = dncInsertOptions' {sioMultiAsset = MultiAssetDisable} + } testLabel = "conwayConfigMultiAssetsDisabled" cfgDir = conwayConfigDir diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Stake.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Stake.hs index d415c6dc8..e1c8d9fb2 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Stake.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Stake.hs @@ -412,7 +412,7 @@ registerStakeCreds = do registerStakeCredsNoShelley :: IOManager -> [(Text, Text)] -> Assertion registerStakeCredsNoShelley = do - withCustomConfig args Nothing cfgDir testLabel $ \interpreter mockServer dbSync -> do + withCustomConfigAndDropDB args Nothing cfgDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync -- These should not be saved when shelley is disabled diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Tx.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Tx.hs index 48a19da3a..5b5f27118 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Tx.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Tx.hs @@ -7,7 +7,6 @@ module Test.Cardano.Db.Mock.Unit.Conway.Tx ( consumeSameBlock, addTxMetadata, addTxMetadataDisabled, - addTxMetadataWhitelist, ) where import Cardano.DbSync.Config (SyncNodeConfig (..)) @@ -132,31 +131,6 @@ addTxMetadata ioManager metadata = do { dncInsertOptions = dncInsertOptions' {sioMetadata = MetadataEnable} } -addTxMetadataWhitelist :: IOManager -> [(Text, Text)] -> Assertion -addTxMetadataWhitelist = do - withCustomConfigAndDropDB args Nothing cfgDir testLabel $ \interpreter mockServer dbSync -> do - startDBSync dbSync - - -- Add blocks with transactions - void $ do - UnifiedApi.withConwayFindLeaderAndSubmitTx interpreter mockServer $ \_ -> - let txBody = Conway.mkDummyTxBody - auxData = Map.fromList [(1, I 1), (2, I 2)] - in Right (Conway.mkAuxDataTx True txBody auxData) - - -- Wait for it to sync - assertBlockNoBackoff dbSync 1 - -- Should have tx metadata - assertEqBackoff dbSync queryTxMetadataCount 1 [] "Expected tx metadata" - where - args = - initCommandLineArgs - { claConfigFilename = "test-db-sync-config-keep-metadata.json" - , claFullMode = False - } - testLabel = "conwayConfigMetadataKeep" - cfgDir = conwayConfigDir - addTxMetadataDisabled :: IOManager -> [(Text, Text)] -> Assertion addTxMetadataDisabled ioManager metadata = do syncNodeConfig <- mksNodeConfig diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Whitelist.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Whitelist.hs new file mode 100644 index 000000000..5d057b328 --- /dev/null +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Whitelist.hs @@ -0,0 +1,287 @@ +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} + +module Test.Cardano.Db.Mock.Unit.Conway.Whitelist ( + addTxMultiAssetsWhitelist, + addTxMetadataWhitelist, + addTxMetadataWhitelistMultiple, + addSimpleTxStakeAddrsWhitelist, + fullTxStakeAddressWhitelist, +) +where + +import Cardano.DbSync.Config (SyncNodeConfig (..)) +import Cardano.DbSync.Config.Types (MetadataConfig (..), MultiAssetConfig (..), ShelleyInsertConfig (..), SyncInsertOptions (..)) +import Cardano.Ledger.Coin (Coin (..)) +import Cardano.Ledger.Mary.Value (MaryValue (..), MultiAsset (..), PolicyID (..)) +import Cardano.Ledger.Shelley.TxAuxData (Metadatum (..)) +import Cardano.Mock.ChainSync.Server (IOManager ()) +import qualified Cardano.Mock.Forging.Tx.Alonzo.ScriptsExamples as Examples +import qualified Cardano.Mock.Forging.Tx.Conway as Conway +import Cardano.Mock.Forging.Types +import qualified Cardano.Mock.Query as MockQ +import Cardano.Prelude hiding (head) +import Data.ByteString.Short (toShort) +import Data.List.NonEmpty (fromList) +import qualified Data.Map as Map +import Test.Cardano.Db.Mock.Config +import Test.Cardano.Db.Mock.UnifiedApi (withConwayFindLeaderAndSubmit) +import qualified Test.Cardano.Db.Mock.UnifiedApi as Api +import qualified Test.Cardano.Db.Mock.UnifiedApi as UnifiedApi +import Test.Cardano.Db.Mock.Validate +import Test.Tasty.HUnit (Assertion ()) +import Prelude (head, (!!)) + +addTxMultiAssetsWhitelist :: IOManager -> [(Text, Text)] -> Assertion +addTxMultiAssetsWhitelist ioManager metadata = do + syncNodeConfig <- mksNodeConfig + withCustomConfig args (Just syncNodeConfig) cfgDir testLabel action ioManager metadata + where + action = \interpreter mockServer dbSync -> do + startDBSync dbSync + -- Forge a block with multiple multi-asset scripts + void $ Api.withConwayFindLeaderAndSubmit interpreter mockServer $ \state' -> do + let assetsMinted = + Map.fromList [(head Examples.assetNames, 10), (Examples.assetNames !! 1, 4)] + policy0 = PolicyID $ Examples.alwaysMintScriptHashRandomPolicyVal 1 + policy1 = PolicyID $ Examples.alwaysMintScriptHashRandomPolicyVal 2 + mintValue = + MultiAsset $ + Map.fromList [(policy0, assetsMinted), (policy1, assetsMinted)] + assets = + Map.fromList [(head Examples.assetNames, 5), (Examples.assetNames !! 1, 2)] + outValue = + MaryValue (Coin 20) $ + MultiAsset $ + Map.fromList [(policy0, assets), (policy1, assets)] + + -- Forge a multi-asset script + tx0 <- + Conway.mkMultiAssetsScriptTx + [UTxOIndex 0] + (UTxOIndex 1) + [ (UTxOAddress Examples.alwaysSucceedsScriptAddr, outValue) + , (UTxOAddress Examples.alwaysMintScriptAddr, outValue) + ] + [] + mintValue + True + 100 + state' + + -- Consume the outputs from tx0 + let utxos = Conway.mkUTxOConway tx0 + tx1 <- + Conway.mkMultiAssetsScriptTx + [UTxOPair (head utxos), UTxOPair (utxos !! 1), UTxOIndex 2] + (UTxOIndex 3) + [ (UTxOAddress Examples.alwaysSucceedsScriptAddr, outValue) + , (UTxOAddress Examples.alwaysMintScriptAddr, outValue) + , (UTxOAddressNew 0, outValue) + , (UTxOAddressNew 0, outValue) + ] + [] + mintValue + True + 200 + state' + pure [tx0, tx1] + + -- Verify script counts + assertBlockNoBackoff dbSync 1 + assertAlonzoCounts dbSync (2, 4, 1, 2, 4, 2, 0, 0) + -- create 4 multi-assets but only 2 should be added due to the whitelist + assertEqBackoff dbSync MockQ.queryMultiAssetCount 2 [] "Expected 2 multi-assets" + -- do the policy match the whitelist + assertEqBackoff dbSync MockQ.queryMultiAssetMetadataPolicy (Just policyShortBs) [] "Expected correct policy in db" + + args = initCommandLineArgs {claFullMode = False} + testLabel = "conwayConfigMultiAssetsWhitelist" + + cfgDir = conwayConfigDir + + policyShortBs = toShort "4509cdddad21412c22c9164e10bc6071340ba235562f1575a35ded4d" + + mksNodeConfig :: IO SyncNodeConfig + mksNodeConfig = do + initConfigFile <- mkSyncNodeConfig cfgDir args + let dncInsertOptions' = dncInsertOptions initConfigFile + pure $ + initConfigFile + { dncInsertOptions = + dncInsertOptions' + { sioMultiAsset = + MultiAssetPolicies $ + fromList [policyShortBs] + } + } + +-- 2 blocks each with 4 metadata entries. +-- The whitelist has one tx metadata key which is in the first block +-- so only the TX in the first block should have tx metadata kept. +addTxMetadataWhitelist :: IOManager -> [(Text, Text)] -> Assertion +addTxMetadataWhitelist ioManager metadata = do + syncNodeConfig <- mksNodeConfig + withCustomConfigAndDropDB args (Just syncNodeConfig) cfgDir testLabel action ioManager metadata + where + action = \interpreter mockServer dbSync -> do + startDBSync dbSync + -- Add transactions with metadata + void $ do + UnifiedApi.withConwayFindLeaderAndSubmitTx interpreter mockServer $ \_ -> + let txBody = Conway.mkDummyTxBodyWithFee $ Coin 1_000 + auxData = Map.fromList [(1, I 1), (2, I 2), (3, I 3), (4, I 4)] + in Right (Conway.mkAuxDataTx True txBody auxData) + void $ do + UnifiedApi.withConwayFindLeaderAndSubmitTx interpreter mockServer $ \_ -> + let txBody = Conway.mkDummyTxBodyWithFee $ Coin 2_000 + auxData = Map.fromList [(5, I 5), (6, I 6), (7, I 7), (8, I 8)] + in Right (Conway.mkAuxDataTx True txBody auxData) + + assertBlockNoBackoff dbSync 2 + -- Should have first block's tx metadata + assertEqBackoff dbSync MockQ.queryTxMetadataCount 4 [] "Expected tx metadata" + + args = initCommandLineArgs {claFullMode = False} + testLabel = "conwayConfigMetadataWhitelist" + + cfgDir = conwayConfigDir + + -- match all metadata keys of value 1 + mksNodeConfig :: IO SyncNodeConfig + mksNodeConfig = do + initConfigFile <- mkSyncNodeConfig cfgDir args + let dncInsertOptions' = dncInsertOptions initConfigFile + pure $ + initConfigFile + { dncInsertOptions = dncInsertOptions' {sioMetadata = MetadataKeys $ fromList [1]} + } + +-- 2 blocks each with 4 metadata entries +-- The whitelist is set to keys [1,6] each key in in different TX +-- so all TxMetadata should be kept from both blocks. +addTxMetadataWhitelistMultiple :: IOManager -> [(Text, Text)] -> Assertion +addTxMetadataWhitelistMultiple ioManager metadata = do + syncNodeConfig <- mksNodeConfig + withCustomConfigAndDropDB args (Just syncNodeConfig) cfgDir testLabel action ioManager metadata + where + action = \interpreter mockServer dbSync -> do + startDBSync dbSync + -- Add transactions with metadata + void $ do + UnifiedApi.withConwayFindLeaderAndSubmitTx interpreter mockServer $ \_ -> + let txBody = Conway.mkDummyTxBodyWithFee $ Coin 1_000 + auxData = Map.fromList [(1, I 1), (2, I 2), (3, I 3), (4, I 4)] + in Right (Conway.mkAuxDataTx True txBody auxData) + void $ do + UnifiedApi.withConwayFindLeaderAndSubmitTx interpreter mockServer $ \_ -> + let txBody = Conway.mkDummyTxBodyWithFee $ Coin 2_000 + auxData = Map.fromList [(5, I 5), (6, I 6), (7, I 7), (8, I 8)] + in Right (Conway.mkAuxDataTx True txBody auxData) + + assertBlockNoBackoff dbSync 2 + -- Should have both block's tx metadata + assertEqBackoff dbSync MockQ.queryTxMetadataCount 8 [] "Expected tx metadata" + + args = initCommandLineArgs {claFullMode = False} + testLabel = "conwayConfigMetadataWhitelist" + + cfgDir = conwayConfigDir + + -- match all metadata keys of value 1 + mksNodeConfig :: IO SyncNodeConfig + mksNodeConfig = do + initConfigFile <- mkSyncNodeConfig cfgDir args + let dncInsertOptions' = dncInsertOptions initConfigFile + pure $ + initConfigFile + { dncInsertOptions = dncInsertOptions' {sioMetadata = MetadataKeys $ fromList [1, 6]} + } + +addSimpleTxStakeAddrsWhitelist :: IOManager -> [(Text, Text)] -> Assertion +addSimpleTxStakeAddrsWhitelist ioManager metadata = do + syncNodeConfig <- mksNodeConfig + withCustomConfigAndDropDB args (Just syncNodeConfig) cfgDir testLabel action ioManager metadata + where + action = \interpreter mockServer dbSync -> do + -- Forge a block + void $ + UnifiedApi.withConwayFindLeaderAndSubmitTx interpreter mockServer $ + Conway.mkPaymentTx (UTxOIndex 0) (UTxOIndex 1) 10_000 500 + + startDBSync dbSync + -- Verify it syncs + assertBlockNoBackoff dbSync 1 + assertTxCount dbSync 12 + + assertEqBackoff dbSync MockQ.queryStakeAddressHashRaw (Just shelleyStakeAddrShortBs) [] "Expected matching stake address" + + testLabel = "conwayAddSimpleTx" + args = initCommandLineArgs {claFullMode = False} + cfgDir = conwayConfigDir + shelleyStakeAddrShortBs = toShort "e0921c25093b263793a1baf36166b819543f5822c62f72571111111111" + -- match all metadata keys of value 1 + mksNodeConfig :: IO SyncNodeConfig + mksNodeConfig = do + initConfigFile <- mkSyncNodeConfig cfgDir args + let dncInsertOptions' = dncInsertOptions initConfigFile + pure $ + initConfigFile + { dncInsertOptions = + dncInsertOptions' + { sioShelley = + ShelleyStakeAddrs $ + fromList [shelleyStakeAddrShortBs] + } + } + +fullTxStakeAddressWhitelist :: IOManager -> [(Text, Text)] -> Assertion +fullTxStakeAddressWhitelist ioManager metadata = do + syncNodeConfig <- mksNodeConfig + withCustomConfig args (Just syncNodeConfig) cfgDir testLabel action ioManager metadata + where + action = + \interpreter mockServer dbSync -> do + startDBSync dbSync + -- Add some blocks with transactions + void $ withConwayFindLeaderAndSubmit interpreter mockServer $ \state' -> + sequence + [ Conway.mkFullTx 0 100 state' + , Conway.mkFullTx 1 200 state' + ] + -- Wait for them to sync + assertBlockNoBackoff dbSync 1 + assertTxCount dbSync 13 + -- Check all tables that stake addresses effect + assertEqBackoff dbSync MockQ.queryStakeAddressCount 5 [] "Expected 5 stake addresses" + assertEqBackoff dbSync MockQ.queryCollateralTxOutCount 2 [] "Expected 1 collateral tx out" + assertEqBackoff dbSync MockQ.queryPoolUpdateCount 5 [] "Expected 3 pool updates" + assertEqBackoff dbSync MockQ.queryStakeDeRegCount 2 [] "Expected 1 stake deregistration" + assertEqBackoff dbSync MockQ.queryStakeRegCount 2 [] "Expected 1 stake registration" + assertEqBackoff dbSync MockQ.countTxOutNonNullStakeAddrIds 2 [] "Expected 1 non-null stake address id" + -- TODO: Cmdv: Missing tables checks that are currently blank in tests: + -- delegation_vote, gov_action_proposal, instant_reward, reserve, + -- treasury, treasury_withdrawl. + + testLabel = "fullTxStakeAddressWhitelist" + args = initCommandLineArgs {claFullMode = True} + cfgDir = conwayConfigDir + shelleyStakeAddr0 = toShort "e0addfa484e8095ff53f45b25cf337923cf79abe6ec192fdf288d621f9" + shelleyStakeAddr1 = toShort "e0921c25093b263793a1baf36166b819543f5822c62f72571111111111" + shelleyStakeAddr2 = toShort "e0921c25093b263793a1baf36166b819543f5822c62f72573333333333" + shelleyStakeAddr3 = toShort "e0000131350ac206583290486460934394208654903261221230945870" + shelleyStakeAddr4 = toShort "e022236827154873624578632414768234573268457923654973246472" + + mksNodeConfig :: IO SyncNodeConfig + mksNodeConfig = do + initConfigFile <- mkSyncNodeConfig cfgDir args + let dncInsertOptions' = dncInsertOptions initConfigFile + pure $ + initConfigFile + { dncInsertOptions = + dncInsertOptions' + { sioShelley = + ShelleyStakeAddrs $ + fromList [shelleyStakeAddr0, shelleyStakeAddr1, shelleyStakeAddr2, shelleyStakeAddr3, shelleyStakeAddr4] + } + } diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs index 9e1576055..8ce442541 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs @@ -40,7 +40,8 @@ module Test.Cardano.Db.Mock.Validate ( assertPoolCounters, poolCountersQuery, checkStillRuns, -) where +) +where import Cardano.Db import qualified Cardano.Db as DB @@ -216,7 +217,7 @@ assertAddrValues env ix expected sta = do q = queryAddressOutputs address assertEqBackoff env q expected defaultDelays "Unexpected Balance" -assertRight :: Show err => Either err a -> IO a +assertRight :: (Show err) => Either err a -> IO a assertRight ei = case ei of Right a -> pure a @@ -243,7 +244,7 @@ assertCertCounts env expected = pure (registr - 5, deregistr, deleg - 5, withdrawal) assertRewardCounts :: - EraCrypto era ~ StandardCrypto => + (EraCrypto era ~ StandardCrypto) => DBSyncEnv -> LedgerState (ShelleyBlock p era) -> Bool -> @@ -477,11 +478,11 @@ poolCountersQuery = do <$> (select . from $ \(_a :: SqlExpr (Entity PoolRelay)) -> pure countRows) pure (poolHash, poolMetadataRef, poolUpdate, poolOwner, poolRetire, poolRelay) -addPoolCounters :: Num a => (a, a, a, a, a, a) -> (a, a, a, a, a, a) -> (a, a, a, a, a, a) +addPoolCounters :: (Num a) => (a, a, a, a, a, a) -> (a, a, a, a, a, a) -> (a, a, a, a, a, a) addPoolCounters (a, b, c, d, e, f) (a', b', c', d', e', f') = (a + a', b + b', c + c', d + d', e + e', f + f') assertPoolLayerCounters :: - EraCrypto era ~ StandardCrypto => + (EraCrypto era ~ StandardCrypto) => DBSyncEnv -> (Word64, Word64) -> [(PoolIndex, (Either DBFail Bool, Bool, Bool))] -> diff --git a/cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-keep-metadata.json b/cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-keep-metadata.json deleted file mode 100644 index 670309b67..000000000 --- a/cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-keep-metadata.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "EnableLogMetrics": false, - "EnableLogging": true, - "NetworkName": "testing", - "NodeConfigFile": "test-config.json", - "PrometheusPort": 8080, - "RequiresNetworkMagic": "RequiresMagic", - "defaultBackends": [ - "KatipBK" - ], - "defaultScribes": [ - [ - "StdoutSK", - "stdout" - ] - ], - "minSeverity": "Info", - "options": { - "cfokey": { - "value": "Release-1.0.0" - }, - "mapBackends": {}, - "mapSeverity": { - "db-sync-node": "Info", - "db-sync-node.Mux": "Error", - "db-sync-node.Subscription": "Error" - }, - "mapSubtrace": { - "#ekgview": { - "contents": [ - [ - { - "contents": "cardano.epoch-validation.benchmark", - "tag": "Contains" - }, - [ - { - "contents": ".monoclock.basic.", - "tag": "Contains" - } - ] - ], - [ - { - "contents": "cardano.epoch-validation.benchmark", - "tag": "Contains" - }, - [ - { - "contents": "diff.RTS.cpuNs.timed.", - "tag": "Contains" - } - ] - ], - [ - { - "contents": "#ekgview.#aggregation.cardano.epoch-validation.benchmark", - "tag": "StartsWith" - }, - [ - { - "contents": "diff.RTS.gcNum.timed.", - "tag": "Contains" - } - ] - ] - ], - "subtrace": "FilterTrace" - }, - "#messagecounters.aggregation": { - "subtrace": "NoTrace" - }, - "#messagecounters.ekgview": { - "subtrace": "NoTrace" - }, - "#messagecounters.katip": { - "subtrace": "NoTrace" - }, - "#messagecounters.monitoring": { - "subtrace": "NoTrace" - }, - "#messagecounters.switchboard": { - "subtrace": "NoTrace" - }, - "benchmark": { - "contents": [ - "GhcRtsStats", - "MonotonicClock" - ], - "subtrace": "ObservableTrace" - }, - "cardano.epoch-validation.utxo-stats": { - "subtrace": "NoTrace" - } - } - }, - "rotation": { - "rpKeepFilesNum": 10, - "rpLogLimitBytes": 5000000, - "rpMaxAgeHours": 24 - }, - "setupBackends": [ - "AggregationBK", - "KatipBK" - ], - "setupScribes": [ - { - "scFormat": "ScText", - "scKind": "StdoutSK", - "scName": "stdout", - "scRotation": null - } - ], - "insert_options": { - "metadata": { - "enable": true, - "keys": [1] - } - } -} diff --git a/cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-no-metadata.json b/cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-no-metadata.json deleted file mode 100644 index 730e364de..000000000 --- a/cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-no-metadata.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "EnableLogMetrics": false, - "EnableLogging": true, - "NetworkName": "testing", - "NodeConfigFile": "test-config.json", - "PrometheusPort": 8080, - "RequiresNetworkMagic": "RequiresMagic", - "defaultBackends": [ - "KatipBK" - ], - "defaultScribes": [ - [ - "StdoutSK", - "stdout" - ] - ], - "minSeverity": "Info", - "options": { - "cfokey": { - "value": "Release-1.0.0" - }, - "mapBackends": {}, - "mapSeverity": { - "db-sync-node": "Info", - "db-sync-node.Mux": "Error", - "db-sync-node.Subscription": "Error" - }, - "mapSubtrace": { - "#ekgview": { - "contents": [ - [ - { - "contents": "cardano.epoch-validation.benchmark", - "tag": "Contains" - }, - [ - { - "contents": ".monoclock.basic.", - "tag": "Contains" - } - ] - ], - [ - { - "contents": "cardano.epoch-validation.benchmark", - "tag": "Contains" - }, - [ - { - "contents": "diff.RTS.cpuNs.timed.", - "tag": "Contains" - } - ] - ], - [ - { - "contents": "#ekgview.#aggregation.cardano.epoch-validation.benchmark", - "tag": "StartsWith" - }, - [ - { - "contents": "diff.RTS.gcNum.timed.", - "tag": "Contains" - } - ] - ] - ], - "subtrace": "FilterTrace" - }, - "#messagecounters.aggregation": { - "subtrace": "NoTrace" - }, - "#messagecounters.ekgview": { - "subtrace": "NoTrace" - }, - "#messagecounters.katip": { - "subtrace": "NoTrace" - }, - "#messagecounters.monitoring": { - "subtrace": "NoTrace" - }, - "#messagecounters.switchboard": { - "subtrace": "NoTrace" - }, - "benchmark": { - "contents": [ - "GhcRtsStats", - "MonotonicClock" - ], - "subtrace": "ObservableTrace" - }, - "cardano.epoch-validation.utxo-stats": { - "subtrace": "NoTrace" - } - } - }, - "rotation": { - "rpKeepFilesNum": 10, - "rpLogLimitBytes": 5000000, - "rpMaxAgeHours": 24 - }, - "setupBackends": [ - "AggregationBK", - "KatipBK" - ], - "setupScribes": [ - { - "scFormat": "ScText", - "scKind": "StdoutSK", - "scName": "stdout", - "scRotation": null - } - ], - "insert_options": { - "metadata": { - "enable": false - } - } -} diff --git a/cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-no-multi-assets.json b/cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-no-multi-assets.json deleted file mode 100644 index ef96054a6..000000000 --- a/cardano-chain-gen/test/testfiles/config-conway/test-db-sync-config-no-multi-assets.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "EnableLogMetrics": false, - "EnableLogging": true, - "NetworkName": "testing", - "NodeConfigFile": "test-config.json", - "PrometheusPort": 8080, - "RequiresNetworkMagic": "RequiresMagic", - "defaultBackends": [ - "KatipBK" - ], - "defaultScribes": [ - [ - "StdoutSK", - "stdout" - ] - ], - "minSeverity": "Info", - "options": { - "cfokey": { - "value": "Release-1.0.0" - }, - "mapBackends": {}, - "mapSeverity": { - "db-sync-node": "Info", - "db-sync-node.Mux": "Error", - "db-sync-node.Subscription": "Error" - }, - "mapSubtrace": { - "#ekgview": { - "contents": [ - [ - { - "contents": "cardano.epoch-validation.benchmark", - "tag": "Contains" - }, - [ - { - "contents": ".monoclock.basic.", - "tag": "Contains" - } - ] - ], - [ - { - "contents": "cardano.epoch-validation.benchmark", - "tag": "Contains" - }, - [ - { - "contents": "diff.RTS.cpuNs.timed.", - "tag": "Contains" - } - ] - ], - [ - { - "contents": "#ekgview.#aggregation.cardano.epoch-validation.benchmark", - "tag": "StartsWith" - }, - [ - { - "contents": "diff.RTS.gcNum.timed.", - "tag": "Contains" - } - ] - ] - ], - "subtrace": "FilterTrace" - }, - "#messagecounters.aggregation": { - "subtrace": "NoTrace" - }, - "#messagecounters.ekgview": { - "subtrace": "NoTrace" - }, - "#messagecounters.katip": { - "subtrace": "NoTrace" - }, - "#messagecounters.monitoring": { - "subtrace": "NoTrace" - }, - "#messagecounters.switchboard": { - "subtrace": "NoTrace" - }, - "benchmark": { - "contents": [ - "GhcRtsStats", - "MonotonicClock" - ], - "subtrace": "ObservableTrace" - }, - "cardano.epoch-validation.utxo-stats": { - "subtrace": "NoTrace" - } - } - }, - "rotation": { - "rpKeepFilesNum": 10, - "rpLogLimitBytes": 5000000, - "rpMaxAgeHours": 24 - }, - "setupBackends": [ - "AggregationBK", - "KatipBK" - ], - "setupScribes": [ - { - "scFormat": "ScText", - "scKind": "StdoutSK", - "scName": "stdout", - "scRotation": null - } - ], - "insert_options": { - "multi_asset": { - "enable": false - } - } -} diff --git a/cardano-chain-gen/test/testfiles/fingerprint/conwayConfigMetadataWhitelist b/cardano-chain-gen/test/testfiles/fingerprint/conwayConfigMetadataWhitelist new file mode 100644 index 000000000..c40a0a0ca --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/conwayConfigMetadataWhitelist @@ -0,0 +1 @@ +[12,16] \ No newline at end of file diff --git a/cardano-chain-gen/test/testfiles/fingerprint/conwayConfigMultiAssetsDisabled b/cardano-chain-gen/test/testfiles/fingerprint/conwayConfigMultiAssetsWhitelist similarity index 100% rename from cardano-chain-gen/test/testfiles/fingerprint/conwayConfigMultiAssetsDisabled rename to cardano-chain-gen/test/testfiles/fingerprint/conwayConfigMultiAssetsWhitelist diff --git a/cardano-chain-gen/test/testfiles/fingerprint/fullTxStakeAddressWhitelist b/cardano-chain-gen/test/testfiles/fingerprint/fullTxStakeAddressWhitelist new file mode 100644 index 000000000..a81fc6a84 --- /dev/null +++ b/cardano-chain-gen/test/testfiles/fingerprint/fullTxStakeAddressWhitelist @@ -0,0 +1 @@ +[12] \ No newline at end of file diff --git a/cardano-db-sync/cardano-db-sync.cabal b/cardano-db-sync/cardano-db-sync.cabal index bbd569375..5c56511a3 100644 --- a/cardano-db-sync/cardano-db-sync.cabal +++ b/cardano-db-sync/cardano-db-sync.cabal @@ -83,12 +83,11 @@ library Cardano.DbSync.Era.Shelley.Generic.Tx.Types Cardano.DbSync.Era.Shelley.Generic.Util Cardano.DbSync.Era.Shelley.Generic.Witness - Cardano.DbSync.Era.Shelley.Genesis Cardano.DbSync.Era.Shelley.Query Cardano.DbSync.Era.Universal.Adjust Cardano.DbSync.Era.Universal.Block Cardano.DbSync.Era.Universal.Epoch - Cardano.DbSync.Era.Universal.Validate + Cardano.DbSync.Era.Universal.Genesis Cardano.DbSync.Era.Universal.Insert.Certificate Cardano.DbSync.Era.Universal.Insert.GovAction Cardano.DbSync.Era.Universal.Insert.Grouped @@ -96,6 +95,7 @@ library Cardano.DbSync.Era.Universal.Insert.Other Cardano.DbSync.Era.Universal.Insert.Pool Cardano.DbSync.Era.Universal.Insert.Tx + Cardano.DbSync.Era.Universal.Validate -- Temporary debugging validation @@ -141,6 +141,7 @@ library Cardano.DbSync.Util.Bech32 Cardano.DbSync.Util.Cbor Cardano.DbSync.Util.Constraint + Cardano.DbSync.Util.Whitelist Paths_cardano_db_sync diff --git a/cardano-db-sync/src/Cardano/DbSync.hs b/cardano-db-sync/src/Cardano/DbSync.hs index 5ac873b73..76c62b6d6 100644 --- a/cardano-db-sync/src/Cardano/DbSync.hs +++ b/cardano-db-sync/src/Cardano/DbSync.hs @@ -47,7 +47,6 @@ import Cardano.Prelude hiding (Nat, (%)) import Cardano.Slotting.Slot (EpochNo (..)) import Control.Concurrent.Async import Control.Monad.Extra (whenJust) -import qualified Data.Strict.Maybe as Strict import qualified Data.Text as Text import Data.Version (showVersion) import Database.Persist.Postgresql (ConnectionString, withPostgresqlConn) @@ -160,8 +159,7 @@ runSyncNode :: IO () runSyncNode metricsSetters trce iomgr dbConnString ranMigrations runMigrationFnc syncNodeConfigFromFile syncNodeParams syncOptions = do whenJust maybeLedgerDir $ - \enpLedgerStateDir -> do - createDirectoryIfMissing True (unLedgerStateDir enpLedgerStateDir) + \enpLedgerStateDir -> createDirectoryIfMissing True (unLedgerStateDir enpLedgerStateDir) logInfo trce $ "Using byron genesis file from: " <> (show . unGenesisFile $ dncByronGenesisFile syncNodeConfigFromFile) logInfo trce $ "Using shelley genesis file from: " <> (show . unGenesisFile $ dncShelleyGenesisFile syncNodeConfigFromFile) logInfo trce $ "Using alonzo genesis file from: " <> (show . unGenesisFile $ dncAlonzoGenesisFile syncNodeConfigFromFile) @@ -256,24 +254,17 @@ extractSyncOptions snp aop snc = , snapshotEveryLagging = enpSnEveryLagging snp } where - maybeKeepMNames = - case sioMetadata (dncInsertOptions snc) of - MetadataKeys ks -> Strict.Just (map fromIntegral $ toList ks) - MetadataEnable -> Strict.Nothing - MetadataDisable -> Strict.Nothing - iopts = InsertOptions { ioInOut = isTxOutEnabled' , ioTxCBOR = isTxCBOREnabled (sioTxCBOR (dncInsertOptions snc)) , ioUseLedger = useLedger - , ioShelley = isShelleyEnabled (sioShelley (dncInsertOptions snc)) + , ioShelley = sioShelley (dncInsertOptions snc) , -- Rewards are only disabled on "disable_all" and "only_gov" presets ioRewards = True - , ioMultiAssets = isMultiAssetEnabled (sioMultiAsset (dncInsertOptions snc)) - , ioMetadata = isMetadataEnabled (sioMetadata (dncInsertOptions snc)) - , ioKeepMetadataNames = maybeKeepMNames - , ioPlutusExtra = isPlutusEnabled (sioPlutus (dncInsertOptions snc)) + , ioMultiAssets = sioMultiAsset (dncInsertOptions snc) + , ioMetadata = sioMetadata (dncInsertOptions snc) + , ioPlutus = sioPlutus (dncInsertOptions snc) , ioOffChainPoolData = useOffchainPoolData , ioGov = useGovernance , ioRemoveJsonbFromSchema = isRemoveJsonbFromSchemaEnabled (sioRemoveJsonbFromSchema (dncInsertOptions snc)) diff --git a/cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs b/cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs index 86e6e92a4..8d4bb3817 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs @@ -40,6 +40,7 @@ import Data.ByteString (ByteString) import Data.List.Extra import Data.Map (Map) import qualified Data.Map.Strict as Map +import Data.Maybe (catMaybes) import qualified Data.Text as Text import Database.Persist.Sql (SqlBackend) import Lens.Micro @@ -148,7 +149,7 @@ storePage :: ExceptT SyncNodeError (ReaderT SqlBackend m) () storePage syncEnv cache percQuantum (n, ls) = do when (n `mod` 10 == 0) $ liftIO $ logInfo trce $ "Bootstrap in progress " <> prc <> "%" - txOuts <- mapM (prepareTxOut syncEnv cache) ls + txOuts <- catMaybes <$> mapM (prepareTxOut syncEnv cache) ls txOutIds <- lift . DB.insertManyTxOutPlex True False $ etoTxOut . fst <$> txOuts let maTxOuts = concatMap mkmaTxOuts $ zip txOutIds (snd <$> txOuts) void . lift $ DB.insertManyMaTxOut maTxOuts @@ -170,14 +171,13 @@ prepareTxOut :: SyncEnv -> TxCache -> (TxIn StandardCrypto, BabbageTxOut era) -> - ExceptT SyncNodeError (ReaderT SqlBackend m) (ExtendedTxOut, [MissingMaTxOut]) + ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe (ExtendedTxOut, [MissingMaTxOut])) prepareTxOut syncEnv txCache (TxIn txHash (TxIx index), txOut) = do let txHashByteString = Generic.safeHashToByteString $ unTxId txHash let genTxOut = fromTxOut index txOut txId <- queryTxIdWithCache txCache txHashByteString - insertTxOut trce cache iopts (txId, txHashByteString) genTxOut + insertTxOut syncEnv cache iopts (txId, txHashByteString) genTxOut where - trce = getTrace syncEnv cache = envCache syncEnv iopts = soptInsertOptions $ envOptions syncEnv diff --git a/cardano-db-sync/src/Cardano/DbSync/Api/Types.hs b/cardano-db-sync/src/Cardano/DbSync/Api/Types.hs index cef302752..90b2db7e2 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Api/Types.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Api/Types.hs @@ -15,7 +15,7 @@ module Cardano.DbSync.Api.Types ( import qualified Cardano.Db as DB import Cardano.DbSync.Cache.Types (CacheStatus) -import Cardano.DbSync.Config.Types (SyncNodeConfig) +import Cardano.DbSync.Config.Types (MetadataConfig, MultiAssetConfig, PlutusConfig, ShelleyInsertConfig, SyncNodeConfig) import Cardano.DbSync.Ledger.Types (HasLedgerEnv) import Cardano.DbSync.LocalStateQuery (NoLedgerEnv) import Cardano.DbSync.Types ( @@ -74,18 +74,17 @@ data SyncOptions = SyncOptions deriving (Show) data InsertOptions = InsertOptions - { ioTxCBOR :: !Bool + { ioGov :: !Bool , ioInOut :: !Bool - , ioUseLedger :: !Bool - , ioShelley :: !Bool - , ioRewards :: !Bool - , ioMultiAssets :: !Bool - , ioMetadata :: !Bool - , ioKeepMetadataNames :: Strict.Maybe [Word64] - , ioPlutusExtra :: !Bool + , ioMetadata :: !MetadataConfig + , ioMultiAssets :: !MultiAssetConfig , ioOffChainPoolData :: !Bool - , ioGov :: !Bool + , ioPlutus :: !PlutusConfig , ioRemoveJsonbFromSchema :: !Bool + , ioRewards :: !Bool + , ioShelley :: !ShelleyInsertConfig + , ioTxCBOR :: !Bool + , ioUseLedger :: !Bool } deriving (Show) diff --git a/cardano-db-sync/src/Cardano/DbSync/Cache.hs b/cardano-db-sync/src/Cardano/DbSync/Cache.hs index ccafe230b..f92477af3 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Cache.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Cache.hs @@ -11,23 +11,26 @@ module Cardano.DbSync.Cache ( insertBlockAndCache, insertDatumAndCache, insertPoolKeyWithCache, + insertStakeAddress, queryDatum, queryMAWithCache, + queryOrInsertRewardAccount, + queryOrInsertStakeAddress, queryPoolKeyOrInsert, queryPoolKeyWithCache, queryPrevBlockWithCache, - queryOrInsertStakeAddress, - queryOrInsertRewardAccount, - insertStakeAddress, queryStakeAddrWithCache, rollbackCache, -- * CacheStatistics getCacheStatistics, -) where +) +where import Cardano.BM.Trace import qualified Cardano.Db as DB +import Cardano.DbSync.Api (getTrace) +import Cardano.DbSync.Api.Types (SyncEnv (..)) import Cardano.DbSync.Cache.Epoch (rollbackMapEpochInCache) import qualified Cardano.DbSync.Cache.LRU as LRU import Cardano.DbSync.Cache.Types (CacheAction (..), CacheInternal (..), CacheStatistics (..), CacheStatus (..), initCacheStatistics, isCacheActionUpdate) @@ -83,36 +86,37 @@ getCacheStatistics cs = queryOrInsertRewardAccount :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> CacheAction -> Ledger.RewardAccount StandardCrypto -> ReaderT SqlBackend m DB.StakeAddressId -queryOrInsertRewardAccount trce cache cacheUA rewardAddr = do - eiAddrId <- queryRewardAccountWithCacheRetBs trce cache cacheUA rewardAddr +queryOrInsertRewardAccount syncEnv cache cacheNew rewardAddr = do + eiAddrId <- queryRewardAccountWithCacheRetBs syncEnv cache cacheNew rewardAddr case eiAddrId of - Left (_err, bs) -> insertStakeAddress rewardAddr (Just bs) + Left (_err, bs) -> insertStakeAddress syncEnv rewardAddr (Just bs) Right addrId -> pure addrId queryOrInsertStakeAddress :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> CacheAction -> Network -> StakeCred -> ReaderT SqlBackend m DB.StakeAddressId -queryOrInsertStakeAddress trce cache cacheUA nw cred = - queryOrInsertRewardAccount trce cache cacheUA $ Ledger.RewardAccount nw cred +queryOrInsertStakeAddress syncEnv cache cacheUA nw cred = + queryOrInsertRewardAccount syncEnv cache cacheUA $ Ledger.RewardAccount nw cred -- If the address already exists in the table, it will not be inserted again (due to -- the uniqueness constraint) but the function will return the 'StakeAddressId'. insertStakeAddress :: (MonadBaseControl IO m, MonadIO m) => + SyncEnv -> Ledger.RewardAccount StandardCrypto -> Maybe ByteString -> ReaderT SqlBackend m DB.StakeAddressId -insertStakeAddress rewardAddr stakeCredBs = do +insertStakeAddress _syncEnv rewardAddr stakeCredBs = do DB.insertStakeAddress $ DB.StakeAddress { DB.stakeAddressHashRaw = addrBs @@ -122,39 +126,40 @@ insertStakeAddress rewardAddr stakeCredBs = do where addrBs = fromMaybe (Ledger.serialiseRewardAccount rewardAddr) stakeCredBs +------------------------------------------------------------------------------------------------ queryRewardAccountWithCacheRetBs :: forall m. MonadIO m => - Trace IO Text -> + SyncEnv -> CacheStatus -> CacheAction -> Ledger.RewardAccount StandardCrypto -> ReaderT SqlBackend m (Either (DB.LookupFail, ByteString) DB.StakeAddressId) -queryRewardAccountWithCacheRetBs trce cache cacheUA rwdAcc = - queryStakeAddrWithCacheRetBs trce cache cacheUA (Ledger.raNetwork rwdAcc) (Ledger.raCredential rwdAcc) +queryRewardAccountWithCacheRetBs syncEnv cache cacheUA rwdAcc = + queryStakeAddrWithCacheRetBs syncEnv cache cacheUA (Ledger.raNetwork rwdAcc) (Ledger.raCredential rwdAcc) queryStakeAddrWithCache :: forall m. MonadIO m => - Trace IO Text -> + SyncEnv -> CacheStatus -> CacheAction -> Network -> StakeCred -> ReaderT SqlBackend m (Either DB.LookupFail DB.StakeAddressId) -queryStakeAddrWithCache trce cache cacheUA nw cred = - mapLeft fst <$> queryStakeAddrWithCacheRetBs trce cache cacheUA nw cred +queryStakeAddrWithCache syncEnv cache cacheUA nw cred = + mapLeft fst <$> queryStakeAddrWithCacheRetBs syncEnv cache cacheUA nw cred queryStakeAddrWithCacheRetBs :: forall m. MonadIO m => - Trace IO Text -> + SyncEnv -> CacheStatus -> CacheAction -> Network -> StakeCred -> ReaderT SqlBackend m (Either (DB.LookupFail, ByteString) DB.StakeAddressId) -queryStakeAddrWithCacheRetBs trce cache cacheUA nw cred = do +queryStakeAddrWithCacheRetBs syncEnv cache cacheUA nw cred = do let !bs = Ledger.serialiseRewardAccount (Ledger.RewardAccount nw cred) case cache of NoCache -> do @@ -166,10 +171,10 @@ queryStakeAddrWithCacheRetBs trce cache cacheUA nw cred = do currentCache <- if isNewCache then do - liftIO $ logInfo trce "Stake Raw Hashes cache is new and empty. Populating with addresses from db..." + liftIO $ logInfo (getTrace syncEnv) "Stake Raw Hashes cache is new and empty. Populating with addresses from db..." queryRes <- DB.queryAddressWithReward (fromIntegral $ LRU.getCapacity prevCache) liftIO $ atomically $ writeTVar (cStakeRawHashes ci) $ LRU.fromList queryRes prevCache - liftIO $ logInfo trce "Population of cache complete." + liftIO $ logInfo (getTrace syncEnv) "Population of cache complete." liftIO $ readTVarIO (cStakeRawHashes ci) else pure prevCache @@ -278,20 +283,20 @@ insertPoolKeyWithCache cache cacheUA pHash = queryPoolKeyOrInsert :: (MonadBaseControl IO m, MonadIO m) => Text -> - Trace IO Text -> + SyncEnv -> CacheStatus -> CacheAction -> Bool -> PoolKeyHash -> ReaderT SqlBackend m DB.PoolHashId -queryPoolKeyOrInsert txt trce cache cacheUA logsWarning hsh = do +queryPoolKeyOrInsert txt syncEnv cache cacheUA logsWarning hsh = do pk <- queryPoolKeyWithCache cache cacheUA hsh case pk of Right poolHashId -> pure poolHashId Left err -> do when logsWarning $ liftIO $ - logWarning trce $ + logWarning (getTrace syncEnv) $ mconcat [ "Failed with " , textShow err @@ -329,7 +334,10 @@ queryMAWithCache cache policyId asset = let !assetNameBs = Generic.unAssetName asset maId <- maybe (Left (policyBs, assetNameBs)) Right <$> DB.queryMultiAssetId policyBs assetNameBs whenRight maId $ - liftIO . atomically . modifyTVar (cMultiAssets ci) . LRU.insert (policyId, asset) + liftIO + . atomically + . modifyTVar (cMultiAssets ci) + . LRU.insert (policyId, asset) pure maId queryPrevBlockWithCache :: diff --git a/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs b/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs index 5053f142f..f01f3b1f4 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs @@ -46,10 +46,11 @@ module Cardano.DbSync.Config.Types ( isTxOutEnabled, hasLedger, shouldUseLedger, - isShelleyEnabled, - isMultiAssetEnabled, - isMetadataEnabled, - isPlutusEnabled, + isShelleyModeActive, + isShelleyWhitelistModeActive, + isMultiAssetModeActive, + isMetadataModeActive, + isPlutusModeActive, isTxOutBootstrap, isTxOutConsumed, isTxOutPrune, @@ -70,6 +71,7 @@ import qualified Data.Aeson as Aeson import Data.Aeson.Types (Parser, typeMismatch) import Data.ByteString.Short (ShortByteString (), fromShort, toShort) import Data.Default.Class (Default (..)) +import qualified Data.Text as Text import Ouroboros.Consensus.Cardano.CanHardFork (TriggerHardFork (..)) newtype LogFileDir = LogFileDir @@ -199,7 +201,8 @@ data LedgerInsertConfig data ShelleyInsertConfig = ShelleyEnable | ShelleyDisable - | ShelleyStakeAddrs (NonEmpty ShortByteString) + | -- | Whitelist of Shelley stake addresses + ShelleyStakeAddrs (NonEmpty ShortByteString) deriving (Eq, Show) newtype RewardsConfig = RewardsConfig @@ -209,19 +212,22 @@ newtype RewardsConfig = RewardsConfig data MultiAssetConfig = MultiAssetEnable | MultiAssetDisable - | MultiAssetPolicies (NonEmpty ShortByteString) + | -- | Whitelist of multiAsset policy IDs + MultiAssetPolicies (NonEmpty ShortByteString) deriving (Eq, Show) data MetadataConfig = MetadataEnable | MetadataDisable - | MetadataKeys (NonEmpty Word) + | -- | Whitelist of metadata keys + MetadataKeys (NonEmpty Word) deriving (Eq, Show) data PlutusConfig = PlutusEnable | PlutusDisable - | PlutusScripts (NonEmpty ShortByteString) + | -- | Whitelist of plutus script hashes + PlutusScripts (NonEmpty ShortByteString) deriving (Eq, Show) newtype GovernanceConfig = GovernanceConfig @@ -335,25 +341,29 @@ shouldUseLedger LedgerDisable = False shouldUseLedger LedgerEnable = True shouldUseLedger LedgerIgnore = False -isShelleyEnabled :: ShelleyInsertConfig -> Bool -isShelleyEnabled ShelleyDisable = False -isShelleyEnabled ShelleyEnable = True -isShelleyEnabled (ShelleyStakeAddrs _) = True +isShelleyModeActive :: ShelleyInsertConfig -> Bool +isShelleyModeActive ShelleyDisable = False +isShelleyModeActive ShelleyEnable = True +isShelleyModeActive (ShelleyStakeAddrs _) = True -isMultiAssetEnabled :: MultiAssetConfig -> Bool -isMultiAssetEnabled MultiAssetDisable = False -isMultiAssetEnabled MultiAssetEnable = True -isMultiAssetEnabled (MultiAssetPolicies _) = True +isShelleyWhitelistModeActive :: ShelleyInsertConfig -> Bool +isShelleyWhitelistModeActive (ShelleyStakeAddrs _) = True +isShelleyWhitelistModeActive _other = False -isMetadataEnabled :: MetadataConfig -> Bool -isMetadataEnabled MetadataDisable = False -isMetadataEnabled MetadataEnable = True -isMetadataEnabled (MetadataKeys _) = True +isMultiAssetModeActive :: MultiAssetConfig -> Bool +isMultiAssetModeActive MultiAssetDisable = False +isMultiAssetModeActive MultiAssetEnable = True +isMultiAssetModeActive (MultiAssetPolicies _) = True -isPlutusEnabled :: PlutusConfig -> Bool -isPlutusEnabled PlutusDisable = False -isPlutusEnabled PlutusEnable = True -isPlutusEnabled (PlutusScripts _) = True +isMetadataModeActive :: MetadataConfig -> Bool +isMetadataModeActive MetadataDisable = False +isMetadataModeActive MetadataEnable = True +isMetadataModeActive (MetadataKeys _) = True + +isPlutusModeActive :: PlutusConfig -> Bool +isPlutusModeActive PlutusDisable = False +isPlutusModeActive PlutusEnable = True +isPlutusModeActive (PlutusScripts _) = True -- ------------------------------------------------------------------------------------------------- @@ -487,7 +497,7 @@ instance FromJSON LedgerInsertConfig where instance ToJSON ShelleyInsertConfig where toJSON cfg = Aeson.object - [ "enable" .= isShelleyEnabled cfg + [ "enable" .= isShelleyModeActive cfg , "stake_addresses" .= stakeAddrs cfg ] where @@ -499,16 +509,17 @@ instance FromJSON ShelleyInsertConfig where enable <- obj .: "enable" stakeAddrs <- obj .:? "stake_addresses" - pure $ - case (enable, stakeAddrs) of - (False, _) -> ShelleyDisable - (True, Nothing) -> ShelleyEnable - (True, Just addrs) -> ShelleyStakeAddrs (map parseShortByteString addrs) + case (enable, stakeAddrs) of + (False, _) -> pure ShelleyDisable + (True, Nothing) -> pure ShelleyEnable + (True, Just addrs) -> do + addrsParsed <- traverse parseValidateHash addrs + pure $ ShelleyStakeAddrs addrsParsed instance ToJSON MultiAssetConfig where toJSON cfg = Aeson.object - [ "enable" .= isMultiAssetEnabled cfg + [ "enable" .= isMultiAssetModeActive cfg , "policies" .= policies cfg ] where @@ -520,16 +531,17 @@ instance FromJSON MultiAssetConfig where enable <- obj .: "enable" policies <- obj .:? "policies" - pure $ - case (enable, policies) of - (False, _) -> MultiAssetDisable - (True, Nothing) -> MultiAssetEnable - (True, Just ps) -> MultiAssetPolicies (map parseShortByteString ps) + case (enable, policies) of + (False, _) -> pure MultiAssetDisable + (True, Nothing) -> pure MultiAssetEnable + (True, Just ps) -> do + policiesParsed <- traverse parseValidateHash ps + pure $ MultiAssetPolicies policiesParsed instance ToJSON MetadataConfig where toJSON cfg = Aeson.object - [ "enable" .= isMetadataEnabled cfg + [ "enable" .= isMetadataModeActive cfg , "keys" .= keys cfg ] where @@ -550,7 +562,7 @@ instance FromJSON MetadataConfig where instance ToJSON PlutusConfig where toJSON cfg = Aeson.object - [ "enable" .= isPlutusEnabled cfg + [ "enable" .= isPlutusModeActive cfg , "script_hashes" .= scriptHashes cfg ] where @@ -562,11 +574,12 @@ instance FromJSON PlutusConfig where enable <- obj .: "enable" scriptHashes <- obj .:? "script_hashes" - pure $ - case (enable, scriptHashes) of - (False, _) -> PlutusDisable - (True, Nothing) -> PlutusEnable - (True, Just hs) -> PlutusScripts (map parseShortByteString hs) + case (enable, scriptHashes) of + (False, _) -> pure PlutusDisable + (True, Nothing) -> pure PlutusEnable + (True, Just hs) -> do + hsParsed <- traverse parseValidateHash hs + pure $ PlutusScripts hsParsed instance ToJSON GovernanceConfig where toJSON = boolToEnableDisable . isGovernanceEnabled @@ -637,8 +650,11 @@ enableDisableToBool = \case "disable" -> Just False _ -> Nothing -parseShortByteString :: Text -> ShortByteString -parseShortByteString = toShort . encodeUtf8 +parseValidateHash :: Text -> Parser ShortByteString +parseValidateHash txt = + if "\\x" `Text.isPrefixOf` txt + then fail $ "Invalid Hash: starts with \\x please adjust it: " <> show txt + else pure $ toShort $ encodeUtf8 txt shortByteStringToJSON :: ShortByteString -> Aeson.Value shortByteStringToJSON = toJSON . decodeUtf8 . fromShort diff --git a/cardano-db-sync/src/Cardano/DbSync/Default.hs b/cardano-db-sync/src/Cardano/DbSync/Default.hs index 0285533c1..554248db7 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Default.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Default.hs @@ -18,6 +18,7 @@ import Cardano.DbSync.Api.Ledger import Cardano.DbSync.Api.Types (ConsistentLevel (..), InsertOptions (..), LedgerEnv (..), SyncEnv (..), SyncOptions (..)) import Cardano.DbSync.Epoch (epochHandler) import Cardano.DbSync.Era.Byron.Insert (insertByronBlock) + import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Era.Universal.Block (insertBlockUniversal) import Cardano.DbSync.Era.Universal.Epoch (hasEpochStartEvent, hasNewEpochEvent) @@ -165,15 +166,15 @@ insertBlock syncEnv cblk applyRes firstAfterRollback tookSnapshot = do BlockAlonzo blk -> newExceptT $ insertBlockUniversal' $ - Generic.fromAlonzoBlock (ioPlutusExtra iopts) (getPrices applyResult) blk + Generic.fromAlonzoBlock (ioPlutus iopts) (getPrices applyResult) blk BlockBabbage blk -> newExceptT $ insertBlockUniversal' $ - Generic.fromBabbageBlock (ioPlutusExtra iopts) (getPrices applyResult) blk + Generic.fromBabbageBlock (ioPlutus iopts) (getPrices applyResult) blk BlockConway blk -> newExceptT $ insertBlockUniversal' $ - Generic.fromConwayBlock (ioPlutusExtra iopts) (getPrices applyResult) blk + Generic.fromConwayBlock (ioPlutus iopts) (getPrices applyResult) blk -- update the epoch updateEpoch details isNewEpochEvent whenPruneTxOut syncEnv $ diff --git a/cardano-db-sync/src/Cardano/DbSync/Era.hs b/cardano-db-sync/src/Cardano/DbSync/Era.hs index 32c203f20..90bba42fb 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era.hs @@ -9,7 +9,7 @@ module Cardano.DbSync.Era ( import Cardano.DbSync.Api.Types (SyncEnv) import Cardano.DbSync.Config import qualified Cardano.DbSync.Era.Byron.Genesis as Byron -import qualified Cardano.DbSync.Era.Shelley.Genesis as Shelley +import qualified Cardano.DbSync.Era.Universal.Genesis as Shelley import Cardano.DbSync.Error import Cardano.Prelude diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs index e74620297..ed8db311a 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs @@ -25,6 +25,7 @@ module Cardano.DbSync.Era.Shelley.Generic.Block ( import qualified Cardano.Crypto.Hash as Crypto import qualified Cardano.Crypto.KES.Class as KES +import Cardano.DbSync.Config.Types (PlutusConfig) import Cardano.DbSync.Era.Shelley.Generic.Tx import Cardano.DbSync.Types import Cardano.DbSync.Util.Bech32 (serialiseVerKeyVrfToBech32) @@ -120,8 +121,8 @@ fromMaryBlock blk = , blkTxs = map fromMaryTx (getTxs blk) } -fromAlonzoBlock :: Bool -> Maybe Prices -> ShelleyBlock TPraosStandard StandardAlonzo -> Block -fromAlonzoBlock iope mprices blk = +fromAlonzoBlock :: PlutusConfig -> Maybe Prices -> ShelleyBlock TPraosStandard StandardAlonzo -> Block +fromAlonzoBlock plutusConfig mprices blk = Block { blkEra = Alonzo , blkHash = blockHash blk @@ -134,11 +135,11 @@ fromAlonzoBlock iope mprices blk = , blkVrfKey = blockVrfKeyView $ blockVrfVkTPraos blk , blkOpCert = blockOpCertKeyTPraos blk , blkOpCertCounter = blockOpCertCounterTPraos blk - , blkTxs = map (fromAlonzoTx iope mprices) (getTxs blk) + , blkTxs = map (fromAlonzoTx plutusConfig mprices) (getTxs blk) } -fromBabbageBlock :: Bool -> Maybe Prices -> ShelleyBlock PraosStandard StandardBabbage -> Block -fromBabbageBlock iope mprices blk = +fromBabbageBlock :: PlutusConfig -> Maybe Prices -> ShelleyBlock PraosStandard StandardBabbage -> Block +fromBabbageBlock plutusConfig mprices blk = Block { blkEra = Babbage , blkHash = blockHash blk @@ -151,11 +152,11 @@ fromBabbageBlock iope mprices blk = , blkVrfKey = blockVrfKeyView $ blockVrfVkPraos blk , blkOpCert = blockOpCertKeyPraos blk , blkOpCertCounter = blockOpCertCounterPraos blk - , blkTxs = map (fromBabbageTx iope mprices) (getTxs blk) + , blkTxs = map (fromBabbageTx plutusConfig mprices) (getTxs blk) } -fromConwayBlock :: Bool -> Maybe Prices -> ShelleyBlock PraosStandard StandardConway -> Block -fromConwayBlock iope mprices blk = +fromConwayBlock :: PlutusConfig -> Maybe Prices -> ShelleyBlock PraosStandard StandardConway -> Block +fromConwayBlock plutusConfig mprices blk = Block { blkEra = Conway , blkHash = blockHash blk @@ -168,7 +169,7 @@ fromConwayBlock iope mprices blk = , blkVrfKey = blockVrfKeyView $ blockVrfVkPraos blk , blkOpCert = blockOpCertKeyPraos blk , blkOpCertCounter = blockOpCertCounterPraos blk - , blkTxs = map (fromConwayTx iope mprices) (getTxs blk) + , blkTxs = map (fromConwayTx plutusConfig mprices) (getTxs blk) } -- ------------------------------------------------------------------------------------------------- diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Alonzo.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Alonzo.hs index 3529779c9..a98562348 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Alonzo.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Alonzo.hs @@ -27,6 +27,7 @@ module Cardano.DbSync.Era.Shelley.Generic.Tx.Alonzo ( import qualified Cardano.Crypto.Hash as Crypto import qualified Cardano.Db as DB +import Cardano.DbSync.Config.Types (PlutusConfig, isPlutusModeActive) import Cardano.DbSync.Era.Shelley.Generic.Metadata import Cardano.DbSync.Era.Shelley.Generic.Script (fromTimelock) import Cardano.DbSync.Era.Shelley.Generic.ScriptData (ScriptData (..)) @@ -67,8 +68,8 @@ import qualified Data.Set as Set import Lens.Micro import Ouroboros.Consensus.Cardano.Block (EraCrypto, StandardAlonzo, StandardCrypto) -fromAlonzoTx :: Bool -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardAlonzo) -> Tx -fromAlonzoTx ioExtraPlutus mprices (blkIndex, tx) = +fromAlonzoTx :: PlutusConfig -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardAlonzo) -> Tx +fromAlonzoTx plutusConfig mprices (blkIndex, tx) = Tx { txHash = txHashId tx , txBlockIndex = blkIndex @@ -131,7 +132,7 @@ fromAlonzoTx ioExtraPlutus mprices (blkIndex, tx) = MaryValue ada (MultiAsset maMap) = txOut ^. Core.valueTxOutL mDataHash = txOut ^. Alonzo.dataHashTxOutL - (finalMaps, redeemers) = resolveRedeemers ioExtraPlutus mprices tx (Left . toShelleyCert) + (finalMaps, redeemers) = resolveRedeemers plutusConfig mprices tx (Left . toShelleyCert) -- This is true if second stage contract validation passes or there are no contracts. isValid2 :: Bool @@ -181,13 +182,13 @@ resolveRedeemers :: , Core.EraTx era , DBScriptPurpose era ) => - Bool -> + PlutusConfig -> Maybe Alonzo.Prices -> Core.Tx era -> (TxCert era -> Cert) -> (RedeemerMaps, [(Word64, TxRedeemer)]) -resolveRedeemers ioExtraPlutus mprices tx toCert = - if not ioExtraPlutus +resolveRedeemers plutusConfig mprices tx toCert = + if not $ isPlutusModeActive plutusConfig then (initRedeemersMaps, []) else mkRdmrAndUpdateRec (initRedeemersMaps, []) $ diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Babbage.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Babbage.hs index f062d5791..c34cddfed 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Babbage.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Babbage.hs @@ -6,6 +6,7 @@ {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeOperators #-} {-# LANGUAGE NoImplicitPrelude #-} +{-# OPTIONS_GHC -Wno-unused-matches #-} module Cardano.DbSync.Era.Shelley.Generic.Tx.Babbage ( fromBabbageTx, @@ -13,6 +14,7 @@ module Cardano.DbSync.Era.Shelley.Generic.Tx.Babbage ( fromTxOut, ) where +import Cardano.DbSync.Config.Types (PlutusConfig) import Cardano.DbSync.Era.Shelley.Generic.Metadata import Cardano.DbSync.Era.Shelley.Generic.Tx.Allegra (getInterval) import Cardano.DbSync.Era.Shelley.Generic.Tx.Alonzo @@ -35,8 +37,8 @@ import qualified Data.Map.Strict as Map import Lens.Micro import Ouroboros.Consensus.Shelley.Eras (StandardBabbage, StandardCrypto) -fromBabbageTx :: Bool -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardBabbage) -> Tx -fromBabbageTx ioExtraPlutus mprices (blkIndex, tx) = +fromBabbageTx :: PlutusConfig -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardBabbage) -> Tx +fromBabbageTx plutusConfig mprices (blkIndex, tx) = Tx { txHash = txHashId tx , txBlockIndex = blkIndex @@ -103,7 +105,7 @@ fromBabbageTx ioExtraPlutus mprices (blkIndex, tx) = case Alonzo.isValid tx of Alonzo.IsValid x -> x - (finalMaps, redeemers) = resolveRedeemers ioExtraPlutus mprices tx (Left . toShelleyCert) + (finalMaps, redeemers) = resolveRedeemers plutusConfig mprices tx (Left . toShelleyCert) (invalidBef, invalidAfter) = getInterval txBody collInputs = mkCollTxIn txBody diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Conway.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Conway.hs index 7eb95025b..49fecef63 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Conway.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Tx/Conway.hs @@ -8,6 +8,7 @@ module Cardano.DbSync.Era.Shelley.Generic.Tx.Conway ( fromConwayTx, ) where +import Cardano.DbSync.Config.Types (PlutusConfig) import Cardano.DbSync.Era.Shelley.Generic.Metadata import Cardano.DbSync.Era.Shelley.Generic.Tx.Allegra (getInterval) import Cardano.DbSync.Era.Shelley.Generic.Tx.Alonzo @@ -27,8 +28,8 @@ import qualified Data.Map.Strict as Map import Lens.Micro import Ouroboros.Consensus.Cardano.Block (StandardConway, StandardCrypto) -fromConwayTx :: Bool -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardConway) -> Tx -fromConwayTx ioExtraPlutus mprices (blkIndex, tx) = +fromConwayTx :: PlutusConfig -> Maybe Alonzo.Prices -> (Word64, Core.Tx StandardConway) -> Tx +fromConwayTx plutusConfig mprices (blkIndex, tx) = Tx { txHash = txHashId tx , txBlockIndex = blkIndex @@ -98,7 +99,7 @@ fromConwayTx ioExtraPlutus mprices (blkIndex, tx) = case Alonzo.isValid tx of Alonzo.IsValid x -> x - (finalMaps, redeemers) = resolveRedeemers ioExtraPlutus mprices tx Right + (finalMaps, redeemers) = resolveRedeemers plutusConfig mprices tx Right (invalidBef, invalidAfter) = getInterval txBody collInputs = mkCollTxIn txBody diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Adjust.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Adjust.hs index 942e6fc82..2092b99ea 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Adjust.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Adjust.hs @@ -5,15 +5,18 @@ module Cardano.DbSync.Era.Universal.Adjust ( adjustEpochRewards, -) where +) +where -import Cardano.BM.Trace (Trace, logInfo) +import Cardano.BM.Trace (logInfo) import qualified Cardano.Db as Db +import Cardano.DbSync.Api (getTrace) +import Cardano.DbSync.Api.Types (SyncEnv (..)) import Cardano.DbSync.Cache ( queryPoolKeyWithCache, queryStakeAddrWithCache, ) -import Cardano.DbSync.Cache.Types (CacheAction (..), CacheStatus) +import Cardano.DbSync.Cache.Types (CacheAction (..)) import qualified Cardano.DbSync.Era.Shelley.Generic.Rewards as Generic import Cardano.DbSync.Types (StakeCred) import Cardano.Ledger.BaseTypes (Network) @@ -48,17 +51,17 @@ import Database.Esqueleto.Experimental ( adjustEpochRewards :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> Network -> - CacheStatus -> EpochNo -> Generic.Rewards -> Set StakeCred -> ReaderT SqlBackend m () -adjustEpochRewards trce nw cache epochNo rwds creds = do +adjustEpochRewards syncEnv nw epochNo rwds creds = do let eraIgnored = Map.toList $ Generic.unRewards rwds - liftIO . logInfo trce $ - mconcat + liftIO + . Cardano.BM.Trace.logInfo (getTrace syncEnv) + $ mconcat [ "Removing " , if null eraIgnored then "" else textShow (length eraIgnored) <> " rewards and " , show (length creds) @@ -66,20 +69,19 @@ adjustEpochRewards trce nw cache epochNo rwds creds = do ] forM_ eraIgnored $ \(cred, rewards) -> forM_ (Set.toList rewards) $ \rwd -> - deleteReward trce nw cache epochNo (cred, rwd) - crds <- rights <$> forM (Set.toList creds) (queryStakeAddrWithCache trce cache DoNotUpdateCache nw) + deleteReward syncEnv nw epochNo (cred, rwd) + crds <- rights <$> forM (Set.toList creds) (queryStakeAddrWithCache syncEnv (envCache syncEnv) DoNotUpdateCache nw) deleteOrphanedRewards epochNo crds deleteReward :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> Network -> - CacheStatus -> EpochNo -> (StakeCred, Generic.Reward) -> ReaderT SqlBackend m () -deleteReward trce nw cache epochNo (cred, rwd) = do - mAddrId <- queryStakeAddrWithCache trce cache DoNotUpdateCache nw cred +deleteReward syncEnv nw epochNo (cred, rwd) = do + mAddrId <- queryStakeAddrWithCache syncEnv (envCache syncEnv) DoNotUpdateCache nw cred eiPoolId <- queryPoolKeyWithCache cache DoNotUpdateCache (Generic.rewardPool rwd) case (mAddrId, eiPoolId) of (Right addrId, Right poolId) -> do @@ -89,9 +91,11 @@ deleteReward trce nw cache epochNo (cred, rwd) = do where_ (rwdDb ^. Db.RewardType ==. val (Generic.rewardSource rwd)) where_ (rwdDb ^. Db.RewardSpendableEpoch ==. val (unEpochNo epochNo)) where_ (rwdDb ^. Db.RewardPoolId ==. val poolId) - _ -> pure () + _other -> pure () + where + cache = envCache syncEnv -deleteOrphanedRewards :: MonadIO m => EpochNo -> [Db.StakeAddressId] -> ReaderT SqlBackend m () +deleteOrphanedRewards :: (MonadIO m) => EpochNo -> [Db.StakeAddressId] -> ReaderT SqlBackend m () deleteOrphanedRewards (EpochNo epochNo) xs = delete $ do rwd <- from $ table @Db.Reward diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Block.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Block.hs index 5a8f9bb62..2352436d6 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Block.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Block.hs @@ -15,17 +15,16 @@ import Cardano.BM.Trace (Trace, logDebug, logInfo) import qualified Cardano.Db as DB import Cardano.DbSync.Api import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..), SyncOptions (..)) -import Cardano.DbSync.Cache ( - insertBlockAndCache, - queryPoolKeyWithCache, - queryPrevBlockWithCache, - ) +import Cardano.DbSync.Cache (insertBlockAndCache, queryPoolKeyWithCache, queryPrevBlockWithCache) import Cardano.DbSync.Cache.Epoch (writeEpochBlockDiffToCache) import Cardano.DbSync.Cache.Types (CacheAction (..), CacheStatus (..), EpochBlockDiff (..)) - +import Cardano.DbSync.Config.Types (isShelleyModeActive) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Era.Universal.Epoch import Cardano.DbSync.Era.Universal.Insert.Grouped +import Cardano.DbSync.Era.Universal.Insert.Pool ( + IsPoolMember, + ) import Cardano.DbSync.Era.Universal.Insert.Tx (insertTx) import Cardano.DbSync.Era.Util (liftLookupFail) import Cardano.DbSync.Error @@ -33,13 +32,10 @@ import Cardano.DbSync.Ledger.Types (ApplyResult (..)) import Cardano.DbSync.OffChain import Cardano.DbSync.Types import Cardano.DbSync.Util - -import Cardano.DbSync.Era.Universal.Insert.Pool (IsPoolMember) import Cardano.Ledger.BaseTypes import qualified Cardano.Ledger.BaseTypes as Ledger import Cardano.Ledger.Keys import Cardano.Prelude - import Control.Monad.Trans.Control (MonadBaseControl) import Control.Monad.Trans.Except.Extra (newExceptT) import Data.Either.Extra (eitherToMaybe) @@ -47,7 +43,6 @@ import Database.Persist.Sql (SqlBackend) -------------------------------------------------------------------------------------------- -- Insert a universal Block. --- This is the entry point for inserting a block into the database, used for all eras appart from Byron. -------------------------------------------------------------------------------------------- insertBlockUniversal :: (MonadBaseControl IO m, MonadIO m) => @@ -71,7 +66,7 @@ insertBlockUniversal syncEnv shouldLog withinTwoMins withinHalfHour blk details mPhid <- lift $ queryPoolKeyWithCache cache UpdateCache $ coerceKeyRole $ Generic.blkSlotLeader blk let epochNo = sdEpochNo details - slid <- lift . DB.insertSlotLeader $ Generic.mkSlotLeader (ioShelley iopts) (Generic.unKeyHashRaw $ Generic.blkSlotLeader blk) (eitherToMaybe mPhid) + slid <- lift . DB.insertSlotLeader $ Generic.mkSlotLeader (isShelleyModeActive $ ioShelley iopts) (Generic.unKeyHashRaw $ Generic.blkSlotLeader blk) (eitherToMaybe mPhid) blkId <- lift . insertBlockAndCache cache $ DB.Block @@ -147,7 +142,7 @@ insertBlockUniversal syncEnv shouldLog withinTwoMins withinHalfHour blk details ] whenStrictJust (apNewEpoch applyResult) $ \newEpoch -> do - insertOnNewEpoch tracer iopts blkId (Generic.blkSlotNo blk) epochNo newEpoch + insertOnNewEpoch syncEnv blkId (Generic.blkSlotNo blk) epochNo newEpoch insertStakeSlice syncEnv $ apStakeSlice applyResult @@ -175,8 +170,8 @@ insertBlockUniversal syncEnv shouldLog withinTwoMins withinHalfHour blk details renderErrorMessage :: Generic.BlockEra -> Text renderErrorMessage eraText = case eraText of - Generic.Shelley -> "insertBlockForEra" - other -> mconcat ["insertBlockForEra(", textShow other, ")"] + Generic.Shelley -> "insertBlockUniversal" + other -> mconcat ["insertBlockUniversal(", textShow other, ")"] tracer :: Trace IO Text tracer = getTrace syncEnv diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Epoch.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Epoch.hs index f0790e193..aefbbad05 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Epoch.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Epoch.hs @@ -19,7 +19,8 @@ module Cardano.DbSync.Era.Universal.Epoch ( insertPoolDepositRefunds, insertStakeSlice, sumRewardTotal, -) where +) +where import Cardano.BM.Trace (Trace, logInfo) import qualified Cardano.Db as DB @@ -27,6 +28,7 @@ import Cardano.DbSync.Api import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..)) import Cardano.DbSync.Cache (queryOrInsertStakeAddress, queryPoolKeyOrInsert) import Cardano.DbSync.Cache.Types (CacheAction (..), CacheStatus) +import Cardano.DbSync.Config.Types (isShelleyModeActive) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Era.Universal.Insert.Certificate (insertPots) import Cardano.DbSync.Era.Universal.Insert.GovAction (insertCostModel, insertDrepDistr, insertUpdateEnacted, updateExpired, updateRatified) @@ -34,9 +36,10 @@ import Cardano.DbSync.Era.Universal.Insert.Other (toDouble) import Cardano.DbSync.Error import Cardano.DbSync.Ledger.Event import Cardano.DbSync.Types -import Cardano.DbSync.Util (whenStrictJust) +import Cardano.DbSync.Util (whenFalseEmpty, whenStrictJust) import Cardano.DbSync.Util.Constraint (constraintNameEpochStake, constraintNameReward) -import Cardano.Ledger.Address (RewardAccount (..)) +import Cardano.DbSync.Util.Whitelist (shelleyStakeAddrWhitelistCheck) +import qualified Cardano.Ledger.Address as Ledger import Cardano.Ledger.BaseTypes (Network, unEpochInterval) import qualified Cardano.Ledger.BaseTypes as Ledger import Cardano.Ledger.Binary.Version (getVersion) @@ -48,26 +51,24 @@ import Cardano.Ledger.Conway.Rules (RatifyState (..)) import Cardano.Prelude import Cardano.Slotting.Slot (EpochNo (..), SlotNo) import Control.Concurrent.Class.MonadSTM.Strict (readTVarIO) +import Control.Monad.Extra (mapMaybeM) import Control.Monad.Trans.Control (MonadBaseControl) import qualified Data.Map.Strict as Map import qualified Data.Set as Set import Database.Persist.Sql (SqlBackend) -{- HLINT ignore "Use readTVarIO" -} - -------------------------------------------------------------------------------------------- -- Insert Epoch -------------------------------------------------------------------------------------------- insertOnNewEpoch :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> - InsertOptions -> + SyncEnv -> DB.BlockId -> SlotNo -> EpochNo -> Generic.NewEpoch -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertOnNewEpoch tracer iopts blkId slotNo epochNo newEpoch = do +insertOnNewEpoch syncEnv blkId slotNo epochNo newEpoch = do whenStrictJust (Generic.euProtoParams epochUpdate) $ \params -> lift $ insertEpochParam tracer blkId epochNo params (Generic.euNonce epochUpdate) whenStrictJust (Generic.neAdaPots newEpoch) $ \pots -> @@ -75,14 +76,16 @@ insertOnNewEpoch tracer iopts blkId slotNo epochNo newEpoch = do whenStrictJust (Generic.neDRepState newEpoch) $ \dreps -> when (ioGov iopts) $ do let (drepSnapshot, ratifyState) = finishDRepPulser dreps lift $ insertDrepDistr epochNo drepSnapshot - updateRatified epochNo (toList $ rsEnacted ratifyState) - updateExpired epochNo (toList $ rsExpired ratifyState) + updateRatified syncEnv epochNo (toList $ rsEnacted ratifyState) + updateExpired syncEnv epochNo (toList $ rsExpired ratifyState) whenStrictJust (Generic.neEnacted newEpoch) $ \enactedSt -> do when (ioGov iopts) $ do - insertUpdateEnacted tracer blkId epochNo enactedSt + insertUpdateEnacted syncEnv blkId epochNo enactedSt where epochUpdate :: Generic.EpochUpdate epochUpdate = Generic.neEpochUpdate newEpoch + tracer = getTrace syncEnv + iopts = getInsertOptions syncEnv insertEpochParam :: (MonadBaseControl IO m, MonadIO m) => @@ -203,7 +206,7 @@ insertEpochStake :: insertEpochStake syncEnv nw epochNo stakeChunk = do let cache = envCache syncEnv DB.ManualDbConstraints {..} <- liftIO $ readTVarIO $ envDbConstraints syncEnv - dbStakes <- mapM (mkStake cache) stakeChunk + dbStakes <- mapMaybeM (mkStake cache) stakeChunk let chunckDbStakes = splittRecordsEvery 100000 dbStakes -- minimising the bulk inserts into hundred thousand chunks to improve performance forM_ chunckDbStakes $ \dbs -> lift $ DB.insertManyEpochStakes dbConstraintEpochStake constraintNameEpochStake dbs @@ -212,19 +215,23 @@ insertEpochStake syncEnv nw epochNo stakeChunk = do (MonadBaseControl IO m, MonadIO m) => CacheStatus -> (StakeCred, (Shelley.Coin, PoolKeyHash)) -> - ExceptT SyncNodeError (ReaderT SqlBackend m) DB.EpochStake - mkStake cache (saddr, (coin, pool)) = do - saId <- lift $ queryOrInsertStakeAddress trce cache UpdateCache nw saddr - poolId <- lift $ queryPoolKeyOrInsert "insertEpochStake" trce cache UpdateCache (ioShelley iopts) pool - pure $ - DB.EpochStake - { DB.epochStakeAddrId = saId - , DB.epochStakePoolId = poolId - , DB.epochStakeAmount = Generic.coinToDbLovelace coin - , DB.epochStakeEpochNo = unEpochNo epochNo -- The epoch where this delegation becomes valid. - } - - trce = getTrace syncEnv + ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe DB.EpochStake) + mkStake cache (saddr, (coin, pool)) = + whenFalseEmpty + (shelleyStakeAddrWhitelistCheck syncEnv $ Ledger.RewardAccount nw saddr) + Nothing + ( do + saId <- lift $ queryOrInsertStakeAddress syncEnv cache UpdateCache nw saddr + poolId <- lift $ queryPoolKeyOrInsert "insertEpochStake" syncEnv cache UpdateCache (isShelleyModeActive $ ioShelley iopts) pool + pure $ + Just $ + DB.EpochStake + { DB.epochStakeAddrId = saId + , DB.epochStakePoolId = poolId + , DB.epochStakeAmount = Generic.coinToDbLovelace coin + , DB.epochStakeEpochNo = unEpochNo epochNo -- The epoch where this delegation becomes valid. + } + ) iopts = getInsertOptions syncEnv insertRewards :: @@ -248,8 +255,12 @@ insertRewards syncEnv nw earnedEpoch spendableEpoch cache rewardsChunk = do (StakeCred, Set Generic.Reward) -> ExceptT SyncNodeError (ReaderT SqlBackend m) [DB.Reward] mkRewards (saddr, rset) = do - saId <- lift $ queryOrInsertStakeAddress trce cache UpdateCache nw saddr - mapM (prepareReward saId) (Set.toList rset) + -- Check if the stake address is in the shelley whitelist + if shelleyStakeAddrWhitelistCheck syncEnv $ Ledger.RewardAccount nw saddr + then do + saId <- lift $ queryOrInsertStakeAddress syncEnv cache UpdateCache nw saddr + mapM (prepareReward saId) (Set.toList rset) + else pure [] prepareReward :: (MonadBaseControl IO m, MonadIO m) => @@ -273,21 +284,20 @@ insertRewards syncEnv nw earnedEpoch spendableEpoch cache rewardsChunk = do PoolKeyHash -> ExceptT SyncNodeError (ReaderT SqlBackend m) DB.PoolHashId queryPool poolHash = - lift (queryPoolKeyOrInsert "insertRewards" trce cache UpdateCache (ioShelley iopts) poolHash) + lift (queryPoolKeyOrInsert "insertRewards" syncEnv cache UpdateCache (isShelleyModeActive $ ioShelley iopts) poolHash) - trce = getTrace syncEnv iopts = getInsertOptions syncEnv insertRewardRests :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> Network -> EpochNo -> EpochNo -> CacheStatus -> [(StakeCred, Set Generic.RewardRest)] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertRewardRests trce nw earnedEpoch spendableEpoch cache rewardsChunk = do +insertRewardRests syncEnv nw earnedEpoch spendableEpoch cache rewardsChunk = do dbRewards <- concatMapM mkRewards rewardsChunk let chunckDbRewards = splittRecordsEvery 100000 dbRewards -- minimising the bulk inserts into hundred thousand chunks to improve performance @@ -298,8 +308,12 @@ insertRewardRests trce nw earnedEpoch spendableEpoch cache rewardsChunk = do (StakeCred, Set Generic.RewardRest) -> ExceptT SyncNodeError (ReaderT SqlBackend m) [DB.RewardRest] mkRewards (saddr, rset) = do - saId <- lift $ queryOrInsertStakeAddress trce cache UpdateCache nw saddr - pure $ map (prepareReward saId) (Set.toList rset) + -- Check if the stake address is in the shelley whitelist + if shelleyStakeAddrWhitelistCheck syncEnv $ Ledger.RewardAccount nw saddr + then do + saId <- lift $ queryOrInsertStakeAddress syncEnv cache UpdateCache nw saddr + pure $ map (prepareReward saId) (Set.toList rset) + else pure [] prepareReward :: DB.StakeAddressId -> @@ -316,14 +330,14 @@ insertRewardRests trce nw earnedEpoch spendableEpoch cache rewardsChunk = do insertProposalRefunds :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> Network -> EpochNo -> EpochNo -> CacheStatus -> [GovActionRefunded] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertProposalRefunds trce nw earnedEpoch spendableEpoch cache refunds = do +insertProposalRefunds syncEnv nw earnedEpoch spendableEpoch cache refunds = do dbRewards <- mapM mkReward refunds lift $ DB.insertManyRewardRests dbRewards where @@ -332,7 +346,7 @@ insertProposalRefunds trce nw earnedEpoch spendableEpoch cache refunds = do GovActionRefunded -> ExceptT SyncNodeError (ReaderT SqlBackend m) DB.RewardRest mkReward refund = do - saId <- lift $ queryOrInsertStakeAddress trce cache UpdateCache nw (raCredential $ garReturnAddr refund) + saId <- lift $ queryOrInsertStakeAddress syncEnv cache UpdateCache nw (Ledger.raCredential $ garReturnAddr refund) pure $ DB.RewardRest { DB.rewardRestAddrId = saId diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Genesis.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Genesis.hs similarity index 93% rename from cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Genesis.hs rename to cardano-db-sync/src/Cardano/DbSync/Era/Universal/Genesis.hs index d833549bd..750bfe034 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Genesis.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Genesis.hs @@ -7,7 +7,7 @@ {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE NoImplicitPrelude #-} -module Cardano.DbSync.Era.Shelley.Genesis ( +module Cardano.DbSync.Era.Universal.Genesis ( insertValidateGenesisDist, ) where @@ -15,9 +15,12 @@ import Cardano.BM.Trace (Trace, logError, logInfo) import qualified Cardano.Db as DB import Cardano.DbSync.Api import Cardano.DbSync.Api.Types (SyncEnv (envBackend)) -import Cardano.DbSync.Cache.Types (CacheStatus (..), useNoCache) +import Cardano.DbSync.Cache.Types (useNoCache) import qualified Cardano.DbSync.Era.Shelley.Generic.Util as Generic -import Cardano.DbSync.Era.Universal.Insert.Certificate (insertDelegation, insertStakeRegistration) +import Cardano.DbSync.Era.Universal.Insert.Certificate ( + insertDelegation, + insertStakeRegistration, + ) import Cardano.DbSync.Era.Universal.Insert.Other (insertStakeAddressRefIfMissing) import Cardano.DbSync.Era.Universal.Insert.Pool (insertPoolRegister) import Cardano.DbSync.Era.Util (liftLookupFail) @@ -149,12 +152,12 @@ insertValidateGenesisDist syncEnv networkName cfg shelleyInitiation = do , DB.blockOpCertCounter = Nothing } disInOut <- liftIO $ getDisableInOutState syncEnv - lift $ mapM_ (insertTxOuts tracer hasConsumed disInOut bid) $ genesisUtxOs cfg + lift $ mapM_ (insertTxOuts syncEnv hasConsumed disInOut bid) $ genesisUtxOs cfg liftIO . logInfo tracer $ "Initial genesis distribution populated. Hash " <> renderByteArray (configGenesisHash cfg) when hasStakes $ - insertStaking tracer useNoCache bid cfg + insertStaking syncEnv bid cfg supply <- lift DB.queryTotalSupply liftIO $ logInfo tracer ("Total genesis supply of Ada: " <> DB.renderAda supply) @@ -218,13 +221,13 @@ validateGenesisDistribution prunes tracer networkName cfg bid expectedTxCount = insertTxOuts :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> Bool -> Bool -> DB.BlockId -> (TxIn StandardCrypto, ShelleyTxOut StandardShelley) -> ReaderT SqlBackend m () -insertTxOuts trce hasConsumed disInOut blkId (TxIn txInId _, txOut) = do +insertTxOuts syncEnv hasConsumed disInOut blkId (TxIn txInId _, txOut) = do -- Each address/value pair of the initial coin distribution comes from an artifical transaction -- with a hash generated by hashing the address. txId <- @@ -242,7 +245,7 @@ insertTxOuts trce hasConsumed disInOut blkId (TxIn txInId _, txOut) = do , DB.txValidContract = True , DB.txScriptSize = 0 } - _ <- insertStakeAddressRefIfMissing trce useNoCache (txOut ^. Core.addrTxOutL) + _ <- insertStakeAddressRefIfMissing syncEnv useNoCache (txOut ^. Core.addrTxOutL) DB.insertTxOutPlex hasConsumed disInOut $ DB.TxOut { DB.txOutTxId = txId @@ -263,12 +266,11 @@ insertTxOuts trce hasConsumed disInOut blkId (TxIn txInId _, txOut) = do -- Insert pools and delegations coming from Genesis. insertStaking :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> - CacheStatus -> + SyncEnv -> DB.BlockId -> ShelleyGenesis StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertStaking tracer cache blkId genesis = do +insertStaking syncEnv blkId genesis = do -- All Genesis staking comes from an artifical transaction -- with a hash generated by hashing the address. txId <- @@ -290,12 +292,12 @@ insertStaking tracer cache blkId genesis = do let params = zip [0 ..] $ ListMap.elems $ sgsPools $ sgStaking genesis let network = sgNetworkId genesis -- TODO: add initial deposits for genesis pools. - forM_ params $ uncurry (insertPoolRegister tracer useNoCache (const False) Nothing network (EpochNo 0) blkId txId) + forM_ params $ uncurry (insertPoolRegister syncEnv useNoCache (const False) Nothing network (EpochNo 0) blkId txId) let stakes = zip [0 ..] $ ListMap.toList (sgsStake $ sgStaking genesis) forM_ stakes $ \(n, (keyStaking, keyPool)) -> do -- TODO: add initial deposits for genesis stake keys. - insertStakeRegistration tracer cache (EpochNo 0) Nothing txId (2 * n) (Generic.annotateStakingCred network (KeyHashObj keyStaking)) - insertDelegation tracer cache network (EpochNo 0) 0 txId (2 * n + 1) Nothing (KeyHashObj keyStaking) keyPool + insertStakeRegistration syncEnv (EpochNo 0) Nothing txId (2 * n) (Generic.annotateStakingCred network (KeyHashObj keyStaking)) + insertDelegation syncEnv useNoCache network (EpochNo 0) 0 txId (2 * n + 1) Nothing (KeyHashObj keyStaking) keyPool -- ----------------------------------------------------------------------------- diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Certificate.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Certificate.hs index a8a37482c..04fb85f5f 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Certificate.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Certificate.hs @@ -22,7 +22,7 @@ module Cardano.DbSync.Era.Universal.Insert.Certificate ( mkAdaPots, ) where -import Cardano.BM.Trace (Trace, logWarning) +import Cardano.BM.Trace (logWarning) import qualified Cardano.Db as DB import Cardano.DbSync.Api import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..)) @@ -32,12 +32,20 @@ import Cardano.DbSync.Cache ( queryPoolKeyOrInsert, ) import Cardano.DbSync.Cache.Types (CacheAction (..), CacheStatus (..)) +import Cardano.DbSync.Config.Types (isShelleyModeActive) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic -import Cardano.DbSync.Era.Universal.Insert.GovAction (insertCommitteeHash, insertCredDrepHash, insertDrep, insertVotingAnchor) +import Cardano.DbSync.Era.Universal.Insert.GovAction ( + insertCommitteeHash, + insertCredDrepHash, + insertDrep, + insertVotingAnchor, + ) import Cardano.DbSync.Era.Universal.Insert.Pool (IsPoolMember, insertPoolCert) import Cardano.DbSync.Error import Cardano.DbSync.Types import Cardano.DbSync.Util +import Cardano.DbSync.Util.Whitelist (shelleyCustomStakeWhitelistCheck, shelleyStakeAddrWhitelistCheck) +import qualified Cardano.Ledger.Address as Ledger import Cardano.Ledger.BaseTypes import qualified Cardano.Ledger.BaseTypes as Ledger import Cardano.Ledger.CertState @@ -57,6 +65,9 @@ import qualified Data.Map.Strict as Map import Database.Persist.Sql (SqlBackend) import Ouroboros.Consensus.Cardano.Block (StandardCrypto) +-------------------------------------------------------------------------------------------- +-- Insert Certificates +-------------------------------------------------------------------------------------------- insertCertificate :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> @@ -72,19 +83,20 @@ insertCertificate :: insertCertificate syncEnv isMember mDeposits blkId txId epochNo slotNo redeemers (Generic.TxCertificate ridx idx cert) = case cert of Left (ShelleyTxCertDelegCert deleg) -> - when (ioShelley iopts) $ insertDelegCert tracer cache mDeposits network txId idx mRedeemerId epochNo slotNo deleg + when (isShelleyModeActive $ ioShelley iopts) $ insertDelegCert syncEnv mDeposits network txId idx mRedeemerId epochNo slotNo deleg Left (ShelleyTxCertPool pool) -> - when (ioShelley iopts) $ insertPoolCert tracer cache isMember mDeposits network epochNo blkId txId idx pool + when (isShelleyModeActive $ ioShelley iopts) $ insertPoolCert syncEnv cache isMember mDeposits network epochNo blkId txId idx pool Left (ShelleyTxCertMir mir) -> - when (ioShelley iopts) $ insertMirCert tracer cache network txId idx mir + when (isShelleyModeActive $ ioShelley iopts) $ insertMirCert syncEnv network txId idx mir Left (ShelleyTxCertGenesisDeleg _gen) -> - when (ioShelley iopts) $ + when (isShelleyModeActive $ ioShelley iopts) $ liftIO $ logWarning tracer "insertCertificate: Unhandled DCertGenesis certificate" Right (ConwayTxCertDeleg deleg) -> - insertConwayDelegCert syncEnv mDeposits txId idx mRedeemerId epochNo slotNo deleg + when (isShelleyModeActive $ ioShelley iopts) $ + insertConwayDelegCert syncEnv mDeposits txId idx mRedeemerId epochNo slotNo deleg Right (ConwayTxCertPool pool) -> - when (ioShelley iopts) $ insertPoolCert tracer cache isMember mDeposits network epochNo blkId txId idx pool + when (isShelleyModeActive $ ioShelley iopts) $ insertPoolCert syncEnv cache isMember mDeposits network epochNo blkId txId idx pool Right (ConwayTxCertGov c) -> when (ioGov iopts) $ case c of ConwayRegDRep cred coin anchor -> @@ -98,16 +110,15 @@ insertCertificate syncEnv isMember mDeposits blkId txId epochNo slotNo redeemers ConwayUpdateDRep cred anchor -> lift $ insertDrepRegistration blkId txId idx cred Nothing (strictMaybeToMaybe anchor) where - tracer = getTrace syncEnv cache = envCache syncEnv + tracer = getTrace syncEnv iopts = getInsertOptions syncEnv network = getNetwork syncEnv mRedeemerId = mlookup ridx redeemers insertDelegCert :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> - CacheStatus -> + SyncEnv -> Maybe Generic.Deposits -> Ledger.Network -> DB.TxId -> @@ -117,11 +128,11 @@ insertDelegCert :: SlotNo -> ShelleyDelegCert StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertDelegCert tracer cache mDeposits network txId idx mRedeemerId epochNo slotNo dCert = +insertDelegCert syncEnv mDeposits network txId idx mRedeemerId epochNo slotNo dCert = case dCert of - ShelleyRegCert cred -> insertStakeRegistration tracer cache epochNo mDeposits txId idx $ Generic.annotateStakingCred network cred - ShelleyUnRegCert cred -> insertStakeDeregistration tracer cache network epochNo txId idx mRedeemerId cred - ShelleyDelegCert cred poolkh -> insertDelegation tracer cache network epochNo slotNo txId idx mRedeemerId cred poolkh + ShelleyRegCert cred -> insertStakeRegistration syncEnv epochNo mDeposits txId idx $ Generic.annotateStakingCred network cred + ShelleyUnRegCert cred -> insertStakeDeregistration syncEnv network epochNo txId idx mRedeemerId cred + ShelleyDelegCert cred poolkh -> insertDelegation syncEnv (envCache syncEnv) network epochNo slotNo txId idx mRedeemerId cred poolkh insertConwayDelegCert :: (MonadBaseControl IO m, MonadIO m) => @@ -137,47 +148,44 @@ insertConwayDelegCert :: insertConwayDelegCert syncEnv mDeposits txId idx mRedeemerId epochNo slotNo dCert = case dCert of ConwayRegCert cred _dep -> - when (ioShelley iopts) $ - insertStakeRegistration trce cache epochNo mDeposits txId idx $ + when (isShelleyModeActive $ ioShelley iopts) $ + insertStakeRegistration syncEnv epochNo mDeposits txId idx $ Generic.annotateStakingCred network cred ConwayUnRegCert cred _dep -> - when (ioShelley iopts) $ - insertStakeDeregistration trce cache network epochNo txId idx mRedeemerId cred + when (isShelleyModeActive $ ioShelley iopts) $ + insertStakeDeregistration syncEnv network epochNo txId idx mRedeemerId cred ConwayDelegCert cred delegatee -> insertDeleg cred delegatee ConwayRegDelegCert cred delegatee _dep -> do - when (ioShelley iopts) $ - insertStakeRegistration trce cache epochNo mDeposits txId idx $ + when (isShelleyModeActive $ ioShelley iopts) $ + insertStakeRegistration syncEnv epochNo mDeposits txId idx $ Generic.annotateStakingCred network cred insertDeleg cred delegatee where insertDeleg cred = \case DelegStake poolkh -> - when (ioShelley iopts) $ - insertDelegation trce cache network epochNo slotNo txId idx mRedeemerId cred poolkh + when (isShelleyModeActive $ ioShelley iopts) $ + insertDelegation syncEnv cache network epochNo slotNo txId idx mRedeemerId cred poolkh DelegVote drep -> when (ioGov iopts) $ - insertDelegationVote trce cache network txId idx cred drep + insertDelegationVote syncEnv network txId idx cred drep DelegStakeVote poolkh drep -> do - when (ioShelley iopts) $ - insertDelegation trce cache network epochNo slotNo txId idx mRedeemerId cred poolkh + when (isShelleyModeActive $ ioShelley iopts) $ + insertDelegation syncEnv cache network epochNo slotNo txId idx mRedeemerId cred poolkh when (ioGov iopts) $ - insertDelegationVote trce cache network txId idx cred drep - - trce = getTrace syncEnv + insertDelegationVote syncEnv network txId idx cred drep cache = envCache syncEnv iopts = getInsertOptions syncEnv network = getNetwork syncEnv insertMirCert :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> - CacheStatus -> + SyncEnv -> Ledger.Network -> DB.TxId -> Word16 -> MIRCert StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertMirCert tracer cache network txId idx mcert = do +insertMirCert syncEnv network txId idx mcert = do case mirPot mcert of ReservesMIR -> case mirRewards mcert of @@ -192,29 +200,33 @@ insertMirCert tracer cache network txId idx mcert = do (MonadBaseControl IO m, MonadIO m) => (StakeCred, Ledger.DeltaCoin) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () - insertMirReserves (cred, dcoin) = do - addrId <- lift $ queryOrInsertStakeAddress tracer cache UpdateCache network cred - void . lift . DB.insertReserve $ - DB.Reserve - { DB.reserveAddrId = addrId - , DB.reserveCertIndex = idx - , DB.reserveTxId = txId - , DB.reserveAmount = DB.deltaCoinToDbInt65 dcoin - } + insertMirReserves (cred, dcoin) = + -- Check if the stake address is in the shelley whitelist + when (shelleyStakeAddrWhitelistCheck syncEnv $ Ledger.RewardAccount network cred) $ do + addrId <- lift $ queryOrInsertStakeAddress syncEnv (envCache syncEnv) UpdateCache network cred + void . lift . DB.insertReserve $ + DB.Reserve + { DB.reserveAddrId = addrId + , DB.reserveCertIndex = idx + , DB.reserveTxId = txId + , DB.reserveAmount = DB.deltaCoinToDbInt65 dcoin + } insertMirTreasury :: (MonadBaseControl IO m, MonadIO m) => (StakeCred, Ledger.DeltaCoin) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () - insertMirTreasury (cred, dcoin) = do - addrId <- lift $ queryOrInsertStakeAddress tracer cache UpdateCache network cred - void . lift . DB.insertTreasury $ - DB.Treasury - { DB.treasuryAddrId = addrId - , DB.treasuryCertIndex = idx - , DB.treasuryTxId = txId - , DB.treasuryAmount = DB.deltaCoinToDbInt65 dcoin - } + insertMirTreasury (cred, dcoin) = + -- Check if the stake address is in the shelley whitelist + when (shelleyStakeAddrWhitelistCheck syncEnv $ Ledger.RewardAccount network cred) $ do + addrId <- lift $ queryOrInsertStakeAddress syncEnv (envCache syncEnv) UpdateCache network cred + void . lift . DB.insertTreasury $ + DB.Treasury + { DB.treasuryAddrId = addrId + , DB.treasuryCertIndex = idx + , DB.treasuryTxId = txId + , DB.treasuryAmount = DB.deltaCoinToDbInt65 dcoin + } insertPotTransfer :: (MonadBaseControl IO m, MonadIO m) => @@ -316,8 +328,7 @@ insertCommitteeDeRegistration blockId txId idx khCold mAnchor = do insertStakeDeregistration :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> - CacheStatus -> + SyncEnv -> Ledger.Network -> EpochNo -> DB.TxId -> @@ -325,37 +336,40 @@ insertStakeDeregistration :: Maybe DB.RedeemerId -> StakeCred -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertStakeDeregistration trce cache network epochNo txId idx mRedeemerId cred = do - scId <- lift $ queryOrInsertStakeAddress trce cache EvictAndUpdateCache network cred - void . lift . DB.insertStakeDeregistration $ - DB.StakeDeregistration - { DB.stakeDeregistrationAddrId = scId - , DB.stakeDeregistrationCertIndex = idx - , DB.stakeDeregistrationEpochNo = unEpochNo epochNo - , DB.stakeDeregistrationTxId = txId - , DB.stakeDeregistrationRedeemerId = mRedeemerId - } +insertStakeDeregistration syncEnv network epochNo txId idx mRedeemerId cred = do + -- Check if the stake address is in the shelley whitelist + when (shelleyCustomStakeWhitelistCheck syncEnv $ Ledger.RewardAccount network cred) $ do + scId <- lift $ queryOrInsertStakeAddress syncEnv (envCache syncEnv) EvictAndUpdateCache network cred + void . lift . DB.insertStakeDeregistration $ + DB.StakeDeregistration + { DB.stakeDeregistrationAddrId = scId + , DB.stakeDeregistrationCertIndex = idx + , DB.stakeDeregistrationEpochNo = unEpochNo epochNo + , DB.stakeDeregistrationTxId = txId + , DB.stakeDeregistrationRedeemerId = mRedeemerId + } insertStakeRegistration :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> - CacheStatus -> + SyncEnv -> EpochNo -> Maybe Generic.Deposits -> DB.TxId -> Word16 -> Shelley.RewardAccount StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertStakeRegistration tracer cache epochNo mDeposits txId idx rewardAccount = do - saId <- lift $ queryOrInsertRewardAccount tracer cache UpdateCache rewardAccount - void . lift . DB.insertStakeRegistration $ - DB.StakeRegistration - { DB.stakeRegistrationAddrId = saId - , DB.stakeRegistrationCertIndex = idx - , DB.stakeRegistrationEpochNo = unEpochNo epochNo - , DB.stakeRegistrationDeposit = Generic.coinToDbLovelace . Generic.stakeKeyDeposit <$> mDeposits - , DB.stakeRegistrationTxId = txId - } +insertStakeRegistration syncEnv epochNo mDeposits txId idx rewardAccount = do + -- Check if the stake address is in the shelley whitelist + when (shelleyCustomStakeWhitelistCheck syncEnv rewardAccount) $ do + saId <- lift $ queryOrInsertRewardAccount syncEnv (envCache syncEnv) UpdateCache rewardAccount + void . lift . DB.insertStakeRegistration $ + DB.StakeRegistration + { DB.stakeRegistrationAddrId = saId + , DB.stakeRegistrationCertIndex = idx + , DB.stakeRegistrationEpochNo = unEpochNo epochNo + , DB.stakeRegistrationDeposit = Generic.coinToDbLovelace . Generic.stakeKeyDeposit <$> mDeposits + , DB.stakeRegistrationTxId = txId + } -------------------------------------------------------------------------------------------- -- Insert Pots @@ -401,7 +415,7 @@ mkAdaPots blockId slotNo epochNo pots = -------------------------------------------------------------------------------------------- insertDelegation :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> Ledger.Network -> EpochNo -> @@ -412,40 +426,43 @@ insertDelegation :: StakeCred -> Ledger.KeyHash 'Ledger.StakePool StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertDelegation trce cache network (EpochNo epoch) slotNo txId idx mRedeemerId cred poolkh = do - addrId <- lift $ queryOrInsertStakeAddress trce cache UpdateCache network cred - poolHashId <- lift $ queryPoolKeyOrInsert "insertDelegation" trce cache UpdateCache True poolkh - void . lift . DB.insertDelegation $ - DB.Delegation - { DB.delegationAddrId = addrId - , DB.delegationCertIndex = idx - , DB.delegationPoolHashId = poolHashId - , DB.delegationActiveEpochNo = epoch + 2 -- The first epoch where this delegation is valid. - , DB.delegationTxId = txId - , DB.delegationSlotNo = unSlotNo slotNo - , DB.delegationRedeemerId = mRedeemerId - } +insertDelegation syncEnv cache network (EpochNo epoch) slotNo txId idx mRedeemerId cred poolkh = + -- Check if the stake address is in the shelley whitelist + when (shelleyCustomStakeWhitelistCheck syncEnv $ Ledger.RewardAccount network cred) $ do + addrId <- lift $ queryOrInsertStakeAddress syncEnv cache UpdateCache network cred + poolHashId <- lift $ queryPoolKeyOrInsert "insertDelegation" syncEnv cache UpdateCache True poolkh + void . lift . DB.insertDelegation $ + DB.Delegation + { DB.delegationAddrId = addrId + , DB.delegationCertIndex = idx + , DB.delegationPoolHashId = poolHashId + , DB.delegationActiveEpochNo = epoch + 2 -- The first epoch where this delegation is valid. + , DB.delegationTxId = txId + , DB.delegationSlotNo = unSlotNo slotNo + , DB.delegationRedeemerId = mRedeemerId + } insertDelegationVote :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> - CacheStatus -> + SyncEnv -> Ledger.Network -> DB.TxId -> Word16 -> StakeCred -> DRep StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertDelegationVote trce cache network txId idx cred drep = do - addrId <- lift $ queryOrInsertStakeAddress trce cache UpdateCache network cred - drepId <- lift $ insertDrep drep - void - . lift - . DB.insertDelegationVote - $ DB.DelegationVote - { DB.delegationVoteAddrId = addrId - , DB.delegationVoteCertIndex = idx - , DB.delegationVoteDrepHashId = drepId - , DB.delegationVoteTxId = txId - , DB.delegationVoteRedeemerId = Nothing - } +insertDelegationVote syncEnv network txId idx cred drep = + -- Check if the stake address is in the shelley whitelist + when (shelleyStakeAddrWhitelistCheck syncEnv $ Ledger.RewardAccount network cred) $ do + addrId <- lift $ queryOrInsertStakeAddress syncEnv (envCache syncEnv) UpdateCache network cred + drepId <- lift $ insertDrep drep + void + . lift + . DB.insertDelegationVote + $ DB.DelegationVote + { DB.delegationVoteAddrId = addrId + , DB.delegationVoteCertIndex = idx + , DB.delegationVoteDrepHashId = drepId + , DB.delegationVoteTxId = txId + , DB.delegationVoteRedeemerId = Nothing + } diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/GovAction.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/GovAction.hs index 0a320d7bf..d04917594 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/GovAction.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/GovAction.hs @@ -6,6 +6,7 @@ {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE TypeOperators #-} {-# LANGUAGE NoImplicitPrelude #-} module Cardano.DbSync.Era.Universal.Insert.GovAction ( @@ -28,12 +29,15 @@ module Cardano.DbSync.Era.Universal.Insert.GovAction ( ) where -import Cardano.BM.Trace (Trace, logWarning) +import Cardano.BM.Trace (logWarning) import qualified Cardano.Crypto as Crypto import Cardano.Db (DbWord64 (..)) import qualified Cardano.Db as DB +import Cardano.DbSync.Api (getTrace) +import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..), SyncOptions (..)) import Cardano.DbSync.Cache (queryOrInsertRewardAccount, queryPoolKeyOrInsert) -import Cardano.DbSync.Cache.Types (CacheAction (..), CacheStatus (..)) +import Cardano.DbSync.Cache.Types (CacheAction (..)) +import Cardano.DbSync.Config.Types (ShelleyInsertConfig (..)) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Era.Shelley.Generic.ParamProposal import Cardano.DbSync.Era.Universal.Insert.Other (toDouble) @@ -42,13 +46,15 @@ import Cardano.DbSync.Error import Cardano.DbSync.Ledger.State import Cardano.DbSync.Util import Cardano.DbSync.Util.Bech32 (serialiseDrepToBech32) +import Cardano.DbSync.Util.Whitelist (shelleyStakeAddrWhitelistCheck) +import qualified Cardano.Ledger.Address as Ledger import Cardano.Ledger.BaseTypes import qualified Cardano.Ledger.BaseTypes as Ledger import Cardano.Ledger.CertState (DRep (..)) import Cardano.Ledger.Coin (Coin) import qualified Cardano.Ledger.Coin as Ledger import Cardano.Ledger.Compactible (Compactible (..)) -import Cardano.Ledger.Conway.Core (DRepVotingThresholds (..), PoolVotingThresholds (..)) +import Cardano.Ledger.Conway.Core (DRepVotingThresholds (..), Era (..), PoolVotingThresholds (..)) import Cardano.Ledger.Conway.Governance import qualified Cardano.Ledger.Credential as Ledger import Cardano.Ledger.DRep (DRepState (..)) @@ -65,55 +71,57 @@ import qualified Data.Map.Strict as Map import qualified Data.Text.Encoding as Text import Database.Persist.Sql (SqlBackend) import Ouroboros.Consensus.Cardano.Block (StandardConway, StandardCrypto) +import Prelude (zip3) insertGovActionProposal :: forall m. (MonadIO m, MonadBaseControl IO m) => - Trace IO Text -> - CacheStatus -> + SyncEnv -> DB.BlockId -> DB.TxId -> Maybe EpochNo -> Maybe (ConwayGovState StandardConway) -> (Word64, (GovActionId StandardCrypto, ProposalProcedure StandardConway)) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertGovActionProposal trce cache blkId txId govExpiresAt mcgs (index, (govId, pp)) = do - addrId <- - lift $ queryOrInsertRewardAccount trce cache UpdateCache $ pProcReturnAddr pp - votingAnchorId <- lift $ insertVotingAnchor blkId DB.GovActionAnchor $ pProcAnchor pp - mParamProposalId <- lift $ +insertGovActionProposal syncEnv blkId txId govExpiresAt mcgs (index, (govId, pp)) = do + -- check if shelley stake address is in the whitelist + when (shelleyStakeAddrWhitelistCheck syncEnv $ pProcReturnAddr pp) $ do + addrId <- lift $ queryOrInsertRewardAccount syncEnv cache UpdateCache $ pProcReturnAddr pp + votingAnchorId <- lift $ insertVotingAnchor blkId DB.GovActionAnchor $ pProcAnchor pp + mParamProposalId <- lift $ + case pProcGovAction pp of + ParameterChange _ pparams _ -> + Just <$> insertParamProposal blkId txId (convertConwayParamProposal pparams) + _ -> pure Nothing + prevGovActionDBId <- case mprevGovAction of + Nothing -> pure Nothing + Just prevGovActionId -> resolveGovActionProposal syncEnv prevGovActionId + govActionProposalId <- + lift $ + DB.insertGovActionProposal $ + DB.GovActionProposal + { DB.govActionProposalTxId = txId + , DB.govActionProposalIndex = index + , DB.govActionProposalPrevGovActionProposal = prevGovActionDBId + , DB.govActionProposalDeposit = Generic.coinToDbLovelace $ pProcDeposit pp + , DB.govActionProposalReturnAddress = addrId + , DB.govActionProposalExpiration = unEpochNo <$> govExpiresAt + , DB.govActionProposalVotingAnchorId = Just votingAnchorId + , DB.govActionProposalType = Generic.toGovAction $ pProcGovAction pp + , DB.govActionProposalDescription = Text.decodeUtf8 $ LBS.toStrict $ Aeson.encode (pProcGovAction pp) + , DB.govActionProposalParamProposal = mParamProposalId + , DB.govActionProposalRatifiedEpoch = Nothing + , DB.govActionProposalEnactedEpoch = Nothing + , DB.govActionProposalDroppedEpoch = Nothing + , DB.govActionProposalExpiredEpoch = Nothing + } case pProcGovAction pp of - ParameterChange _ pparams _ -> - Just <$> insertParamProposal blkId txId (convertConwayParamProposal pparams) - _ -> pure Nothing - prevGovActionDBId <- case mprevGovAction of - Nothing -> pure Nothing - Just prevGovActionId -> Just <$> resolveGovActionProposal prevGovActionId - govActionProposalId <- - lift $ - DB.insertGovActionProposal $ - DB.GovActionProposal - { DB.govActionProposalTxId = txId - , DB.govActionProposalIndex = index - , DB.govActionProposalPrevGovActionProposal = prevGovActionDBId - , DB.govActionProposalDeposit = Generic.coinToDbLovelace $ pProcDeposit pp - , DB.govActionProposalReturnAddress = addrId - , DB.govActionProposalExpiration = (\epochNum -> unEpochNo epochNum + 1) <$> govExpiresAt - , DB.govActionProposalVotingAnchorId = Just votingAnchorId - , DB.govActionProposalType = Generic.toGovAction $ pProcGovAction pp - , DB.govActionProposalDescription = Text.decodeUtf8 $ LBS.toStrict $ Aeson.encode (pProcGovAction pp) - , DB.govActionProposalParamProposal = mParamProposalId - , DB.govActionProposalRatifiedEpoch = Nothing - , DB.govActionProposalEnactedEpoch = Nothing - , DB.govActionProposalDroppedEpoch = Nothing - , DB.govActionProposalExpiredEpoch = Nothing - } - case pProcGovAction pp of - TreasuryWithdrawals mp _ -> lift $ mapM_ (insertTreasuryWithdrawal govActionProposalId) (Map.toList mp) - UpdateCommittee {} -> lift $ insertNewCommittee govActionProposalId - NewConstitution _ constitution -> lift $ void $ insertConstitution blkId (Just govActionProposalId) constitution - _ -> pure () + TreasuryWithdrawals mp _ -> lift $ mapM_ (insertTreasuryWithdrawal govActionProposalId) (Map.toList mp) + UpdateCommittee {} -> lift $ insertNewCommittee govActionProposalId + NewConstitution _ constitution -> void $ lift $ insertConstitution blkId (Just govActionProposalId) constitution + _ -> pure () where + cache = envCache syncEnv mprevGovAction :: Maybe (GovActionId StandardCrypto) = case pProcGovAction pp of ParameterChange prv _ _ -> unGovPurposeId <$> strictMaybeToMaybe prv HardForkInitiation prv _ -> unGovPurposeId <$> strictMaybeToMaybe prv @@ -122,9 +130,13 @@ insertGovActionProposal trce cache blkId txId govExpiresAt mcgs (index, (govId, NewConstitution prv _ -> unGovPurposeId <$> strictMaybeToMaybe prv _ -> Nothing + insertTreasuryWithdrawal :: + DB.GovActionProposalId -> + (Ledger.RewardAccount StandardCrypto, Coin) -> + ReaderT SqlBackend m DB.TreasuryWithdrawalId insertTreasuryWithdrawal gaId (rwdAcc, coin) = do addrId <- - queryOrInsertRewardAccount trce cache UpdateCache rwdAcc + queryOrInsertRewardAccount syncEnv cache UpdateCache rwdAcc DB.insertTreasuryWithdrawal $ DB.TreasuryWithdrawal { DB.treasuryWithdrawalGovActionProposalId = gaId @@ -140,7 +152,7 @@ insertGovActionProposal trce cache blkId txId govExpiresAt mcgs (index, (govId, case findProposedCommittee govId cgs of Right (Just committee) -> void $ insertCommittee (Just govActionProposalId) committee other -> - liftIO $ logWarning trce $ textShow other <> ": Failed to find committee for " <> textShow pp + liftIO $ logWarning (getTrace syncEnv) $ textShow other <> ": Failed to find committee for " <> textShow pp insertCommittee :: (MonadIO m, MonadBaseControl IO m) => Maybe DB.GovActionProposalId -> Committee StandardConway -> ReaderT SqlBackend m DB.CommitteeId insertCommittee mgapId committee = do @@ -171,17 +183,26 @@ insertCommittee mgapId committee = do -------------------------------------------------------------------------------------- resolveGovActionProposal :: MonadIO m => + SyncEnv -> GovActionId StandardCrypto -> - ExceptT SyncNodeError (ReaderT SqlBackend m) DB.GovActionProposalId -resolveGovActionProposal gaId = do + ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe DB.GovActionProposalId) +resolveGovActionProposal syncEnv gaId = do gaTxId <- liftLookupFail "resolveGovActionProposal.queryTxId" $ DB.queryTxId $ Generic.unTxHash $ gaidTxId gaId let (GovActionIx index) = gaidGovActionIx gaId - liftLookupFail "resolveGovActionProposal.queryGovActionProposalId" $ - DB.queryGovActionProposalId gaTxId (fromIntegral index) -- TODO: Use Word32? + case ioShelley insertOpts of + -- if we have whitelist for stake addresses then don't input the proposal + ShelleyStakeAddrs _ -> pure Nothing + _ -> do + result <- + liftLookupFail "resolveGovActionProposal.queryGovActionProposalId" $ + DB.queryGovActionProposalId gaTxId (fromIntegral index) -- TODO: Use Word32? + pure $ Just result + where + insertOpts = soptInsertOptions $ envOptions syncEnv insertParamProposal :: (MonadBaseControl IO m, MonadIO m) => @@ -266,51 +287,56 @@ insertConstitution blockId mgapId constitution = do -------------------------------------------------------------------------------------- insertVotingProcedures :: (MonadIO m, MonadBaseControl IO m) => - Trace IO Text -> - CacheStatus -> + SyncEnv -> DB.BlockId -> DB.TxId -> + [(GovActionId StandardCrypto, ProposalProcedure StandardConway)] -> (Voter StandardCrypto, [(GovActionId StandardCrypto, VotingProcedure StandardConway)]) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertVotingProcedures trce cache blkId txId (voter, actions) = - mapM_ (insertVotingProcedure trce cache blkId txId voter) (zip [0 ..] actions) +insertVotingProcedures syncEnv blkId txId proposalPs (voter, actions) = + -- TODO: cmdv will actions & proposalPs always be the same length? + mapM_ (insertVotingProcedure syncEnv blkId txId voter) (zip3 [0 ..] actions (map snd proposalPs)) insertVotingProcedure :: (MonadIO m, MonadBaseControl IO m) => - Trace IO Text -> - CacheStatus -> + SyncEnv -> DB.BlockId -> DB.TxId -> Voter StandardCrypto -> - (Word16, (GovActionId StandardCrypto, VotingProcedure StandardConway)) -> + (Word16, (GovActionId StandardCrypto, VotingProcedure StandardConway), ProposalProcedure StandardConway) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertVotingProcedure trce cache blkId txId voter (index, (gaId, vp)) = do - govActionId <- resolveGovActionProposal gaId - votingAnchorId <- whenMaybe (strictMaybeToMaybe $ vProcAnchor vp) $ lift . insertVotingAnchor blkId DB.OtherAnchor - (mCommitteeVoterId, mDRepVoter, mStakePoolVoter) <- case voter of - CommitteeVoter cred -> do - khId <- lift $ insertCommitteeHash cred - pure (Just khId, Nothing, Nothing) - DRepVoter cred -> do - drep <- lift $ insertCredDrepHash cred - pure (Nothing, Just drep, Nothing) - StakePoolVoter poolkh -> do - poolHashId <- lift $ queryPoolKeyOrInsert "insertVotingProcedure" trce cache UpdateCache False poolkh - pure (Nothing, Nothing, Just poolHashId) - void - . lift - . DB.insertVotingProcedure - $ DB.VotingProcedure - { DB.votingProcedureTxId = txId - , DB.votingProcedureIndex = index - , DB.votingProcedureGovActionProposalId = govActionId - , DB.votingProcedureCommitteeVoter = mCommitteeVoterId - , DB.votingProcedureDrepVoter = mDRepVoter - , DB.votingProcedurePoolVoter = mStakePoolVoter - , DB.votingProcedureVoterRole = Generic.toVoterRole voter - , DB.votingProcedureVote = Generic.toVote $ vProcVote vp - , DB.votingProcedureVotingAnchorId = votingAnchorId - } +insertVotingProcedure syncEnv blkId txId voter (index, (gaId, vp), proposalP) = do + -- check if shelley stake address is in the whitelist + when (shelleyStakeAddrWhitelistCheck syncEnv $ pProcReturnAddr proposalP) $ do + maybeGovActionId <- resolveGovActionProposal syncEnv gaId + case maybeGovActionId of + Nothing -> pure () + Just govActionId -> do + votingAnchorId <- whenMaybe (strictMaybeToMaybe $ vProcAnchor vp) $ lift . insertVotingAnchor blkId DB.OtherAnchor + (mCommitteeVoterId, mDRepVoter, mStakePoolVoter) <- case voter of + CommitteeVoter cred -> do + khId <- lift $ insertCommitteeHash cred + pure (Just khId, Nothing, Nothing) + DRepVoter cred -> do + drep <- lift $ insertCredDrepHash cred + pure (Nothing, Just drep, Nothing) + StakePoolVoter poolkh -> do + poolHashId <- lift $ queryPoolKeyOrInsert "insertVotingProcedure" syncEnv (envCache syncEnv) UpdateCache False poolkh + pure (Nothing, Nothing, Just poolHashId) + void + . lift + . DB.insertVotingProcedure + $ DB.VotingProcedure + { DB.votingProcedureTxId = txId + , DB.votingProcedureIndex = index + , DB.votingProcedureGovActionProposalId = govActionId + , DB.votingProcedureCommitteeVoter = mCommitteeVoterId + , DB.votingProcedureDrepVoter = mDRepVoter + , DB.votingProcedurePoolVoter = mStakePoolVoter + , DB.votingProcedureVoterRole = Generic.toVoterRole voter + , DB.votingProcedureVote = Generic.toVote $ vProcVote vp + , DB.votingProcedureVotingAnchorId = votingAnchorId + } insertVotingAnchor :: (MonadIO m, MonadBaseControl IO m) => DB.BlockId -> DB.AnchorType -> Anchor StandardCrypto -> ReaderT SqlBackend m DB.VotingAnchorId insertVotingAnchor blockId anchorType anchor = @@ -387,56 +413,65 @@ insertCostModel _blkId cms = updateRatified :: forall m. MonadIO m => + SyncEnv -> EpochNo -> [GovActionState StandardConway] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -updateRatified epochNo ratifiedActions = do +updateRatified syncEnv epochNo ratifiedActions = do forM_ ratifiedActions $ \action -> do - gaId <- resolveGovActionProposal $ gasId action - lift $ DB.updateGovActionRatified gaId (unEpochNo epochNo) + mGaId <- resolveGovActionProposal syncEnv $ gasId action + whenJust mGaId $ \gaId -> + lift $ DB.updateGovActionRatified gaId (unEpochNo epochNo) updateExpired :: forall m. MonadIO m => + SyncEnv -> EpochNo -> [GovActionId StandardCrypto] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -updateExpired epochNo ratifiedActions = do +updateExpired syncEnv epochNo ratifiedActions = do forM_ ratifiedActions $ \action -> do - gaId <- resolveGovActionProposal action - lift $ DB.updateGovActionExpired gaId (unEpochNo epochNo) + mGaId <- resolveGovActionProposal syncEnv action + whenJust mGaId $ \gaId -> + lift $ DB.updateGovActionExpired gaId (unEpochNo epochNo) updateDropped :: forall m. MonadIO m => + SyncEnv -> EpochNo -> [GovActionId StandardCrypto] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -updateDropped epochNo ratifiedActions = do +updateDropped syncEnv epochNo ratifiedActions = do forM_ ratifiedActions $ \action -> do - gaId <- resolveGovActionProposal action - lift $ DB.updateGovActionDropped gaId (unEpochNo epochNo) + mGaId <- resolveGovActionProposal syncEnv action + whenJust mGaId $ \gaId -> + lift $ DB.updateGovActionDropped gaId (unEpochNo epochNo) insertUpdateEnacted :: forall m. (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> DB.BlockId -> EpochNo -> ConwayGovState StandardConway -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertUpdateEnacted trce blkId epochNo enactedState = do +insertUpdateEnacted syncEnv blkId epochNo enactedState = do whenJust (strictMaybeToMaybe (grPParamUpdate govIds)) $ \prevId -> do - gaId <- resolveGovActionProposal $ unGovPurposeId prevId - void $ lift $ DB.updateGovActionEnacted gaId (unEpochNo epochNo) + maybeGaId <- resolveGovActionProposal syncEnv $ unGovPurposeId prevId + case maybeGaId of + Nothing -> pure () + Just gaId -> void $ lift $ DB.updateGovActionEnacted gaId (unEpochNo epochNo) whenJust (strictMaybeToMaybe (grHardFork govIds)) $ \prevId -> do - gaId <- resolveGovActionProposal $ unGovPurposeId prevId - void $ lift $ DB.updateGovActionEnacted gaId (unEpochNo epochNo) - - (mcommitteeId, mnoConfidenceGaId) <- handleCommittee + maybeGaId <- resolveGovActionProposal syncEnv $ unGovPurposeId prevId + case maybeGaId of + Nothing -> pure () + Just gaId -> void $ lift $ DB.updateGovActionEnacted gaId (unEpochNo epochNo) + (mcommitteeId, mnoConfidenceGaId) <- handleCommittee syncEnv govIds epochNo enactedState - constitutionId <- handleConstitution + constitutionId <- handleConstitution syncEnv blkId govIds epochNo enactedState void $ lift $ @@ -450,67 +485,88 @@ insertUpdateEnacted trce blkId epochNo enactedState = do where govIds = govStatePrevGovActionIds enactedState - handleCommittee = do - mCommitteeGaId <- case strictMaybeToMaybe (grCommittee govIds) of +handleCommittee :: + (EraCrypto era ~ StandardCrypto, MonadIO m, MonadBaseControl IO m) => + SyncEnv -> + GovRelation StrictMaybe era -> + EpochNo -> + ConwayGovState StandardConway -> + ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe DB.CommitteeId, Maybe DB.GovActionProposalId) +handleCommittee syncEnv govIds epochNo enactedState = do + mCommitteeGaId <- case strictMaybeToMaybe (grCommittee govIds) of + Nothing -> pure Nothing + Just prevId -> do + mGaId <- resolveGovActionProposal syncEnv $ unGovPurposeId prevId + case mGaId of Nothing -> pure Nothing - Just prevId -> do - gaId <- resolveGovActionProposal $ unGovPurposeId prevId + Just gaId -> do _nCommittee <- lift $ DB.updateGovActionEnacted gaId (unEpochNo epochNo) pure $ Just gaId - case (mCommitteeGaId, strictMaybeToMaybe (cgsCommittee enactedState)) of - (Nothing, Nothing) -> pure (Nothing, Nothing) - (Nothing, Just committee) -> do - -- No enacted proposal means we're after conway genesis territory - committeeIds <- lift $ DB.queryProposalCommittee Nothing - case committeeIds of - [] -> do - committeeId <- lift $ insertCommittee Nothing committee - pure (Just committeeId, Nothing) - (committeeId : _rest) -> - pure (Just committeeId, Nothing) - (Just committeeGaId, Nothing) -> - -- No committee with enacted action means it's a no confidence action. - pure (Nothing, Just committeeGaId) - (Just committeeGaId, Just committee) -> do - committeeIds <- lift $ DB.queryProposalCommittee (Just committeeGaId) - case committeeIds of - [] -> do - -- This should never happen. Having a committee and an enacted action, means - -- the committee came from a proposal which should be returned from the query. - liftIO $ - logWarning trce $ - mconcat - [ "The impossible happened! Couldn't find the committee " - , textShow committee - , " which was enacted by a proposal " - , textShow committeeGaId - ] - pure (Nothing, Nothing) - (committeeId : _rest) -> - pure (Just committeeId, Nothing) + case (mCommitteeGaId, strictMaybeToMaybe (cgsCommittee enactedState)) of + (Nothing, Nothing) -> pure (Nothing, Nothing) + (Nothing, Just committee) -> do + -- No enacted proposal means we're after conway genesis territory + committeeIds <- lift $ DB.queryProposalCommittee Nothing + case committeeIds of + [] -> do + committeeId <- lift $ insertCommittee Nothing committee + pure (Just committeeId, Nothing) + (committeeId : _rest) -> + pure (Just committeeId, Nothing) + (Just committeeGaId, Nothing) -> + -- No committee with enacted action means it's a no confidence action. + pure (Nothing, Just committeeGaId) + (Just committeeGaId, Just committee) -> do + committeeIds <- lift $ DB.queryProposalCommittee (Just committeeGaId) + case committeeIds of + [] -> do + -- This should never happen. Having a committee and an enacted action, means + -- the committee came from a proposal which should be returned from the query. + liftIO $ + logWarning (getTrace syncEnv) $ + mconcat + [ "The impossible happened! Couldn't find the committee " + , textShow committee + , " which was enacted by a proposal " + , textShow committeeGaId + ] + pure (Nothing, Nothing) + (committeeId : _rest) -> + pure (Just committeeId, Nothing) - handleConstitution = do - mConstitutionGaId <- case strictMaybeToMaybe (grConstitution govIds) of +handleConstitution :: + (EraCrypto era ~ StandardCrypto, MonadIO m, MonadBaseControl IO m) => + SyncEnv -> + DB.BlockId -> + GovRelation StrictMaybe era -> + EpochNo -> + ConwayGovState StandardConway -> + ExceptT SyncNodeError (ReaderT SqlBackend m) DB.ConstitutionId +handleConstitution syncEnv blkId govIds epochNo enactedState = do + mConstitutionGaId <- case strictMaybeToMaybe (grConstitution govIds) of + Nothing -> pure Nothing + Just prevId -> do + mGaId <- resolveGovActionProposal syncEnv $ unGovPurposeId prevId + case mGaId of Nothing -> pure Nothing - Just prevId -> do - gaId <- resolveGovActionProposal $ unGovPurposeId prevId + Just gaId -> do _nConstitution <- lift $ DB.updateGovActionEnacted gaId (unEpochNo epochNo) pure $ Just gaId - constitutionIds <- lift $ DB.queryProposalConstitution mConstitutionGaId - case constitutionIds of - -- The first case can only happen once on the first Conway epoch. - -- On next epochs there will be at least one constitution, so the query will return something. - [] -> lift $ insertConstitution blkId Nothing (cgsConstitution enactedState) - constitutionId : rest -> do - unless (null rest) $ - liftIO $ - logWarning trce $ - mconcat - [ "Found multiple constitutions for proposal " - , textShow mConstitutionGaId - , ": " - , textShow constitutionIds - ] - pure constitutionId + constitutionIds <- lift $ DB.queryProposalConstitution mConstitutionGaId + case constitutionIds of + -- The first case can only happen once on the first Conway epoch. + -- On next epochs there will be at least one constitution, so the query will return something. + [] -> lift $ insertConstitution blkId Nothing (cgsConstitution enactedState) + constitutionId : rest -> do + unless (null rest) $ + liftIO $ + logWarning (getTrace syncEnv) $ + mconcat + [ "Found multiple constitutions for proposal " + , textShow mConstitutionGaId + , ": " + , textShow constitutionIds + ] + pure constitutionId diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Grouped.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Grouped.hs index 1be48a539..624909e96 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Grouped.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Grouped.hs @@ -62,6 +62,7 @@ data ExtendedTxOut = ExtendedTxOut { etoTxHash :: !ByteString , etoTxOut :: !DB.TxOut } + deriving (Show) data ExtendedTxIn = ExtendedTxIn { etiTxIn :: !DB.TxIn @@ -144,12 +145,15 @@ insertReverseIndex blockId minIds = -- This happens the input consumes an output introduced in the same block. resolveTxInputs :: MonadIO m => + SyncEnv -> + -- | Has the output been consumed? Bool -> + -- | Does the output need a value? Bool -> [ExtendedTxOut] -> Generic.TxIn -> ExceptT SyncNodeError (ReaderT SqlBackend m) (Generic.TxIn, DB.TxId, Either Generic.TxIn DB.TxOutId, Maybe DbLovelace) -resolveTxInputs hasConsumed needsValue groupedOutputs txIn = +resolveTxInputs _syncEnv hasConsumed needsValue groupedOutputs txIn = liftLookupFail ("resolveTxInputs " <> textShow txIn <> " ") $ do qres <- case (hasConsumed, needsValue) of diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs index db30f90d2..02c5edb61 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs @@ -89,9 +89,9 @@ insertNewEpochLedgerEvents syncEnv currentEpochNo@(EpochNo curEpoch) = let rewards = Map.toList $ Generic.unRewards rwd insertRewards syncEnv ntw (subFromCurrentEpoch 1) (EpochNo $ curEpoch + 1) cache rewards LedgerRestrainedRewards e rwd creds -> - lift $ adjustEpochRewards tracer ntw cache e rwd creds + lift $ adjustEpochRewards syncEnv ntw e rwd creds LedgerTotalRewards _e rwd -> - lift $ validateEpochRewards tracer ntw (subFromCurrentEpoch 2) currentEpochNo rwd + lift $ validateEpochRewards syncEnv tracer ntw (subFromCurrentEpoch 2) currentEpochNo rwd LedgerAdaPots _ -> pure () -- These are handled separately by insertBlock LedgerGovInfo en ex uncl -> do @@ -99,19 +99,20 @@ insertNewEpochLedgerEvents syncEnv currentEpochNo@(EpochNo curEpoch) = liftIO $ logInfo tracer $ "Found " <> textShow (Set.size uncl) <> " unclaimed proposal refunds" - updateDropped (EpochNo curEpoch) (garGovActionId <$> (en <> ex)) + updateDropped syncEnv (EpochNo curEpoch) (garGovActionId <$> (en <> ex)) let en' = filter (\e -> Set.notMember (garGovActionId e) uncl) en ex' = filter (\e -> Set.notMember (garGovActionId e) uncl) ex - insertProposalRefunds tracer ntw (subFromCurrentEpoch 1) currentEpochNo cache (en' <> ex') -- TODO: check if they are disjoint to avoid double entries. + insertProposalRefunds syncEnv ntw (subFromCurrentEpoch 1) currentEpochNo cache (en' <> ex') -- TODO: check if they are disjoint to avoid double entries. forM_ en $ \gar -> whenJust (garMTreasury gar) $ \treasuryMap -> do - gaId <- resolveGovActionProposal (garGovActionId gar) - lift $ void $ DB.updateGovActionEnacted gaId (unEpochNo currentEpochNo) + mGaId <- resolveGovActionProposal syncEnv (garGovActionId gar) + whenJust mGaId $ \gaId -> + lift $ void $ DB.updateGovActionEnacted gaId (unEpochNo currentEpochNo) let rewards = Map.mapKeys Ledger.raCredential $ Map.map (Set.singleton . mkTreasuryReward) treasuryMap - insertRewardRests tracer ntw (subFromCurrentEpoch 1) currentEpochNo cache (Map.toList rewards) + insertRewardRests syncEnv ntw (subFromCurrentEpoch 1) currentEpochNo cache (Map.toList rewards) LedgerMirDist rwd -> do unless (Map.null rwd) $ do let rewards = Map.toList rwd - insertRewardRests tracer ntw (subFromCurrentEpoch 1) currentEpochNo cache rewards + insertRewardRests syncEnv ntw (subFromCurrentEpoch 1) currentEpochNo cache rewards liftIO . logInfo tracer $ "Inserted " <> show (length rewards) <> " Mir rewards" LedgerPoolReap en drs -> unless (Map.null $ Generic.unRewards drs) $ do diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Other.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Other.hs index 22fe83482..7efd695ca 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Other.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Other.hs @@ -14,19 +14,23 @@ module Cardano.DbSync.Era.Universal.Insert.Other ( insertRedeemerData, insertStakeAddressRefIfMissing, insertMultiAsset, + insertScriptWithWhitelist, insertScript, insertExtraKeyWitness, -) where +) +where -import Cardano.BM.Trace (Trace) import qualified Cardano.Db as DB +import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..), SyncOptions (..)) import Cardano.DbSync.Cache (insertDatumAndCache, queryDatum, queryMAWithCache, queryOrInsertRewardAccount, queryOrInsertStakeAddress) import Cardano.DbSync.Cache.Types (CacheAction (..), CacheStatus (..)) +import Cardano.DbSync.Config.Types (isShelleyWhitelistModeActive) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Era.Universal.Insert.Grouped import Cardano.DbSync.Era.Util (safeDecodeToJson) import Cardano.DbSync.Error import Cardano.DbSync.Util +import Cardano.DbSync.Util.Whitelist (isSimplePlutusScriptHashInWhitelist, shelleyStakeAddrWhitelistCheck) import qualified Cardano.Ledger.Address as Ledger import qualified Cardano.Ledger.BaseTypes as Ledger import Cardano.Ledger.Coin (Coin (..)) @@ -34,6 +38,7 @@ import qualified Cardano.Ledger.Credential as Ledger import Cardano.Ledger.Mary.Value (AssetName (..), PolicyID (..)) import Cardano.Prelude import Control.Monad.Trans.Control (MonadBaseControl) +import Data.ByteString.Short (ShortByteString) import Database.Persist.Sql (SqlBackend) import Ouroboros.Consensus.Cardano.Block (StandardCrypto) @@ -42,14 +47,14 @@ import Ouroboros.Consensus.Cardano.Block (StandardCrypto) -------------------------------------------------------------------------------------------- insertRedeemer :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> Bool -> [ExtendedTxOut] -> DB.TxId -> (Word64, Generic.TxRedeemer) -> ExceptT SyncNodeError (ReaderT SqlBackend m) (Word64, DB.RedeemerId) -insertRedeemer tracer disInOut groupedOutputs txId (rix, redeemer) = do - tdId <- insertRedeemerData tracer txId $ Generic.txRedeemerData redeemer +insertRedeemer syncEnv disInOut groupedOutputs txId (rix, redeemer) = do + tdId <- insertRedeemerData syncEnv txId $ Generic.txRedeemerData redeemer scriptHash <- findScriptHash rid <- lift @@ -70,24 +75,27 @@ insertRedeemer tracer disInOut groupedOutputs txId (rix, redeemer) = do (MonadBaseControl IO m, MonadIO m) => ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe ByteString) findScriptHash = - case (disInOut, Generic.txRedeemerScriptHash redeemer) of - (True, _) -> pure Nothing - (_, Nothing) -> pure Nothing - (_, Just (Right bs)) -> pure $ Just bs - (_, Just (Left txIn)) -> resolveScriptHash groupedOutputs txIn + -- If we are in shelley whitelist mode, we don't need to resolve the script hash + if isShelleyWhitelistModeActive $ ioShelley $ soptInsertOptions $ envOptions syncEnv + then pure Nothing + else case (disInOut, Generic.txRedeemerScriptHash redeemer) of + (True, _) -> pure Nothing + (_, Nothing) -> pure Nothing + (_, Just (Right bs)) -> pure $ Just bs + (_, Just (Left txIn)) -> resolveScriptHash groupedOutputs txIn insertRedeemerData :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> DB.TxId -> Generic.PlutusData -> ExceptT SyncNodeError (ReaderT SqlBackend m) DB.RedeemerDataId -insertRedeemerData tracer txId txd = do +insertRedeemerData syncEnv txId txd = do mRedeemerDataId <- lift $ DB.queryRedeemerData $ Generic.dataHashToBytes $ Generic.txDataHash txd case mRedeemerDataId of Just redeemerDataId -> pure redeemerDataId Nothing -> do - value <- safeDecodeToJson tracer "insertDatum: Column 'value' in table 'datum' " $ Generic.txDataValue txd + value <- safeDecodeToJson syncEnv "insertRedeemerData: Column 'value' in table 'datum' " $ Generic.txDataValue txd lift . DB.insertRedeemerData $ DB.RedeemerData @@ -102,17 +110,17 @@ insertRedeemerData tracer txId txd = do -------------------------------------------------------------------------------------------- insertDatum :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> DB.TxId -> Generic.PlutusData -> ExceptT SyncNodeError (ReaderT SqlBackend m) DB.DatumId -insertDatum tracer cache txId txd = do +insertDatum syncEnv cache txId txd = do mDatumId <- lift $ queryDatum cache $ Generic.txDataHash txd case mDatumId of Just datumId -> pure datumId Nothing -> do - value <- safeDecodeToJson tracer "insertRedeemerData: Column 'value' in table 'redeemer' " $ Generic.txDataValue txd + value <- safeDecodeToJson syncEnv "insertDatum: Column 'value' in table 'redeemer' " $ Generic.txDataValue txd lift $ insertDatumAndCache cache (Generic.txDataHash txd) $ DB.Datum @@ -124,38 +132,46 @@ insertDatum tracer cache txId txd = do insertWithdrawals :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> DB.TxId -> Map Word64 DB.RedeemerId -> Generic.TxWithdrawal -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertWithdrawals tracer cache txId redeemers txWdrl = do - addrId <- - lift $ queryOrInsertRewardAccount tracer cache UpdateCache $ Generic.txwRewardAccount txWdrl - void . lift . DB.insertWithdrawal $ - DB.Withdrawal - { DB.withdrawalAddrId = addrId - , DB.withdrawalTxId = txId - , DB.withdrawalAmount = Generic.coinToDbLovelace $ Generic.txwAmount txWdrl - , DB.withdrawalRedeemerId = mlookup (Generic.txwRedeemerIndex txWdrl) redeemers - } +insertWithdrawals syncEnv cache txId redeemers txWdrl = do + -- check if shelley stake address is in the whitelist + when (shelleyStakeAddrWhitelistCheck syncEnv $ Generic.txwRewardAccount txWdrl) $ do + addrId <- + lift $ queryOrInsertRewardAccount syncEnv cache UpdateCache $ Generic.txwRewardAccount txWdrl + void + . lift + . DB.insertWithdrawal + $ DB.Withdrawal + { DB.withdrawalAddrId = addrId + , DB.withdrawalTxId = txId + , DB.withdrawalAmount = Generic.coinToDbLovelace $ Generic.txwAmount txWdrl + , DB.withdrawalRedeemerId = mlookup (Generic.txwRedeemerIndex txWdrl) redeemers + } -- | Insert a stake address if it is not already in the `stake_address` table. Regardless of -- whether it is newly inserted or it is already there, we retrun the `StakeAddressId`. insertStakeAddressRefIfMissing :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> Ledger.Addr StandardCrypto -> ReaderT SqlBackend m (Maybe DB.StakeAddressId) -insertStakeAddressRefIfMissing trce cache addr = +insertStakeAddressRefIfMissing syncEnv cache addr = case addr of Ledger.AddrBootstrap {} -> pure Nothing Ledger.Addr nw _pcred sref -> case sref of Ledger.StakeRefBase cred -> do - Just <$> queryOrInsertStakeAddress trce cache DoNotUpdateCache nw cred + -- Check if the stake address is in the shelley whitelist + whenFalseEmpty + (shelleyStakeAddrWhitelistCheck syncEnv $ Ledger.RewardAccount nw cred) + Nothing + (Just <$> queryOrInsertStakeAddress syncEnv cache DoNotUpdateCache nw cred) Ledger.StakeRefPtr ptr -> do DB.queryStakeRefPtr ptr Ledger.StakeRefNull -> pure Nothing @@ -163,14 +179,24 @@ insertStakeAddressRefIfMissing trce cache addr = insertMultiAsset :: (MonadBaseControl IO m, MonadIO m) => CacheStatus -> + Maybe (NonEmpty ShortByteString) -> PolicyID StandardCrypto -> AssetName -> - ReaderT SqlBackend m DB.MultiAssetId -insertMultiAsset cache policy aName = do + ReaderT SqlBackend m (Maybe DB.MultiAssetId) +insertMultiAsset cache mWhitelist policy aName = do mId <- queryMAWithCache cache policy aName case mId of - Right maId -> pure maId + Right maId -> pure $ Just maId Left (policyBs, assetNameBs) -> + case mWhitelist of + -- we want to check the whitelist at the begining + Just whitelist -> + if shortBsBase16Encode policyBs `elem` whitelist + then Just <$> insertAssettIntoDB policyBs assetNameBs + else pure Nothing + Nothing -> Just <$> insertAssettIntoDB policyBs assetNameBs + where + insertAssettIntoDB policyBs assetNameBs = DB.insertMultiAssetUnchecked $ DB.MultiAsset { DB.multiAssetPolicy = policyBs @@ -178,13 +204,24 @@ insertMultiAsset cache policy aName = do , DB.multiAssetFingerprint = DB.unAssetFingerprint (DB.mkAssetFingerprint policyBs assetNameBs) } +insertScriptWithWhitelist :: + (MonadBaseControl IO m, MonadIO m) => + SyncEnv -> + DB.TxId -> + Generic.TxScript -> + ReaderT SqlBackend m (Maybe DB.ScriptId) +insertScriptWithWhitelist syncEnv txId script = do + if isSimplePlutusScriptHashInWhitelist syncEnv $ Generic.txScriptHash script + then insertScript syncEnv txId script <&> Just + else pure Nothing + insertScript :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> DB.TxId -> Generic.TxScript -> ReaderT SqlBackend m DB.ScriptId -insertScript tracer txId script = do +insertScript syncEnv txId script = do mScriptId <- DB.queryScript $ Generic.txScriptHash script case mScriptId of Just scriptId -> pure scriptId @@ -200,17 +237,16 @@ insertScript tracer txId script = do , DB.scriptBytes = Generic.txScriptCBOR script } where - scriptConvert :: MonadIO m => Generic.TxScript -> m (Maybe Text) + scriptConvert :: (MonadIO m) => Generic.TxScript -> m (Maybe Text) scriptConvert s = - maybe (pure Nothing) (safeDecodeToJson tracer "insertScript: Column 'json' in table 'script' ") (Generic.txScriptJson s) + maybe (pure Nothing) (safeDecodeToJson syncEnv "insertScript: Column 'json' in table 'script' ") (Generic.txScriptJson s) insertExtraKeyWitness :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> DB.TxId -> ByteString -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertExtraKeyWitness _tracer txId keyHash = do +insertExtraKeyWitness txId keyHash = do void . lift . DB.insertExtraKeyWitness diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Pool.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Pool.hs index 6fbe0bcdc..0c9e8a3b9 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Pool.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Pool.hs @@ -5,6 +5,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE NoImplicitPrelude #-} +{-# OPTIONS_GHC -Wno-deprecations #-} module Cardano.DbSync.Era.Universal.Insert.Pool ( IsPoolMember, @@ -16,10 +17,10 @@ module Cardano.DbSync.Era.Universal.Insert.Pool ( insertPoolCert, ) where -import Cardano.BM.Trace (Trace) import Cardano.Crypto.Hash (hashToBytes) import Cardano.Db (PoolUrl (..)) import qualified Cardano.Db as DB +import Cardano.DbSync.Api.Types (SyncEnv (..)) import Cardano.DbSync.Cache ( insertPoolKeyWithCache, queryOrInsertRewardAccount, @@ -31,6 +32,7 @@ import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Error import Cardano.DbSync.Types (PoolKeyHash) import Cardano.DbSync.Util +import Cardano.DbSync.Util.Whitelist (shelleyStakeAddrWhitelistCheck) import qualified Cardano.Ledger.Address as Ledger import Cardano.Ledger.BaseTypes import qualified Cardano.Ledger.BaseTypes as Ledger @@ -48,7 +50,7 @@ type IsPoolMember = PoolKeyHash -> Bool insertPoolRegister :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> IsPoolMember -> Maybe Generic.Deposits -> @@ -59,36 +61,35 @@ insertPoolRegister :: Word16 -> PoolP.PoolParams StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertPoolRegister trce cache isMember mdeposits network (EpochNo epoch) blkId txId idx params = do - poolHashId <- lift $ insertPoolKeyWithCache cache UpdateCache (PoolP.ppId params) - mdId <- case strictMaybeToMaybe $ PoolP.ppMetadata params of - Just md -> Just <$> insertPoolMetaDataRef poolHashId txId md - Nothing -> pure Nothing - - isRegistration <- isPoolRegistration poolHashId - let epochActivationDelay = if isRegistration then 2 else 3 - deposit = if isRegistration then Generic.coinToDbLovelace . Generic.poolDeposit <$> mdeposits else Nothing - - saId <- lift $ queryOrInsertRewardAccount trce cache UpdateCache (adjustNetworkTag $ PoolP.ppRewardAccount params) - poolUpdateId <- - lift - . DB.insertPoolUpdate - $ DB.PoolUpdate - { DB.poolUpdateHashId = poolHashId - , DB.poolUpdateCertIndex = idx - , DB.poolUpdateVrfKeyHash = hashToBytes (PoolP.ppVrf params) - , DB.poolUpdatePledge = Generic.coinToDbLovelace (PoolP.ppPledge params) - , DB.poolUpdateRewardAddrId = saId - , DB.poolUpdateActiveEpochNo = epoch + epochActivationDelay - , DB.poolUpdateMetaId = mdId - , DB.poolUpdateMargin = realToFrac $ Ledger.unboundRational (PoolP.ppMargin params) - , DB.poolUpdateFixedCost = Generic.coinToDbLovelace (PoolP.ppCost params) - , DB.poolUpdateDeposit = deposit - , DB.poolUpdateRegisteredTxId = txId - } - - mapM_ (insertPoolOwner trce cache network poolUpdateId) $ toList (PoolP.ppOwners params) - mapM_ (insertPoolRelay poolUpdateId) $ toList (PoolP.ppRelays params) +insertPoolRegister syncEnv cache isMember mdeposits network (EpochNo epoch) blkId txId idx params = + -- Check if the stake address is in the shelley whitelist + when (shelleyStakeAddrWhitelistCheck syncEnv $ adjustNetworkTag (PoolP.ppRewardAccount params)) $ do + poolHashId <- lift $ insertPoolKeyWithCache cache UpdateCache (PoolP.ppId params) + mdId <- case strictMaybeToMaybe $ PoolP.ppMetadata params of + Just md -> Just <$> insertPoolMetaDataRef poolHashId txId md + Nothing -> pure Nothing + isRegistration <- isPoolRegistration poolHashId + let epochActivationDelay = if isRegistration then 2 else 3 + deposit = if isRegistration then Generic.coinToDbLovelace . Generic.poolDeposit <$> mdeposits else Nothing + saId <- lift $ queryOrInsertRewardAccount syncEnv cache UpdateCache (adjustNetworkTag $ PoolP.ppRewardAcnt params) + poolUpdateId <- + lift + . DB.insertPoolUpdate + $ DB.PoolUpdate + { DB.poolUpdateHashId = poolHashId + , DB.poolUpdateCertIndex = idx + , DB.poolUpdateVrfKeyHash = hashToBytes (PoolP.ppVrf params) + , DB.poolUpdatePledge = Generic.coinToDbLovelace (PoolP.ppPledge params) + , DB.poolUpdateRewardAddrId = saId + , DB.poolUpdateActiveEpochNo = epoch + epochActivationDelay + , DB.poolUpdateMetaId = mdId + , DB.poolUpdateMargin = realToFrac $ Ledger.unboundRational (PoolP.ppMargin params) + , DB.poolUpdateFixedCost = Generic.coinToDbLovelace (PoolP.ppCost params) + , DB.poolUpdateDeposit = deposit + , DB.poolUpdateRegisteredTxId = txId + } + mapM_ (insertPoolOwner syncEnv cache network poolUpdateId) $ toList (PoolP.ppOwners params) + mapM_ (insertPoolRelay poolUpdateId) $ toList (PoolP.ppRelays params) where isPoolRegistration :: MonadIO m => DB.PoolHashId -> ExceptT SyncNodeError (ReaderT SqlBackend m) Bool isPoolRegistration poolHashId = @@ -108,15 +109,15 @@ insertPoolRegister trce cache isMember mdeposits network (EpochNo epoch) blkId t insertPoolRetire :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> - DB.TxId -> + SyncEnv -> CacheStatus -> + DB.TxId -> EpochNo -> Word16 -> Ledger.KeyHash 'Ledger.StakePool StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertPoolRetire trce txId cache epochNum idx keyHash = do - poolId <- lift $ queryPoolKeyOrInsert "insertPoolRetire" trce cache UpdateCache True keyHash +insertPoolRetire syncEnv cache txId epochNum idx keyHash = do + poolId <- lift $ queryPoolKeyOrInsert "insertPoolRetire" syncEnv cache UpdateCache True keyHash void . lift . DB.insertPoolRetire $ DB.PoolRetire { DB.poolRetireHashId = poolId @@ -143,19 +144,21 @@ insertPoolMetaDataRef poolId txId md = insertPoolOwner :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> Ledger.Network -> DB.PoolUpdateId -> Ledger.KeyHash 'Ledger.Staking StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertPoolOwner trce cache network poolUpdateId skh = do - saId <- lift $ queryOrInsertStakeAddress trce cache UpdateCache network (Ledger.KeyHashObj skh) - void . lift . DB.insertPoolOwner $ - DB.PoolOwner - { DB.poolOwnerAddrId = saId - , DB.poolOwnerPoolUpdateId = poolUpdateId - } +insertPoolOwner syncEnv cache network poolUpdateId skh = + -- Check if the stake address is in the shelley whitelist + when (shelleyStakeAddrWhitelistCheck syncEnv $ Ledger.RewardAccount network (Ledger.KeyHashObj skh)) $ do + saId <- lift $ queryOrInsertStakeAddress syncEnv cache UpdateCache network (Ledger.KeyHashObj skh) + void . lift . DB.insertPoolOwner $ + DB.PoolOwner + { DB.poolOwnerAddrId = saId + , DB.poolOwnerPoolUpdateId = poolUpdateId + } insertPoolRelay :: (MonadBaseControl IO m, MonadIO m) => @@ -197,7 +200,7 @@ insertPoolRelay updateId relay = insertPoolCert :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> IsPoolMember -> Maybe Generic.Deposits -> @@ -208,7 +211,7 @@ insertPoolCert :: Word16 -> PoolCert StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertPoolCert tracer cache isMember mdeposits network epoch blkId txId idx pCert = +insertPoolCert syncEnv cache isMember mdeposits network epoch blkId txId idx pCert = case pCert of - RegPool pParams -> insertPoolRegister tracer cache isMember mdeposits network epoch blkId txId idx pParams - RetirePool keyHash epochNum -> insertPoolRetire tracer txId cache epochNum idx keyHash + RegPool pParams -> insertPoolRegister syncEnv (envCache syncEnv) isMember mdeposits network epoch blkId txId idx pParams + RetirePool keyHash epochNum -> insertPoolRetire syncEnv cache txId epochNum idx keyHash diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Tx.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Tx.hs index 16c9b94af..b7f52175c 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Tx.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Tx.hs @@ -11,7 +11,8 @@ module Cardano.DbSync.Era.Universal.Insert.Tx ( insertTx, insertTxOut, -) where +) +where import Cardano.BM.Trace (Trace) import Cardano.Db (DbLovelace (..), DbWord64 (..)) @@ -19,7 +20,7 @@ import qualified Cardano.Db as DB import Cardano.DbSync.Api import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..)) import Cardano.DbSync.Cache.Types (CacheStatus (..)) - +import Cardano.DbSync.Config.Types (MetadataConfig (..), MultiAssetConfig (..), PlutusConfig (..), isPlutusModeActive, isShelleyModeActive) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Era.Shelley.Generic.Metadata (TxMetadataValue (..), metadataValueToJsonNoSchema) import Cardano.DbSync.Era.Universal.Insert.Certificate (insertCertificate) @@ -35,6 +36,7 @@ import Cardano.DbSync.Era.Universal.Insert.Other ( insertMultiAsset, insertRedeemer, insertScript, + insertScriptWithWhitelist, insertStakeAddressRefIfMissing, insertWithdrawals, ) @@ -44,6 +46,7 @@ import Cardano.DbSync.Error import Cardano.DbSync.Ledger.Types (ApplyResult (..), getGovExpiresAt, lookupDepositsMap) import Cardano.DbSync.Util import Cardano.DbSync.Util.Cbor (serialiseTxMetadataToCbor) +import Cardano.DbSync.Util.Whitelist (isPlutusScriptHashesInWhitelist, plutusMultiAssetWhitelistCheck, shelleyStkAddrWhitelistCheckWithAddr) import Cardano.Ledger.BaseTypes import Cardano.Ledger.Coin (Coin (..)) import Cardano.Ledger.Mary.Value (AssetName (..), MultiAsset (..), PolicyID (..)) @@ -52,8 +55,8 @@ import Control.Monad.Extra (mapMaybeM) import Control.Monad.Trans.Control (MonadBaseControl) import qualified Data.Aeson as Aeson import qualified Data.ByteString.Lazy.Char8 as LBS +import Data.ByteString.Short (ShortByteString) import qualified Data.Map.Strict as Map -import qualified Data.Strict.Maybe as Strict import Database.Persist.Sql (SqlBackend) import Ouroboros.Consensus.Cardano.Block (StandardCrypto) @@ -78,29 +81,30 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped let !outSum = fromIntegral $ unCoin $ Generic.txOutSum tx !withdrawalSum = fromIntegral $ unCoin $ Generic.txWithdrawalSum tx hasConsumed = getHasConsumedOrPruneTxOut syncEnv + disInOut <- liftIO $ getDisableInOutState syncEnv -- In some txs and with specific configuration we may be able to find necessary data within the tx body. -- In these cases we can avoid expensive queries. - (resolvedInputs, fees', deposits) <- case (disInOut, mdeposits, unCoin <$> Generic.txFees tx) of + (resolvedInputs, resolvedFees', resolvedDeposits) <- case (disInOut, mdeposits, unCoin <$> Generic.txFees tx) of (True, _, _) -> pure ([], 0, unCoin <$> mdeposits) (_, Just deposits, Just fees) -> do - (resolvedInputs, _) <- splitLast <$> mapM (resolveTxInputs hasConsumed False (fst <$> groupedTxOut grouped)) (Generic.txInputs tx) + (resolvedInputs, _) <- splitLast <$> mapM (resolveTxInputs syncEnv hasConsumed False (fst <$> groupedTxOut grouped)) (Generic.txInputs tx) pure (resolvedInputs, fees, Just (unCoin deposits)) (_, Nothing, Just fees) -> do - (resolvedInputs, amounts) <- splitLast <$> mapM (resolveTxInputs hasConsumed False (fst <$> groupedTxOut grouped)) (Generic.txInputs tx) + (resolvedInputs, amounts) <- splitLast <$> mapM (resolveTxInputs syncEnv hasConsumed False (fst <$> groupedTxOut grouped)) (Generic.txInputs tx) if any isNothing amounts then pure (resolvedInputs, fees, Nothing) else let !inSum = sum $ map unDbLovelace $ catMaybes amounts in pure (resolvedInputs, fees, Just $ fromIntegral (inSum + withdrawalSum) - fromIntegral outSum - fromIntegral fees) + -- Nothing in fees means a phase 2 failure (_, _, Nothing) -> do - -- Nothing in fees means a phase 2 failure - (resolvedInsFull, amounts) <- splitLast <$> mapM (resolveTxInputs hasConsumed True (fst <$> groupedTxOut grouped)) (Generic.txInputs tx) + (resolvedInsFull, amounts) <- splitLast <$> mapM (resolveTxInputs syncEnv hasConsumed True (fst <$> groupedTxOut grouped)) (Generic.txInputs tx) let !inSum = sum $ map unDbLovelace $ catMaybes amounts !diffSum = if inSum >= outSum then inSum - outSum else 0 !fees = maybe diffSum (fromIntegral . unCoin) (Generic.txFees tx) pure (resolvedInsFull, fromIntegral fees, Just 0) - let fees = fromIntegral fees' + let resolvedFees = fromIntegral resolvedFees' -- Insert transaction and get txId from the DB. !txId <- lift @@ -110,8 +114,8 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped , DB.txBlockId = blkId , DB.txBlockIndex = blockIndex , DB.txOutSum = DB.DbLovelace outSum - , DB.txFee = DB.DbLovelace fees - , DB.txDeposit = fromIntegral <$> deposits + , DB.txFee = DB.DbLovelace resolvedFees + , DB.txDeposit = fromIntegral <$> resolvedDeposits , DB.txSize = Generic.txSize tx , DB.txInvalidBefore = DbWord64 . unSlotNo <$> Generic.txInvalidBefore tx , DB.txInvalidHereafter = DbWord64 . unSlotNo <$> Generic.txInvalidHereafter tx @@ -130,166 +134,194 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped if not (Generic.txValidContract tx) then do - !txOutsGrouped <- mapM (insertTxOut tracer cache iopts (txId, txHash)) (Generic.txOutputs tx) + !txOutsGrouped <- do + if isplutusMultiAssetInWhitelist + then mapMaybeM (insertTxOut syncEnv cache iopts (txId, txHash)) txOuts + else pure mempty let !txIns = map (prepareTxIn txId Map.empty) resolvedInputs -- There is a custom semigroup instance for BlockGroupedData which uses addition for the values `fees` and `outSum`. -- Same happens bellow on last line of this function. - pure (grouped <> BlockGroupedData txIns txOutsGrouped [] [] fees outSum) + pure (grouped <> BlockGroupedData txIns txOutsGrouped [] [] resolvedFees outSum) else do -- The following operations only happen if the script passes stage 2 validation (or the tx has -- no script). - !txOutsGrouped <- mapM (insertTxOut tracer cache iopts (txId, txHash)) (Generic.txOutputs tx) + !txOutsGrouped <- do + if isplutusMultiAssetInWhitelist + then mapMaybeM (insertTxOut syncEnv cache iopts (txId, txHash)) txOuts + else pure mempty !redeemers <- Map.fromList <$> whenFalseMempty - (ioPlutusExtra iopts) - (mapM (insertRedeemer tracer disInOut (fst <$> groupedTxOut grouped) txId) (Generic.txRedeemer tx)) + (isPlutusModeActive $ ioPlutus iopts) + (mapM (insertRedeemer syncEnv disInOut (fst <$> groupedTxOut grouped) txId) (Generic.txRedeemer tx)) - when (ioPlutusExtra iopts) $ do - mapM_ (insertDatum tracer cache txId) (Generic.txData tx) + when (isPlutusModeActive $ ioPlutus iopts) $ do + mapM_ (insertDatum syncEnv cache txId) (Generic.txData tx) mapM_ (insertCollateralTxIn tracer txId) (Generic.txCollateralInputs tx) mapM_ (insertReferenceTxIn tracer txId) (Generic.txReferenceInputs tx) - mapM_ (insertCollateralTxOut tracer cache iopts (txId, txHash)) (Generic.txCollateralOutputs tx) + mapM_ (insertExtraKeyWitness txId) $ Generic.txExtraKeyWitnesses tx + mapM_ (lift . insertScriptWithWhitelist syncEnv txId) $ Generic.txScripts tx + mapM_ (insertCollateralTxOut syncEnv cache (txId, txHash)) (Generic.txCollateralOutputs tx) + + txMetadata <- do + case ioMetadata iopts of + MetadataDisable -> pure mempty + MetadataEnable -> prepareTxMetadata syncEnv Nothing txId (Generic.txMetadata tx) + MetadataKeys whitelist -> prepareTxMetadata syncEnv (Just whitelist) txId (Generic.txMetadata tx) - txMetadata <- - whenFalseMempty (ioMetadata iopts) $ - insertTxMetadata - tracer - txId - iopts - (Generic.txMetadata tx) mapM_ (insertCertificate syncEnv isMember mDeposits blkId txId epochNo slotNo redeemers) $ Generic.txCertificates tx - when (ioShelley iopts) $ - mapM_ (insertWithdrawals tracer cache txId redeemers) $ - Generic.txWithdrawals tx - when (ioShelley iopts) $ - mapM_ (lift . insertParamProposal blkId txId) $ - Generic.txParamProposal tx - - maTxMint <- - whenFalseMempty (ioMultiAssets iopts) $ - insertMaTxMint tracer cache txId $ - Generic.txMint tx - when (ioPlutusExtra iopts) $ - mapM_ (lift . insertScript tracer txId) $ - Generic.txScripts tx + when (isShelleyModeActive $ ioShelley iopts) $ do + mapM_ (insertWithdrawals syncEnv cache txId redeemers) $ Generic.txWithdrawals tx + mapM_ (lift . insertParamProposal blkId txId) $ Generic.txParamProposal tx - when (ioPlutusExtra iopts) $ - mapM_ (insertExtraKeyWitness tracer txId) $ - Generic.txExtraKeyWitnesses tx + maTxMint <- + case ioMultiAssets iopts of + MultiAssetDisable -> pure mempty + MultiAssetEnable -> insertMaTxMint cache Nothing txId $ Generic.txMint tx + MultiAssetPolicies whitelist -> insertMaTxMint cache (Just whitelist) txId $ Generic.txMint tx when (ioGov iopts) $ do - mapM_ (insertGovActionProposal tracer cache blkId txId (getGovExpiresAt applyResult epochNo) (apGovActionState applyResult)) $ zip [0 ..] (Generic.txProposalProcedure tx) - mapM_ (insertVotingProcedures tracer cache blkId txId) (Generic.txVotingProcedure tx) + mapM_ (insertGovActionProposal syncEnv blkId txId (getGovExpiresAt applyResult epochNo) (apGovActionState applyResult)) $ zip [0 ..] (Generic.txProposalProcedure tx) + mapM_ (insertVotingProcedures syncEnv blkId txId (Generic.txProposalProcedure tx)) (Generic.txVotingProcedure tx) let !txIns = map (prepareTxIn txId redeemers) resolvedInputs - pure (grouped <> BlockGroupedData txIns txOutsGrouped txMetadata maTxMint fees outSum) + pure (grouped <> BlockGroupedData txIns txOutsGrouped txMetadata maTxMint resolvedFees outSum) where + txOuts = Generic.txOutputs tx + txMints = Generic.txMint tx tracer = getTrace syncEnv cache = envCache syncEnv iopts = getInsertOptions syncEnv mDeposits = maybeFromStrict $ apDeposits applyResult + isplutusMultiAssetInWhitelist = plutusMultiAssetWhitelistCheck syncEnv txMints txOuts -------------------------------------------------------------------------------------- -- INSERT TXOUT -------------------------------------------------------------------------------------- insertTxOut :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> InsertOptions -> (DB.TxId, ByteString) -> Generic.TxOut -> - ExceptT SyncNodeError (ReaderT SqlBackend m) (ExtendedTxOut, [MissingMaTxOut]) -insertTxOut tracer cache iopts (txId, txHash) (Generic.TxOut index addr value maMap mScript dt) = do - mSaId <- lift $ insertStakeAddressRefIfMissing tracer cache addr - mDatumId <- - whenFalseEmpty (ioPlutusExtra iopts) Nothing $ - Generic.whenInlineDatum dt $ - insertDatum tracer cache txId - mScriptId <- - whenFalseEmpty (ioPlutusExtra iopts) Nothing $ - whenMaybe mScript $ - lift . insertScript tracer txId - let !txOut = - DB.TxOut - { DB.txOutTxId = txId - , DB.txOutIndex = index - , DB.txOutAddress = Generic.renderAddress addr - , DB.txOutAddressHasScript = hasScript - , DB.txOutPaymentCred = Generic.maybePaymentCred addr - , DB.txOutStakeAddressId = mSaId - , DB.txOutValue = Generic.coinToDbLovelace value - , DB.txOutDataHash = Generic.dataHashToBytes <$> Generic.getTxOutDatumHash dt - , DB.txOutInlineDatumId = mDatumId - , DB.txOutReferenceScriptId = mScriptId - } - let !eutxo = ExtendedTxOut txHash txOut - !maTxOuts <- whenFalseMempty (ioMultiAssets iopts) $ insertMaTxOuts tracer cache maMap - pure (eutxo, maTxOuts) + ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe (ExtendedTxOut, [MissingMaTxOut])) +insertTxOut syncEnv cache iopts (txId, txHash) (Generic.TxOut index addr value maMap mScript dt) = + case ioPlutus iopts of + PlutusDisable -> buildExtendedTxOutPart2 Nothing Nothing + _other -> buildExtendedTxOutPart1 where + buildExtendedTxOutPart1 :: + (MonadBaseControl IO m, MonadIO m) => + ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe (ExtendedTxOut, [MissingMaTxOut])) + buildExtendedTxOutPart1 = do + mDatumId <- Generic.whenInlineDatum dt $ insertDatum syncEnv cache txId + mScriptId <- case mScript of + Just script -> lift $ Just <$> insertScript syncEnv txId script + Nothing -> pure Nothing + buildExtendedTxOutPart2 mDatumId mScriptId + + buildExtendedTxOutPart2 :: + (MonadBaseControl IO m, MonadIO m) => + Maybe DB.DatumId -> + Maybe DB.ScriptId -> + ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe (ExtendedTxOut, [MissingMaTxOut])) + buildExtendedTxOutPart2 mDatumId mScriptId = do + mSaId <- lift $ insertStakeAddressRefIfMissing syncEnv cache addr + let !txOut = + DB.TxOut + { DB.txOutTxId = txId + , DB.txOutIndex = index + , DB.txOutAddress = Generic.renderAddress addr + , DB.txOutAddressHasScript = hasScript + , DB.txOutPaymentCred = Generic.maybePaymentCred addr + , DB.txOutStakeAddressId = mSaId + , DB.txOutValue = Generic.coinToDbLovelace value + , DB.txOutDataHash = Generic.dataHashToBytes <$> Generic.getTxOutDatumHash dt + , DB.txOutInlineDatumId = mDatumId + , DB.txOutReferenceScriptId = mScriptId + } + let !eutxo = ExtendedTxOut txHash txOut + case ioMultiAssets iopts of + MultiAssetDisable -> pure $ Just (eutxo, mempty) + MultiAssetEnable -> do + !maTxOuts <- insertMaTxOuts cache Nothing maMap + pure $ Just (eutxo, maTxOuts) + MultiAssetPolicies whitelist -> do + !maTxOuts <- insertMaTxOuts cache (Just whitelist) maMap + pure $ Just (eutxo, maTxOuts) + hasScript :: Bool hasScript = maybe False Generic.hasCredScript (Generic.getPaymentCred addr) -insertTxMetadata :: - (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> +prepareTxMetadata :: + (MonadIO m) => + SyncEnv -> + Maybe (NonEmpty Word) -> DB.TxId -> - InsertOptions -> Maybe (Map Word64 TxMetadataValue) -> - ExceptT SyncNodeError (ReaderT SqlBackend m) [DB.TxMetadata] -insertTxMetadata tracer txId inOpts mmetadata = do + m [DB.TxMetadata] +prepareTxMetadata syncEnv mWhitelist txId mmetadata = case mmetadata of Nothing -> pure [] - Just metadata -> mapMaybeM prepare $ Map.toList metadata + Just metadata -> do + whitelistAndPrepare $ Map.toList metadata where - prepare :: - (MonadBaseControl IO m, MonadIO m) => - (Word64, TxMetadataValue) -> - ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe DB.TxMetadata) - prepare (key, md) = do - case ioKeepMetadataNames inOpts of - Strict.Just metadataNames -> do - let isMatchingKey = key `elem` metadataNames - if isMatchingKey - then mkDbTxMetadata (key, md) - else pure Nothing - -- if we have TxMetadata and keepMetadataNames is Nothing then we want to keep all metadata - Strict.Nothing -> mkDbTxMetadata (key, md) + whitelistAndPrepare :: + (MonadIO m) => + [(Word64, TxMetadataValue)] -> + m [DB.TxMetadata] + whitelistAndPrepare metadataList = + case mWhitelist of + -- if we have any metadata key in the whitelist then keep all metadata + -- otherwise discard all metadata. + Just whitelist -> + if isAnyInWhitelist whitelist metadataList + then mapM mkDbTxMetadata metadataList + else pure [] + -- not using a whitelist, keep all metadata + Nothing -> mapM mkDbTxMetadata metadataList + + isAnyInWhitelist :: + NonEmpty Word -> + [(Word64, TxMetadataValue)] -> + Bool + isAnyInWhitelist whitelist metaDataList = do + let results = map (\(key, _) -> fromIntegral key `elem` whitelist) metaDataList + or results mkDbTxMetadata :: - (MonadBaseControl IO m, MonadIO m) => + (MonadIO m) => (Word64, TxMetadataValue) -> - ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe DB.TxMetadata) + m DB.TxMetadata mkDbTxMetadata (key, md) = do let jsonbs = LBS.toStrict $ Aeson.encode (metadataValueToJsonNoSchema md) singleKeyCBORMetadata = serialiseTxMetadataToCbor $ Map.singleton key md - mjson <- safeDecodeToJson tracer "prepareTxMetadata: Column 'json' in table 'metadata' " jsonbs + mjson <- safeDecodeToJson syncEnv "prepareTxMetadata: Column 'json' in table 'metadata' " jsonbs pure $ - Just $ - DB.TxMetadata - { DB.txMetadataKey = DbWord64 key - , DB.txMetadataJson = mjson - , DB.txMetadataBytes = singleKeyCBORMetadata - , DB.txMetadataTxId = txId - } + DB.TxMetadata + { DB.txMetadataKey = DbWord64 key + , DB.txMetadataJson = mjson + , DB.txMetadataBytes = singleKeyCBORMetadata + , DB.txMetadataTxId = txId + } -------------------------------------------------------------------------------------- -- INSERT MULTI ASSET -------------------------------------------------------------------------------------- insertMaTxMint :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> CacheStatus -> + Maybe (NonEmpty ShortByteString) -> DB.TxId -> MultiAsset StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) [DB.MaTxMint] -insertMaTxMint _tracer cache txId (MultiAsset mintMap) = +insertMaTxMint cache mWhitelist txId (MultiAsset mintMap) = concatMapM (lift . prepareOuter) $ Map.toList mintMap where prepareOuter :: @@ -297,29 +329,32 @@ insertMaTxMint _tracer cache txId (MultiAsset mintMap) = (PolicyID StandardCrypto, Map AssetName Integer) -> ReaderT SqlBackend m [DB.MaTxMint] prepareOuter (policy, aMap) = - mapM (prepareInner policy) $ Map.toList aMap + mapMaybeM (prepareInner policy) $ Map.toList aMap prepareInner :: (MonadBaseControl IO m, MonadIO m) => PolicyID StandardCrypto -> (AssetName, Integer) -> - ReaderT SqlBackend m DB.MaTxMint + ReaderT SqlBackend m (Maybe DB.MaTxMint) prepareInner policy (aname, amount) = do - maId <- insertMultiAsset cache policy aname - pure $ - DB.MaTxMint - { DB.maTxMintIdent = maId - , DB.maTxMintQuantity = DB.integerToDbInt65 amount - , DB.maTxMintTxId = txId - } + maybeMaId <- insertMultiAsset cache mWhitelist policy aname + pure $ case maybeMaId of + Just maId -> + Just $ + DB.MaTxMint + { DB.maTxMintIdent = maId + , DB.maTxMintQuantity = DB.integerToDbInt65 amount + , DB.maTxMintTxId = txId + } + Nothing -> Nothing insertMaTxOuts :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> CacheStatus -> + Maybe (NonEmpty ShortByteString) -> Map (PolicyID StandardCrypto) (Map AssetName Integer) -> ExceptT SyncNodeError (ReaderT SqlBackend m) [MissingMaTxOut] -insertMaTxOuts _tracer cache maMap = +insertMaTxOuts cache mWhitelist maMap = concatMapM (lift . prepareOuter) $ Map.toList maMap where prepareOuter :: @@ -327,62 +362,66 @@ insertMaTxOuts _tracer cache maMap = (PolicyID StandardCrypto, Map AssetName Integer) -> ReaderT SqlBackend m [MissingMaTxOut] prepareOuter (policy, aMap) = - mapM (prepareInner policy) $ Map.toList aMap + mapMaybeM (prepareInner policy) $ Map.toList aMap prepareInner :: (MonadBaseControl IO m, MonadIO m) => PolicyID StandardCrypto -> (AssetName, Integer) -> - ReaderT SqlBackend m MissingMaTxOut + ReaderT SqlBackend m (Maybe MissingMaTxOut) prepareInner policy (aname, amount) = do - maId <- insertMultiAsset cache policy aname - pure $ - MissingMaTxOut - { mmtoIdent = maId - , mmtoQuantity = DbWord64 (fromIntegral amount) - } + mMaId <- insertMultiAsset cache mWhitelist policy aname + pure $ case mMaId of + Just maId -> + Just $ + MissingMaTxOut + { mmtoIdent = maId + , mmtoQuantity = DbWord64 (fromIntegral amount) + } + Nothing -> Nothing --------------------------------------------------------------------------------------- --- INSERT COLLATERAL --------------------------------------------------------------------------------------- insertCollateralTxOut :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> CacheStatus -> - InsertOptions -> (DB.TxId, ByteString) -> Generic.TxOut -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertCollateralTxOut tracer cache iopts (txId, _txHash) (Generic.TxOut index addr value maMap mScript dt) = do - mSaId <- lift $ insertStakeAddressRefIfMissing tracer cache addr - mDatumId <- - whenFalseEmpty (ioPlutusExtra iopts) Nothing $ - Generic.whenInlineDatum dt $ - insertDatum tracer cache txId - mScriptId <- - whenFalseEmpty (ioPlutusExtra iopts) Nothing $ - whenMaybe mScript $ - lift . insertScript tracer txId - _ <- - lift - . DB.insertCollateralTxOut - $ DB.CollateralTxOut - { DB.collateralTxOutTxId = txId - , DB.collateralTxOutIndex = index - , DB.collateralTxOutAddress = Generic.renderAddress addr - , DB.collateralTxOutAddressHasScript = hasScript - , DB.collateralTxOutPaymentCred = Generic.maybePaymentCred addr - , DB.collateralTxOutStakeAddressId = mSaId - , DB.collateralTxOutValue = Generic.coinToDbLovelace value - , DB.collateralTxOutDataHash = Generic.dataHashToBytes <$> Generic.getTxOutDatumHash dt - , DB.collateralTxOutMultiAssetsDescr = textShow maMap - , DB.collateralTxOutInlineDatumId = mDatumId - , DB.collateralTxOutReferenceScriptId = mScriptId - } - pure () +insertCollateralTxOut syncEnv cache (txId, _txHash) txout@(Generic.TxOut index addr value maMap mScript dt) = do + -- check if shelley stake address is in the whitelist + when (shelleyStkAddrWhitelistCheckWithAddr syncEnv addr) $ do + -- check plutus script hash is in the whitelist + if isPlutusScriptHashesInWhitelist syncEnv [txout] + then insertColTxOutPart1 + else void $ insertColTxOutPart2 Nothing Nothing where - -- TODO: Is there any reason to add new tables for collateral multi-assets/multi-asset-outputs + insertColTxOutPart1 = do + mDatumId <- Generic.whenInlineDatum dt $ insertDatum syncEnv cache txId + mScriptId <- case mScript of + Just script -> lift $ Just <$> insertScript syncEnv txId script + Nothing -> pure Nothing + insertColTxOutPart2 mDatumId mScriptId + pure () + insertColTxOutPart2 mDatumId mScriptId = do + mSaId <- lift $ insertStakeAddressRefIfMissing syncEnv cache addr + _ <- + lift + . DB.insertCollateralTxOut + $ DB.CollateralTxOut + { DB.collateralTxOutTxId = txId + , DB.collateralTxOutIndex = index + , DB.collateralTxOutAddress = Generic.renderAddress addr + , DB.collateralTxOutAddressHasScript = hasScript + , DB.collateralTxOutPaymentCred = Generic.maybePaymentCred addr + , DB.collateralTxOutStakeAddressId = mSaId + , DB.collateralTxOutValue = Generic.coinToDbLovelace value + , DB.collateralTxOutDataHash = Generic.dataHashToBytes <$> Generic.getTxOutDatumHash dt + , DB.collateralTxOutMultiAssetsDescr = textShow maMap + , DB.collateralTxOutInlineDatumId = mDatumId + , DB.collateralTxOutReferenceScriptId = mScriptId + } + pure () hasScript :: Bool hasScript = maybe False Generic.hasCredScript (Generic.getPaymentCred addr) diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Validate.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Validate.hs index d155df128..353805d5b 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Validate.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Validate.hs @@ -12,6 +12,9 @@ module Cardano.DbSync.Era.Universal.Validate ( import Cardano.BM.Trace (Trace, logError, logInfo, logWarning) import Cardano.Db (DbLovelace, RewardSource) import qualified Cardano.Db as Db +import Cardano.DbSync.Api (getTrace) +import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..), SyncOptions (..)) +import Cardano.DbSync.Config.Types (ShelleyInsertConfig (..)) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Ledger.Event import Cardano.DbSync.Types @@ -50,13 +53,14 @@ import GHC.Err (error) validateEpochRewards :: (MonadBaseControl IO m, MonadIO m) => + SyncEnv -> Trace IO Text -> Network -> EpochNo -> EpochNo -> Map StakeCred (Set (Ledger.Reward StandardCrypto)) -> ReaderT SqlBackend m () -validateEpochRewards tracer network _earnedEpochNo spendableEpochNo rmap = do +validateEpochRewards syncEnv tracer network _earnedEpochNo spendableEpochNo rmap = do actualCount <- Db.queryNormalEpochRewardCount (unEpochNo spendableEpochNo) if actualCount /= expectedCount then do @@ -69,7 +73,7 @@ validateEpochRewards tracer network _earnedEpochNo spendableEpochNo rmap = do , " but got " , textShow actualCount ] - logFullRewardMap tracer spendableEpochNo network (convertPoolRewards rmap) + logFullRewardMap syncEnv spendableEpochNo network (convertPoolRewards rmap) else do liftIO . logInfo tracer $ mconcat @@ -84,16 +88,16 @@ validateEpochRewards tracer network _earnedEpochNo spendableEpochNo rmap = do logFullRewardMap :: (MonadBaseControl IO m, MonadIO m) => - Trace IO Text -> + SyncEnv -> EpochNo -> Network -> Generic.Rewards -> ReaderT SqlBackend m () -logFullRewardMap tracer epochNo network ledgerMap = do +logFullRewardMap syncEnv epochNo network ledgerMap = do dbMap <- queryRewardMap epochNo when (Map.size dbMap > 0 && Map.size (Generic.unRewards ledgerMap) > 0) $ liftIO $ - diffRewardMap tracer network dbMap (Map.mapKeys (Generic.stakingCredHash network) $ Map.map convert $ Generic.unRewards ledgerMap) + diffRewardMap syncEnv network dbMap (Map.mapKeys (Generic.stakingCredHash network) $ Map.map convert $ Generic.unRewards ledgerMap) where convert :: Set Generic.Reward -> [(RewardSource, Coin)] convert = map (\rwd -> (Generic.rewardSource rwd, Generic.rewardAmount rwd)) . Set.toList @@ -130,15 +134,18 @@ queryRewardMap (EpochNo epochNo) = do x : _ -> (fst x, List.sort $ map snd xs) diffRewardMap :: - Trace IO Text -> + SyncEnv -> Network -> Map ByteString [(RewardSource, DbLovelace)] -> Map ByteString [(RewardSource, Coin)] -> IO () -diffRewardMap tracer _nw dbMap ledgerMap = do +diffRewardMap syncEnv _nw dbMap ledgerMap = do when (Map.size diffMap > 0) $ do - logError tracer "diffRewardMap:" - mapM_ (logError tracer . render) $ Map.toList diffMap + case ioShelley $ soptInsertOptions (envOptions syncEnv) of + ShelleyStakeAddrs _ -> pure () + _ -> do + logError tracer "diffRewardMap:" + mapM_ (logError tracer . render) $ Map.toList diffMap where keys :: [ByteString] keys = List.nubOrd (Map.keys dbMap ++ Map.keys ledgerMap) @@ -162,3 +169,5 @@ diffRewardMap tracer _nw dbMap ledgerMap = do render :: (ByteString, ([(RewardSource, DbLovelace)], [(RewardSource, Coin)])) -> Text render (cred, (xs, ys)) = mconcat [" ", show cred, ": ", show xs, " /= ", show ys] + + tracer = getTrace syncEnv diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Util.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Util.hs index e9a4a5430..a060d995b 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Util.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Util.hs @@ -6,10 +6,13 @@ module Cardano.DbSync.Era.Util ( containsUnicodeNul, safeDecodeUtf8, safeDecodeToJson, -) where +) +where -import Cardano.BM.Trace (Trace, logWarning) +import Cardano.BM.Trace (logWarning) import qualified Cardano.Db as DB +import Cardano.DbSync.Api (getTrace) +import Cardano.DbSync.Api.Types (SyncEnv) import Cardano.DbSync.Error import Cardano.Prelude import Control.Monad.Trans.Except.Extra (firstExceptT, newExceptT) @@ -18,7 +21,7 @@ import qualified Data.Text as Text import qualified Data.Text.Encoding as Text import qualified Data.Text.Encoding.Error as Text -liftLookupFail :: Monad m => Text -> m (Either DB.LookupFail a) -> ExceptT SyncNodeError m a +liftLookupFail :: (Monad m) => Text -> m (Either DB.LookupFail a) -> ExceptT SyncNodeError m a liftLookupFail loc = firstExceptT (\lf -> SNErrDefault $ mconcat [loc, " ", show lf]) . newExceptT @@ -33,13 +36,14 @@ safeDecodeUtf8 bs containsUnicodeNul :: Text -> Bool containsUnicodeNul = Text.isInfixOf "\\u000" -safeDecodeToJson :: MonadIO m => Trace IO Text -> Text -> ByteString -> m (Maybe Text) -safeDecodeToJson tracer tracePrefix jsonBs = do +safeDecodeToJson :: (MonadIO m) => SyncEnv -> Text -> ByteString -> m (Maybe Text) +safeDecodeToJson syncEnv tracePrefix jsonBs = do ejson <- liftIO $ safeDecodeUtf8 jsonBs case ejson of Left err -> do - liftIO . logWarning tracer $ - mconcat + liftIO + . logWarning tracer + $ mconcat [tracePrefix, ": Could not decode to UTF8: ", textShow err] -- We have to insert pure Nothing @@ -50,3 +54,5 @@ safeDecodeToJson tracer tracePrefix jsonBs = do liftIO $ logWarning tracer $ tracePrefix <> "was recorded as null, due to a Unicode NUL character found when trying to parse the json." pure Nothing else pure $ Just json + where + tracer = getTrace syncEnv diff --git a/cardano-db-sync/src/Cardano/DbSync/Error.hs b/cardano-db-sync/src/Cardano/DbSync/Error.hs index e01a3d3ba..6ea2ba2bd 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Error.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Error.hs @@ -9,6 +9,7 @@ module Cardano.DbSync.Error ( NodeConfigError (..), annotateInvariantTx, bsBase16Encode, + shortBsBase16Encode, dbSyncNodeError, dbSyncInvariant, renderSyncInvariant, @@ -28,6 +29,7 @@ import Cardano.DbSync.Util import Cardano.Prelude import Control.Monad.Trans.Except.Extra (left) import qualified Data.ByteString.Base16 as Base16 +import Data.ByteString.Short (ShortByteString, toShort) import Data.String (String) import qualified Data.Text as Text import qualified Data.Text.Encoding as Text @@ -183,6 +185,9 @@ bsBase16Encode bs = Left _ -> Text.pack $ "UTF-8 decode failed for " ++ Show.show bs Right txt -> txt +shortBsBase16Encode :: ByteString -> ShortByteString +shortBsBase16Encode bs = toShort (Base16.encode bs) + runOrThrowIO :: forall e a m. (MonadIO m) => (Exception e) => m (Either e a) -> m a runOrThrowIO ioEither = do et <- ioEither diff --git a/cardano-db-sync/src/Cardano/DbSync/Util/Whitelist.hs b/cardano-db-sync/src/Cardano/DbSync/Util/Whitelist.hs new file mode 100644 index 000000000..aec4ec341 --- /dev/null +++ b/cardano-db-sync/src/Cardano/DbSync/Util/Whitelist.hs @@ -0,0 +1,129 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Cardano.DbSync.Util.Whitelist where + +import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..), SyncOptions (..)) +import Cardano.DbSync.Config.Types (MultiAssetConfig (..), PlutusConfig (..), ShelleyInsertConfig (..)) +import qualified Cardano.DbSync.Era.Shelley.Generic as Generic +import Cardano.DbSync.Error (shortBsBase16Encode) +import qualified Cardano.Ledger.Address as Ledger +import qualified Cardano.Ledger.Credential as Ledger +import Cardano.Ledger.Crypto (StandardCrypto) +import Cardano.Ledger.Mary.Value (MultiAsset (..), PolicyID (..)) +import Cardano.Prelude (NonEmpty) +import Data.ByteString (ByteString) +import Data.ByteString.Short (ShortByteString, toShort) +import Data.Map (keys) + +-- check both whitelist but also checking plutus Maybes first +plutusMultiAssetWhitelistCheck :: + SyncEnv -> + -- | TxMint + MultiAsset StandardCrypto -> + -- | TxOuts + [Generic.TxOut] -> + Bool +plutusMultiAssetWhitelistCheck syncEnv txMints txOuts = + isPlutusScriptHashesInWhitelist syncEnv txOuts || isMAPoliciesInWhitelist syncEnv txMints txOuts + +-- | Check if any script hash or address is in the whitelist +isPlutusScriptHashesInWhitelist :: SyncEnv -> [Generic.TxOut] -> Bool +isPlutusScriptHashesInWhitelist syncEnv txOuts = do + case ioPlutus iopts of + PlutusEnable -> True + PlutusDisable -> False + PlutusScripts whitelist -> + any (\txOut -> isScriptHashWhitelisted whitelist txOut || isAddressWhitelisted whitelist txOut) txOuts + where + iopts = soptInsertOptions $ envOptions syncEnv + -- check if the script hash is in the whitelist + isScriptHashWhitelisted :: NonEmpty ShortByteString -> Generic.TxOut -> Bool + isScriptHashWhitelisted whitelist txOut = + maybe False ((`elem` whitelist) . toShort . Generic.txScriptHash) (Generic.txOutScript txOut) + -- check if the address is in the whitelist + isAddressWhitelisted :: NonEmpty ShortByteString -> Generic.TxOut -> Bool + isAddressWhitelisted whitelist txOut = + maybe False ((`elem` whitelist) . toShort) (Generic.maybePaymentCred $ Generic.txOutAddress txOut) + +isSimplePlutusScriptHashInWhitelist :: SyncEnv -> ByteString -> Bool +isSimplePlutusScriptHashInWhitelist syncEnv scriptHash = do + case ioPlutus iopts of + PlutusEnable -> True + PlutusDisable -> False + PlutusScripts plutusWhitelist -> toShort scriptHash `elem` plutusWhitelist + where + iopts = soptInsertOptions $ envOptions syncEnv + +isMAPoliciesInWhitelist :: + SyncEnv -> + -- | TxMint + MultiAsset StandardCrypto -> + -- | TxOuts + [Generic.TxOut] -> + Bool +isMAPoliciesInWhitelist syncEnv (MultiAsset mintMap) txOuts = do + let iopts = soptInsertOptions $ envOptions syncEnv + case ioMultiAssets iopts of + MultiAssetEnable -> True + MultiAssetDisable -> True + MultiAssetPolicies multiAssetWhitelist -> + mintPoliciesCheck || txOutPoliciesCheck + where + mintPoliciesCheck :: Bool + mintPoliciesCheck = any (checkMAValueMap multiAssetWhitelist) mintPolicies + + txOutPoliciesCheck :: Bool + txOutPoliciesCheck = + any + ( \txout -> + any (checkMAValueMap multiAssetWhitelist) (keys $ Generic.txOutMaValue txout) + ) + txOuts + + checkMAValueMap :: NonEmpty ShortByteString -> PolicyID StandardCrypto -> Bool + checkMAValueMap maWhitelist policyId = + toShort (Generic.unScriptHash (policyID policyId)) `elem` maWhitelist + + mintPolicies :: [PolicyID StandardCrypto] + mintPolicies = keys mintMap + +shelleyStkAddrWhitelistCheckWithAddr :: + SyncEnv -> + Ledger.Addr StandardCrypto -> + Bool +shelleyStkAddrWhitelistCheckWithAddr syncEnv addr = do + case addr of + Ledger.AddrBootstrap {} -> False + Ledger.Addr network _pcred stakeRef -> + case stakeRef of + Ledger.StakeRefBase cred -> shelleyStakeAddrWhitelistCheck syncEnv $ Ledger.RewardAccount network cred + Ledger.StakeRefPtr _ -> True + Ledger.StakeRefNull -> True + +-- | This allows ShelleyDisabled to also pass through for specific cases. +shelleyCustomStakeWhitelistCheck :: SyncEnv -> Ledger.RewardAccount StandardCrypto -> Bool +shelleyCustomStakeWhitelistCheck syncEnv rwdAcc = do + case ioShelley iopts of + ShelleyDisable -> True + ShelleyEnable -> True + ShelleyStakeAddrs shelleyWhitelist -> checkShelleyWhitelist shelleyWhitelist rwdAcc + where + iopts = soptInsertOptions $ envOptions syncEnv + +shelleyStakeAddrWhitelistCheck :: SyncEnv -> Ledger.RewardAccount StandardCrypto -> Bool +shelleyStakeAddrWhitelistCheck syncEnv rwdAcc = do + case ioShelley iopts of + ShelleyDisable -> False + ShelleyEnable -> True + ShelleyStakeAddrs shelleyWhitelist -> checkShelleyWhitelist shelleyWhitelist rwdAcc + where + iopts = soptInsertOptions $ envOptions syncEnv + +-- | Check Shelley is enabled and if the stake address is in the whitelist +checkShelleyWhitelist :: NonEmpty ShortByteString -> Ledger.RewardAccount StandardCrypto -> Bool +checkShelleyWhitelist shelleyWhitelist rwdAcc = do + shortBsBase16Encode stakeAddress `elem` shelleyWhitelist + where + network = Ledger.raNetwork rwdAcc + rewardCred = Ledger.raCredential rwdAcc + stakeAddress = Ledger.serialiseRewardAccount (Ledger.RewardAccount network rewardCred) diff --git a/cardano-db-sync/test/Cardano/DbSync/Config/TypesTest.hs b/cardano-db-sync/test/Cardano/DbSync/Config/TypesTest.hs index 4aa68c285..6c119bd87 100644 --- a/cardano-db-sync/test/Cardano/DbSync/Config/TypesTest.hs +++ b/cardano-db-sync/test/Cardano/DbSync/Config/TypesTest.hs @@ -24,9 +24,9 @@ tests = , ("hasLedger", prop_hasLedger) , ("shouldUseLedger", prop_shouldUseLedger) , ("isShelleyEnabled", prop_isShelleyEnabled) - , ("isMultiAssetEnabled", prop_isMultiAssetEnabled) - , ("isMetadataEnabled", prop_isMetadataEnabled) - , ("isPlutusEnabled", prop_isPlutusEnabled) + , ("isMultiAssetModeActive", prop_isMultiAssetModeActive) + , ("isMetadataModeActive", prop_isMetadataModeActive) + , ("isPlutusModeActive", prop_isPlutusModeActive) ] prop_syncInsertConfigFromJSON :: Property @@ -82,31 +82,31 @@ prop_isShelleyEnabled = property $ do let shelleyCfg = sioShelley cfg -- Shelley is enabled if it is not ShelleyDisable - isShelleyEnabled shelleyCfg === (shelleyCfg /= ShelleyDisable) + isShelleyModeActive shelleyCfg === (shelleyCfg /= ShelleyDisable) -prop_isMultiAssetEnabled :: Property -prop_isMultiAssetEnabled = property $ do +prop_isMultiAssetModeActive :: Property +prop_isMultiAssetModeActive = property $ do cfg <- forAll Gen.syncInsertOptions let multiAssetCfg = sioMultiAsset cfg -- MultiAsset is enabled if it is not MultiAssetDisable - isMultiAssetEnabled multiAssetCfg === (multiAssetCfg /= MultiAssetDisable) + isMultiAssetModeActive multiAssetCfg === (multiAssetCfg /= MultiAssetDisable) -prop_isMetadataEnabled :: Property -prop_isMetadataEnabled = property $ do +prop_isMetadataModeActive :: Property +prop_isMetadataModeActive = property $ do cfg <- forAll Gen.syncInsertOptions let metadataCfg = sioMetadata cfg -- Metadata is enabled if it is not MetadataDisable - isMetadataEnabled metadataCfg === (metadataCfg /= MetadataDisable) + isMetadataModeActive metadataCfg === (metadataCfg /= MetadataDisable) -prop_isPlutusEnabled :: Property -prop_isPlutusEnabled = property $ do +prop_isPlutusModeActive :: Property +prop_isPlutusModeActive = property $ do cfg <- forAll Gen.syncInsertOptions let plutusCfg = sioPlutus cfg -- Plutus is enabled if it is not PlutusDisable - isPlutusEnabled plutusCfg === (plutusCfg /= PlutusDisable) + isPlutusModeActive plutusCfg === (plutusCfg /= PlutusDisable) -- | Various JSON values that should generate the default config genDefaultJson :: Gen Aeson.Value diff --git a/cardano-db/src/Cardano/Db/Schema.hs b/cardano-db/src/Cardano/Db/Schema.hs index 1665a41dd..3553fa5ff 100644 --- a/cardano-db/src/Cardano/Db/Schema.hs +++ b/cardano-db/src/Cardano/Db/Schema.hs @@ -157,6 +157,7 @@ share inlineDatumId DatumId Maybe noreference referenceScriptId ScriptId Maybe noreference UniqueTxout txId index -- The (tx_id, index) pair must be unique. + deriving Eq Show CollateralTxOut txId TxId noreference -- This type is the primary key for the 'tx' table. diff --git a/doc/configuration.md b/doc/configuration.md index d497317cc..4bad246c5 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -343,9 +343,11 @@ Maintains the ledger state, but doesn't use any of its data, except to load UTxO Shelley Properties: -| Property | Type | Required | -| :---------------- | :-------- | :------- | -| [enable](#enable) | `boolean` | Optional | +| Property | Type | Required | +| :----------------------------------- | :-------- | :------- | +| [enable](#enable) | `boolean` | Optional | +| [stake\_addresses](#stake-addresses) | `array` | Optional | + #### Enable @@ -356,6 +358,54 @@ proposals. Does not control `epoch_stake` and `rewards`, For this check `ledger` * Type: `boolean` +##### Stake Addresses + +Whitelist stake addresses. Only set values will be kept in the database, all others will be ignored. + +`stake_addresses` + + * Type: `string` + +The string will be validated and needs to omit `\x` from the start of the hash eg: + +`\x6c969320597b755454ff3653ad09725d590c570827a129aeb4385526` +should be entered as: +`6c969320597b755454ff3653ad09725d590c570827a129aeb4385526` + +Be mindfull that whitelisting stake address will omit data that does not match being present from the following tables. + +| table name | +| :-------------------- | +| collateral_tx_out | +| delegation | +| delegation_vote | +| epoch_stake | +| gov_action_proposal | +| instant_reward | +| pool_owner | +| pool_relay | +| pool_update | +| reserve | +| reward | +| stake_deregistration | +| stake_registration | +| treasury | +| treasury_withdrawal | +| tx_out | +| withdrawal | + +#### Example + +```json +"shelley": { + "enable": true + "stake_addresses": + ["6c969320597b755454ff3653ad09725d590c570827a129aeb4385526" + ,"994cf4c18f5613ca49c275f63d464b6d95123bfa8985e82b24b5680b" + ] + } +``` + ### Multi Asset `multi_asset` @@ -364,9 +414,10 @@ proposals. Does not control `epoch_stake` and `rewards`, For this check `ledger` Multi Asset Properties: -| Property | Type | Required | -| :------------------ | :-------- | :------- | -| [enable](#enable-1) | `boolean` | Optional | +| Property | Type | Required | +| :-------------------- | :-------- | :------- | +| [enable](#enable-1) | `boolean` | Optional | +| [policies](#policies) | `array` | Optional | #### Enable @@ -376,6 +427,43 @@ Enables or disables multi assets tables and entries. * Type: `boolean` +#### Policies + +Whitelist for multi asset policies hash. Only set values will be kept in the database, all others will be ignored. + +`policies` + + * Type: `string` + +The string will be validated and needs to omit `\x` from the start of the hash eg: + +`\x6c969320597b755454ff3653ad09725d590c570827a129aeb4385526` +should be entered as: +`6c969320597b755454ff3653ad09725d590c570827a129aeb4385526` + +Be mindfull that whitelisting policies will omit data that does not match whitelist being present from the following tables. + +| table name | +| :-------------------- | +| datum | +| ma_tx_mint | +| ma_tx_out | +| multi_assets | +| script | +| tx_out | + +#### Example + +```json +"multi_asset": { + "enable": true + "policies": + ["6c969320597b755454ff3653ad09725d590c570827a129aeb4385526" + ,"994cf4c18f5613ca49c275f63d464b6d95123bfa8985e82b24b5680b" + ] + } +``` + ### Metadata `metadata` @@ -405,6 +493,15 @@ If set, only keep metadata with the specified keys. * Type: `integer[]` +#### Example + +```json +"metadata": { + "enable": true + "keys": [12345, 6789] + } +``` + ### Plutus `plutus` @@ -413,9 +510,10 @@ If set, only keep metadata with the specified keys. Plutus Properties: -| Property | Type | Required | -| :------------------ | :-------- | :------- | -| [enable](#enable-3) | `boolean` | Optional | +| Property | Type | Required | +| :------------------------------- | :-------- | :------- | +| [enable](#enable-3) | `boolean` | Optional | +| [script\_hashes](#script-hashes) | `string` | Optional | #### Enable @@ -425,6 +523,40 @@ Enables or disables most tables and entries related to plutus and scripts. * Type: `boolean` +#### Script Hashes + +Whitelist for plutus hash. Only set values will be kept in the database, all others will be ignored. + +`script_hashes` + + * Type: `string` + +The string will be validated and needs to omit `\x` from the start of the hash eg: + +`\x6c969320597b755454ff3653ad09725d590c570827a129aeb4385526` +should be entered as: +`6c969320597b755454ff3653ad09725d590c570827a129aeb4385526` + +Be mindfull that whitelisting script hashes will be Null in the following tables. + + +| table name | +| :-------------------- | +| tx_out | +| collateral_tx_out | + +#### Example + +```json +"plutus": { + "enable": true + "script_hashes": + ["6c969320597b755454ff3653ad09725d590c570827a129aeb4385526" + ,"994cf4c18f5613ca49c275f63d464b6d95123bfa8985e82b24b5680b" + ] + } +``` + ### Governance `governance`