Extendend whitelist options

IntersectMBO · Mar 6, 2024 · b1ff17f · b1ff17f
1 parent 77cd794
commit b1ff17f
Show file tree

Hide file tree

Showing 20 changed files with 342 additions and 598 deletions.
diff --git a/cardano-db-sync/cardano-db-sync.cabal b/cardano-db-sync/cardano-db-sync.cabal
@@ -63,6 +63,7 @@ library
                         Cardano.DbSync.Era.Byron.Util
                         Cardano.DbSync.Era.Cardano.Insert
                         Cardano.DbSync.Era.Cardano.Util
+                        Cardano.DbSync.Era.Conway.Insert.GovAction
                         Cardano.DbSync.Era.Shelley.Generic
                         Cardano.DbSync.Era.Shelley.Generic.Block
                         Cardano.DbSync.Era.Shelley.Generic.EpochUpdate
@@ -83,19 +84,18 @@ library
                         Cardano.DbSync.Era.Shelley.Generic.Tx.Types
                         Cardano.DbSync.Era.Shelley.Generic.Util
                         Cardano.DbSync.Era.Shelley.Generic.Witness
-                        Cardano.DbSync.Era.Shelley.Genesis
                         Cardano.DbSync.Era.Shelley.Query
                         Cardano.DbSync.Era.Universal.Adjust
                         Cardano.DbSync.Era.Universal.Block
                         Cardano.DbSync.Era.Universal.Epoch
-                        Cardano.DbSync.Era.Universal.Validate
+                        Cardano.DbSync.Era.Universal.Genesis
                         Cardano.DbSync.Era.Universal.Insert.Certificate
-                        Cardano.DbSync.Era.Universal.Insert.GovAction
                         Cardano.DbSync.Era.Universal.Insert.Grouped
                         Cardano.DbSync.Era.Universal.Insert.LedgerEvent
                         Cardano.DbSync.Era.Universal.Insert.Other
                         Cardano.DbSync.Era.Universal.Insert.Pool
                         Cardano.DbSync.Era.Universal.Insert.Tx
+                        Cardano.DbSync.Era.Universal.Validate
 
 
                         -- Temporary debugging validation
@@ -140,6 +140,7 @@ library
                         Cardano.DbSync.Util.Bech32
                         Cardano.DbSync.Util.Cbor
                         Cardano.DbSync.Util.Constraint
+                        Cardano.DbSync.Util.Whitelist
 
                         Paths_cardano_db_sync
 

diff --git a/cardano-db-sync/src/Cardano/DbSync.hs b/cardano-db-sync/src/Cardano/DbSync.hs
@@ -47,7 +47,6 @@ import Cardano.Prelude hiding (Nat, (%))
 import Cardano.Slotting.Slot (EpochNo (..))
 import Control.Concurrent.Async
 import Control.Monad.Extra (whenJust)
-import qualified Data.Strict.Maybe as Strict
 import qualified Data.Text as Text
 import Data.Version (showVersion)
 import Database.Persist.Postgresql (ConnectionString, withPostgresqlConn)
@@ -160,8 +159,7 @@ runSyncNode ::
   IO ()
 runSyncNode metricsSetters trce iomgr dbConnString ranMigrations runMigrationFnc syncNodeConfigFromFile syncNodeParams syncOptions = do
   whenJust maybeLedgerDir $
-    \enpLedgerStateDir -> do
-      createDirectoryIfMissing True (unLedgerStateDir enpLedgerStateDir)
+    \enpLedgerStateDir -> createDirectoryIfMissing True (unLedgerStateDir enpLedgerStateDir)
   logInfo trce $ "Using byron genesis file from: " <> (show . unGenesisFile $ dncByronGenesisFile syncNodeConfigFromFile)
   logInfo trce $ "Using shelley genesis file from: " <> (show . unGenesisFile $ dncShelleyGenesisFile syncNodeConfigFromFile)
   logInfo trce $ "Using alonzo genesis file from: " <> (show . unGenesisFile $ dncAlonzoGenesisFile syncNodeConfigFromFile)
@@ -170,39 +168,38 @@ runSyncNode metricsSetters trce iomgr dbConnString ranMigrations runMigrationFnc
 
   Db.runIohkLogging trce $
     withPostgresqlConn dbConnString $
-      \backend -> liftIO $ do
-        runOrThrowIO $ runExceptT $ do
-          genCfg <- readCardanoGenesisConfig syncNodeConfigFromFile
-          logProtocolMagicId trce $ genesisProtocolMagicId genCfg
-          syncEnv <-
-            ExceptT $
-              mkSyncEnvFromConfig
-                trce
-                backend
-                dbConnString
-                syncOptions
-                genCfg
-                syncNodeConfigFromFile
-                syncNodeParams
-                ranMigrations
-                runMigrationFnc
-          liftIO $ runExtraMigrationsMaybe syncEnv
-          unless useLedger $ liftIO $ do
-            logInfo trce "Migrating to a no ledger schema"
-            Db.noLedgerMigrations backend trce
-          insertValidateGenesisDist syncEnv (dncNetworkName syncNodeConfigFromFile) genCfg (useShelleyInit syncNodeConfigFromFile)
+      \backend -> liftIO $ runOrThrowIO $ runExceptT $ do
+        genCfg <- readCardanoGenesisConfig syncNodeConfigFromFile
+        logProtocolMagicId trce $ genesisProtocolMagicId genCfg
+        syncEnv <-
+          ExceptT $
+            mkSyncEnvFromConfig
+              trce
+              backend
+              dbConnString
+              syncOptions
+              genCfg
+              syncNodeConfigFromFile
+              syncNodeParams
+              ranMigrations
+              runMigrationFnc
+        liftIO $ runExtraMigrationsMaybe syncEnv
+        unless useLedger $ liftIO $ do
+          logInfo trce "Migrating to a no ledger schema"
+          Db.noLedgerMigrations backend trce
+        insertValidateGenesisDist syncEnv (dncNetworkName syncNodeConfigFromFile) genCfg (useShelleyInit syncNodeConfigFromFile)
 
-          -- communication channel between datalayer thread and chainsync-client thread
-          threadChannels <- liftIO newThreadChannels
-          liftIO $
-            mapConcurrently_
-              id
-              [ runDbThread syncEnv metricsSetters threadChannels
-              , runSyncNodeClient metricsSetters syncEnv iomgr trce threadChannels (enpSocketPath syncNodeParams)
-              , runFetchOffChainPoolThread syncEnv
-              , runFetchOffChainVoteThread syncEnv
-              , runLedgerStateWriteThread (getTrace syncEnv) (envLedgerEnv syncEnv)
-              ]
+        -- communication channel between datalayer thread and chainsync-client thread
+        threadChannels <- liftIO newThreadChannels
+        liftIO $
+          mapConcurrently_
+            id
+            [ runDbThread syncEnv metricsSetters threadChannels
+            , runSyncNodeClient metricsSetters syncEnv iomgr trce threadChannels (enpSocketPath syncNodeParams)
+            , runFetchOffChainPoolThread syncEnv
+            , runFetchOffChainVoteThread syncEnv
+            , runLedgerStateWriteThread (getTrace syncEnv) (envLedgerEnv syncEnv)
+            ]
   where
     useShelleyInit :: SyncNodeConfig -> Bool
     useShelleyInit cfg =
@@ -245,23 +242,16 @@ extractSyncOptions snp aop snc =
     , snapshotEveryLagging = enpSnEveryLagging snp
     }
   where
-    maybeKeepMNames =
-      case sioMetadata (dncInsertOptions snc) of
-        MetadataKeys ks -> Strict.Just (map fromIntegral $ toList ks)
-        MetadataEnable -> Strict.Nothing
-        MetadataDisable -> Strict.Nothing
-
     iopts =
       InsertOptions
         { ioInOut = isTxOutEnabled'
         , ioUseLedger = useLedger
         , ioShelley = isShelleyEnabled (sioShelley (dncInsertOptions snc))
         , -- Rewards are only disabled on "disable_all" and "only_gov" presets
           ioRewards = True
-        , ioMultiAssets = isMultiAssetEnabled (sioMultiAsset (dncInsertOptions snc))
-        , ioMetadata = isMetadataEnabled (sioMetadata (dncInsertOptions snc))
-        , ioKeepMetadataNames = maybeKeepMNames
-        , ioPlutusExtra = isPlutusEnabled (sioPlutus (dncInsertOptions snc))
+        , ioMultiAssets = sioMultiAsset (dncInsertOptions snc)
+        , ioMetadata = sioMetadata (dncInsertOptions snc)
+        , ioPlutus = sioPlutus (dncInsertOptions snc)
         , ioOffChainPoolData = useOffchainPoolData
         , ioGov = useGovernance
         }

diff --git a/cardano-db-sync/src/Cardano/DbSync/Api/Types.hs b/cardano-db-sync/src/Cardano/DbSync/Api/Types.hs
@@ -15,7 +15,7 @@ module Cardano.DbSync.Api.Types (
 
 import qualified Cardano.Db as DB
 import Cardano.DbSync.Cache.Types (Cache)
-import Cardano.DbSync.Config.Types (SyncNodeConfig)
+import Cardano.DbSync.Config.Types (MetadataConfig, MultiAssetConfig, PlutusConfig, SyncNodeConfig)
 import Cardano.DbSync.Ledger.Types (HasLedgerEnv)
 import Cardano.DbSync.LocalStateQuery (NoLedgerEnv)
 import Cardano.DbSync.Types (
@@ -74,16 +74,15 @@ data SyncOptions = SyncOptions
   deriving (Show)
 
 data InsertOptions = InsertOptions
-  { ioInOut :: !Bool
-  , ioUseLedger :: !Bool
-  , ioShelley :: !Bool
-  , ioRewards :: !Bool
-  , ioMultiAssets :: !Bool
-  , ioMetadata :: !Bool
-  , ioKeepMetadataNames :: Strict.Maybe [Word64]
-  , ioPlutusExtra :: !Bool
+  { ioGov :: !Bool
+  , ioInOut :: !Bool
+  , ioMetadata :: !MetadataConfig
+  , ioMultiAssets :: !MultiAssetConfig
   , ioOffChainPoolData :: !Bool
-  , ioGov :: !Bool
+  , ioPlutus :: !PlutusConfig
+  , ioRewards :: !Bool
+  , ioShelley :: !Bool
+  , ioUseLedger :: !Bool
   }
   deriving (Show)
 

diff --git a/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs b/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs
@@ -189,7 +189,8 @@ data LedgerInsertConfig
 data ShelleyInsertConfig
   = ShelleyEnable
   | ShelleyDisable
-  | ShelleyStakeAddrs (NonEmpty ShortByteString)
+  | -- Whitelist of Shelley stake addresses
+    ShelleyStakeAddrs (NonEmpty ShortByteString)
   deriving (Eq, Show)
 
 newtype RewardsConfig = RewardsConfig
@@ -199,19 +200,22 @@ newtype RewardsConfig = RewardsConfig
 data MultiAssetConfig
   = MultiAssetEnable
   | MultiAssetDisable
-  | MultiAssetPolicies (NonEmpty ShortByteString)
+  | -- | Whitelist of multiAsset policy IDs
+    MultiAssetPolicies (NonEmpty ShortByteString)
   deriving (Eq, Show)
 
 data MetadataConfig
   = MetadataEnable
   | MetadataDisable
-  | MetadataKeys (NonEmpty Word)
+  | -- | Whitelist of metadata keys
+    MetadataKeys (NonEmpty Word)
   deriving (Eq, Show)
 
 data PlutusConfig
   = PlutusEnable
   | PlutusDisable
-  | PlutusScripts (NonEmpty ShortByteString)
+  | -- | Whitelist of plutus script hashes
+    PlutusScripts (NonEmpty ShortByteString)
   deriving (Eq, Show)
 
 newtype GovernanceConfig = GovernanceConfig

diff --git a/cardano-db-sync/src/Cardano/DbSync/Default.hs b/cardano-db-sync/src/Cardano/DbSync/Default.hs
@@ -18,6 +18,7 @@ import Cardano.DbSync.Api.Ledger
 import Cardano.DbSync.Api.Types (ConsistentLevel (..), InsertOptions (..), LedgerEnv (..), SyncEnv (..), SyncOptions (..))
 import Cardano.DbSync.Epoch (epochHandler)
 import Cardano.DbSync.Era.Byron.Insert (insertByronBlock)
+
 import qualified Cardano.DbSync.Era.Shelley.Generic as Generic
 import Cardano.DbSync.Era.Universal.Block (insertBlockUniversal)
 import Cardano.DbSync.Era.Universal.Epoch (hasEpochStartEvent, hasNewEpochEvent)
@@ -165,15 +166,15 @@ insertBlock syncEnv cblk applyRes firstAfterRollback tookSnapshot = do
     BlockAlonzo blk ->
       newExceptT $
         insertBlockUniversal' $
-          Generic.fromAlonzoBlock (ioPlutusExtra iopts) (getPrices applyResult) blk
+          Generic.fromAlonzoBlock (ioPlutus iopts) (getPrices applyResult) blk
     BlockBabbage blk ->
       newExceptT $
         insertBlockUniversal' $
-          Generic.fromBabbageBlock (ioPlutusExtra iopts) (getPrices applyResult) blk
+          Generic.fromBabbageBlock (ioPlutus iopts) (getPrices applyResult) blk
     BlockConway blk ->
       newExceptT $
         insertBlockUniversal' $
-          Generic.fromConwayBlock (ioPlutusExtra iopts) (getPrices applyResult) blk
+          Generic.fromConwayBlock (ioPlutus iopts) (getPrices applyResult) blk
   -- update the epoch
   updateEpoch details isNewEpochEvent
   whenPruneTxOut syncEnv $

diff --git a/cardano-db-sync/src/Cardano/DbSync/Era.hs b/cardano-db-sync/src/Cardano/DbSync/Era.hs
@@ -9,7 +9,7 @@ module Cardano.DbSync.Era (
 import Cardano.DbSync.Api.Types (SyncEnv)
 import Cardano.DbSync.Config
 import qualified Cardano.DbSync.Era.Byron.Genesis as Byron
-import qualified Cardano.DbSync.Era.Shelley.Genesis as Shelley
+import qualified Cardano.DbSync.Era.Universal.Genesis as Shelley
 import Cardano.DbSync.Error
 import Cardano.Prelude
 

diff --git a/...rc/Cardano/DbSync/Era/Conway/GovAction.hs → ...ano/DbSync/Era/Conway/Insert/GovAction.hs b/...rc/Cardano/DbSync/Era/Conway/GovAction.hs → ...ano/DbSync/Era/Conway/Insert/GovAction.hs
@@ -8,7 +8,7 @@
 {-# LANGUAGE TypeFamilies #-}
 {-# LANGUAGE NoImplicitPrelude #-}
 
-module Cardano.DbSync.Era.Conway.GovAction (
+module Cardano.DbSync.Era.Conway.Insert.GovAction (
   insertConstitution,
   insertCostModel,
   insertCredDrepHash,
@@ -47,10 +47,12 @@ import Cardano.Ledger.Conway.Governance
 import qualified Cardano.Ledger.Credential as Ledger
 import Cardano.Ledger.DRep (DRepState (..))
 import Cardano.Ledger.Keys (KeyRole (..))
-import qualified Cardano.Ledger.Plutus.CostModels as Ledger
+import Cardano.Ledger.Plutus.CostModels (CostModel)
 import Cardano.Ledger.Plutus.Language (Language)
 import Cardano.Ledger.Shelley.API (Coin (..))
 import Cardano.Prelude
+
+import Cardano.Ledger.Plutus (mkCostModels)
 import Control.Monad.Extra (whenJust)
 import Control.Monad.Trans.Control (MonadBaseControl)
 import qualified Data.Aeson as Aeson
@@ -72,7 +74,7 @@ insertGovActionProposal ::
 insertGovActionProposal cache blkId txId govExpiresAt (index, pp) = do
   addrId <-
     lift $ queryOrInsertRewardAccount cache CacheNew $ pProcReturnAddr pp
-  votingAnchorId <- lift $ insertAnchor txId $ pProcAnchor pp
+  votingAnchorId <- lift $ insertVotingAnchor txId $ pProcAnchor pp
   mParamProposalId <- lift $
     case pProcGovAction pp of
       ParameterChange _ pparams _ ->
@@ -286,15 +288,6 @@ insertVotingAnchor txId anchor =
       , DB.votingAnchorDataHash = Generic.safeHashToByteString $ anchorDataHash anchor
       }
 
-insertAnchor :: (MonadIO m, MonadBaseControl IO m) => DB.TxId -> Anchor StandardCrypto -> ReaderT SqlBackend m DB.VotingAnchorId
-insertAnchor txId anchor =
-  DB.insertAnchor $
-    DB.VotingAnchor
-      { DB.votingAnchorTxId = txId
-      , DB.votingAnchorUrl = DB.VoteUrl $ Ledger.urlToText $ anchorUrl anchor -- TODO: Conway check unicode and size of URL
-      , DB.votingAnchorDataHash = Generic.safeHashToByteString $ anchorDataHash anchor
-      }
-
 --------------------------------------------------------------------------------------
 -- DREP
 --------------------------------------------------------------------------------------
@@ -340,21 +333,19 @@ insertDrepDistr e pSnapshot = do
 insertCostModel ::
   (MonadBaseControl IO m, MonadIO m) =>
   DB.BlockId ->
-  Map Language Ledger.CostModel ->
+  Map Language CostModel ->
   ReaderT SqlBackend m DB.CostModelId
 insertCostModel _blkId cms =
   DB.insertCostModel $
     DB.CostModel
-      { DB.costModelHash = Crypto.abstractHashToBytes $ Crypto.serializeCborHash $ Ledger.mkCostModels cms
+      { DB.costModelHash = Crypto.abstractHashToBytes $ Crypto.serializeCborHash $ mkCostModels cms
       , DB.costModelCosts = Text.decodeUtf8 $ LBS.toStrict $ Aeson.encode cms
       }
 
 updateEnacted :: forall m. (MonadBaseControl IO m, MonadIO m) => Bool -> EpochNo -> EnactState StandardConway -> ExceptT SyncNodeError (ReaderT SqlBackend m) ()
 updateEnacted isEnacted epochNo enactedState = do
   whenJust (strictMaybeToMaybe (enactedState ^. ensPrevPParamUpdateL)) $ \prevId -> do
-    gaId <- resolveGovActionProposal $ getPrevId prevId
+    gaId <- resolveGovActionProposal $ unGovPurposeId prevId
     if isEnacted
       then lift $ DB.updateGovActionEnacted gaId (unEpochNo epochNo)
       else lift $ DB.updateGovActionRatified gaId (unEpochNo epochNo)
-  where
-    getPrevId = unGovPurposeId
diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Generic/Block.hs
@@ -25,6 +25,7 @@ module Cardano.DbSync.Era.Shelley.Generic.Block (
 
 import qualified Cardano.Crypto.Hash as Crypto
 import qualified Cardano.Crypto.KES.Class as KES
+import Cardano.DbSync.Config.Types (PlutusConfig)
 import Cardano.DbSync.Era.Shelley.Generic.Tx
 import Cardano.DbSync.Types
 import Cardano.DbSync.Util.Bech32 (serialiseVerKeyVrfToBech32)
@@ -120,8 +121,8 @@ fromMaryBlock blk =
     , blkTxs = map fromMaryTx (getTxs blk)
     }
 
-fromAlonzoBlock :: Bool -> Maybe Prices -> ShelleyBlock TPraosStandard StandardAlonzo -> Block
-fromAlonzoBlock iope mprices blk =
+fromAlonzoBlock :: PlutusConfig -> Maybe Prices -> ShelleyBlock TPraosStandard StandardAlonzo -> Block
+fromAlonzoBlock plutusConfig mprices blk =
   Block
     { blkEra = Alonzo
     , blkHash = blockHash blk
@@ -134,11 +135,11 @@ fromAlonzoBlock iope mprices blk =
     , blkVrfKey = blockVrfKeyView $ blockVrfVkTPraos blk
     , blkOpCert = blockOpCertKeyTPraos blk
     , blkOpCertCounter = blockOpCertCounterTPraos blk
-    , blkTxs = map (fromAlonzoTx iope mprices) (getTxs blk)
+    , blkTxs = map (fromAlonzoTx plutusConfig mprices) (getTxs blk)
     }
 
-fromBabbageBlock :: Bool -> Maybe Prices -> ShelleyBlock PraosStandard StandardBabbage -> Block
-fromBabbageBlock iope mprices blk =
+fromBabbageBlock :: PlutusConfig -> Maybe Prices -> ShelleyBlock PraosStandard StandardBabbage -> Block
+fromBabbageBlock plutusConfig mprices blk =
   Block
     { blkEra = Babbage
     , blkHash = blockHash blk
@@ -151,11 +152,11 @@ fromBabbageBlock iope mprices blk =
     , blkVrfKey = blockVrfKeyView $ blockVrfVkPraos blk
     , blkOpCert = blockOpCertKeyPraos blk
     , blkOpCertCounter = blockOpCertCounterPraos blk
-    , blkTxs = map (fromBabbageTx iope mprices) (getTxs blk)
+    , blkTxs = map (fromBabbageTx plutusConfig mprices) (getTxs blk)
     }
 
-fromConwayBlock :: Bool -> Maybe Prices -> ShelleyBlock PraosStandard StandardConway -> Block
-fromConwayBlock iope mprices blk =
+fromConwayBlock :: PlutusConfig -> Maybe Prices -> ShelleyBlock PraosStandard StandardConway -> Block
+fromConwayBlock plutusConfig mprices blk =
   Block
     { blkEra = Conway
     , blkHash = blockHash blk
@@ -168,7 +169,7 @@ fromConwayBlock iope mprices blk =
     , blkVrfKey = blockVrfKeyView $ blockVrfVkPraos blk
     , blkOpCert = blockOpCertKeyPraos blk
     , blkOpCertCounter = blockOpCertCounterPraos blk
-    , blkTxs = map (fromConwayTx iope mprices) (getTxs blk)
+    , blkTxs = map (fromConwayTx plutusConfig mprices) (getTxs blk)
     }
 
 -- -------------------------------------------------------------------------------------------------