Merge #3852 #3880 #3882

3852: cardano-tracer: init RTView r=deepfire a=denisshevchenko

This is pre-MVP for RTView.

3880: Old peers tracing was erroneously called in new tracing r=deepfire a=jutaro

/nix/store/qaplqccmisqy8n7ai65nssafzkxyyc7p-cabal-install-exe-cabal-3.6.2.0/bin/cabal --project-file=/home/deepfire/cardano-node/.nix-shell-cabal.project run exe:cardano-node -- +RTS -sghc-rts-report.txt -RTS run --config config.json --database-path run/current/node-0/db-testnet --topology topology.json --host-addr 127.0.0.1 --port 30000 --socket-path node.socket +RTS -N2 -I0 -A16m -qg -qb --disable-delayed-os-memory-return -RTS
cardano-node: ExceptionInLinkedThread (ThreadId 11) The name ""peersFromNodeKernel"" is already taken by a metric.
CallStack (from HasCallStack):
  error, called at ./System/Metrics.hs:214:5 in ekg-core-0.1.1.7-FjoslY1tzknIAl90c73kOZ:System.Metrics

3882: Fix datum in tx and ref scripts r=Jimbo4350 a=ch1bo

(Couldn't reopen #3881, so created this one)

:snowflake: Add a roundtrip property `TxBodyContent -> TxBody -> TxBodyContent`

This helped in fixing the :bug: and uncover the two additional gaps in the code. I'm not 100% happy with the current implementation of the property though! I needed to accept two exceptions to the general `===`:

  1. `SimpleScriptV1` reference scripts may become `SimpleScriptV2`
  2. A `TxOutDatumHash` + a matching `ScriptData` may become a `TxOutDatumTx`

:snowflake: Resolve datum hash + matching datum in transaction to `TxOutDatumInTx`, fixes #3866 

❄️ Add missing script languages to `scriptLanguageSupportedInEra` for `BabbageEra`

❄️ Allow scripts in any language as refeference scripts

Co-authored-by: Denis Shevchenko <denis.shevchenko@iohk.io>
Co-authored-by: Kosyrev Serge <serge.kosyrev@iohk.io>
Co-authored-by: Yupanqui <jnf@arcor.de>
Co-authored-by: Sebastian Nagel <sebastian.nagel@ncoding.at>

cardano-api/cardano-api.cabal

@@ -233,6 +233,7 @@ test-suite cardano-api-test
                         Test.Cardano.Api.Typed.Orphans
                         Test.Cardano.Api.Typed.RawBytes
                         Test.Cardano.Api.Typed.Script
+                        Test.Cardano.Api.Typed.TxBody
                         Test.Cardano.Api.Typed.Value
 
   ghc-options:          -threaded -rtsopts -with-rtsopts=-N -with-rtsopts=-T

cardano-api/gen/Gen/Cardano/Api/Typed.hs

@@ -44,6 +44,7 @@ module Gen.Cardano.Api.Typed
   , genStakeAddress
   , genTx
   , genTxBody
+  , genTxBodyContent
   , genLovelace
   , genValue
   , genValueDefault

cardano-api/src/Cardano/Api/Script.hs

@@ -619,6 +619,15 @@ scriptLanguageSupportedInEra era lang =
       (AlonzoEra, PlutusScriptLanguage PlutusScriptV1) ->
         Just PlutusScriptV1InAlonzo
 
+      (BabbageEra, SimpleScriptLanguage SimpleScriptV1) ->
+        Just SimpleScriptV1InBabbage
+
+      (BabbageEra, SimpleScriptLanguage SimpleScriptV2) ->
+        Just SimpleScriptV2InBabbage
+
+      (BabbageEra, PlutusScriptLanguage PlutusScriptV1) ->
+        Just PlutusScriptV1InBabbage
+
       (BabbageEra, PlutusScriptLanguage PlutusScriptV2) ->
         Just PlutusScriptV2InBabbage
 
@@ -1432,21 +1441,14 @@ fromShelleyScriptToReferenceScript sbe script =
    scriptInEraToRefScript $ fromShelleyBasedScript sbe script
 
 scriptInEraToRefScript :: ScriptInEra era -> ReferenceScript era
-scriptInEraToRefScript sIne@(ScriptInEra langInEra s) =
-  let sLang = languageOfScriptLanguageInEra langInEra
-      era = shelleyBasedToCardanoEra $ eraOfScriptInEra sIne
-  in case refInsScriptsAndInlineDatsSupportedInEra era of
-       Nothing -> ReferenceScriptNone
-       Just supp ->
-         case sLang of
-           PlutusScriptLanguage PlutusScriptV2 ->
-             ReferenceScript supp $ toScriptInAnyLang s
-           SimpleScriptLanguage SimpleScriptV1 ->
-             ReferenceScriptNone
-           SimpleScriptLanguage SimpleScriptV2 ->
-             ReferenceScriptNone
-           PlutusScriptLanguage PlutusScriptV1 ->
-             ReferenceScriptNone
+scriptInEraToRefScript sIne@(ScriptInEra _ s) =
+  case refInsScriptsAndInlineDatsSupportedInEra era of
+    Nothing -> ReferenceScriptNone
+    Just supp ->
+      -- Any script can be a reference script
+      ReferenceScript supp $ toScriptInAnyLang s
+ where
+  era = shelleyBasedToCardanoEra $ eraOfScriptInEra sIne
 
 -- Helpers
 

cardano-api/src/Cardano/Api/TxBody.hs

@@ -2248,8 +2248,15 @@ fromLedgerTxOuts era body scriptdata =
       , txout <- toList (Alonzo.outputs' body) ]
 
     ShelleyBasedEraBabbage ->
-      map (fromBabbageTxOut ScriptDataInBabbageEra ReferenceTxInsScriptsInlineDatumsInBabbageEra  MultiAssetInBabbageEra)
-          (toList $ Babbage.outputs body)
+      [ fromBabbageTxOut
+          MultiAssetInBabbageEra
+          ScriptDataInBabbageEra
+          ReferenceTxInsScriptsInlineDatumsInBabbageEra
+          txdatums
+          txouts
+      | let txdatums = selectTxDatums scriptdata
+      , txouts <- toList (Babbage.outputs body)
+      ]
   where
     selectTxDatums  TxBodyNoScriptData                            = Map.empty
     selectTxDatums (TxBodyScriptData _ (Alonzo.TxDats' datums) _) = datums
@@ -2288,20 +2295,41 @@ fromBabbageTxOut
   => ShelleyLedgerEra era ~ ledgerera
   => Ledger.Crypto ledgerera ~ StandardCrypto
   => Ledger.Value ledgerera ~ Mary.Value StandardCrypto
-  => ScriptDataSupportedInEra era
+  => MultiAssetSupportedInEra era
+  -> ScriptDataSupportedInEra era
   -> ReferenceTxInsScriptsInlineDatumsSupportedInEra era
-  -> MultiAssetSupportedInEra era
+  -> Map (Alonzo.DataHash StandardCrypto)
+         (Alonzo.Data ledgerera)
   -> Babbage.TxOut ledgerera
   -> TxOut CtxTx era
-fromBabbageTxOut s i m (Babbage.TxOut addr val datum mRefScript) =
+fromBabbageTxOut multiAssetInEra scriptDataInEra inlineDatumsInEra txdatums txout =
    TxOut
      (fromShelleyAddr shelleyBasedEra addr)
-     (TxOutValue m (fromMaryValue val))
-     (fromBabbageTxOutDatum s i datum)
+     (TxOutValue multiAssetInEra (fromMaryValue val))
+     babbageTxOutDatum
      (case mRefScript of
        SNothing -> ReferenceScriptNone
        SJust rScript -> fromShelleyScriptToReferenceScript shelleyBasedEra rScript
      )
+ where
+   -- NOTE: This is different to 'fromBabbageTxOutDatum' as it may resolve
+   -- 'DatumHash' values using the datums included in the transaction.
+   babbageTxOutDatum :: TxOutDatum CtxTx era
+   babbageTxOutDatum =
+     case datum of
+       Babbage.NoDatum -> TxOutDatumNone
+       Babbage.DatumHash dh -> resolveDatumInTx dh
+       Babbage.Datum d ->
+         TxOutDatumInline inlineDatumsInEra $
+           binaryDataToScriptData inlineDatumsInEra d
+
+   resolveDatumInTx :: Alonzo.DataHash StandardCrypto -> TxOutDatum CtxTx era
+   resolveDatumInTx dh
+      | Just d <- Map.lookup dh txdatums
+                  = TxOutDatumInTx' scriptDataInEra (ScriptDataHash dh) (fromAlonzoData d)
+      | otherwise = TxOutDatumHash scriptDataInEra (ScriptDataHash dh)
+
+   (Babbage.TxOut addr val datum mRefScript) = txout
 
 fromLedgerTxTotalCollateral
   :: ShelleyBasedEra era

cardano-api/test/Test/Cardano/Api/Typed/TxBody.hs

@@ -0,0 +1,81 @@
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE TemplateHaskell #-}
+
+module Test.Cardano.Api.Typed.TxBody
+  ( tests
+  ) where
+
+import           Cardano.Prelude
+
+import           Hedgehog (Property, annotateShow, failure, (===), MonadTest)
+import qualified Hedgehog as H
+import           Test.Tasty (TestTree)
+import           Test.Tasty.Hedgehog (testProperty)
+import           Test.Tasty.TH (testGroupGenerator)
+
+import           Cardano.Api
+import           Cardano.Api.Shelley (ReferenceScript (..), refScriptToShelleyScript)
+import           Data.Type.Equality (TestEquality (testEquality))
+import           Gen.Cardano.Api.Typed
+import           Test.Cardano.Api.Typed.Orphans ()
+
+{- HLINT ignore "Use camelCase" -}
+
+-- | Check the txOuts in a TxBodyContent after a ledger roundtrip.
+prop_roundtrip_txbodycontent_txouts:: Property
+prop_roundtrip_txbodycontent_txouts =
+  H.property $ do
+    content <- H.forAll $ genTxBodyContent BabbageEra
+    -- Create the ledger body & auxiliaries
+    body <- case makeTransactionBody content of
+      Left err -> annotateShow err >> failure
+      Right body -> pure body
+    annotateShow body
+    -- Convert ledger body back via 'getTxBodyContent' and 'fromLedgerTxBody'
+    let (TxBody content') = body
+    matchTxOuts (txOuts content) (txOuts content')
+ where
+  matchTxOuts :: MonadTest m => [TxOut CtxTx BabbageEra] -> [TxOut CtxTx BabbageEra] -> m ()
+  matchTxOuts as bs =
+    mapM_ matchTxOut $ zip as bs
+
+  matchTxOut :: MonadTest m => (TxOut CtxTx BabbageEra, TxOut CtxTx BabbageEra) -> m ()
+  matchTxOut (a, b) = do
+    let TxOut aAddress aValue aDatum aRefScript = a
+    let TxOut bAddress bValue bDatum bRefScript = b
+    aAddress === bAddress
+    aValue === bValue
+    matchDatum (aDatum, bDatum)
+    matchRefScript (aRefScript, bRefScript)
+
+  -- NOTE: We accept TxOutDatumInTx instead of TxOutDatumHash as it may be
+  -- correctly resolved given a datum matching the hash was generated.
+  matchDatum :: MonadTest m => (TxOutDatum CtxTx era, TxOutDatum CtxTx era) -> m ()
+  matchDatum = \case
+    (TxOutDatumHash _ dh, TxOutDatumInTx _ d) ->
+      dh === hashScriptData d
+    (a, b) ->
+      a === b
+
+  -- NOTE: After Allegra, all eras interpret SimpleScriptV1 as SimpleScriptV2
+  -- because V2 is a superset of V1. So we accept that as a valid conversion.
+  matchRefScript :: MonadTest m => (ReferenceScript BabbageEra, ReferenceScript BabbageEra) -> m ()
+  matchRefScript (a, b)
+    | isSimpleScriptV1 a && isSimpleScriptV2 b =
+      refScriptToShelleyScript BabbageEra a === refScriptToShelleyScript BabbageEra b
+    | otherwise =
+      a === b
+
+  isSimpleScriptV1 :: ReferenceScript era -> Bool
+  isSimpleScriptV1 = isLang (SimpleScriptLanguage SimpleScriptV1)
+
+  isSimpleScriptV2 :: ReferenceScript era -> Bool
+  isSimpleScriptV2 = isLang (SimpleScriptLanguage SimpleScriptV2)
+
+  isLang :: ScriptLanguage a -> ReferenceScript era -> Bool
+  isLang expected = \case
+    (ReferenceScript _ (ScriptInAnyLang actual _)) -> isJust $ testEquality expected actual
+    _ -> False
+
+tests :: TestTree
+tests = $testGroupGenerator

cardano-api/test/cardano-api-test.hs

@@ -17,6 +17,7 @@ import qualified Test.Cardano.Api.Typed.JSON
 import qualified Test.Cardano.Api.Typed.Ord
 import qualified Test.Cardano.Api.Typed.RawBytes
 import qualified Test.Cardano.Api.Typed.Script
+import qualified Test.Cardano.Api.Typed.TxBody
 import qualified Test.Cardano.Api.Typed.Value
 
 main :: IO ()
@@ -41,5 +42,6 @@ tests =
     , Test.Cardano.Api.Typed.Ord.tests
     , Test.Cardano.Api.Typed.RawBytes.tests
     , Test.Cardano.Api.Typed.Script.tests
+    , Test.Cardano.Api.Typed.TxBody.tests
     , Test.Cardano.Api.Typed.Value.tests
     ]

cardano-node/src/Cardano/Node/Run.hs

@@ -6,7 +6,6 @@
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE PackageImports #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE TupleSections #-}
 {-# LANGUAGE TypeApplications #-}
 
 #if !defined(mingw32_HOST_OS)
@@ -192,59 +191,63 @@ handleNodeWithTracers cmdPc nc p networkMagic runP = do
       let fp = maybe  "No file path found!"
                       unConfigPath
                       (getLast (pncConfigFile cmdPc))
-      (tracers, mLoggingLayer) <-
-        case ncTraceConfig nc of
-          TraceDispatcher{} -> do
-            (, Nothing) <$>
-              initTraceDispatcher
-                nc
-                p
-                networkMagic
-                nodeKernelData
-                p2pMode
-          _ -> do
-            eLoggingLayer <- runExceptT $ createLoggingLayer
-              (Text.pack (showVersion version))
+      case ncTraceConfig nc of
+        TraceDispatcher{} -> do
+          tracers <-
+            initTraceDispatcher
               nc
               p
-
-            loggingLayer <- case eLoggingLayer of
-              Left err  -> putTextLn (Text.pack $ show err) >> exitFailure
-              Right res -> return res
-            !trace <- setupTrace loggingLayer
-            let tracer = contramap pack $ toLogObject trace
-            logTracingVerbosity nc tracer
-
-            -- Legacy logging infrastructure must trace 'nodeStartTime' and 'nodeBasicInfo'.
-            startTime <- getCurrentTime
-            traceCounter "nodeStartTime" trace (ceiling $ utcTimeToPOSIXSeconds startTime)
-            nbi <- nodeBasicInfo nc p startTime
-            forM_ nbi $ \(LogObject nm mt content) ->
-              traceNamedObject (appendName nm trace) (mt, content)
-
-            (,Just loggingLayer) <$>
-              mkTracers
-                (Consensus.configBlock cfg)
-                (ncTraceConfig nc)
-                trace
-                nodeKernelData
-                (llEKGDirect loggingLayer)
-                p2pMode
-
-      getStartupInfo nc p fp
-        >>= mapM_ (traceWith $ startupTracer tracers)
-
-      Async.withAsync (handlePeersListSimple (error "Implement Tracer IO [Peer blk]") nodeKernelData)
-          $ \_peerLoggingThread ->
-            -- We ignore peer logging thread if it dies, but it will be killed
-            -- when 'handleSimpleNode' terminates.
-                handleSimpleNode runP p2pMode tracers nc
-                  (\nk -> do
-                      setNodeKernel nodeKernelData nk
-                      traceWith (nodeStateTracer tracers) NodeKernelOnline)
-                `finally`
-                forM_ mLoggingLayer
-                  shutdownLoggingLayer
+              networkMagic
+              nodeKernelData
+              p2pMode
+          handleSimpleNode runP p2pMode tracers nc
+            (\nk -> do
+                setNodeKernel nodeKernelData nk
+                traceWith (nodeStateTracer tracers) NodeKernelOnline)
+
+        _ -> do
+          eLoggingLayer <- runExceptT $ createLoggingLayer
+            (Text.pack (showVersion version))
+            nc
+            p
+
+          loggingLayer <- case eLoggingLayer of
+            Left err  -> putTextLn (Text.pack $ show err) >> exitFailure
+            Right res -> return res
+          !trace <- setupTrace loggingLayer
+          let tracer = contramap pack $ toLogObject trace
+          logTracingVerbosity nc tracer
+
+          -- Legacy logging infrastructure must trace 'nodeStartTime' and 'nodeBasicInfo'.
+          startTime <- getCurrentTime
+          traceCounter "nodeStartTime" trace (ceiling $ utcTimeToPOSIXSeconds startTime)
+          nbi <- nodeBasicInfo nc p startTime
+          forM_ nbi $ \(LogObject nm mt content) ->
+            traceNamedObject (appendName nm trace) (mt, content)
+
+          tracers <-
+            mkTracers
+              (Consensus.configBlock cfg)
+              (ncTraceConfig nc)
+              trace
+              nodeKernelData
+              (llEKGDirect loggingLayer)
+              p2pMode
+
+          getStartupInfo nc p fp
+            >>= mapM_ (traceWith $ startupTracer tracers)
+
+          Async.withAsync (handlePeersListSimple (error "Implement Tracer IO [Peer blk]") nodeKernelData)
+              $ \_peerLoggingThread ->
+                -- We ignore peer logging thread if it dies, but it will be killed
+                -- when 'handleSimpleNode' terminates.
+                    handleSimpleNode runP p2pMode tracers nc
+                      (\nk -> do
+                          setNodeKernel nodeKernelData nk
+                          traceWith (nodeStateTracer tracers) NodeKernelOnline)
+                    `finally`
+                    forM_ eLoggingLayer
+                      shutdownLoggingLayer
 
 
 logTracingVerbosity :: NodeConfiguration -> Tracer IO String -> IO ()