[next-node]: merge updates from the main branch.

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,8 +8,7 @@ Pre-submit checklist:
     - [ ] Tests are provided (if possible)
     - [ ] Commit sequence broadly makes sense
     - [ ] Key commits have useful messages
-    - [ ] Relevant tickets are mentioned in commit messages
-    - [ ] Formatting, materialized Nix files, PNG optimization, etc. are updated
+    - [ ] Formatting, PNG optimization, etc. are updated
 - PR
     - [ ] Self-reviewed the diff
     - [ ] Useful pull request description

.github/workflows/test.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
-      - uses: nixbuild/nix-quick-install-action@v13
+      - uses: nixbuild/nix-quick-install-action@v14
       - run: nix-instantiate release.nix --arg supportedSystems '[ builtins.currentSystem ]' --restrict-eval -I . --allowed-uris 'https://github.com/NixOS/nixpkgs https://github.com/input-output-hk https://github.com/NixOS/nixpkgs-channels' --option trusted-public-keys "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= iohk.cachix.org-1:DpRUyj7h7V830dp/i6Nti+NEO2/nhblbov/8MW7Rqoo= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" --option substituters "https://hydra.iohk.io https://iohk.cachix.org https://cache.nixos.org/" --show-trace
   nix-tests:
     strategy:
@@ -18,7 +18,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
-      - uses: nixbuild/nix-quick-install-action@v13
+      - uses: nixbuild/nix-quick-install-action@v14
       - run: nix-build -A tests.nixpkgsFmt -A tests.purs-tidy -A tests.shellcheck -A tests.stylishHaskell -A tests.generated --arg supportedSystems '[ builtins.currentSystem ]' --restrict-eval -I . --allowed-uris 'https://github.com/NixOS/nixpkgs https://github.com/input-output-hk https://github.com/NixOS/nixpkgs-channels' --option trusted-public-keys "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= iohk.cachix.org-1:DpRUyj7h7V830dp/i6Nti+NEO2/nhblbov/8MW7Rqoo= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" --option substituters "https://hydra.iohk.io https://iohk.cachix.org https://cache.nixos.org/"
   check-for-updates:
     strategy:
@@ -27,7 +27,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
-      - uses: nixbuild/nix-quick-install-action@v13
+      - uses: nixbuild/nix-quick-install-action@v14
       - run: |
           nix --extra-experimental-features 'nix-command flakes' --extra-experimental-features flakes flake lock --option trusted-public-keys "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= iohk.cachix.org-1:DpRUyj7h7V830dp/i6Nti+NEO2/nhblbov/8MW7Rqoo= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" --option substituters "https://hydra.iohk.io https://iohk.cachix.org https://cache.nixos.org/"
           nix-shell --extra-experimental-features 'nix-command flakes' --command "cd plutus-playground-client && (update-client-deps || update-client-deps)" --option trusted-public-keys "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= iohk.cachix.org-1:DpRUyj7h7V830dp/i6Nti+NEO2/nhblbov/8MW7Rqoo= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" --option substituters "https://hydra.iohk.io https://iohk.cachix.org https://cache.nixos.org/" # Double-call to work around bug in spago2nix on first fetch

cabal.project

@@ -16,6 +16,7 @@ packages: doc
           plutus-playground-server
           plutus-script-utils
           plutus-use-cases
+          plutus-streaming
           quickcheck-dynamic
           web-ghc
 
@@ -315,3 +316,9 @@ source-repository-package
     location: https://github.com/haskell-works/hw-aeson
     tag: d99d2f3e39a287607418ae605b132a3deb2b753f
     --sha256: 1vxqcwjg9q37wbwi27y9ba5163lzfz51f1swbi0rp681yg63zvn4
+
+-- Temporary indexing
+source-repository-package
+  type: git
+  location: https://github.com/raduom/hysterical-screams
+  tag: f3bbd38a19f99de5c8ddc650c94330b2d09a865b

doc/plutus/tutorials/Escrow6.hs

@@ -275,6 +275,6 @@ prop_CrashTolerance = CM.propRunActions_
 
 check_propEscrowWithCoverage :: IO ()
 check_propEscrowWithCoverage = do
-  cr <- CM.quickCheckWithCoverage stdArgs (set coverageIndex covIdx defaultCoverageOptions) $ \covopts ->
+  cr <- CM.quickCheckWithCoverage stdArgs (set coverageIndex covIdx $ defaultCoverageOptions) $ \covopts ->
     withMaxSuccess 1000 $ CM.propRunActionsWithOptions @EscrowModel CM.defaultCheckOptionsContractModel covopts (const (pure True))
   writeCoverageReport "Escrow" covIdx cr

doc/plutus/tutorials/EscrowImpl.hs

@@ -69,8 +69,9 @@ import Plutus.V1.Ledger.Api (Datum (Datum), DatumHash, ValidatorHash)
 import Plutus.V1.Ledger.Contexts (ScriptContext (ScriptContext, scriptContextTxInfo), TxInfo (txInfoValidRange))
 
 import Plutus.Contract (AsContractError (_ContractError), Contract, ContractError, Endpoint, HasEndpoint, Promise,
-                        awaitTime, currentTime, endpoint, mapError, mkTxConstraints, ownPaymentPubKeyHash, promiseMap,
-                        selectList, submitUnbalancedTx, type (.\/), utxosAt, waitNSlots)
+                        adjustUnbalancedTx, awaitTime, currentTime, endpoint, mapError, mkTxConstraints,
+                        ownPaymentPubKeyHash, promiseMap, selectList, submitUnbalancedTx, type (.\/), utxosAt,
+                        waitNSlots)
 import Plutus.Contract.Typed.Tx qualified as Typed
 import PlutusTx qualified
 {- START imports -}
@@ -80,7 +81,7 @@ import PlutusTx.Coverage qualified as PlutusTx
 import PlutusTx.Prelude (Bool (False), Either (Left, Right), all, either, foldl, id, mempty, traceIfFalse, ($), (&&),
                          (+), (-), (.), (<$>), (==), (>=))
 
-import Prelude (Semigroup ((<>)), foldMap)
+import Prelude (Semigroup ((<>)), foldMap, (>>=))
 import Prelude qualified as Haskell
 
 type EscrowSchema =
@@ -269,8 +270,8 @@ pay inst escrow vl = do
     pk <- ownPaymentPubKeyHash
     let tx = Constraints.mustPayToTheScript pk vl
           <> Constraints.mustValidateIn (Ledger.interval 1 (escrowDeadline escrow))
-    utx <- mkTxConstraints (Constraints.typedValidatorLookups inst) tx
-    getCardanoTxId <$> submitUnbalancedTx (Constraints.adjustUnbalancedTx utx)
+    utx <- mkTxConstraints (Constraints.typedValidatorLookups inst) tx >>= adjustUnbalancedTx
+    getCardanoTxId <$> submitUnbalancedTx utx
 
 newtype RedeemSuccess = RedeemSuccess TxId
     deriving (Haskell.Eq, Haskell.Show)
@@ -312,8 +313,8 @@ redeem inst escrow = mapError (review _EscrowError) $ do
          else do
            utx <- mkTxConstraints ( Constraints.typedValidatorLookups inst
                                  <> Constraints.unspentOutputs unspentOutputs
-                                  ) tx
-           RedeemSuccess . getCardanoTxId <$> submitUnbalancedTx (Constraints.adjustUnbalancedTx utx)
+                                  ) tx >>= adjustUnbalancedTx
+           RedeemSuccess . getCardanoTxId <$> submitUnbalancedTx utx
 
 newtype RefundSuccess = RefundSuccess TxId
     deriving newtype (Haskell.Eq, Haskell.Show, Generic)
@@ -344,8 +345,8 @@ refund inst escrow = do
     then do
         utx <- mkTxConstraints ( Constraints.typedValidatorLookups inst
                               <> Constraints.unspentOutputs unspentOutputs
-                               ) tx'
-        RefundSuccess . getCardanoTxId <$> submitUnbalancedTx (Constraints.adjustUnbalancedTx utx)
+                               ) tx' >>= adjustUnbalancedTx
+        RefundSuccess . getCardanoTxId <$> submitUnbalancedTx utx
     else throwing _RefundFailed ()
 
 -- | Pay some money into the escrow contract. Then release all funds to their

nix/pkgs/haskell/default.nix

@@ -25,10 +25,8 @@ let
     in
     parseIndexState (builtins.readFile ../../../cabal.project);
 
-  # The compiler that we are using. We are using a patched version so we need to specify it explicitly.
-  # This version has the experimental core interface files patch, and a fix for unboxed tuples in
-  # GHCi, which helps with HLS.
-  compiler-nix-name = "ghc810420210212";
+  # The compiler that we are using
+  compiler-nix-name = "ghc8107";
 
   # The haskell project created by haskell-nix.stackProject'
   baseProject =

nix/pkgs/haskell/extra.nix

@@ -44,8 +44,6 @@ let
       "https://github.com/haskell/lsp.git"."ef59c28b41ed4c5775f0ab0c1e985839359cec96" = "1whcgw4hhn2aplrpy9w8q6rafwy7znnp0rczgr6py15fqyw2fwb5";
     };
     modules = [{
-      # for compatibility with the GHC patch for extensible interfaces, not needed on mainline GHC.
-      packages.ghcide.patches = [ ../../patches/ghcide_partial_iface.patch ];
       # Workaround for https://github.com/haskell/haskell-language-server/issues/1160
       packages.haskell-language-server.patches = lib.mkIf stdenv.isDarwin [ ../../patches/haskell-language-server-dynamic.patch ];
       # See https://github.com/haskell/haskell-language-server/pull/1382#issuecomment-780472005

nix/pkgs/haskell/haskell.nix

@@ -61,6 +61,7 @@ let
             plutus-pab-executables.package.buildable = false;
             plutus-playground-server.package.buildable = false; # Would also require libpq
             plutus-script-utils.package.buildable = false;
+            plutus-streaming.package.buildable = false;
             plutus-tx-plugin.package.buildable = false;
             plutus-use-cases.package.buildable = false;
             plutus-example.package.buildable = false;

nix/pkgs/haskell/sha256map.nix

@@ -19,4 +19,5 @@
   "https://github.com/input-output-hk/Win32-network"."3825d3abf75f83f406c1f7161883c438dac7277d" = "19wahfv726fa3mqajpqdqhnl9ica3xmf68i254q45iyjcpj1psqx";
   "https://github.com/Quid2/flat"."ee59880f47ab835dbd73bea0847dab7869fc20d8" = "1lrzknw765pz2j97nvv9ip3l1mcpf2zr4n56hwlz0rk7wq7ls4cm";
   "https://github.com/vshabanov/ekg-json"."00ebe7211c981686e65730b7144fbf5350462608" = "sha256-VT8Ur585TCn03P2TVi6t92v2Z6tl8vKijICjse6ocv8=";
+  "https://github.com/raduom/hysterical-screams"."f3bbd38a19f99de5c8ddc650c94330b2d09a865b" = "1sy9l3sz1vxw7si9sdhh30p3zzadxh346856kdk4qg45j0m0gf1l";
 }

plutus-chain-index/app/Indexer.hs

@@ -0,0 +1,146 @@
+{-# LANGUAGE ExplicitForAll #-}
+{-# LANGUAGE GADTs          #-}
+{-# LANGUAGE LambdaCase     #-}
+{-# LANGUAGE TupleSections  #-}
+
+module Main where
+
+import Cardano.Api (Block (Block), BlockHeader (BlockHeader), BlockInMode (BlockInMode), BlockNo (BlockNo), CardanoMode,
+                    ChainPoint (ChainPoint), Hash, NetworkId (Mainnet), SlotNo (SlotNo), chainPointToSlotNo,
+                    deserialiseFromRawBytesHex, proxyToAsType)
+import Cardano.Api qualified as C
+import Cardano.BM.Trace (nullTracer)
+import Cardano.Protocol.Socket.Client (ChainSyncEvent (Resume, RollBackward, RollForward), runChainSync)
+import Control.Concurrent (threadDelay)
+import Control.Lens.Operators ((^.))
+import Control.Monad (forever, when)
+import Data.IORef (IORef, newIORef, readIORef, writeIORef)
+import Data.List (findIndex)
+import Data.Map (assocs)
+import Data.Maybe (fromJust, fromMaybe)
+import Data.Proxy (Proxy (Proxy))
+import Data.Text (pack)
+import Data.Text.Encoding (encodeUtf8)
+import Index.VSplit qualified as Ix
+import Ledger.TimeSlot (SlotConfig (..))
+import Marconi.Index.Datum (DatumIndex)
+import Marconi.Index.Datum qualified as Ix
+import Options.Applicative (Parser, execParser, fullDesc, header, help, helper, info, long, metavar, progDesc,
+                            strOption, (<**>))
+import Plutus.ChainIndex.Tx (ChainIndexTx (..))
+import Plutus.Contract.CardanoAPI (fromCardanoTx)
+import Plutus.Script.Utils.V1.Scripts (Datum, DatumHash)
+
+{- | This executable is meant to exercise a set of indexers (for now datumhash -> datum)
+     against the mainnet (meant to be used for testing).
+
+     In case you want to access the results of the datumhash indexer you need to query
+     the resulting database:
+     $ sqlite3 datums.sqlite
+     > select slotNo, datumHash, datum from kv_datumhsh_datum where slotNo = 39920450;
+     39920450|679a55b523ff8d61942b2583b76e5d49498468164802ef1ebe513c685d6fb5c2|X(002f9787436835852ea78d3c45fc3d436b324184
+-}
+
+-- Options
+data Options = Options
+  { socketPath :: FilePath
+  , dbPath     :: FilePath
+  }
+
+options :: Parser Options
+options = Options
+      <$> strOption
+          ( long "socket"
+         <> metavar "SOCKET"
+         <> help "Path to node socket." )
+      <*> strOption
+          ( long "database"
+         <> metavar "DATABASE"
+         <> help "Path to database." )
+
+-- We only care about the mainnet
+slotConfig :: SlotConfig
+slotConfig =
+  SlotConfig
+    { scSlotZeroTime = 1596059091000
+    , scSlotLength   = 1000
+    }
+
+networkId :: NetworkId
+networkId = Mainnet
+
+-- We don't generally need to sync blocks earlier than the Goguen era (other than
+-- testing for memory leaks) so we may want to start synchronising from a slot that
+-- is closer to Goguen era.
+closeToGoguen :: ChainPoint
+closeToGoguen =
+  ChainPoint
+    (SlotNo 39795032)
+    (fromJust $ parseHash "3e6f6450f85962d651654ee66091980b2332166f5505fd10b97b0520c9efac90")
+
+parseHash :: String -> Maybe (Hash BlockHeader)
+parseHash hash =
+  deserialiseFromRawBytesHex (proxyToAsType Proxy) (encodeUtf8 $ pack hash)
+
+getDatums :: BlockInMode CardanoMode -> [(SlotNo, (DatumHash, Datum))]
+getDatums (BlockInMode (Block (BlockHeader slotNo _ _) txs) era) =
+  case era of
+    C.ByronEraInCardanoMode   -> concatMap (go era) txs
+    C.ShelleyEraInCardanoMode -> concatMap (go era) txs
+    C.AllegraEraInCardanoMode -> concatMap (go era) txs
+    C.MaryEraInCardanoMode    -> concatMap (go era) txs
+    C.AlonzoEraInCardanoMode  -> concatMap (go era) txs
+    C.BabbageEraInCardanoMode -> concatMap (go era) txs
+  where
+    go :: C.IsCardanoEra era
+       => C.EraInMode era C.CardanoMode
+       -> C.Tx era
+       -> [(SlotNo, (DatumHash, Datum))]
+    go era' tx =
+      let hashes = either (const []) (assocs . _citxData) $ fromCardanoTx era' tx
+      in  map (slotNo,) hashes
+
+processBlock :: IORef DatumIndex -> ChainSyncEvent -> IO ()
+processBlock ixref = \case
+  -- Not really supported
+  Resume point         -> putStrLn ("resume " <> show point) >> pure ()
+  RollForward blk@(BlockInMode (Block (BlockHeader slotNo _ blockNo@(BlockNo b)) _txs) _era) _tip -> do
+    when (b `rem` 1000 == 0) $
+      putStrLn $ show slotNo <> " / " <> show blockNo
+    ix  <- readIORef ixref
+    ix' <- Ix.insert (getDatums blk) ix
+    writeIORef ixref ix'
+  RollBackward point tip -> do
+    putStrLn ("rollback to " <> show tip)
+    rollbackToPoint point ixref
+
+rollbackToPoint
+  :: ChainPoint -> IORef DatumIndex -> IO ()
+rollbackToPoint point ixref = do
+  ix     <- readIORef ixref
+  events <- Ix.getEvents (ix ^. Ix.storage)
+  let ix' = fromMaybe ix $ rollbackOffset events ix
+  writeIORef ixref ix'
+  where
+    rollbackOffset :: [Ix.Event] -> DatumIndex -> Maybe DatumIndex
+    rollbackOffset events ix = do
+      slot   <- chainPointToSlotNo point
+      offset <- findIndex (any (\(s, _) -> s < slot)) events
+      Ix.rewind offset ix
+
+main :: IO ()
+main = do
+  options' <- execParser opts
+  tix <- Ix.open (dbPath options') (Ix.Depth 2160) >>= newIORef
+  _ <- runChainSync (socketPath options')
+                    nullTracer
+                    slotConfig
+                    networkId
+                    [closeToGoguen]
+                    (processBlock tix)
+  forever $ threadDelay 1000000000
+  where
+    opts = info (options <**> helper)
+             ( fullDesc
+            <> progDesc "Synchronise datums with mainnet"
+            <> header "indexer - an indexing proof of concept" )

plutus-chain-index/plutus-chain-index.cabal

@@ -37,7 +37,9 @@ library
         Plutus.ChainIndex.Lib
         Plutus.ChainIndex.Logging
         Plutus.ChainIndex.SyncStats
-    other-modules: Control.Concurrent.STM.TBMQueue
+        Marconi.Index.Datum
+    other-modules:
+        Control.Concurrent.STM.TBMQueue
     hs-source-dirs: src
     build-depends:
         plutus-ledger -any,
@@ -65,6 +67,9 @@ library
         stm -any,
         time-units -any,
         yaml -any,
+        serialise -any,
+        hysterical-screams -any,
+        bytestring -any,
 
 executable plutus-chain-index
   main-is: Main.hs
@@ -78,3 +83,26 @@ executable plutus-chain-index
   build-depends:
     base >=4.9 && <5,
     plutus-chain-index -any
+
+executable plutus-indexer
+  main-is: Indexer.hs
+  hs-source-dirs: app
+  default-language: Haskell2010
+  default-extensions: ImportQualifiedPost
+  ghc-options:
+        -threaded -rtsopts -with-rtsopts=-N -Wall -Wcompat
+        -Wincomplete-uni-patterns -Wincomplete-record-updates
+        -Wno-missing-import-lists -Wredundant-constraints -O0
+  build-depends:
+    base >=4.9 && <5,
+    plutus-chain-index -any,
+    iohk-monitoring -any,
+    cardano-api -any,
+    plutus-ledger -any,
+    plutus-chain-index-core -any,
+    plutus-script-utils -any,
+    containers -any,
+    lens -any,
+    hysterical-screams -any,
+    text -any,
+    optparse-applicative -any,