Add Para threads to MMR Root

relayer/chain/relaychain/connection.go

@@ -6,6 +6,7 @@ package relaychain
 import (
 	"context"
 	"fmt"
+	"sort"
 
 	gsrpc "github.com/snowfork/go-substrate-rpc-client/v4"
 	"github.com/snowfork/go-substrate-rpc-client/v4/types"
@@ -130,37 +131,6 @@ type ParaHead struct {
 	Data   types.Bytes
 }
 
-// Fetches heads for each parachain Id filtering out para threads.
-func (conn *Connection) FetchParachainHeads(relayChainBlockHash types.Hash) ([]ParaHead, error) {
-	// Fetch para heads
-	paraHeads, err := conn.fetchParaHeads(relayChainBlockHash)
-	if err != nil {
-		log.WithError(err).Error("Cannot fetch para heads.")
-		return nil, err
-	}
-
-	// fetch ids of parachains (not including parathreads)
-	var parachainIDs []uint32
-	parachainsKey, err := types.CreateStorageKey(conn.Metadata(), "Paras", "Parachains", nil, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	_, err = conn.API().RPC.State.GetStorage(parachainsKey, &parachainIDs, relayChainBlockHash)
-	if err != nil {
-		return nil, err
-	}
-
-	// filter out parathreads
-	var parachainHeads []ParaHead
-	for _, v := range parachainIDs {
-		if head, ok := paraHeads[v]; ok {
-			parachainHeads = append(parachainHeads, head)
-		}
-	}
-	return parachainHeads, nil
-}
-
 func (co *Connection) FetchParachainHead(relayBlockhash types.Hash, paraID uint32, header *types.Header) (bool, error) {
 	encodedParaID, err := types.EncodeToBytes(paraID)
 	if err != nil {
@@ -272,7 +242,8 @@ func (co *Connection) fetchKeys(keyPrefix []byte, blockHash types.Hash) ([]types
 //	Key: hash_twox_128("Paras") + hash_twox_128("Heads") + hash_twox_64(ParaId) + Encode(ParaId)
 const ParaIDOffset = 16 + 16 + 8
 
-func (co *Connection) fetchParaHeads(blockHash types.Hash) (map[uint32]ParaHead, error) {
+// Fetch heads for all Paras. Included are parachains and parathreads.
+func (co *Connection) FetchParasHeads(blockHash types.Hash) ([]ParaHead, error) {
 	keyPrefix := types.CreateStorageKeyPrefix("Paras", "Heads")
 	keys, err := co.fetchKeys(keyPrefix, blockHash)
 	if err != nil {
@@ -292,7 +263,7 @@ func (co *Connection) fetchParaHeads(blockHash types.Hash) (map[uint32]ParaHead,
 		return nil, err
 	}
 
-	heads := make(map[uint32]ParaHead)
+	heads := make([]ParaHead, 0, 32)
 	for _, changeSet := range changeSets {
 		for _, change := range changeSet.Changes {
 			if change.StorageData.IsNone() {
@@ -313,12 +284,47 @@ func (co *Connection) fetchParaHeads(blockHash types.Hash) (map[uint32]ParaHead,
 				return nil, err
 			}
 
-			heads[paraID] = ParaHead{
+			heads = append(heads, ParaHead{
 				ParaID: paraID,
 				Data:   headData,
-			}
+			})
 		}
 	}
 
+	sort.SliceStable(heads, func(i int, j int) bool {
+		return heads[i].ParaID < heads[j].ParaID
+	})
+
+	return heads, nil
+}
+
+// Filters para heads to parachains only.
+func (conn *Connection) FilterParachainHeads(paraHeads []ParaHead, relayChainBlockHash types.Hash) ([]ParaHead, error) {
+
+	// fetch ids of parachains (not including parathreads)
+	var parachainIDs []uint32
+	parachainsKey, err := types.CreateStorageKey(conn.Metadata(), "Paras", "Parachains", nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = conn.API().RPC.State.GetStorage(parachainsKey, &parachainIDs, relayChainBlockHash)
+	if err != nil {
+		return nil, err
+	}
+
+	// create a set of parachains
+	parachains := make(map[uint32]struct{}, len(paraHeads))
+	for _, parachain := range parachainIDs {
+		parachains[parachain] = struct{}{}
+	}
+
+	// filter to return parachains
+	heads := make([]ParaHead, 0, len(paraHeads))
+	for _, head := range paraHeads {
+		if _, ok := parachains[head.ParaID]; ok {
+			heads = append(heads, head)
+		}
+	}
 	return heads, nil
 }

relayer/cmd/parachain_head_proof.go

@@ -92,7 +92,7 @@ func ParachainHeadProofFn(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	paraHeadsAsSlice, err := conn.FetchParachainHeads(relayChainBlockHash)
+	paraHeadsAsSlice, err := conn.FetchParasHeads(relayChainBlockHash)
 	if err != nil {
 		log.WithError(err).Error("Cannot fetch parachain headers")
 		return err

relayer/relays/parachain/beefy-listener.go

@@ -217,6 +217,10 @@ func (li *BeefyListener) fetchLatestBeefyBlock(ctx context.Context) (uint64, typ
 	return number, hash, nil
 }
 
+// The maximum paras that will be included in the proof.
+// https://github.com/paritytech/polkadot-sdk/blob/d66dee3c3da836bcf41a12ca4e1191faee0b6a5b/polkadot/runtime/parachains/src/paras/mod.rs#L1225-L1232
+const MaxParaHeads = 1024
+
 // Generates a proof for an MMR leaf, and then generates a merkle proof for our parachain header, which should be verifiable against the
 // parachains root in the mmr leaf.
 func (li *BeefyListener) generateProof(ctx context.Context, input *ProofInput, header *types.Header) (*ProofOutput, error) {
@@ -255,21 +259,10 @@ func (li *BeefyListener) generateProof(ctx context.Context, input *ProofInput, h
 		return nil, fmt.Errorf("retrieve MMR root hash at block %v: %w", latestBeefyBlockHash.Hex(), err)
 	}
 
-	// Generate a merkle proof for the parachain head with input ParaId
-	// and verify with merkle root hash of all parachain heads
-	// Polkadot uses the following code to generate merkle root from parachain headers:
-	// https://github.com/paritytech/polkadot/blob/2eb7672905d99971fc11ad7ff4d57e68967401d2/runtime/rococo/src/lib.rs#L706-L709
-	merkleProofData, err := CreateParachainMerkleProof(input.ParaHeads, input.ParaID)
+	var merkleProofData *MerkleProofData
+	merkleProofData, input.ParaHeads, err = li.generateAndValidateParasHeadsMerkleProof(input, &mmrProof)
 	if err != nil {
-		return nil, fmt.Errorf("create parachain header proof: %w", err)
-	}
-
-	// Verify merkle root generated is same as value generated in relaychain
-	if merkleProofData.Root.Hex() != mmrProof.Leaf.ParachainHeads.Hex() {
-		return nil, fmt.Errorf("MMR parachain merkle root does not match calculated parachain merkle root (mmr: %s, computed: %s)",
-			mmrProof.Leaf.ParachainHeads.Hex(),
-			merkleProofData.Root.String(),
-		)
+		return nil, err
 	}
 
 	log.Debug("Created all parachain merkle proof data")
@@ -278,12 +271,54 @@ func (li *BeefyListener) generateProof(ctx context.Context, input *ProofInput, h
 		MMRProof:        simplifiedProof,
 		MMRRootHash:     mmrRootHash,
 		Header:          *header,
-		MerkleProofData: merkleProofData,
+		MerkleProofData: *merkleProofData,
 	}
 
 	return &output, nil
 }
 
+// Generate a merkle proof for the parachain head with input ParaId and verify with merkle root hash of all parachain heads
+func (li *BeefyListener) generateAndValidateParasHeadsMerkleProof(input *ProofInput, mmrProof *types.GenerateMMRProofResponse) (*MerkleProofData, []relaychain.ParaHead, error) {
+	// Polkadot uses the following code to generate merkle root from parachain headers:
+	// https://github.com/paritytech/polkadot-sdk/blob/d66dee3c3da836bcf41a12ca4e1191faee0b6a5b/polkadot/runtime/westend/src/lib.rs#L453-L460
+	// Truncate the ParaHeads to the 1024
+	// https://github.com/paritytech/polkadot-sdk/blob/d66dee3c3da836bcf41a12ca4e1191faee0b6a5b/polkadot/runtime/parachains/src/paras/mod.rs#L1305-L1311
+	paraHeads := input.ParaHeads
+	numParas := min(MaxParaHeads, len(paraHeads))
+	merkleProofData, err := CreateParachainMerkleProof(paraHeads[:numParas], input.ParaID)
+	if err != nil {
+		return nil, paraHeads, fmt.Errorf("create parachain header proof: %w", err)
+	}
+
+	// Verify merkle root generated is same as value generated in relaychain and if so exit early
+	if merkleProofData.Root.Hex() == mmrProof.Leaf.ParachainHeads.Hex() {
+		return &merkleProofData, paraHeads, nil
+	}
+
+	// Try a filtering out parathreads
+	log.WithFields(log.Fields{
+		"beefyBlock": merkleProofData.Root.Hex(),
+		"leafIndex":  mmrProof.Leaf.ParachainHeads.Hex(),
+	}).Warn("MMR parachain merkle root does not match calculated merkle root. Trying to filtering out parathreads.")
+
+	paraHeads, err = li.relaychainConn.FilterParachainHeads(paraHeads, input.RelayBlockHash)
+	if err != nil {
+		return nil, paraHeads, fmt.Errorf("could not filter out parathreads: %w", err)
+	}
+
+	merkleProofData, err = CreateParachainMerkleProof(paraHeads[:numParas], input.ParaID)
+	if err != nil {
+		return nil, paraHeads, fmt.Errorf("create parachain header proof: %w", err)
+	}
+	if merkleProofData.Root.Hex() != mmrProof.Leaf.ParachainHeads.Hex() {
+		return nil, paraHeads, fmt.Errorf("MMR parachain merkle root does not match calculated parachain merkle root (mmr: %s, computed: %s)",
+			mmrProof.Leaf.ParachainHeads.Hex(),
+			merkleProofData.Root.String(),
+		)
+	}
+	return &merkleProofData, paraHeads, nil
+}
+
 func (li *BeefyListener) waitAndSend(ctx context.Context, task *Task, waitingPeriod uint64) error {
 	paraNonce := (*task.MessageProofs)[0].Message.Nonce
 	log.Info(fmt.Sprintf("waiting for nonce %d to be picked up by another relayer", paraNonce))

relayer/relays/parachain/scanner.go

@@ -253,14 +253,15 @@ func (s *Scanner) gatherProofInputs(
 			return fmt.Errorf("fetch relaychain block hash: %w", err)
 		}
 
-		parachainHeads, err := s.relayConn.FetchParachainHeads(relayBlockHash)
+		parachainHeads, err := s.relayConn.FetchParasHeads(relayBlockHash)
 		if err != nil {
 			return fmt.Errorf("fetch parachain heads: %w", err)
 		}
 
 		task.ProofInput = &ProofInput{
 			ParaID:           s.paraID,
 			RelayBlockNumber: relayBlockNumber,
+			RelayBlockHash:   relayBlockHash,
 			ParaHeads:        parachainHeads,
 		}
 	}

relayer/relays/parachain/types.go

@@ -28,6 +28,8 @@ type ProofInput struct {
 	ParaID uint32
 	// Relay chain block number in which our parachain head was included
 	RelayBlockNumber uint64
+	// Relay chain block hash in which our parachain head was included
+	RelayBlockHash types.Hash
 	// All included paraheads in RelayBlockNumber
 	ParaHeads []relaychain.ParaHead
 }

smoketest/.envrc-westend

@@ -0,0 +1,26 @@
+source_up_if_exists
+
+export ETH_NETWORK=sepolia
+export POLKADOT_NETWORK=westend
+
+# Endpoints
+export BRIDGE_HUB_WS_URL=wss://westend-bridge-hub-rpc.polkadot.io
+export ASSET_HUB_WS_URL=wss://westend-asset-hub-rpc.polkadot.io
+export RELAY_CHAIN_WS_URL=wss://westend-rpc.polkadot.io
+export ETHEREUM_HTTP_API=https://sepolia.infura.io/v3/***
+export ETHEREUM_API=wss://sepolia.infura.io/ws/v3/***
+
+# Contract address
+export GATEWAY_PROXY_CONTRACT=9ed8b47bc3417e3bd0507adc06e56e2fa360a4e9
+export WETH_CONTRACT=fff9976782d46cc05630d1f6ebab18b2324d6b14
+
+# Receiver Accounts
+export SUBSTRATE_RECEIVER=5827013ddc4082f8252f8729bd2f06e77e7863dea9202a6f0e7a2c34e356e85a
+export ETHEREUM_RECEIVER=302F0B71B8aD3CF6dD90aDb668E49b2168d652fd
+
+# Sender Keys
+export ETHEREUM_KEY=*
+export SUBSTRATE_KEY=*
+
+# Wait period in blocks
+WAIT_PERIOD=1000

smoketest/.gitignore

@@ -16,3 +16,4 @@ src/parachains/bridgehub.rs
 src/parachains/penpal.rs
 src/parachains/relaychain.rs
 src/contracts
+.envrc