feat: [ADR-070] Unordered Transactions (1/2)

client/flags/flags.go

@@ -74,6 +74,7 @@ const (
 	FlagOffset           = "offset"
 	FlagCountTotal       = "count-total"
 	FlagTimeoutHeight    = "timeout-height"
+	FlagUnordered        = "unordered"
 	FlagKeyAlgorithm     = "algo"
 	FlagKeyType          = "key-type"
 	FlagFeePayer         = "fee-payer"
@@ -136,6 +137,7 @@ func AddTxFlagsToCmd(cmd *cobra.Command) {
 	f.BoolP(FlagSkipConfirmation, "y", false, "Skip tx broadcasting prompt confirmation")
 	f.String(FlagSignMode, "", "Choose sign mode (direct|amino-json|direct-aux|textual), this is an advanced feature")
 	f.Uint64(FlagTimeoutHeight, 0, "Set a block timeout height to prevent the tx from being committed past a certain height")
+	f.Bool(FlagUnordered, false, "Enable unordered transaction delivery; must be used in conjunction with --timeout-height")
 	f.String(FlagFeePayer, "", "Fee payer pays fees for the transaction instead of deducting from the signer")
 	f.String(FlagFeeGranter, "", "Fee granter grants fees for the transaction")
 	f.String(FlagTip, "", "Tip is the amount that is going to be transferred to the fee payer on the target chain. This flag is only valid when used with --aux, and is ignored if the target chain didn't enable the TipDecorator")

client/tx/factory.go

@@ -36,6 +36,7 @@ type Factory struct {
 	gasAdjustment      float64
 	chainID            string
 	fromName           string
+	unordered          bool
 	offline            bool
 	generateOnly       bool
 	memo               string
@@ -86,6 +87,7 @@ func NewFactoryCLI(clientCtx client.Context, flagSet *pflag.FlagSet) (Factory, e
 	gasAdj := clientCtx.Viper.GetFloat64(flags.FlagGasAdjustment)
 	memo := clientCtx.Viper.GetString(flags.FlagNote)
 	timeoutHeight := clientCtx.Viper.GetUint64(flags.FlagTimeoutHeight)
+	unordered := clientCtx.Viper.GetBool(flags.FlagUnordered)
 
 	gasStr := clientCtx.Viper.GetString(flags.FlagGas)
 	gasSetting, _ := flags.ParseGasSetting(gasStr)
@@ -103,6 +105,7 @@ func NewFactoryCLI(clientCtx client.Context, flagSet *pflag.FlagSet) (Factory, e
 		accountNumber:      accNum,
 		sequence:           accSeq,
 		timeoutHeight:      timeoutHeight,
+		unordered:          unordered,
 		gasAdjustment:      gasAdj,
 		memo:               memo,
 		signMode:           signMode,
@@ -132,6 +135,7 @@ func (f Factory) Fees() sdk.Coins                           { return f.fees }
 func (f Factory) GasPrices() sdk.DecCoins                   { return f.gasPrices }
 func (f Factory) AccountRetriever() client.AccountRetriever { return f.accountRetriever }
 func (f Factory) TimeoutHeight() uint64                     { return f.timeoutHeight }
+func (f Factory) Unordered() bool                           { return f.unordered }
 
 // SimulateAndExecute returns the option to simulate and then execute the transaction
 // using the gas from the simulation results
@@ -237,6 +241,12 @@ func (f Factory) WithTimeoutHeight(height uint64) Factory {
 	return f
 }
 
+// WithUnordered returns a copy of the Factory with an updated unordered field.
+func (f Factory) WithUnordered(v bool) Factory {
+	f.unordered = v
+	return f
+}
+
 // WithFeeGranter returns a copy of the Factory with an updated fee granter.
 func (f Factory) WithFeeGranter(fg sdk.AccAddress) Factory {
 	f.feeGranter = fg

client/tx_config.go

@@ -48,6 +48,7 @@ type (
 		SetFeePayer(feePayer sdk.AccAddress)
 		SetGasLimit(limit uint64)
 		SetTimeoutHeight(height uint64)
+		SetUnordered(v bool)
 		SetFeeGranter(feeGranter sdk.AccAddress)
 		AddAuxSignerData(tx.AuxSignerData) error
 	}

docs/architecture/adr-070-unordered-account.md

@@ -2,7 +2,7 @@
 
 ## Changelog
 
-* Dec 4, 2023: Initial Draft
+* Dec 4, 2023: Initial Draft (@yihuang, @tac0turtle, @alexanderbez)
 
 ## Status
 
@@ -53,108 +53,147 @@ message TxBody {
 }
 ```
 
-### `DedupTxHashManager`
+### Replay Protection
+
+In order to provide replay protection, a user should ensure that the transaction's
+TTL value is relatively short-lived but long enough to provide enough time to be
+included in a block, e.g. ~H+50. 
+
+We facilitate this by storing the transaction's hash in a durable map, `UnorderedTxManager`,
+to prevent duplicates, i.e. replay attacks. Upon transaction ingress during `CheckTx`,
+we check if the transaction's hash exists in this map or if the TTL value is stale,
+i.e. before the current block. If so, we reject it. Upon inclusion in a block
+during `DeliverTx`, the transaction's hash is set in the map along with it's TTL
+value.
+
+This map is evaluated at the end of each block, e.g. ABCI `Commit`, and all stale
+transactions, i.e. transactions's TTL value who's now beyond the committed block,
+are purged from the map.
+
+An important point to note is that in theory, it may be possible to submit an unordered
+transaction twice, or multiple times, before the transaction is included in a block.
+However, we'll note a few important layers of protection and mitigation:
+
+* Assuming CometBFT is used as the underlying consensus engine and a non-noop mempool
+  is used, CometBFT will reject the duplicate for you.
+* For applications that leverage ABCI++, `ProcessProposal` should evaluate and reject
+  malicious proposals with duplicate transactions.
+* For applications that leverage their own application mempool, their mempool should
+  reject the duplicate for you.
+* Finally, worst case if the duplicate transaction is somehow selected for a block
+  proposal, 2nd and all further attempts to evaluate it, will fail during `DeliverTx`,
+  so worst case you just end up filling up block space with a duplicate transaction.
 
 ```golang
+type TxHash [32]byte
+
 const PurgeLoopSleepMS = 500
 
-// DedupTxHashManager contains the tx hash dictionary for duplicates checking,
-// and expire them when block number progresses.
-type DedupTxHashManager struct {
-  mutex sync.RWMutex
-  // tx hash -> expire block number
-  // for duplicates checking and expiration
-  hashes map[TxHash]uint64
-  // channel to receive latest block numbers
+// UnorderedTxManager contains the tx hash dictionary for duplicates checking,
+// and expire them when block production progresses.
+type UnorderedTxManager struct {
+  // blockCh defines a channel to receive newly committed block heights
   blockCh chan uint64
+
+  mu sync.RWMutex
+	// txHashes defines a map from tx hash -> TTL value, which is used for duplicate
+	// checking and replay protection, as well as purging the map when the TTL is
+	// expired.
+	txHashes map[TxHash]uint64
 }
 
-func NewDedupTxHashManager() *DedupTxHashManager {
-  m := &DedupTxHashManager{
-    hashes: make(map[TxHash]uint64),
-    blockCh: make(ch *uint64, 16),
+func NewUnorderedTxManager() *UnorderedTxManager {
+  m := &UnorderedTxManager{
+		blockCh:  make(chan uint64, 16),
+		txHashes: make(map[TxHash]uint64),
   }
+
+ return m
+}
+
+func (m *UnorderedTxManager) Start() {
   go m.purgeLoop()
-  return m
 }
 
-func (dtm *DedupTxHashManager) Close() error {
-  close(dtm.blockCh)
-  dtm.blockCh = nil
+func (m *UnorderedTxManager) Close() error {
+  close(m.blockCh)
+  m.blockCh = nil
   return nil
 }
 
-func (dtm *DedupTxHashManager) Contains(hash TxHash) (ok bool) {
-  dtm.mutex.RLock()
-  defer dtm.mutex.RUnlock()
+func (m *UnorderedTxManager) Contains(hash TxHash)  bool{
+  m.mu.RLock()
+  defer m.mu.RUnlock()
 
-  _, ok = dtm.hashes[hash]
-  return
+  _, ok := m.txHashes[hash]
+  return ok
 }
 
-func (dtm *DedupTxHashManager) Size() int {
-  dtm.mutex.RLock()
-  defer dtm.mutex.RUnlock()
+func (m *UnorderedTxManager) Size() int {
+  m.mu.RLock()
+  defer m.mu.RUnlock()
 
-  return len(dtm.hashes)
+  return len(m.txHashes)
 }
 
-func (dtm *DedupTxHashManager) Add(hash TxHash, expire uint64) (ok bool) {
-  dtm.mutex.Lock()
-  defer dtm.mutex.Unlock()
+func (m *UnorderedTxManager) Add(hash TxHash, expire uint64) {
+  m.mu.Lock()
+  defer m.mu.Unlock()
+
+  m.txHashes[hash] = expire
+}
 
-  dtm.hashes[hash] = expire
-  return
+// OnNewBlock send the latest block number to the background purge loop, which
+// should be called in ABCI Commit event.
+func (m *UnorderedTxManager) OnNewBlock(blockHeight uint64) {
+  m.blockCh <- blockHeight
+}
+
+// expiredTxs returns expired tx hashes based on the provided block height.
+func (m *UnorderedTxManager) expiredTxs(blockHeight uint64) []TxHash {
+  m.mu.RLock()
+  defer m.mu.RUnlock()
+
+  var result []TxHash
+  for txHash, expire := range m.txHashes {
+    if blockHeight > expire {
+      result = append(result, txHash)
+    }
+  }
+
+  return result
 }
 
-// OnNewBlock send the latest block number to the background purge loop,
-// it should be called in abci commit event.
-func (dtm *DedupTxHashManager) OnNewBlock(blockNumber uint64) {
-  dtm.blockCh <- &blockNumber
+func (m *UnorderedTxManager) purge(txHashes []TxHash) {
+  m.mu.Lock()
+  defer m.mu.Unlock()
+
+  for _, txHash := range txHashes {
+    delete(m.txHashes, txHash)
+  }
 }
 
-// purgeLoop removes expired tx hashes at background
-func (dtm *DedupTxHashManager) purgeLoop() error {
+
+// purgeLoop removes expired tx hashes in the background
+func (m *UnorderedTxManager) purgeLoop() error {
   for {
-    blocks := channelBatchRecv(dtm.blockCh)
+    blocks := channelBatchRecv(m.blockCh)
     if len(blocks) == 0 {
       // channel closed
       break
     }
 
     latest := *blocks[len(blocks)-1]
-    hashes := dtm.expired(latest)
+    hashes := m.expired(latest)
     if len(hashes) > 0 {
-      dtm.purge(hashes)
+      m.purge(hashes)
     }
 
     // avoid burning cpu in catching up phase
     time.Sleep(PurgeLoopSleepMS * time.Millisecond)
   }
 }
 
-// expired find out expired tx hashes based on latest block number
-func (dtm *DedupTxHashManager) expired(block uint64) []TxHash {
-  dtm.mutex.RLock()
-  defer dtm.mutex.RUnlock()
-
-  var result []TxHash
-  for h, expire := range dtm.hashes {
-    if block > expire {
-      result = append(result, h)
-    }
-  }
-  return result
-}
-
-func (dtm *DedupTxHashManager) purge(hashes []TxHash) {
-  dtm.mutex.Lock()
-  defer dtm.mutex.Unlock()
-
-  for _, hash := range hashes {
-    delete(dtm.hashes, hash)
-  }
-}
 
 // channelBatchRecv try to exhaust the channel buffer when it's not empty,
 // and block when it's empty.
@@ -176,35 +215,38 @@ func channelBatchRecv[T any](ch <-chan *T) []*T {
 }
 ```
 
-### Ante Handlers
+### AnteHandler Decorator
 
-Bypass the nonce decorator for un-ordered transactions.
+In order to facilitate bypassing nonce verification, we have to modify the existing
+`IncrementSequenceDecorator` AnteHandler decorator to skip the nonce verification
+when the transaction is marked as un-ordered.
 
 ```golang
 func (isd IncrementSequenceDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) {
   if tx.UnOrdered() {
     return next(ctx, tx, simulate)
   }
 
-  // the previous logic
+  // ...
 }
 ```
 
-A decorator for the new logic.
+In addition, we need to introduce a new decorator to perform the un-ordered transaction
+verification and map lookup.
 
 ```golang
-type TxHash [32]byte
-
 const (
-  // MaxUnOrderedTTL defines the maximum ttl an un-order tx can set
-  MaxUnOrderedTTL = 1024
+	// DefaultMaxUnOrderedTTL defines the default maximum TTL an un-ordered transaction
+	// can set.
+	DefaultMaxUnOrderedTTL = 1024
 )
 
 type DedupTxDecorator struct {
-  m *DedupTxHashManager
+  m *UnorderedTxManager
+  maxUnOrderedTTL uint64
 }
 
-func (dtd *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) {
+func (d *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) {
   // only apply to un-ordered transactions
   if !tx.UnOrdered() {
     return next(ctx, tx, simulate)
@@ -214,18 +256,18 @@ func (dtd *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate boo
     return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "unordered tx must set timeout-height")
   }
 
-  if tx.TimeoutHeight() > ctx.BlockHeight() + MaxUnOrderedTTL {
-    return nil, errorsmod.Wrapf(sdkerrors.ErrLogic, "unordered tx ttl exceeds %d", MaxUnOrderedTTL)
+  if tx.TimeoutHeight() > ctx.BlockHeight() + d.maxUnOrderedTTL {
+    return nil, errorsmod.Wrapf(sdkerrors.ErrLogic, "unordered tx ttl exceeds %d", d.maxUnOrderedTTL)
   }
 
   // check for duplicates
-  if dtd.m.Contains(tx.Hash()) {
+  if d.m.Contains(tx.Hash()) {
     return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "tx is duplicated")
   }
 
   if !ctx.IsCheckTx() {
-    // a new tx included in the block, add the hash to the dictionary
-    dtd.m.Add(tx.Hash(), tx.TimeoutHeight())
+    // a new tx included in the block, add the hash to the unordered tx manager
+    d.m.Add(tx.Hash(), tx.TimeoutHeight())
   }
 
   return next(ctx, tx, simulate)
@@ -234,11 +276,11 @@ func (dtd *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate boo
 
 ### `OnNewBlock`
 
-Wire the `OnNewBlock` method of `DedupTxHashManager` into the BaseApp's ABCI Commit event.
+Wire the `OnNewBlock` method of `UnorderedTxManager` into the BaseApp's ABCI `Commit` event.
 
 ### Start Up
 
-On start up, the node needs to re-fill the tx hash dictionary of `DedupTxHashManager`
+On start up, the node needs to re-fill the tx hash dictionary of `UnorderedTxManager`
 by scanning `MaxUnOrderedTTL` number of historical blocks for existing un-expired
 un-ordered transactions.