Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[NONEVM-984][SOAK] - Reorg Detection #969

Draft
wants to merge 106 commits into
base: develop
Choose a base branch
from
Draft
Changes from 1 commit
Commits
Show all changes
106 commits
Select commit Hold shift + click to select a range
2d1a82d
refactor so txm owns blockhash assignment
Farber98 Nov 15, 2024
50dfef0
lastValidBlockHeight shouldn't be exported
Farber98 Nov 15, 2024
4e545e2
better comment
Farber98 Nov 15, 2024
4ded53c
refactor sendWithRetry to make it clearer
Farber98 Nov 15, 2024
9e1be6d
confirm loop refactor
Farber98 Nov 18, 2024
7dd2028
fix infinite loop
Farber98 Nov 18, 2024
6c675f2
move accountID inside msg
Farber98 Nov 19, 2024
b0d9426
lint fix
Farber98 Nov 19, 2024
1b38665
base58 does not contain lower l
Farber98 Nov 19, 2024
6923ddf
fix hash errors
Farber98 Nov 19, 2024
462844b
fix generate random hash
Farber98 Nov 19, 2024
fd785d0
remove blockhash as we only need block height
Farber98 Nov 19, 2024
cf958a4
expired tx changes without tests
Farber98 Nov 19, 2024
c5e957b
add maybe to mocks
Farber98 Nov 19, 2024
a505993
expiration tests
Farber98 Nov 19, 2024
adc8b1c
send txes through queue
Farber98 Nov 19, 2024
7d77f99
revert pendingtx leakage of information. overwrite blockhash
Farber98 Nov 20, 2024
92a280b
fix order of confirm loop and not found signature check
Farber98 Nov 20, 2024
2598e19
fix mocks
Farber98 Nov 20, 2024
42b3da1
prevent confirmation loop to mark tx as errored when it needs to be r…
Farber98 Nov 20, 2024
89af1f3
fix test
Farber98 Nov 20, 2024
5e8a0da
fix pointer
Farber98 Nov 20, 2024
75c1dcd
add comments
Farber98 Nov 21, 2024
4ff2d23
reduce rpc calls + refactors
Farber98 Nov 21, 2024
84e423e
tests + check to save rpc calls
Farber98 Nov 21, 2024
7d8319e
address feedback + remove redundant impl
Farber98 Nov 22, 2024
68f3a3e
iface comment
Farber98 Nov 22, 2024
780179f
address feedback on compute unit limit and lastValidBlockHeight assig…
Farber98 Nov 25, 2024
98f0246
blockhash assignment inside txm.sendWithRetry
Farber98 Nov 25, 2024
cbf55f6
address feedback
Farber98 Nov 26, 2024
90daf33
Merge branch 'develop' into nonevm-706-support-custom-bumping-strateg…
Farber98 Nov 26, 2024
77b28cf
refactors after merge
Farber98 Nov 26, 2024
0c4a7d8
fix interactive rebase
Farber98 Nov 26, 2024
849ac48
fix whitespace diffs
Farber98 Nov 26, 2024
20a1548
fix import
Farber98 Nov 26, 2024
a4d4770
fix mocks
Farber98 Nov 26, 2024
56a64da
add on prebroadcaste error
Farber98 Nov 26, 2024
9148d7d
remove rebroadcast count and fix package
Farber98 Nov 27, 2024
caf2cbf
improve docs
Farber98 Nov 27, 2024
1fbd63f
Merge branch 'develop' into backup-branch-fee-bumping
Farber98 Nov 28, 2024
bdf3457
track status on each signature to detect reorgs
Farber98 Nov 28, 2024
3ad2bc8
move things arround + add reorg detection
Farber98 Nov 29, 2024
4fd327a
linting errors
Farber98 Nov 29, 2024
2b29c33
fix some state tracking instances
Farber98 Nov 29, 2024
a6ce47b
remove redundant sig update
Farber98 Nov 30, 2024
3a6e643
move state from txes to sigs
Farber98 Nov 30, 2024
f4c6069
fix listAllExpiredBroadcastedTxs
Farber98 Nov 30, 2024
f027aeb
handle reorg after confirm cycle
Farber98 Nov 30, 2024
8c18891
associate sigs to retry ctx
Farber98 Dec 2, 2024
2902ec0
remove unused ctx
Farber98 Dec 2, 2024
1c1f723
add errored state and remove finalized
Farber98 Dec 2, 2024
6bc0c62
comment
Farber98 Dec 2, 2024
05442b2
Revert "comment"
Farber98 Dec 3, 2024
9b27a5b
Revert "remove unused ctx"
Farber98 Dec 3, 2024
ee14b60
Revert "associate sigs to retry ctx"
Farber98 Dec 3, 2024
d1f1ae7
Revert "fix listAllExpiredBroadcastedTxs"
Farber98 Dec 3, 2024
8911df2
Revert "move state from txes to sigs"
Farber98 Dec 3, 2024
52ce0e9
fix tx state
Farber98 Dec 3, 2024
fbbe978
address feedback
Farber98 Dec 3, 2024
cef6a91
fix ci
Farber98 Dec 3, 2024
3b3a71b
fix lint
Farber98 Dec 3, 2024
ef782c3
handle multiple sigs case
Farber98 Dec 4, 2024
08b0c6e
improve comment
Farber98 Dec 4, 2024
63a5f3f
improve logic and comments
Farber98 Dec 4, 2024
5c22af2
Merge branch 'develop' into backup-branch-fee-bumping
Farber98 Dec 5, 2024
02ffd1a
fix comparison against blockHeight instead of slotHeight
Farber98 Dec 5, 2024
c00494c
address feedback
Farber98 Dec 5, 2024
6ac30b0
fix lint
Farber98 Dec 5, 2024
36ee4ec
Merge branch 'develop' into backup-branch-fee-bumping
Farber98 Dec 5, 2024
0e38174
fix log
Farber98 Dec 5, 2024
d240021
address feedback
Farber98 Dec 6, 2024
4389e13
remove useless slot height
Farber98 Dec 6, 2024
23f42d1
address feedback
Farber98 Dec 6, 2024
6bcdd8f
Merge branch 'develop' into backup-branch-fee-bumping
Farber98 Dec 9, 2024
4a20622
Merge branch 'backup-branch-fee-bumping' into nonevm-984-reorg
Farber98 Dec 9, 2024
53948e1
add comment
Farber98 Dec 9, 2024
fdc8068
tests and fix some bugs
Farber98 Dec 9, 2024
50dd10f
address feedback
Farber98 Dec 10, 2024
34045ae
Merge branch 'backup-branch-fee-bumping' into nonevm-984-reorg
Farber98 Dec 10, 2024
10e5d9d
address feedback
Farber98 Dec 10, 2024
3e014d7
config change
Farber98 Dec 10, 2024
10b0453
Merge branch 'develop' into backup-branch-fee-bumping
Farber98 Dec 11, 2024
2ea0c50
validate that tx doesn't exist in any of maps when adding new tx
Farber98 Dec 11, 2024
b5a6927
Merge branch 'develop' into backup-branch-fee-bumping
Farber98 Dec 11, 2024
0a4b5aa
Merge branch 'backup-branch-fee-bumping' into nonevm-984-reorg
Farber98 Dec 12, 2024
5b34579
Merge branch 'backup-branch-fee-bumping' into nonevm-984-reorg
Farber98 Dec 12, 2024
17769f7
get height instead of whole block optimization
Farber98 Dec 12, 2024
d6fb891
fix mocks on expiration
Farber98 Dec 12, 2024
b6f4729
fix test
Farber98 Dec 12, 2024
dff36b0
Merge branch 'develop' into nonevm-984-reorg
Farber98 Dec 19, 2024
dbf4e41
rebroadcast with new blockhash + add integration tests
Farber98 Dec 23, 2024
d94a2c9
fix integration tests
Farber98 Dec 23, 2024
da1ee68
Merge branch 'develop' into nonevm-984-reorg
Farber98 Dec 23, 2024
84de493
Merge branch 'nonevm-984-reorg' into nonevm-984-reorg-soak
Farber98 Dec 23, 2024
6c08a75
Merge branch 'develop' into nonevm-984-reorg
Farber98 Jan 2, 2025
cc50224
remove unused params and better comments
Farber98 Jan 2, 2025
afa47a6
handle reorg equally for processed and confirmed at a sig level
Farber98 Jan 5, 2025
cdcfbd9
Merge branch 'nonevm-984-reorg' into nonevm-984-reorg-soak
Farber98 Jan 5, 2025
b087266
add comments and rename txHasReorg to IsTxReorged for better readability
Farber98 Jan 6, 2025
4047083
Merge branch 'develop' into nonevm-984-reorg
Farber98 Jan 6, 2025
ae95e4f
change test name to solve github CI failing check
Farber98 Jan 6, 2025
2515b7a
fix ci
Farber98 Jan 6, 2025
2826ad1
fix tests removing parallel
Farber98 Jan 6, 2025
71662e1
fix integration tests
Farber98 Jan 6, 2025
9987e41
capture range var
Farber98 Jan 6, 2025
46ed2fb
Merge branch 'nonevm-984-reorg' into nonevm-984-reorg-soak
Farber98 Jan 6, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'develop' into nonevm-706-support-custom-bumping-strateg…
…y-rpc-expiration-within-confirmation
  • Loading branch information
Farber98 committed Nov 26, 2024
commit 90daf33c9ade93b340459827729f0dec8774b7c1
103 changes: 90 additions & 13 deletions pkg/solana/txm/pendingtx.go
Original file line number Diff line number Diff line change
@@ -41,37 +41,47 @@ type PendingTxContext interface {
OnFinalized(sig solana.Signature, retentionTimeout time.Duration) (string, error)
// OnPrebroadcastError adds transaction that has not yet been broadcasted to the finalized/errored map as errored, matches err type using enum
OnPrebroadcastError(id string, retentionTimeout time.Duration, txState TxState, errType TxErrType) error
// OnPrebroadcastError adds transaction that has not yet been broadcasted to the finalized/errored map as errored, matches err type using enum
OnPrebroadcastError(id string, retentionTimeout time.Duration, txState TxState, errType TxErrType) error
// OnError marks transaction as errored, matches err type using enum, moves it from the broadcasted or confirmed map to finalized/errored map, removes signatures from signature map to stop confirmation checks
OnError(sig solana.Signature, retentionTimeout time.Duration, txState TxState, errType TxErrType) (string, error)
OnError(sig solana.Signature, retentionTimeout time.Duration, txState TxState, errType TxErrType) (string, error)
// GetTxState returns the transaction state for the provided ID if it exists
GetTxState(id string) (TxState, error)
// TrimFinalizedErroredTxs removes transactions that have reached their retention time
TrimFinalizedErroredTxs()
TrimFinalizedErroredTxs() int
// GetTxRebroadcastCount returns the number of times a transaction has been rebroadcasted if found.
GetTxRebroadcastCount(id string) (int, error)
}

// finishedTx is used to store info required to track transactions to finality or error
type pendingTx struct {
tx solana.Transaction
cfg TxConfig
signatures []solana.Signature
id string
rebroadcastCount int
createTs time.Time
retentionTs time.Time
state TxState
lastValidBlockHeight uint64 // to track expiration
}

// finishedTx is used to store minimal info specifically for finalized or errored transactions for external status checks
type finishedTx struct {
retentionTs time.Time
state TxState
rebroadcastCount int
}

var _ PendingTxContext = &pendingTxContext{}

type pendingTxContext struct {
cancelBy map[string]context.CancelFunc
sigToID map[solana.Signature]string

broadcastedProcessedTxs map[string]pendingTx // broadcasted and processed transactions that may require retry and bumping
confirmedTxs map[string]pendingTx // transactions that require monitoring for re-org
finalizedErroredTxs map[string]pendingTx // finalized and errored transactions held onto for status
broadcastedProcessedTxs map[string]pendingTx // broadcasted and processed transactions that may require retry and bumping
confirmedTxs map[string]pendingTx // transactions that require monitoring for re-org
finalizedErroredTxs map[string]finishedTx // finalized and errored transactions held onto for status

lock sync.RWMutex
}
@@ -83,7 +93,7 @@ func newPendingTxContext() *pendingTxContext {

broadcastedProcessedTxs: map[string]pendingTx{},
confirmedTxs: map[string]pendingTx{},
finalizedErroredTxs: map[string]pendingTx{},
finalizedErroredTxs: map[string]finishedTx{},
}
}

@@ -285,7 +295,6 @@ func (c *pendingTxContext) OnProcessed(sig solana.Signature) (string, error) {
if !exists {
return id, ErrTransactionNotFound
}
tx = c.broadcastedProcessedTxs[id]
// update tx state to Processed
tx.state = Processed
// save updated tx back to the broadcasted map
@@ -321,15 +330,15 @@ func (c *pendingTxContext) OnConfirmed(sig solana.Signature) (string, error) {
if !sigExists {
return id, ErrSigDoesNotExist
}
if _, exists := c.broadcastedProcessedTxs[id]; !exists {
tx, exists := c.broadcastedProcessedTxs[id]
if !exists {
return id, ErrTransactionNotFound
}
// call cancel func + remove from map to stop the retry/bumping cycle for this transaction
if cancel, exists := c.cancelBy[id]; exists {
cancel() // cancel context
delete(c.cancelBy, id)
}
tx := c.broadcastedProcessedTxs[id]
// update tx state to Confirmed
tx.state = Confirmed
// move tx to confirmed map
@@ -398,8 +407,13 @@ func (c *pendingTxContext) OnFinalized(sig solana.Signature, retentionTimeout ti
state: Finalized,
retentionTs: time.Now().Add(retentionTimeout),
}
finalizedTx := finishedTx{
state: Finalized,
retentionTs: time.Now().Add(retentionTimeout),
}
// move transaction from confirmed to finalized map
c.finalizedErroredTxs[id] = finalizedTx
c.finalizedErroredTxs[id] = finalizedTx
return id, nil
})
}
@@ -413,7 +427,7 @@ func (c *pendingTxContext) OnPrebroadcastError(id string, retentionTimeout time.
if tx, exists := c.finalizedErroredTxs[id]; exists && tx.state == txState {
return ErrAlreadyInExpectedState
}
_, broadcastedExists := c.broadcastedTxs[id]
_, broadcastedExists := c.broadcastedProcessedTxs[id]
_, confirmedExists := c.confirmedTxs[id]
if broadcastedExists || confirmedExists {
return ErrIDAlreadyExists
@@ -429,7 +443,7 @@ func (c *pendingTxContext) OnPrebroadcastError(id string, retentionTimeout time.
if tx, exists := c.finalizedErroredTxs[id]; exists && tx.state == txState {
return "", ErrAlreadyInExpectedState
}
_, broadcastedExists := c.broadcastedTxs[id]
_, broadcastedExists := c.broadcastedProcessedTxs[id]
_, confirmedExists := c.confirmedTxs[id]
if broadcastedExists || confirmedExists {
return "", ErrIDAlreadyExists
@@ -497,15 +511,21 @@ func (c *pendingTxContext) OnError(sig solana.Signature, retentionTimeout time.D
delete(c.sigToID, s)
}
// if retention duration is set to 0, skip adding transaction to the errored map
// if retention duration is set to 0, skip adding transaction to the errored map
if retentionTimeout == 0 {
return id, nil
}
erroredTx := finishedTx{
state: txState,
retentionTs: time.Now().Add(retentionTimeout),
}
erroredTx := finishedTx{
state: txState,
retentionTs: time.Now().Add(retentionTimeout),
}
// move transaction from broadcasted to error map
c.finalizedErroredTxs[id] = erroredTx
c.finalizedErroredTxs[id] = erroredTx
return id, nil
})
}
@@ -541,6 +561,31 @@ func (c *pendingTxContext) TrimFinalizedErroredTxs() int {
return 0
}

_, err = c.withWriteLock(func() (string, error) {
for _, id := range expiredIDs {
delete(c.finalizedErroredTxs, id)
}
return "", nil
})
if err != nil {
return 0
}
return len(expiredIDs)
func (c *pendingTxContext) TrimFinalizedErroredTxs() int {
var expiredIDs []string
err := c.withReadLock(func() error {
expiredIDs = make([]string, 0, len(c.finalizedErroredTxs))
for id, tx := range c.finalizedErroredTxs {
if time.Now().After(tx.retentionTs) {
expiredIDs = append(expiredIDs, id)
}
}
return nil
})
if err != nil {
return 0
}

_, err = c.withWriteLock(func() (string, error) {
for _, id := range expiredIDs {
delete(c.finalizedErroredTxs, id)
@@ -590,7 +635,11 @@ type pendingTxContextWithProm struct {

type TxErrType int

type TxErrType int

const (
NoFailure TxErrType = iota
TxFailRevert
NoFailure TxErrType = iota
TxFailRevert
TxFailReject
@@ -663,14 +712,39 @@ func (c *pendingTxContextWithProm) OnError(sig solana.Signature, retentionTimeou

func (c *pendingTxContextWithProm) OnPrebroadcastError(id string, retentionTimeout time.Duration, txState TxState, errType TxErrType) error {
err := c.pendingTx.OnPrebroadcastError(id, retentionTimeout, txState, errType) // err indicates transaction not found so may already be removed
func (c *pendingTxContextWithProm) OnError(sig solana.Signature, retentionTimeout time.Duration, txState TxState, errType TxErrType) (string, error) {
id, err := c.pendingTx.OnError(sig, retentionTimeout, txState, errType) // err indicates transaction not found so may already be removed
if err == nil {
incrementErrorMetrics(errType, c.chainID)
}
return id, err
}

func (c *pendingTxContextWithProm) OnPrebroadcastError(id string, retentionTimeout time.Duration, txState TxState, errType TxErrType) error {
err := c.pendingTx.OnPrebroadcastError(id, retentionTimeout, txState, errType) // err indicates transaction not found so may already be removed
if err == nil {
incrementErrorMetrics(errType, c.chainID)
}
return err
}

func incrementErrorMetrics(errType TxErrType, chainID string) {
incrementErrorMetrics(errType, c.chainID)
}
return err
}

func incrementErrorMetrics(errType TxErrType, chainID string) {
switch errType {
case NoFailure:
// Return early if no failure identified
return
case TxFailReject:
promSolTxmRejectTxs.WithLabelValues(chainID).Inc()
case TxFailRevert:
promSolTxmRevertTxs.WithLabelValues(chainID).Inc()
case TxFailDrop:
promSolTxmDropTxs.WithLabelValues(chainID).Inc()
case NoFailure:
// Return early if no failure identified
return
@@ -682,18 +756,21 @@ func incrementErrorMetrics(errType TxErrType, chainID string) {
promSolTxmDropTxs.WithLabelValues(chainID).Inc()
case TxFailSimRevert:
promSolTxmSimRevertTxs.WithLabelValues(chainID).Inc()
promSolTxmSimRevertTxs.WithLabelValues(chainID).Inc()
case TxFailSimOther:
promSolTxmSimOtherTxs.WithLabelValues(chainID).Inc()
promSolTxmSimOtherTxs.WithLabelValues(chainID).Inc()
}
promSolTxmErrorTxs.WithLabelValues(chainID).Inc()
promSolTxmErrorTxs.WithLabelValues(chainID).Inc()
}

func (c *pendingTxContextWithProm) GetTxState(id string) (TxState, error) {
return c.pendingTx.GetTxState(id)
}

func (c *pendingTxContextWithProm) TrimFinalizedErroredTxs() {
c.pendingTx.TrimFinalizedErroredTxs()
func (c *pendingTxContextWithProm) TrimFinalizedErroredTxs() int {
return c.pendingTx.TrimFinalizedErroredTxs()
}

func (c *pendingTxContextWithProm) GetTxRebroadcastCount(id string) (int, error) {
Loading