Skip to content

Commit

Permalink
Handle context.DeadlineExceeded when retrying a transaction (#1191)
Browse files Browse the repository at this point in the history
Caught these errors on `renterd` shutdown, fixed by adding a
`context.DeadlineExceeded` case in `abortRetry`.
```
2024-04-23T10:49:08Z INFO  bus  successfully saved 392 accounts
2024-04-23T10:49:42Z ERROR SQL  trace {"error": "context deadline exceeded", "elapsed": "34.701151966s", "rows": 0, "sql": "\nDELETE\nFROM slabs\nWHERE NOT EXISTS (SELECT 1 FROM slices WHERE slices.db_slab_id = slabs.id)\nAND slabs.db_buffered_slab_id IS NULL\n"}
2024-04-23T10:49:42Z WARN  sql  transaction attempt 1/7 failed, retry in 200ms,  err: context deadline exceeded
2024-04-23T10:49:42Z WARN  sql  transaction attempt 2/7 failed, retry in 500ms,  err: context deadline exceeded
2024-04-23T10:49:43Z WARN  sql  transaction attempt 3/7 failed, retry in 1s,  err: context deadline exceeded
2024-04-23T10:49:44Z WARN  sql  transaction attempt 4/7 failed, retry in 3s,  err: context deadline exceeded
2024-04-23T10:49:47Z WARN  sql  transaction attempt 5/7 failed, retry in 10s,  err: context deadline exceeded
2024-04-23T10:49:57Z WARN  sql  transaction attempt 6/7 failed, retry in 10s,  err: context deadline exceeded
```
  • Loading branch information
ChrisSchinnerl committed Apr 24, 2024
2 parents bb7b280 + a27712d commit 1777b3a
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 2 deletions.
1 change: 1 addition & 0 deletions stores/sql.go
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,7 @@ func (s *SQLStore) retryTransaction(ctx context.Context, fc func(tx *gorm.DB) er
abortRetry := func(err error) bool {
if err == nil ||
errors.Is(err, context.Canceled) ||
errors.Is(err, context.DeadlineExceeded) ||
errors.Is(err, gorm.ErrRecordNotFound) ||
errors.Is(err, errInvalidNumberOfShards) ||
errors.Is(err, errShardRootChanged) ||
Expand Down
14 changes: 12 additions & 2 deletions stores/sql_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -470,8 +470,18 @@ func TestRetryTransaction(t *testing.T) {
t.Fatal("unexpected logs", cmp.Diff(got, want))
}

// retry transaction that aborts, assert no logs were added
ss.retryTransaction(context.Background(), func(tx *gorm.DB) error { return context.Canceled })
// retry transaction with cancelled context
ctx, cancel := context.WithCancel(context.Background())
cancel()
ss.retryTransaction(ctx, func(tx *gorm.DB) error { return nil })
if len(observedLogs.All()) != len(want) {
t.Fatal("expected no logs")
}

ctx, cancel = context.WithTimeout(context.Background(), 1*time.Microsecond)
defer cancel()
time.Sleep(time.Millisecond)
ss.retryTransaction(ctx, func(tx *gorm.DB) error { return nil })
if len(observedLogs.All()) != len(want) {
t.Fatal("expected no logs")
}
Expand Down

0 comments on commit 1777b3a

Please sign in to comment.