From 15922b7be704ec821010171301e034c42a2dc99c Mon Sep 17 00:00:00 2001 From: Shuo Wu Date: Wed, 19 Feb 2025 17:07:32 -0800 Subject: [PATCH] fix: delay cksum calculation for rebuilding src replica and the new rebuilding snap Longhorn 10335 Signed-off-by: Shuo Wu --- pkg/spdk/replica.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/spdk/replica.go b/pkg/spdk/replica.go index b124c261..1e786647 100644 --- a/pkg/spdk/replica.go +++ b/pkg/spdk/replica.go @@ -39,7 +39,8 @@ import ( const ( restorePeriodicRefreshInterval = 2 * time.Second - checksumWaitPeriodAfterRebuilding = 10 * time.Second + checksumWaitPeriodAfterRebuilding = 10 * time.Second + checksumWaitPeriodForRebuildingSnapshot = 1 * time.Minute ) type Replica struct { @@ -252,6 +253,23 @@ func (r *Replica) Sync(spdkClient *spdkclient.Client) (err error) { if bdevLvol.DriverSpecific.Lvol.Xattrs[spdkclient.SnapshotChecksum] != "" { continue } + // Avoid error "Device or resource busy" by delaying checksum calculation for all src replica snapshot lvols during rebuilding + // as these lvols may be operated by shallow copy calls later. + if r.rebuildingSrcCache.dstReplicaName != "" { + continue + } + // Avoid error "Device or resource busy" by delaying checksum calculation for the newly created rebuilding snapshot lvol + // as this lvol will be exposed to the dst replica soon. + if bdevLvol.DriverSpecific.Lvol.Xattrs[spdkclient.UserCreated] == "false" && bdevLvol.DriverSpecific.Lvol.Xattrs[spdkclient.SnapshotTimestamp] != "" { + snapshotTime, err := time.Parse(time.RFC3339, bdevLvol.DriverSpecific.Lvol.Xattrs[spdkclient.SnapshotTimestamp]) + if err != nil { + logrus.WithError(err).Warnf("Failed to parse snapshot timestamp %v for snapshot lvol %v before registering checksum, will skip it", bdevLvol.DriverSpecific.Lvol.Xattrs[spdkclient.SnapshotTimestamp], bdevLvol.Name) + continue + } + if !time.Now().After(snapshotTime.Add(checksumWaitPeriodForRebuildingSnapshot)) { + continue + } + } parentBdevLvol := bdevLvolMap[bdevLvol.DriverSpecific.Lvol.BaseSnapshot] if bdevLvol.DriverSpecific.Lvol.Xattrs[spdkclient.UserCreated] == "false" || (parentBdevLvol != nil && parentBdevLvol.DriverSpecific.Lvol.Xattrs[spdkclient.UserCreated] == "false") { // Skip the checksum calculation of system created snapshot lvols during rebuilding as they may be purged later.