diff --git a/util/memory/tracker.go b/util/memory/tracker.go
index 3b935360f0fce..eee0b707a41cb 100644
--- a/util/memory/tracker.go
+++ b/util/memory/tracker.go
@@ -95,7 +95,7 @@ func (t *Tracker) remove(oldChild *Tracker) {
 			continue
 		}
 
-		atomic.AddInt64(&t.bytesConsumed, -oldChild.BytesConsumed())
+		t.Consume(-oldChild.BytesConsumed())
 		oldChild.parent = nil
 		t.mu.children = append(t.mu.children[:i], t.mu.children[i+1:]...)
 		break
diff --git a/util/memory/tracker_test.go b/util/memory/tracker_test.go
index bf7ac98ecc506..d3744a991d636 100644
--- a/util/memory/tracker_test.go
+++ b/util/memory/tracker_test.go
@@ -156,6 +156,17 @@ func (s *testSuite) TestReplaceChild(c *C) {
 	c.Assert(len(parent.mu.children), Equals, 0)
 	c.Assert(newChild.parent, IsNil)
 	c.Assert(oldChild.parent, IsNil)
+
+	node1 := NewTracker(stringutil.StringerStr("Node1"), -1)
+	node2 := NewTracker(stringutil.StringerStr("Node2"), -1)
+	node3 := NewTracker(stringutil.StringerStr("Node3"), -1)
+	node2.AttachTo(node1)
+	node3.AttachTo(node2)
+	node3.Consume(100)
+	c.Assert(node1.BytesConsumed(), Equals, int64(100))
+	node2.ReplaceChild(node3, nil)
+	c.Assert(node2.BytesConsumed(), Equals, int64(0))
+	c.Assert(node1.BytesConsumed(), Equals, int64(0))
 }
 
 func (s *testSuite) TestToString(c *C) {