Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dump: reduce memory when dump large directories from redis #4481

Merged
merged 4 commits into from
Mar 12, 2024
Merged

dump: reduce memory when dump large directories from redis #4481

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a399070
dump: reduce memory when dump large directories
davies Mar 11, 2024
098920d
cleanup
davies Mar 11, 2024
7db901f
cleanup
davies Mar 11, 2024
f4d2ace
fix inode
SandyXSD Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/meta/dump.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ type DumpedEntry struct {
Xattrs []*DumpedXattr `json:"xattrs,omitempty"`
Chunks []*DumpedChunk `json:"chunks,omitempty"`
Entries map[string]*DumpedEntry `json:"entries,omitempty"`
keys []string
}

var CHARS = []byte("0123456789ABCDEF")
Expand Down
47 changes: 32 additions & 15 deletions pkg/meta/redis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3646,6 +3646,8 @@ var entryPool = sync.Pool{
},
}

const scanEntryBatch = 1000

func (m *redisMeta) dumpEntries(es ...*DumpedEntry) error {
ctx := Background
var keys []string
Expand All @@ -3658,7 +3660,7 @@ func (m *redisMeta) dumpEntries(es ...*DumpedEntry) error {
var xr = make([]*redis.MapStringStringCmd, len(es))
var sr = make([]*redis.StringCmd, len(es))
var cr = make([]*redis.StringSliceCmd, len(es))
var dr = make([]*redis.MapStringStringCmd, len(es))
var dr = make([]*redis.ScanCmd, len(es))
for i, e := range es {
inode := e.Attr.Inode
ar[i] = p.Get(ctx, m.inodeKey(inode))
Expand All @@ -3667,7 +3669,7 @@ func (m *redisMeta) dumpEntries(es ...*DumpedEntry) error {
case "regular":
cr[i] = p.LRange(ctx, m.chunkKey(inode, 0), 0, -1)
case "directory":
dr[i] = p.HGetAll(ctx, m.entryKey(inode))
dr[i] = p.HScan(ctx, m.entryKey(inode), 0, "*", scanEntryBatch)
case "symlink":
sr[i] = p.Get(ctx, m.symKey(inode))
}
Expand Down Expand Up @@ -3743,18 +3745,11 @@ func (m *redisMeta) dumpEntries(es ...*DumpedEntry) error {
}
}
case TypeDirectory:
dirs, err := dr[i].Result()
keys, _, err := dr[i].Result()
if err != nil {
return err
}
e.Entries = make(map[string]*DumpedEntry)
for name := range dirs {
t, inode := m.parseEntry([]byte(dirs[name]))
ce := entryPool.Get().(*DumpedEntry)
ce.Attr.Inode = inode
ce.Attr.Type = typeToString(t)
e.Entries[name] = ce
}
e.keys = keys
case TypeSymlink:
if e.Symlink, err = sr[i].Result(); err != nil {
if err != redis.Nil {
Expand Down Expand Up @@ -3807,12 +3802,32 @@ func (m *redisMeta) dumpDir(inode Ino, tree *DumpedEntry, bw *bufio.Writer, dept
panic(err)
}
}
var err error
entries := make([]*DumpedEntry, 0, len(tree.Entries))
for name, e := range tree.Entries {

if len(tree.keys) == scanEntryBatch*2 {
// partial scan, retry
tree.keys = tree.keys[:0]
err := m.hscan(Background, m.entryKey(inode), func(keys []string) error {
tree.keys = append(tree.keys, keys...)
return nil
})
if err != nil {
return err
}
}
entries := make([]*DumpedEntry, len(tree.keys)/2)
tree.Entries = make(map[string]*DumpedEntry)
for i := range entries {
name := tree.keys[i*2]
t, inode := m.parseEntry([]byte(tree.keys[i*2+1]))
e := entryPool.Get().(*DumpedEntry)
e.Name = name
entries = append(entries, e)
e.Attr.Inode = inode
e.Attr.Type = typeToString(t)
entries[i] = e
tree.Entries[name] = e
}

var err error
if err = tree.writeJsonWithOutEntry(bw, depth); err != nil {
return err
}
Expand Down Expand Up @@ -3871,10 +3886,12 @@ func (m *redisMeta) dumpDir(inode Ino, tree *DumpedEntry, bw *bufio.Writer, dept
}
entries[i] = nil
delete(tree.Entries, e.Name)
e.Name = ""
e.Xattrs = nil
e.Chunks = nil
e.Entries = nil
e.Symlink = ""
e.keys = nil
entryPool.Put(e)
if err != nil {
return err
Expand Down
Loading