Skip to content

Commit

Permalink
vfs: fix file length after fallocate & copyfilerange (#4427)
Browse files Browse the repository at this point in the history 
Co-authored-by: Davies Liu <davies@juicedata.io>
  • Loading branch information
@SandyXSD @davies
SandyXSD and davies committed Jun 20, 2024
1 parent 468ccae commit 9fda3ae
Show file tree
Hide file tree
Showing 7 changed files with 91 additions and 42 deletions.
2 changes: 1 addition & 1 deletion pkg/fs/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -575,7 +575,7 @@ func (fs *FileSystem) CopyFileRange(ctx meta.Context, src string, soff uint64, d
if err != 0 {
return
}
err = fs.m.CopyFileRange(ctx, sfi.inode, soff, dfi.inode, doff, size, 0, &written)
err = fs.m.CopyFileRange(ctx, sfi.inode, soff, dfi.inode, doff, size, 0, &written, nil)
return
}

Expand Down
32 changes: 16 additions & 16 deletions pkg/meta/base_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,25 +476,25 @@ func testMetaClient(t *testing.T, m Meta) {
if len(slices) != 2 || slices[0].Id != 0 || slices[0].Size != 100 || slices[1].Id != sliceId || slices[1].Size != 100 {
t.Fatalf("slices: %v", slices)
}
if st := m.Fallocate(ctx, inode, fallocPunchHole|fallocKeepSize, 100, 50); st != 0 {
if st := m.Fallocate(ctx, inode, fallocPunchHole|fallocKeepSize, 100, 50, nil); st != 0 {
t.Fatalf("fallocate: %s", st)
}
if st := m.Fallocate(ctx, inode, fallocPunchHole|fallocCollapesRange, 100, 50); st != syscall.EINVAL {
if st := m.Fallocate(ctx, inode, fallocPunchHole|fallocCollapesRange, 100, 50, nil); st != syscall.EINVAL {
t.Fatalf("fallocate: %s", st)
}
if st := m.Fallocate(ctx, inode, fallocPunchHole|fallocInsertRange, 100, 50); st != syscall.EINVAL {
if st := m.Fallocate(ctx, inode, fallocPunchHole|fallocInsertRange, 100, 50, nil); st != syscall.EINVAL {
t.Fatalf("fallocate: %s", st)
}
if st := m.Fallocate(ctx, inode, fallocCollapesRange, 100, 50); st != syscall.ENOTSUP {
if st := m.Fallocate(ctx, inode, fallocCollapesRange, 100, 50, nil); st != syscall.ENOTSUP {
t.Fatalf("fallocate: %s", st)
}
if st := m.Fallocate(ctx, inode, fallocPunchHole, 100, 50); st != syscall.EINVAL {
if st := m.Fallocate(ctx, inode, fallocPunchHole, 100, 50, nil); st != syscall.EINVAL {
t.Fatalf("fallocate: %s", st)
}
if st := m.Fallocate(ctx, inode, fallocPunchHole|fallocKeepSize, 0, 0); st != syscall.EINVAL {
if st := m.Fallocate(ctx, inode, fallocPunchHole|fallocKeepSize, 0, 0, nil); st != syscall.EINVAL {
t.Fatalf("fallocate: %s", st)
}
if st := m.Fallocate(ctx, parent, fallocPunchHole|fallocKeepSize, 100, 50); st != syscall.EPERM {
if st := m.Fallocate(ctx, parent, fallocPunchHole|fallocKeepSize, 100, 50, nil); st != syscall.EPERM {
t.Fatalf("fallocate dir: %s", st)
}
if st := m.Read(ctx, inode, 0, &slices); st != 0 {
Expand Down Expand Up @@ -1413,7 +1413,7 @@ func testCopyFileRange(t *testing.T, m Meta) {
m.Write(ctx, iin, 3, 0, Slice{12, 63 << 20, 10 << 20, 30 << 20}, time.Now())
m.Write(ctx, iout, 2, 10<<20, Slice{13, 50 << 20, 10 << 20, 30 << 20}, time.Now())
var copied uint64
if st := m.CopyFileRange(ctx, iin, 150, iout, 30<<20, 200<<20, 0, &copied); st != 0 {
if st := m.CopyFileRange(ctx, iin, 150, iout, 30<<20, 200<<20, 0, &copied, nil); st != 0 {
t.Fatalf("copy file range: %s", st)
}
var expected uint64 = 200 << 20
Expand Down Expand Up @@ -1964,17 +1964,17 @@ func testAttrFlags(t *testing.T, m Meta) {
if st := m.SetAttr(ctx, fallocFile, SetAttrFlag, 0, attr); st != 0 {
t.Fatalf("setattr f: %s", st)
}
if st := m.Fallocate(ctx, fallocFile, fallocKeepSize, 0, 1024); st != 0 {
if st := m.Fallocate(ctx, fallocFile, fallocKeepSize, 0, 1024, nil); st != 0 {
t.Fatalf("fallocate f: %s", st)
}
if st := m.Fallocate(ctx, fallocFile, fallocKeepSize|fallocZeroRange, 0, 1024); st != syscall.EPERM {
if st := m.Fallocate(ctx, fallocFile, fallocKeepSize|fallocZeroRange, 0, 1024, nil); st != syscall.EPERM {
t.Fatalf("fallocate f: %s", st)
}
attr.Flags = FlagImmutable
if st := m.SetAttr(ctx, fallocFile, SetAttrFlag, 0, attr); st != 0 {
t.Fatalf("setattr f: %s", st)
}
if st := m.Fallocate(ctx, fallocFile, fallocKeepSize, 0, 1024); st != syscall.EPERM {
if st := m.Fallocate(ctx, fallocFile, fallocKeepSize, 0, 1024, nil); st != syscall.EPERM {
t.Fatalf("fallocate f: %s", st)
}

Expand All @@ -1985,21 +1985,21 @@ func testAttrFlags(t *testing.T, m Meta) {
if st := m.Create(ctx, 1, "copydstfile", 0644, 022, 0, &copydstFile, nil); st != 0 {
t.Fatalf("create f: %s", st)
}
if st := m.Fallocate(ctx, copysrcFile, 0, 0, 1024); st != 0 {
if st := m.Fallocate(ctx, copysrcFile, 0, 0, 1024, nil); st != 0 {
t.Fatalf("fallocate f: %s", st)
}
attr.Flags = FlagAppend
if st := m.SetAttr(ctx, copydstFile, SetAttrFlag, 0, attr); st != 0 {
t.Fatalf("setattr f: %s", st)
}
if st := m.CopyFileRange(ctx, copysrcFile, 0, copydstFile, 0, 1024, 0, nil); st != syscall.EPERM {
if st := m.CopyFileRange(ctx, copysrcFile, 0, copydstFile, 0, 1024, 0, nil, nil); st != syscall.EPERM {
t.Fatalf("copy_file_range f: %s", st)
}
attr.Flags = FlagImmutable
if st := m.SetAttr(ctx, copydstFile, SetAttrFlag, 0, attr); st != 0 {
t.Fatalf("setattr f: %s", st)
}
if st := m.CopyFileRange(ctx, copysrcFile, 0, copydstFile, 0, 1024, 0, nil); st != syscall.EPERM {
if st := m.CopyFileRange(ctx, copysrcFile, 0, copydstFile, 0, 1024, 0, nil, nil); st != syscall.EPERM {
t.Fatalf("copy_file_range f: %s", st)
}
}
Expand Down Expand Up @@ -2182,7 +2182,7 @@ func testDirStat(t *testing.T, m Meta) {
checkResult(0, align4K(0), 1)

// test dir with file and fallocate
if st := m.Fallocate(Background, fileInode, 0, 0, 4097); st != 0 {
if st := m.Fallocate(Background, fileInode, 0, 0, 4097, nil); st != 0 {
t.Fatalf("fallocate: %s", st)
}
time.Sleep(500 * time.Millisecond)
Expand Down Expand Up @@ -2313,7 +2313,7 @@ func testClone(t *testing.T, m Meta) {
if eno := m.Mknod(Background, dir3, "file3", TypeFile, 0777, 022, 0, "", &file3, nil); eno != 0 {
t.Fatalf("mknod: %s", eno)
}
if eno := m.Fallocate(Background, file3, 0, 0, 67108864); eno != 0 {
if eno := m.Fallocate(Background, file3, 0, 0, 67108864, nil); eno != 0 {
t.Fatalf("fallocate: %s", eno)
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/meta/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ type Meta interface {
// Truncate changes the length for given file.
Truncate(ctx Context, inode Ino, flags uint8, attrlength uint64, attr *Attr, skipPermCheck bool) syscall.Errno
// Fallocate preallocate given space for given file.
Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64) syscall.Errno
Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, length *uint64) syscall.Errno
// ReadLink returns the target of a symlink.
ReadLink(ctx Context, inode Ino, path *[]byte) syscall.Errno
// Symlink creates a symlink in a directory with given name.
Expand Down Expand Up @@ -391,7 +391,7 @@ type Meta interface {
// InvalidateChunkCache invalidate chunk cache
InvalidateChunkCache(ctx Context, inode Ino, indx uint32) syscall.Errno
// CopyFileRange copies part of a file to another one.
CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied *uint64) syscall.Errno
CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied, outLength *uint64) syscall.Errno
// GetParents returns a map of node parents (> 1 parents if hardlinked)
GetParents(ctx Context, inode Ino) map[Ino]int
// GetDirStat returns the space and inodes usage of a directory.
Expand Down
18 changes: 14 additions & 4 deletions pkg/meta/redis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1032,7 +1032,7 @@ func (m *redisMeta) Truncate(ctx Context, inode Ino, flags uint8, length uint64,
return errno(err)
}

func (m *redisMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64) syscall.Errno {
func (m *redisMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, flength *uint64) syscall.Errno {
if mode&fallocCollapesRange != 0 && mode != fallocCollapesRange {
return syscall.EINVAL
}
Expand Down Expand Up @@ -1090,6 +1090,9 @@ func (m *redisMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, si
if err := m.checkQuota(ctx, newSpace, 0, m.getParents(ctx, tx, inode, t.Parent)...); err != 0 {
return err
}
if flength != nil {
*flength = length
}
t.Length = length
now := time.Now()
t.Mtime = now.Unix()
Expand Down Expand Up @@ -2281,7 +2284,7 @@ func (m *redisMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice
return errno(err)
}

func (m *redisMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied *uint64) syscall.Errno {
func (m *redisMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied, outLength *uint64) syscall.Errno {
defer m.timeit("CopyFileRange", time.Now())
f := m.of.find(fout)
if f != nil {
Expand All @@ -2306,7 +2309,9 @@ func (m *redisMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino,
return syscall.EINVAL
}
if offIn >= sattr.Length {
*copied = 0
if copied != nil {
*copied = 0
}
return nil
}
size := size
Expand Down Expand Up @@ -2336,6 +2341,9 @@ func (m *redisMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino,
attr.Mtimensec = uint32(now.Nanosecond())
attr.Ctime = now.Unix()
attr.Ctimensec = uint32(now.Nanosecond())
if outLength != nil {
*outLength = attr.Length
}

var vals [][]string
for i := offIn / ChunkSize; i <= (offIn+size)/ChunkSize; i++ {
Expand Down Expand Up @@ -2402,7 +2410,9 @@ func (m *redisMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino,
return nil
})
if err == nil {
*copied = size
if copied != nil {
*copied = size
}
}
return err
}, m.inodeKey(fout), m.inodeKey(fin))
Expand Down
18 changes: 14 additions & 4 deletions pkg/meta/sql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1046,7 +1046,7 @@ func (m *dbMeta) Truncate(ctx Context, inode Ino, flags uint8, length uint64, at
return errno(err)
}

func (m *dbMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64) syscall.Errno {
func (m *dbMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, flength *uint64) syscall.Errno {
if mode&fallocCollapesRange != 0 && mode != fallocCollapesRange {
return syscall.EINVAL
}
Expand Down Expand Up @@ -1112,6 +1112,9 @@ func (m *dbMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size
if _, err := s.Cols("length", "mtime", "ctime", "mtimensec", "ctimensec").Update(&nodeAttr, &node{Inode: inode}); err != nil {
return err
}
if flength != nil {
*flength = length
}
if mode&(fallocZeroRange|fallocPunchHole) != 0 && off < old {
off, size := off, size
if off+size > old {
Expand Down Expand Up @@ -2289,7 +2292,7 @@ func (m *dbMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Sl
return errno(err)
}

func (m *dbMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied *uint64) syscall.Errno {
func (m *dbMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied, outLength *uint64) syscall.Errno {
defer m.timeit("CopyFileRange", time.Now())
f := m.of.find(fout)
if f != nil {
Expand All @@ -2311,7 +2314,9 @@ func (m *dbMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, off
return syscall.EINVAL
}
if offIn >= nin.Length {
*copied = 0
if copied != nil {
*copied = 0
}
return nil
}
size := size
Expand Down Expand Up @@ -2339,6 +2344,9 @@ func (m *dbMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, off
nout.Ctime = now / 1e3
nout.Mtimensec = int16(now % 1e3)
nout.Ctimensec = int16(now % 1e3)
if outLength != nil {
*outLength = nout.Length
}

var cs []chunk
err = s.Where("inode = ? AND indx >= ? AND indx <= ?", fin, offIn/ChunkSize, (offIn+size)/ChunkSize).ForUpdate().Find(&cs)
Expand Down Expand Up @@ -2409,7 +2417,9 @@ func (m *dbMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, off
if _, err := s.Cols("length", "mtime", "ctime", "mtimensec", "ctimensec").Update(&nout, &node{Inode: fout}); err != nil {
return err
}
*copied = size
if copied != nil {
*copied = size
}
return nil
}, fout)
if err == nil {
Expand Down
18 changes: 14 additions & 4 deletions pkg/meta/tkv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -965,7 +965,7 @@ func (m *kvMeta) Truncate(ctx Context, inode Ino, flags uint8, length uint64, at
return errno(err)
}

func (m *kvMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64) syscall.Errno {
func (m *kvMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, flength *uint64) syscall.Errno {
if mode&fallocCollapesRange != 0 && mode != fallocCollapesRange {
return syscall.EINVAL
}
Expand Down Expand Up @@ -1020,6 +1020,9 @@ func (m *kvMeta) Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size
if err := m.checkQuota(ctx, newSpace, 0, m.getParents(tx, inode, t.Parent)...); err != 0 {
return err
}
if flength != nil {
*flength = length
}
t.Length = length
now := time.Now()
t.Mtime = now.Unix()
Expand Down Expand Up @@ -1970,7 +1973,7 @@ func (m *kvMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Sl
return errno(err)
}

func (m *kvMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied *uint64) syscall.Errno {
func (m *kvMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied, outLength *uint64) syscall.Errno {
defer m.timeit("CopyFileRange", time.Now())
var newLength, newSpace int64
f := m.of.find(fout)
Expand All @@ -1992,7 +1995,9 @@ func (m *kvMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, off
return syscall.EINVAL
}
if offIn >= sattr.Length {
*copied = 0
if copied != nil {
*copied = 0
}
return nil
}
size := size
Expand Down Expand Up @@ -2022,6 +2027,9 @@ func (m *kvMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, off
attr.Mtimensec = uint32(now.Nanosecond())
attr.Ctime = now.Unix()
attr.Ctimensec = uint32(now.Nanosecond())
if outLength != nil {
*outLength = attr.Length
}

vals := make(map[string][]byte)
tx.scan(m.chunkKey(fin, uint32(offIn/ChunkSize)), m.chunkKey(fin, uint32((offIn+size)/ChunkSize)+1),
Expand Down Expand Up @@ -2084,7 +2092,9 @@ func (m *kvMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, off
}
}
tx.set(m.inodeKey(fout), m.marshal(&attr))
*copied = size
if copied != nil {
*copied = size
}
return nil
}, fout)
if err == nil {
Expand Down
Loading

0 comments on commit 9fda3ae

Please sign in to comment.