From 3cad2f7e6def7374ab7d74ec0f00d5158645e956 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 18 Jul 2023 12:20:21 -0400 Subject: [PATCH] fix: large files support io.SeekCurrent --- file/file_test.go | 115 ++++++++++++++++++++++++++++++++++++++++ file/large_file_test.go | 37 +++++++++++++ file/shard.go | 1 + 3 files changed, 153 insertions(+) diff --git a/file/file_test.go b/file/file_test.go index ee01a71..45372ff 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -7,13 +7,16 @@ import ( "io" "testing" + ipfsutil "github.com/ipfs/go-ipfs-util" "github.com/ipfs/go-unixfsnode" + "github.com/ipfs/go-unixfsnode/data/builder" "github.com/ipfs/go-unixfsnode/directory" "github.com/ipfs/go-unixfsnode/file" "github.com/ipld/go-car/v2/blockstore" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" ) func TestRootV0File(t *testing.T) { @@ -61,6 +64,43 @@ func TestNamedV0File(t *testing.T) { } } +func TestFileSeeker(t *testing.T) { + ls := cidlink.DefaultLinkSystem() + storage := cidlink.Memory{} + ls.StorageReadOpener = storage.OpenRead + ls.StorageWriteOpener = storage.OpenWrite + + // Make random file with 1024 bytes. + buf := make([]byte, 1024) + ipfsutil.NewSeededRand(0xdeadbeef).Read(buf) + r := bytes.NewReader(buf) + + // Build UnixFS File as a single chunk + f, _, err := builder.BuildUnixFSFile(r, "size-1024", &ls) + if err != nil { + t.Fatal(err) + } + + // Load the file. + fr, err := ls.Load(ipld.LinkContext{}, f, basicnode.Prototype.Bytes) + if err != nil { + t.Fatal(err) + } + + // Create it. + ufn, err := file.NewUnixFSFile(context.Background(), fr, &ls) + if err != nil { + t.Fatal(err) + } + + rs, err := ufn.AsLargeBytes() + if err != nil { + t.Fatal(err) + } + + testSeekIn1024ByteFile(t, rs) +} + func open(car string, t *testing.T) (ipld.Node, *ipld.LinkSystem) { baseStore, err := blockstore.OpenReadOnly(car) if err != nil { @@ -88,3 +128,78 @@ func open(car string, t *testing.T) (ipld.Node, *ipld.LinkSystem) { } return root, &ls } + +func testSeekIn1024ByteFile(t *testing.T, rs io.ReadSeeker) { + // Seek from the start and try reading + offset, err := rs.Seek(128, io.SeekStart) + if err != nil { + t.Fatal(err) + } + + if offset != 128 { + t.Fatalf("expected offset %d, got %d", 484, offset) + } + + readBuf := make([]byte, 256) + _, err = io.ReadFull(rs, readBuf) + if err != nil { + t.Fatal(err) + } + + // Validate we can detect the offset with SeekCurrent + offset, err = rs.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + + if offset != 384 { + t.Fatalf("expected offset %d, got %d", 384, offset) + } + + // Validate we can read after moving with SeekCurrent + offset, err = rs.Seek(100, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + if offset != 484 { + t.Fatalf("expected offset %d, got %d", 484, offset) + } + + _, err = io.ReadFull(rs, readBuf) + if err != nil { + t.Fatal(err) + } + + offset, err = rs.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + + if offset != 740 { + t.Fatalf("expected offset %d, got %d", 740, offset) + } + + // Validate we can read after moving with SeekEnd + offset, err = rs.Seek(-400, io.SeekEnd) + if err != nil { + t.Fatal(err) + } + + if offset != 624 { + t.Fatalf("expected offset %d, got %d", 624, offset) + } + + _, err = io.ReadFull(rs, readBuf) + if err != nil { + t.Fatal(err) + } + + offset, err = rs.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + + if offset != 880 { + t.Fatalf("expected offset %d, got %d", 880, offset) + } +} diff --git a/file/large_file_test.go b/file/large_file_test.go index 8d5044b..bbfb9d4 100644 --- a/file/large_file_test.go +++ b/file/large_file_test.go @@ -68,6 +68,43 @@ func TestLargeFileReader(t *testing.T) { } } +func TestLargeFileSeeker(t *testing.T) { + ls := cidlink.DefaultLinkSystem() + storage := cidlink.Memory{} + ls.StorageReadOpener = storage.OpenRead + ls.StorageWriteOpener = storage.OpenWrite + + // Make random file with 1024 bytes. + buf := make([]byte, 1024) + ipfsutil.NewSeededRand(0xdeadbeef).Read(buf) + r := bytes.NewReader(buf) + + // Build UnixFS File chunked in 256 byte parts. + f, _, err := builder.BuildUnixFSFile(r, "size-256", &ls) + if err != nil { + t.Fatal(err) + } + + // Load the file. + fr, err := ls.Load(ipld.LinkContext{}, f, dagpb.Type.PBNode) + if err != nil { + t.Fatal(err) + } + + // Create it. + ufn, err := file.NewUnixFSFile(context.Background(), fr, &ls) + if err != nil { + t.Fatal(err) + } + + rs, err := ufn.AsLargeBytes() + if err != nil { + t.Fatal(err) + } + + testSeekIn1024ByteFile(t, rs) +} + func TestLargeFileReaderReadsOnlyNecessaryBlocks(t *testing.T) { tracker, ls := mockTrackingLinkSystem() diff --git a/file/shard.go b/file/shard.go index dac1878..dc28310 100644 --- a/file/shard.go +++ b/file/shard.go @@ -174,6 +174,7 @@ func (s *shardNodeReader) Read(p []byte) (int, error) { s.rdr = rdr } n, err := s.rdr.Read(p) + s.offset += int64(n) return n, err }