diff --git a/internal/http/services/archiver/archiver_test.go b/internal/http/services/archiver/archiver_test.go index 265629e82b5..d7cc30c465c 100644 --- a/internal/http/services/archiver/archiver_test.go +++ b/internal/http/services/archiver/archiver_test.go @@ -18,7 +18,22 @@ package archiver -import "testing" +import ( + "archive/tar" + "archive/zip" + "bytes" + "context" + "errors" + "io" + "os" + "path" + "strings" + "testing" + + downMock "github.com/cs3org/reva/pkg/storage/utils/downloader/mock" + walkerMock "github.com/cs3org/reva/pkg/storage/utils/walker/mock" + "github.com/cs3org/reva/pkg/test" +) func TestGetDeepestCommonDir(t *testing.T) { tests := []struct { @@ -83,10 +98,882 @@ func TestGetDeepestCommonDir(t *testing.T) { res := getDeepestCommonDir(tt.paths) if res != tt.expected { - t.Errorf("getDeepestCommondDir() failed: paths=%+v expected=%s got=%s", tt.paths, tt.expected, res) + t.Fatalf("getDeepestCommondDir() failed: paths=%+v expected=%s got=%s", tt.paths, tt.expected, res) } }) } } + +func UnTar(dir string, r io.Reader) error { + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + break // finish to read the archive + } + if err != nil { + return err + } + + p := path.Join(dir, hdr.Name) + + switch hdr.Typeflag { + case tar.TypeDir: + err = os.MkdirAll(p, 0755) + if err != nil { + return err + } + case tar.TypeReg: + d := path.Dir(p) + err := os.MkdirAll(d, 0755) + if err != nil { + return err + } + file, err := os.Create(p) + if err != nil { + return err + } + _, err = io.Copy(file, tr) + if err != nil { + return err + } + default: + return errors.New("not supported") + } + } + return nil +} + +func TestCreateTar(t *testing.T) { + + tests := []struct { + name string + src test.TestDir + config ArchiverConfig + files []string + expected test.TestDir + err error + }{ + { + name: "one file", + src: test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + }, + config: ArchiverConfig{ + MaxSize: 3, + MaxNumFiles: 1, + }, + files: []string{"foo"}, + expected: test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + }, + err: nil, + }, + { + name: "one big file", + src: test.TestDir{ + "foo": test.TestFile{ + Content: strings.Repeat("a", 1024*1024*1024), + }, + }, + config: ArchiverConfig{ + MaxSize: 1024 * 1024 * 1024 * 2, + MaxNumFiles: 1000, + }, + files: []string{"foo"}, + expected: test.TestDir{ + "foo": test.TestFile{ + Content: strings.Repeat("a", 1024*1024*1024), + }, + }, + err: nil, + }, + { + name: "one file - error max files reached", + src: test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + }, + config: ArchiverConfig{ + MaxSize: 3, + MaxNumFiles: 0, + }, + files: []string{"foo"}, + expected: nil, + err: ErrMaxFileCount, + }, + { + name: "one file - error max size reached", + src: test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + }, + config: ArchiverConfig{ + MaxSize: 0, + MaxNumFiles: 1, + }, + files: []string{"foo"}, + expected: nil, + err: ErrMaxSize, + }, + { + name: "one folder empty", + src: test.TestDir{ + "foo": test.TestDir{}, + }, + config: ArchiverConfig{ + MaxSize: 1000, + MaxNumFiles: 1, + }, + files: []string{"foo"}, + expected: test.TestDir{ + "foo": test.TestDir{}, + }, + err: nil, + }, + { + name: "one folder empty - error max files reached", + src: test.TestDir{ + "foo": test.TestDir{}, + }, + config: ArchiverConfig{ + MaxSize: 1000, + MaxNumFiles: 0, + }, + files: []string{"foo"}, + expected: nil, + err: ErrMaxFileCount, + }, + { + name: "one folder - one file in", + src: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 1000, + MaxNumFiles: 1000, + }, + files: []string{"foo"}, + expected: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + }, + err: nil, + }, + { + name: "multiple folders/files in root dir - tar all", + src: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + "nested_file": test.TestFile{ + Content: "nested_file", + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 100000, + MaxNumFiles: 1000, + }, + files: []string{"foo", "foobar", "other_dir"}, + expected: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + "nested_file": test.TestFile{ + Content: "nested_file", + }, + }, + }, + err: nil, + }, + { + name: "multiple folders/files in root dir - tar partial", + src: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + "nested_file": test.TestFile{ + Content: "nested_file", + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 100000, + MaxNumFiles: 1000, + }, + files: []string{"foo", "foobar"}, + expected: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + }, + err: nil, + }, + { + name: "multiple folders/files in root dir - tar different levels", + src: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + "nested_file": test.TestFile{ + Content: "nested_file", + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 100000, + MaxNumFiles: 1000, + }, + files: []string{"foobar", "other_dir/nested_dir/foo", "other_dir/nested_dir/bar"}, + expected: test.TestDir{ + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + }, + }, + err: nil, + }, + { + name: "multiple folders/files in root dir with extesions", + src: test.TestDir{ + "foo": test.TestDir{ + "bar.txt": test.TestFile{ + Content: "qwerty\ntest", + }, + }, + "main.py": test.TestFile{ + Content: "print(\"Hello world!\")\n", + }, + "other_dir": test.TestDir{ + "images": test.TestDir{ + "foo.png": test.TestFile{ + Content: "", + }, + "bar.jpg": test.TestFile{ + Content: "", + }, + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 100000, + MaxNumFiles: 1000, + }, + files: []string{"foo/bar.txt", "main.py", "other_dir"}, + expected: test.TestDir{ + "foo": test.TestDir{ + "bar.txt": test.TestFile{ + Content: "qwerty\ntest", + }, + }, + "main.py": test.TestFile{ + Content: "print(\"Hello world!\")\n", + }, + "other_dir": test.TestDir{ + "images": test.TestDir{ + "foo.png": test.TestFile{ + Content: "", + }, + "bar.jpg": test.TestFile{ + Content: "", + }, + }, + }, + }, + err: nil, + }, + } + + for _, tt := range tests { + + t.Run(tt.name, func(t *testing.T) { + + ctx := context.TODO() + + tmpdir, cleanup, err := test.NewTestDir(tt.src) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + filesAbs := []string{} + for _, f := range tt.files { + filesAbs = append(filesAbs, path.Join(tmpdir, f)) + } + + w := walkerMock.NewWalker() + d := downMock.NewDownloader() + + arch, err := NewArchiver(filesAbs, w, d, tt.config) + if err != nil { + t.Fatal(err) + } + + var tarFile bytes.Buffer + + err = arch.CreateTar(ctx, io.Writer(&tarFile)) + if err != tt.err { + t.Fatalf("error result different from expected: got=%v, expected=%v", err, tt.err) + } + + tarTmpDir, cleanup, err := test.TmpDir() + if err != nil { + t.Fatal(err) + } + defer cleanup() + + err = UnTar(tarTmpDir, &tarFile) + if err != nil { + t.Fatal(err) + } + + if tt.expected != nil { + expectedTmp, cleanup, err := test.NewTestDir(tt.expected) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if !test.DirEquals(tarTmpDir, expectedTmp) { + t.Fatalf("untar dir different from expected") + } + } + + }) + + } + +} + +func UnZip(dir string, r io.Reader) error { + // save the file temporarely + tmp, cleanup, err := test.TmpDir() + if err != nil { + return err + } + defer cleanup() + + zipFile := path.Join(tmp, "tmp.zip") + zfile, err := os.Create(zipFile) + if err != nil { + return err + } + + _, err = io.Copy(zfile, r) + if err != nil { + return err + } + zfile.Close() + + // open the tmp zip file and read it + zr, err := zip.OpenReader(zipFile) + if err != nil { + return err + } + defer zr.Close() + + for _, f := range zr.File { + + p := path.Join(dir, f.Name) + + d := path.Dir(p) + err := os.MkdirAll(d, 0755) + if err != nil { + return err + } + + if strings.HasSuffix(f.Name, "/") { + // is a dir + err := os.Mkdir(p, 0755) + if err != nil { + return err + } + } else { + // is a regular file + file, err := os.Create(p) + if err != nil { + return err + } + + rc, err := f.Open() + if err != nil { + return err + } + defer zr.Close() + _, err = io.Copy(file, rc) + if err != nil { + return err + } + } + } + return nil +} + +func TestCreateZip(t *testing.T) { + + tests := []struct { + name string + src test.TestDir + config ArchiverConfig + files []string + expected test.TestDir + err error + }{ + { + name: "one file", + src: test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + }, + config: ArchiverConfig{ + MaxSize: 3, + MaxNumFiles: 1, + }, + files: []string{"foo"}, + expected: test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + }, + err: nil, + }, + { + name: "one big file", + src: test.TestDir{ + "foo": test.TestFile{ + Content: strings.Repeat("a", 1024*1024*1024), + }, + }, + config: ArchiverConfig{ + MaxSize: 1024 * 1024 * 1024 * 2, + MaxNumFiles: 1000, + }, + files: []string{"foo"}, + expected: test.TestDir{ + "foo": test.TestFile{ + Content: strings.Repeat("a", 1024*1024*1024), + }, + }, + err: nil, + }, + { + name: "one file - error max files reached", + src: test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + }, + config: ArchiverConfig{ + MaxSize: 3, + MaxNumFiles: 0, + }, + files: []string{"foo"}, + expected: nil, + err: ErrMaxFileCount, + }, + { + name: "one file - error max size reached", + src: test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + }, + config: ArchiverConfig{ + MaxSize: 0, + MaxNumFiles: 1, + }, + files: []string{"foo"}, + expected: nil, + err: ErrMaxSize, + }, + { + name: "one folder empty", + src: test.TestDir{ + "foo": test.TestDir{}, + }, + config: ArchiverConfig{ + MaxSize: 1000, + MaxNumFiles: 1, + }, + files: []string{"foo"}, + expected: test.TestDir{ + "foo": test.TestDir{}, + }, + err: nil, + }, + { + name: "one folder empty - error max files reached", + src: test.TestDir{ + "foo": test.TestDir{}, + }, + config: ArchiverConfig{ + MaxSize: 1000, + MaxNumFiles: 0, + }, + files: []string{"foo"}, + expected: nil, + err: ErrMaxFileCount, + }, + { + name: "one folder - one file in", + src: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 1000, + MaxNumFiles: 1000, + }, + files: []string{"foo"}, + expected: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + }, + err: nil, + }, + { + name: "multiple folders/files in root dir - tar all", + src: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + "nested_file": test.TestFile{ + Content: "nested_file", + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 100000, + MaxNumFiles: 1000, + }, + files: []string{"foo", "foobar", "other_dir"}, + expected: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + "nested_file": test.TestFile{ + Content: "nested_file", + }, + }, + }, + err: nil, + }, + { + name: "multiple folders/files in root dir - tar partial", + src: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + "nested_file": test.TestFile{ + Content: "nested_file", + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 100000, + MaxNumFiles: 1000, + }, + files: []string{"foo", "foobar"}, + expected: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + }, + err: nil, + }, + { + name: "multiple folders/files in root dir - tar different levels", + src: test.TestDir{ + "foo": test.TestDir{ + "bar": test.TestFile{ + Content: "bar", + }, + }, + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + "nested_file": test.TestFile{ + Content: "nested_file", + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 100000, + MaxNumFiles: 1000, + }, + files: []string{"foobar", "other_dir/nested_dir/foo", "other_dir/nested_dir/bar"}, + expected: test.TestDir{ + "foobar": test.TestFile{ + Content: "foobar", + }, + "other_dir": test.TestDir{ + "nested_dir": test.TestDir{ + "foo": test.TestFile{ + Content: "foo", + }, + "bar": test.TestFile{ + Content: "bar", + }, + }, + }, + }, + err: nil, + }, + { + name: "multiple folders/files in root dir with extesions", + src: test.TestDir{ + "foo": test.TestDir{ + "bar.txt": test.TestFile{ + Content: "qwerty\ntest", + }, + }, + "main.py": test.TestFile{ + Content: "print(\"Hello world!\")\n", + }, + "other_dir": test.TestDir{ + "images": test.TestDir{ + "foo.png": test.TestFile{ + Content: "", + }, + "bar.jpg": test.TestFile{ + Content: "", + }, + }, + }, + }, + config: ArchiverConfig{ + MaxSize: 100000, + MaxNumFiles: 1000, + }, + files: []string{"foo/bar.txt", "main.py", "other_dir"}, + expected: test.TestDir{ + "foo": test.TestDir{ + "bar.txt": test.TestFile{ + Content: "qwerty\ntest", + }, + }, + "main.py": test.TestFile{ + Content: "print(\"Hello world!\")\n", + }, + "other_dir": test.TestDir{ + "images": test.TestDir{ + "foo.png": test.TestFile{ + Content: "", + }, + "bar.jpg": test.TestFile{ + Content: "", + }, + }, + }, + }, + err: nil, + }, + } + + for _, tt := range tests { + + t.Run(tt.name, func(t *testing.T) { + + ctx := context.TODO() + + tmpdir, cleanup, err := test.NewTestDir(tt.src) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + filesAbs := []string{} + for _, f := range tt.files { + filesAbs = append(filesAbs, path.Join(tmpdir, f)) + } + + w := walkerMock.NewWalker() + d := downMock.NewDownloader() + + arch, err := NewArchiver(filesAbs, w, d, tt.config) + if err != nil { + t.Fatal(err) + } + + var zipFile bytes.Buffer + + err = arch.CreateZip(ctx, io.Writer(&zipFile)) + if err != tt.err { + t.Fatalf("error result different from expected: got=%v, expected=%v", err, tt.err) + } + + if tt.expected != nil { + zipTmpDir, cleanup, err := test.TmpDir() + if err != nil { + t.Fatal(err) + } + defer cleanup() + + err = UnZip(zipTmpDir, &zipFile) + if err != nil { + t.Fatal(err) + } + + expectedTmp, cleanup, err := test.NewTestDir(tt.expected) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if !test.DirEquals(zipTmpDir, expectedTmp) { + t.Fatalf("unzip dir different from expected") + } + } + + }) + + } + +}