diff --git a/testhelper/docker/resource/minio/minio.go b/testhelper/docker/resource/minio/minio.go index c8cbf5ed..ee10866e 100644 --- a/testhelper/docker/resource/minio/minio.go +++ b/testhelper/docker/resource/minio/minio.go @@ -8,8 +8,11 @@ import ( "io" "log" "net/http" + "os" + "path/filepath" "slices" "strings" + "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -30,8 +33,10 @@ type Resource struct { } type File struct { - Key string - Content string + Key string + Content string + Etag string + LastModificationTime time.Time } func Setup(pool *dockertest.Pool, d resource.Cleaner, opts ...func(*Config)) (*Resource, error) { @@ -155,8 +160,10 @@ func (r *Resource) Contents(ctx context.Context, prefix string) ([]File, error) } contents = append(contents, File{ - Key: objInfo.Key, - Content: string(b), + Key: objInfo.Key, + Content: string(b), + Etag: objInfo.ETag, + LastModificationTime: objInfo.LastModified, }) } @@ -182,3 +189,23 @@ func (r *Resource) ToFileManagerConfig(prefix string) map[string]any { "region": r.Region, } } + +func (r *Resource) UploadFolder(localPath, prefix string) error { + minioClient := r.Client + localPath = filepath.Clean(localPath) + + return filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + objectName, err := filepath.Rel(localPath, path) + if err != nil { + return err + } + _, err = minioClient.FPutObject(context.TODO(), r.BucketName, filepath.Join(prefix, objectName), path, minio.PutObjectOptions{}) + return err + }) +} diff --git a/testhelper/docker/resource/minio/minio_test.go b/testhelper/docker/resource/minio/minio_test.go index 5c06535e..4879b020 100644 --- a/testhelper/docker/resource/minio/minio_test.go +++ b/testhelper/docker/resource/minio/minio_test.go @@ -4,10 +4,12 @@ import ( "bytes" "compress/gzip" "context" + "os" "testing" "github.com/minio/minio-go/v7" "github.com/ory/dockertest/v3" + "github.com/samber/lo" "github.com/stretchr/testify/require" "github.com/rudderlabs/rudder-go-kit/filemanager" @@ -63,10 +65,11 @@ func TestMinioContents(t *testing.T) { minioResource, err := Setup(pool, t) require.NoError(t, err) - _, err = minioResource.Client.PutObject(context.Background(), + uploadInfo, err := minioResource.Client.PutObject(context.Background(), minioResource.BucketName, "test-bucket/hello.txt", bytes.NewBufferString("hello"), -1, minio.PutObjectOptions{}, ) require.NoError(t, err) + etag1 := uploadInfo.ETag var b bytes.Buffer gz := gzip.NewWriter(&b) @@ -75,24 +78,37 @@ func TestMinioContents(t *testing.T) { err = gz.Close() require.NoError(t, err) - _, err = minioResource.Client.PutObject(context.Background(), + uploadInfo, err = minioResource.Client.PutObject(context.Background(), minioResource.BucketName, "test-bucket/hello.txt.gz", &b, -1, minio.PutObjectOptions{}, ) require.NoError(t, err) + etag2 := uploadInfo.ETag - _, err = minioResource.Client.PutObject(context.Background(), + uploadInfo, err = minioResource.Client.PutObject(context.Background(), minioResource.BucketName, "test-bucket/empty", bytes.NewBuffer([]byte{}), -1, minio.PutObjectOptions{}, ) require.NoError(t, err) + etag3 := uploadInfo.ETag files, err := minioResource.Contents(context.Background(), "test-bucket/") require.NoError(t, err) - require.Equal(t, []File{ - {Key: "test-bucket/empty", Content: ""}, - {Key: "test-bucket/hello.txt", Content: "hello"}, - {Key: "test-bucket/hello.txt.gz", Content: "hello compressed"}, - }, files) + // LastModified is set after the file is uploaded, so we can't compare it + lo.ForEach(files, func(f File, _ int) { + switch f.Key { + case "test-bucket/hello.txt": + require.Equal(t, "hello", f.Content) + require.Equal(t, etag1, f.Etag) + case "test-bucket/hello.txt.gz": + require.Equal(t, "hello compressed", f.Content) + require.Equal(t, etag2, f.Etag) + case "test-bucket/empty": + require.Equal(t, "", f.Content) + require.Equal(t, etag3, f.Etag) + default: + t.Fatalf("unexpected file: %s", f.Key) + } + }) t.Run("canceled context", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) @@ -101,4 +117,36 @@ func TestMinioContents(t *testing.T) { _, err := minioResource.Contents(ctx, "test-bucket/") require.ErrorIs(t, err, context.Canceled) }) + + t.Run("can upload a folder", func(t *testing.T) { + expectedContents := map[string]string{ + "test-upload-folder/file1.txt": "file1", + "test-upload-folder/file2.txt": "file2", + } + + tempDir := t.TempDir() + // add some files to the temp dir + file1, err := os.Create(tempDir + "/file1.txt") + require.NoError(t, err, "could not create file1.txt") + _, err = file1.WriteString(expectedContents["test-upload-folder/file1.txt"]) + require.NoError(t, err, "could not write to file1.txt") + + file2, err := os.Create(tempDir + "/file2.txt") + require.NoError(t, err, "could not create file2.txt") + _, err = file2.WriteString(expectedContents["test-upload-folder/file2.txt"]) + require.NoError(t, err, "could not write to file2.txt") + + err = minioResource.UploadFolder(tempDir, "test-upload-folder") + require.NoError(t, err, "could not upload folder") + + files, err := minioResource.Contents(context.Background(), "test-upload-folder") + require.NoError(t, err) + require.Len(t, files, 2, "should have uploaded 2 files") + + lo.ForEach(files, func(f File, _ int) { + if expectedContent, exists := expectedContents[f.Key]; exists { + require.Equal(t, expectedContent, f.Content) + } + }) + }) }