feat: [PRO-3387] minio enhancements (#630)

testhelper/docker/resource/minio/minio.go

@@ -8,8 +8,11 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"os"
+	"path/filepath"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -30,8 +33,10 @@ type Resource struct {
 }
 
 type File struct {
-	Key     string
-	Content string
+	Key                  string
+	Content              string
+	Etag                 string
+	LastModificationTime time.Time
 }
 
 func Setup(pool *dockertest.Pool, d resource.Cleaner, opts ...func(*Config)) (*Resource, error) {
@@ -155,8 +160,10 @@ func (r *Resource) Contents(ctx context.Context, prefix string) ([]File, error)
 		}
 
 		contents = append(contents, File{
-			Key:     objInfo.Key,
-			Content: string(b),
+			Key:                  objInfo.Key,
+			Content:              string(b),
+			Etag:                 objInfo.ETag,
+			LastModificationTime: objInfo.LastModified,
 		})
 	}
 
@@ -182,3 +189,23 @@ func (r *Resource) ToFileManagerConfig(prefix string) map[string]any {
 		"region":           r.Region,
 	}
 }
+
+func (r *Resource) UploadFolder(localPath, prefix string) error {
+	minioClient := r.Client
+	localPath = filepath.Clean(localPath)
+
+	return filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		objectName, err := filepath.Rel(localPath, path)
+		if err != nil {
+			return err
+		}
+		_, err = minioClient.FPutObject(context.TODO(), r.BucketName, filepath.Join(prefix, objectName), path, minio.PutObjectOptions{})
+		return err
+	})
+}

testhelper/docker/resource/minio/minio_test.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"os"
 	"testing"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/ory/dockertest/v3"
+	"github.com/samber/lo"
 	"github.com/stretchr/testify/require"
 
 	"github.com/rudderlabs/rudder-go-kit/filemanager"
@@ -63,10 +65,11 @@ func TestMinioContents(t *testing.T) {
 	minioResource, err := Setup(pool, t)
 	require.NoError(t, err)
 
-	_, err = minioResource.Client.PutObject(context.Background(),
+	uploadInfo, err := minioResource.Client.PutObject(context.Background(),
 		minioResource.BucketName, "test-bucket/hello.txt", bytes.NewBufferString("hello"), -1, minio.PutObjectOptions{},
 	)
 	require.NoError(t, err)
+	etag1 := uploadInfo.ETag
 
 	var b bytes.Buffer
 	gz := gzip.NewWriter(&b)
@@ -75,24 +78,37 @@ func TestMinioContents(t *testing.T) {
 	err = gz.Close()
 	require.NoError(t, err)
 
-	_, err = minioResource.Client.PutObject(context.Background(),
+	uploadInfo, err = minioResource.Client.PutObject(context.Background(),
 		minioResource.BucketName, "test-bucket/hello.txt.gz", &b, -1, minio.PutObjectOptions{},
 	)
 	require.NoError(t, err)
+	etag2 := uploadInfo.ETag
 
-	_, err = minioResource.Client.PutObject(context.Background(),
+	uploadInfo, err = minioResource.Client.PutObject(context.Background(),
 		minioResource.BucketName, "test-bucket/empty", bytes.NewBuffer([]byte{}), -1, minio.PutObjectOptions{},
 	)
 	require.NoError(t, err)
+	etag3 := uploadInfo.ETag
 
 	files, err := minioResource.Contents(context.Background(), "test-bucket/")
 	require.NoError(t, err)
 
-	require.Equal(t, []File{
-		{Key: "test-bucket/empty", Content: ""},
-		{Key: "test-bucket/hello.txt", Content: "hello"},
-		{Key: "test-bucket/hello.txt.gz", Content: "hello compressed"},
-	}, files)
+	// LastModified is set after the file is uploaded, so we can't compare it
+	lo.ForEach(files, func(f File, _ int) {
+		switch f.Key {
+		case "test-bucket/hello.txt":
+			require.Equal(t, "hello", f.Content)
+			require.Equal(t, etag1, f.Etag)
+		case "test-bucket/hello.txt.gz":
+			require.Equal(t, "hello compressed", f.Content)
+			require.Equal(t, etag2, f.Etag)
+		case "test-bucket/empty":
+			require.Equal(t, "", f.Content)
+			require.Equal(t, etag3, f.Etag)
+		default:
+			t.Fatalf("unexpected file: %s", f.Key)
+		}
+	})
 
 	t.Run("canceled context", func(t *testing.T) {
 		ctx, cancel := context.WithCancel(context.Background())
@@ -101,4 +117,36 @@ func TestMinioContents(t *testing.T) {
 		_, err := minioResource.Contents(ctx, "test-bucket/")
 		require.ErrorIs(t, err, context.Canceled)
 	})
+
+	t.Run("can upload a folder", func(t *testing.T) {
+		expectedContents := map[string]string{
+			"test-upload-folder/file1.txt": "file1",
+			"test-upload-folder/file2.txt": "file2",
+		}
+
+		tempDir := t.TempDir()
+		// add some files to the temp dir
+		file1, err := os.Create(tempDir + "/file1.txt")
+		require.NoError(t, err, "could not create file1.txt")
+		_, err = file1.WriteString(expectedContents["test-upload-folder/file1.txt"])
+		require.NoError(t, err, "could not write to file1.txt")
+
+		file2, err := os.Create(tempDir + "/file2.txt")
+		require.NoError(t, err, "could not create file2.txt")
+		_, err = file2.WriteString(expectedContents["test-upload-folder/file2.txt"])
+		require.NoError(t, err, "could not write to file2.txt")
+
+		err = minioResource.UploadFolder(tempDir, "test-upload-folder")
+		require.NoError(t, err, "could not upload folder")
+
+		files, err := minioResource.Contents(context.Background(), "test-upload-folder")
+		require.NoError(t, err)
+		require.Len(t, files, 2, "should have uploaded 2 files")
+
+		lo.ForEach(files, func(f File, _ int) {
+			if expectedContent, exists := expectedContents[f.Key]; exists {
+				require.Equal(t, expectedContent, f.Content)
+			}
+		})
+	})
 }