add integration test for export

.github/workflows/nessie.yaml

@@ -117,6 +117,8 @@ jobs:
       - name: Run Nessie S3
         env:
           NESSIE_STORAGE_NAMESPACE: s3://nessie-system-testing/${{ github.run_number }}
+          NESSIE_AWS_ACCESS_KEY_ID: ${{ secrets.NESSIE_AWS_ACCESS_KEY_ID }}
+          NESSIE_AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }}
         run: go test -v ./nessie --system-tests
       - name: Check files in S3 bucket
         env:

nessie/export_test.go

@@ -0,0 +1,90 @@
+package nessie
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/spf13/viper"
+	"github.com/treeverse/lakefs/api/gen/client/export"
+	"io/ioutil"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/go-openapi/swag"
+	"github.com/stretchr/testify/require"
+	"github.com/treeverse/lakefs/api/gen/client/commits"
+	"github.com/treeverse/lakefs/api/gen/models"
+)
+
+func NewS3Service() *s3.S3 {
+	awsSession := session.Must(session.NewSession())
+	return s3.New(awsSession,
+		aws.NewConfig().
+			WithRegion("us-east-1").
+			WithCredentials(credentials.NewCredentials(
+				&credentials.StaticProvider{
+					Value: credentials.Value{
+						AccessKeyID:     viper.GetString("aws_access_key_id"),
+						SecretAccessKey: viper.GetString("aws_secret_access_key"),
+					}})))
+
+}
+
+func parsePath(t testing.TB, path string) (string, string) {
+	t.Helper()
+	u, err := url.Parse(path)
+	require.NoError(t, err, "failed to parse path")
+	bucket := u.Host
+	keyPath := strings.TrimLeft(u.Path, "/")
+	return bucket, keyPath
+}
+
+func TestExport(t *testing.T) {
+	ctx, _, repo := setupTest(t)
+
+	// set export configurations
+	exportPath, statusPath := setExportPathForTest(ctx, t, masterBranch)
+
+	// upload and commit
+	objPath := "1.txt"
+	_, objContent := uploadFileRandomData(ctx, t, repo, masterBranch, objPath)
+	commitRes, err := client.Commits.Commit(
+		commits.NewCommitParamsWithContext(ctx).
+			WithRepository(repo).
+			WithBranch(masterBranch).
+			WithCommit(&models.CommitCreation{
+				Message: swag.String("nessie:firstCommitForExport"),
+			}), nil)
+	require.NoError(t, err, "failed to commit changes")
+	commit := commitRes.GetPayload()
+
+	// run single export
+	_, err = client.Export.Run(export.NewRunParamsWithContext(ctx).WithRepository(repo).WithBranch(masterBranch), nil)
+	require.NoError(t, err, "failed to export changes")
+
+	// check exported file exist
+	bucket, keyPath := parsePath(t, exportPath)
+	key := keyPath + "/" + objPath
+	s3Svc := NewS3Service()
+	objectOutput, err := s3Svc.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	require.NoError(t, err, "failed to get exported file")
+	body, err := ioutil.ReadAll(objectOutput.Body)
+	require.NoError(t, err, "failed to read exported file")
+	require.Equal(t, objContent, string(body), "unexpected content at %s", objPath)
+
+	// check exported status file exists
+	bucket, keyPath = parsePath(t, statusPath)
+	statusFilename := fmt.Sprintf("%s-%s-%s", repo, masterBranch, commit.ID)
+	key = keyPath + "/" + statusFilename
+	_, err = s3Svc.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	require.NoError(t, err, "failed to get exported status file")
+}

nessie/system_test.go

@@ -2,6 +2,9 @@ package nessie
 
 import (
 	"context"
+	"fmt"
+	"github.com/go-openapi/strfmt"
+	"github.com/treeverse/lakefs/api/gen/client/export"
 	"strings"
 	"testing"
 
@@ -37,6 +40,15 @@ func createRepositoryForTest(ctx context.Context, t *testing.T) string {
 	return name
 }
 
+func setExportPathForTest(ctx context.Context, t *testing.T, branch string) (string, string) {
+	name := strings.ToLower(t.Name())
+	storageNamespace := viper.GetString("storage_namespace")
+	exportPath := fmt.Sprintf("%s/%s_EXPORT", storageNamespace, name)
+	statusPath := fmt.Sprintf("%s/%s_EXPORT/_status", storageNamespace, name)
+	setExportPath(ctx, t, name, branch, exportPath, statusPath)
+	return exportPath, statusPath
+}
+
 func createRepositoryUnique(ctx context.Context, t *testing.T) string {
 	id := xid.New().String()
 	name := "repo-" + id
@@ -45,6 +57,26 @@ func createRepositoryUnique(ctx context.Context, t *testing.T) string {
 	return name
 }
 
+func setExportPath(ctx context.Context, t *testing.T, repo, branch, path, statusPath string) {
+	logger.WithFields(logging.Fields{
+		"repository":  repo,
+		"branch":      branch,
+		"export-path": path,
+		"status-path": statusPath,
+	}).Debug("Create Export Path for test")
+
+	config := models.ContinuousExportConfiguration{
+		ExportPath:             strfmt.URI(path),
+		ExportStatusPath:       strfmt.URI(statusPath),
+		LastKeysInPrefixRegexp: []string{"^_success$", ".*/_success$"},
+	}
+	_, err := client.Export.SetContinuousExport(export.NewSetContinuousExportParamsWithContext(ctx).
+		WithRepository(repo).
+		WithBranch(branch).
+		WithConfig(&config), nil)
+	require.NoErrorf(t, err, "failed to set export configuration repository '%s', path '%s'", repo, path)
+}
+
 func createRepository(ctx context.Context, t *testing.T, name string, repoStorage string) {
 	logger.WithFields(logging.Fields{
 		"repository":        name,