From a83dc02ab9a85c5f78e3d585232e3c97e32a0201 Mon Sep 17 00:00:00 2001 From: guyhardonag Date: Mon, 16 Nov 2020 09:44:09 +0200 Subject: [PATCH] add integration test for export --- .github/workflows/nessie.yaml | 2 + nessie/export_test.go | 90 +++++++++++++++++++++++++++++++++++ nessie/system_test.go | 32 +++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 nessie/export_test.go diff --git a/.github/workflows/nessie.yaml b/.github/workflows/nessie.yaml index d634a83ba04..8d6c89912cc 100644 --- a/.github/workflows/nessie.yaml +++ b/.github/workflows/nessie.yaml @@ -117,6 +117,8 @@ jobs: - name: Run Nessie S3 env: NESSIE_STORAGE_NAMESPACE: s3://nessie-system-testing/${{ github.run_number }} + NESSIE_AWS_ACCESS_KEY_ID: ${{ secrets.NESSIE_AWS_ACCESS_KEY_ID }} + NESSIE_AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }} run: go test -v ./nessie --system-tests - name: Check files in S3 bucket env: diff --git a/nessie/export_test.go b/nessie/export_test.go new file mode 100644 index 00000000000..d99520962cb --- /dev/null +++ b/nessie/export_test.go @@ -0,0 +1,90 @@ +package nessie + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/spf13/viper" + "github.com/treeverse/lakefs/api/gen/client/export" + "io/ioutil" + "net/url" + "strings" + "testing" + + "github.com/go-openapi/swag" + "github.com/stretchr/testify/require" + "github.com/treeverse/lakefs/api/gen/client/commits" + "github.com/treeverse/lakefs/api/gen/models" +) + +func NewS3Service() *s3.S3 { + awsSession := session.Must(session.NewSession()) + return s3.New(awsSession, + aws.NewConfig(). + WithRegion("us-east-1"). + WithCredentials(credentials.NewCredentials( + &credentials.StaticProvider{ + Value: credentials.Value{ + AccessKeyID: viper.GetString("aws_access_key_id"), + SecretAccessKey: viper.GetString("aws_secret_access_key"), + }}))) + +} + +func parsePath(t testing.TB, path string) (string, string) { + t.Helper() + u, err := url.Parse(path) + require.NoError(t, err, "failed to parse path") + bucket := u.Host + keyPath := strings.TrimLeft(u.Path, "/") + return bucket, keyPath +} + +func TestExport(t *testing.T) { + ctx, _, repo := setupTest(t) + + // set export configurations + exportPath, statusPath := setExportPathForTest(ctx, t, masterBranch) + + // upload and commit + objPath := "1.txt" + _, objContent := uploadFileRandomData(ctx, t, repo, masterBranch, objPath) + commitRes, err := client.Commits.Commit( + commits.NewCommitParamsWithContext(ctx). + WithRepository(repo). + WithBranch(masterBranch). + WithCommit(&models.CommitCreation{ + Message: swag.String("nessie:firstCommitForExport"), + }), nil) + require.NoError(t, err, "failed to commit changes") + commit := commitRes.GetPayload() + + // run single export + _, err = client.Export.Run(export.NewRunParamsWithContext(ctx).WithRepository(repo).WithBranch(masterBranch), nil) + require.NoError(t, err, "failed to export changes") + + // check exported file exist + bucket, keyPath := parsePath(t, exportPath) + key := keyPath + "/" + objPath + s3Svc := NewS3Service() + objectOutput, err := s3Svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + require.NoError(t, err, "failed to get exported file") + body, err := ioutil.ReadAll(objectOutput.Body) + require.NoError(t, err, "failed to read exported file") + require.Equal(t, objContent, string(body), "unexpected content at %s", objPath) + + // check exported status file exists + bucket, keyPath = parsePath(t, statusPath) + statusFilename := fmt.Sprintf("%s-%s-%s", repo, masterBranch, commit.ID) + key = keyPath + "/" + statusFilename + _, err = s3Svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + require.NoError(t, err, "failed to get exported status file") +} diff --git a/nessie/system_test.go b/nessie/system_test.go index 6712990c31c..dc934cc28c9 100644 --- a/nessie/system_test.go +++ b/nessie/system_test.go @@ -2,6 +2,9 @@ package nessie import ( "context" + "fmt" + "github.com/go-openapi/strfmt" + "github.com/treeverse/lakefs/api/gen/client/export" "strings" "testing" @@ -37,6 +40,15 @@ func createRepositoryForTest(ctx context.Context, t *testing.T) string { return name } +func setExportPathForTest(ctx context.Context, t *testing.T, branch string) (string, string) { + name := strings.ToLower(t.Name()) + storageNamespace := viper.GetString("storage_namespace") + exportPath := fmt.Sprintf("%s/%s_EXPORT", storageNamespace, name) + statusPath := fmt.Sprintf("%s/%s_EXPORT/_status", storageNamespace, name) + setExportPath(ctx, t, name, branch, exportPath, statusPath) + return exportPath, statusPath +} + func createRepositoryUnique(ctx context.Context, t *testing.T) string { id := xid.New().String() name := "repo-" + id @@ -45,6 +57,26 @@ func createRepositoryUnique(ctx context.Context, t *testing.T) string { return name } +func setExportPath(ctx context.Context, t *testing.T, repo, branch, path, statusPath string) { + logger.WithFields(logging.Fields{ + "repository": repo, + "branch": branch, + "export-path": path, + "status-path": statusPath, + }).Debug("Create Export Path for test") + + config := models.ContinuousExportConfiguration{ + ExportPath: strfmt.URI(path), + ExportStatusPath: strfmt.URI(statusPath), + LastKeysInPrefixRegexp: []string{"^_success$", ".*/_success$"}, + } + _, err := client.Export.SetContinuousExport(export.NewSetContinuousExportParamsWithContext(ctx). + WithRepository(repo). + WithBranch(branch). + WithConfig(&config), nil) + require.NoErrorf(t, err, "failed to set export configuration repository '%s', path '%s'", repo, path) +} + func createRepository(ctx context.Context, t *testing.T, name string, repoStorage string) { logger.WithFields(logging.Fields{ "repository": name,