From ece653d7a2c4c6dceb276367fee3a45af52c58a9 Mon Sep 17 00:00:00 2001 From: Barak Amar Date: Mon, 7 Sep 2020 13:42:18 +0300 Subject: [PATCH] Chore/Nessie run system tests on Google Storage (#568) --- .github/workflows/nessie.yaml | 22 +++++++++++++++++++--- api/api_controller.go | 4 ++++ block/factory/build.go | 5 ++++- docker-compose.yaml | 16 +++++++++------- nessie/system_test.go | 2 +- 5 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nessie.yaml b/.github/workflows/nessie.yaml index b0f9a573f9f..9a7648478e9 100644 --- a/.github/workflows/nessie.yaml +++ b/.github/workflows/nessie.yaml @@ -18,7 +18,7 @@ jobs: id: go - name: Build docker image run: make build-docker - - name: Run lakeFS + - name: Run lakeFS S3 env: TAG: dev LAKEFS_STATS_ENABLED: "false" @@ -27,11 +27,11 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.NESSIE_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }} run: docker-compose up --quiet-pull --build -d - - name: Run Nessie + - name: Run Nessie S3 env: NESSIE_STORAGE_NAMESPACE: s3://nessie-system-testing/${{ github.run_number }} run: go test -v ./nessie --system-tests - - name: Check files in bucket + - name: Check files in S3 bucket env: AWS_ACCESS_KEY_ID: ${{ secrets.NESSIE_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }} @@ -45,3 +45,19 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }} run: | docker-compose ps -q postgres && docker-compose exec postgres pg_dumpall --username=lakefs | gzip | aws s3 cp - s3://nessie-system-testing/${{ github.run_number }}/dump.gz + - name: Run lakeFS GS + env: + TAG: dev + LAKEFS_STATS_ENABLED: "false" + LAKEFS_BLOCKSTORE_TYPE: gs + LAKEFS_GATEWAYS_S3_DOMAIN_NAME: s3.local.lakefs.io:8000 + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON: ${{ secrets.LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON }} + run: | + docker-compose down + docker-compose up --quiet-pull --build -d + - name: Run Nessie GS + env: + NESSIE_STORAGE_NAMESPACE: gs://nessie-system-testing/${{ github.run_number }} + run: go test -v ./nessie --system-tests diff --git a/api/api_controller.go b/api/api_controller.go index b374763a90b..e6b4e88ff15 100644 --- a/api/api_controller.go +++ b/api/api_controller.go @@ -544,6 +544,10 @@ func (c *Controller) CreateRepositoryHandler() repositories.CreateRepositoryHand err = ensureStorageNamespaceRW(deps.BlockAdapter, swag.StringValue(params.Repository.StorageNamespace)) if err != nil { + c.deps.logger. + WithError(err). + WithField("storage_namespace", swag.StringValue(params.Repository.StorageNamespace)). + Warn("Could not access storage namespace") return repositories.NewCreateRepositoryBadRequest(). WithPayload(responseError("error creating repository: could not access storage namespace")) } diff --git a/block/factory/build.go b/block/factory/build.go index b73e5951194..f9c2bd01b1d 100644 --- a/block/factory/build.go +++ b/block/factory/build.go @@ -22,6 +22,9 @@ import ( "google.golang.org/api/option" ) +// googleAuthCloudPlatform - Cloud Storage authentication https://cloud.google.com/storage/docs/authentication +const googleAuthCloudPlatform = "https://www.googleapis.com/auth/cloud-platform" + var ErrInvalidBlockStoreType = errors.New("invalid blockstore type") func BuildBlockAdapter(c *config.Config) (block.Adapter, error) { @@ -91,7 +94,7 @@ func buildGSAdapter(params params.GS) (*gs.Adapter, error) { opts = append(opts, option.WithCredentialsFile(params.CredentialsFile)) } else if params.CredentialsJSON != "" { ctx := context.Background() - cred, err := google.CredentialsFromJSON(ctx, []byte(params.CredentialsJSON)) + cred, err := google.CredentialsFromJSON(ctx, []byte(params.CredentialsJSON), googleAuthCloudPlatform) if err != nil { return nil, err } diff --git a/docker-compose.yaml b/docker-compose.yaml index d413f2d1e7a..b69f9bbabac 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,13 +7,15 @@ services: depends_on: - "postgres" environment: - LAKEFS_AUTH_ENCRYPT_SECRET_KEY: some random secret string - LAKEFS_DATABASE_CONNECTION_STRING: postgres://lakefs:lakefs@postgres/postgres?sslmode=disable - LAKEFS_BLOCKSTORE_TYPE: ${LAKEFS_BLOCKSTORE_TYPE} - LAKEFS_BLOCKSTORE_LOCAL_PATH: /home/lakefs - LAKEFS_GATEWAYS_S3_DOMAIN_NAME: s3.local.lakefs.io:8000 - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_SECRET_KEY: ${AWS_SECRET_ACCESS_KEY} + - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string + - LAKEFS_DATABASE_CONNECTION_STRING=postgres://lakefs:lakefs@postgres/postgres?sslmode=disable + - LAKEFS_BLOCKSTORE_TYPE=${LAKEFS_BLOCKSTORE_TYPE} + - LAKEFS_BLOCKSTORE_LOCAL_PATH=/home/lakefs + - LAKEFS_GATEWAYS_S3_DOMAIN_NAME=s3.local.lakefs.io:8000 + - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_SECRET_KEY=${AWS_SECRET_ACCESS_KEY} + - LAKEFS_LOGGING_LEVEL + - LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON entrypoint: ["/app/wait-for", "postgres:5432", "--", "/app/lakefs", "run"] postgres: image: "postgres:11" diff --git a/nessie/system_test.go b/nessie/system_test.go index b38d806f011..4502ce79042 100644 --- a/nessie/system_test.go +++ b/nessie/system_test.go @@ -57,7 +57,7 @@ func createRepository(ctx context.Context, t *testing.T, name string, repoStorag ID: swag.String(name), StorageNamespace: swag.String(repoStorage), }), nil) - require.NoErrorf(t, err, "failed to create repository %s, storage %s", name, repoStorage) + require.NoErrorf(t, err, "failed to create repository '%s', storage '%s'", name, repoStorage) } func uploadFileRandomData(ctx context.Context, t *testing.T, repo, branch, objPath string) (checksum, content string) {