Chore/Nessie run system tests on Google Storage (#568)

treeverse · Sep 7, 2020 · ece653d · ece653d
1 parent 4bf9f0b
commit ece653d
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 12 deletions.
diff --git a/.github/workflows/nessie.yaml b/.github/workflows/nessie.yaml
@@ -18,7 +18,7 @@ jobs:
         id: go
       - name: Build docker image
         run: make build-docker
-      - name: Run lakeFS
+      - name: Run lakeFS S3
         env:
           TAG: dev
           LAKEFS_STATS_ENABLED: "false"
@@ -27,11 +27,11 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.NESSIE_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }}
         run: docker-compose up --quiet-pull --build -d
-      - name: Run Nessie
+      - name: Run Nessie S3
         env:
           NESSIE_STORAGE_NAMESPACE: s3://nessie-system-testing/${{ github.run_number }}
         run: go test -v ./nessie --system-tests
-      - name: Check files in bucket
+      - name: Check files in S3 bucket
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.NESSIE_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }}
@@ -45,3 +45,19 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }}
         run: |
             docker-compose ps -q postgres && docker-compose exec postgres pg_dumpall --username=lakefs | gzip | aws s3 cp - s3://nessie-system-testing/${{ github.run_number }}/dump.gz
+      - name: Run lakeFS GS
+        env:
+          TAG: dev
+          LAKEFS_STATS_ENABLED: "false"
+          LAKEFS_BLOCKSTORE_TYPE: gs
+          LAKEFS_GATEWAYS_S3_DOMAIN_NAME: s3.local.lakefs.io:8000
+          AWS_ACCESS_KEY_ID: ""
+          AWS_SECRET_ACCESS_KEY: ""
+          LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON: ${{ secrets.LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON }}
+        run: |
+            docker-compose down
+            docker-compose up --quiet-pull --build -d
+      - name: Run Nessie GS
+        env:
+          NESSIE_STORAGE_NAMESPACE: gs://nessie-system-testing/${{ github.run_number }}
+        run: go test -v ./nessie --system-tests
diff --git a/api/api_controller.go b/api/api_controller.go
@@ -544,6 +544,10 @@ func (c *Controller) CreateRepositoryHandler() repositories.CreateRepositoryHand
 
 		err = ensureStorageNamespaceRW(deps.BlockAdapter, swag.StringValue(params.Repository.StorageNamespace))
 		if err != nil {
+			c.deps.logger.
+				WithError(err).
+				WithField("storage_namespace", swag.StringValue(params.Repository.StorageNamespace)).
+				Warn("Could not access storage namespace")
 			return repositories.NewCreateRepositoryBadRequest().
 				WithPayload(responseError("error creating repository: could not access storage namespace"))
 		}

diff --git a/block/factory/build.go b/block/factory/build.go
@@ -22,6 +22,9 @@ import (
 	"google.golang.org/api/option"
 )
 
+// googleAuthCloudPlatform - Cloud Storage authentication https://cloud.google.com/storage/docs/authentication
+const googleAuthCloudPlatform = "https://www.googleapis.com/auth/cloud-platform"
+
 var ErrInvalidBlockStoreType = errors.New("invalid blockstore type")
 
 func BuildBlockAdapter(c *config.Config) (block.Adapter, error) {
@@ -91,7 +94,7 @@ func buildGSAdapter(params params.GS) (*gs.Adapter, error) {
 		opts = append(opts, option.WithCredentialsFile(params.CredentialsFile))
 	} else if params.CredentialsJSON != "" {
 		ctx := context.Background()
-		cred, err := google.CredentialsFromJSON(ctx, []byte(params.CredentialsJSON))
+		cred, err := google.CredentialsFromJSON(ctx, []byte(params.CredentialsJSON), googleAuthCloudPlatform)
 		if err != nil {
 			return nil, err
 		}

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -7,13 +7,15 @@ services:
     depends_on:
       - "postgres"
     environment:
-      LAKEFS_AUTH_ENCRYPT_SECRET_KEY: some random secret string
-      LAKEFS_DATABASE_CONNECTION_STRING: postgres://lakefs:lakefs@postgres/postgres?sslmode=disable
-      LAKEFS_BLOCKSTORE_TYPE: ${LAKEFS_BLOCKSTORE_TYPE}
-      LAKEFS_BLOCKSTORE_LOCAL_PATH: /home/lakefs
-      LAKEFS_GATEWAYS_S3_DOMAIN_NAME: s3.local.lakefs.io:8000
-      LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
-      LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_SECRET_KEY: ${AWS_SECRET_ACCESS_KEY}
+      - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string
+      - LAKEFS_DATABASE_CONNECTION_STRING=postgres://lakefs:lakefs@postgres/postgres?sslmode=disable
+      - LAKEFS_BLOCKSTORE_TYPE=${LAKEFS_BLOCKSTORE_TYPE}
+      - LAKEFS_BLOCKSTORE_LOCAL_PATH=/home/lakefs
+      - LAKEFS_GATEWAYS_S3_DOMAIN_NAME=s3.local.lakefs.io:8000
+      - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_SECRET_KEY=${AWS_SECRET_ACCESS_KEY}
+      - LAKEFS_LOGGING_LEVEL
+      - LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON
     entrypoint: ["/app/wait-for", "postgres:5432", "--", "/app/lakefs", "run"]
   postgres:
     image: "postgres:11"

diff --git a/nessie/system_test.go b/nessie/system_test.go
@@ -57,7 +57,7 @@ func createRepository(ctx context.Context, t *testing.T, name string, repoStorag
 			ID:               swag.String(name),
 			StorageNamespace: swag.String(repoStorage),
 		}), nil)
-	require.NoErrorf(t, err, "failed to create repository %s, storage %s", name, repoStorage)
+	require.NoErrorf(t, err, "failed to create repository '%s', storage '%s'", name, repoStorage)
 }
 
 func uploadFileRandomData(ctx context.Context, t *testing.T, repo, branch, objPath string) (checksum, content string) {