Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Chore/Nessie run system tests on Google Storage #568

Merged
merged 7 commits into from
Sep 7, 2020
+37 −12
Merged

Chore/Nessie run system tests on Google Storage #568

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
eab4590
fix gs credentials from json missing scope
nopcoder Sep 6, 2020
4354e7d
Merge branch 'master' into chore/nessie-gs
nopcoder Sep 6, 2020
25d5338
run nessie flow on gs too
nopcoder Sep 6, 2020
d25f9e5
fix storage namespace for gs
nopcoder Sep 6, 2020
59ac9e1
update secret key name
nopcoder Sep 6, 2020
586daab
fix spell
nopcoder Sep 7, 2020
38c338c
code review changes
nopcoder Sep 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions .github/workflows/nessie.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
id: go
- name: Build docker image
run: make build-docker
- name: Run lakeFS
- name: Run lakeFS S3
env:
TAG: dev
LAKEFS_STATS_ENABLED: "false"
Expand All @@ -27,11 +27,11 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.NESSIE_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }}
run: docker-compose up --quiet-pull --build -d
- name: Run Nessie
- name: Run Nessie S3
env:
NESSIE_STORAGE_NAMESPACE: s3://nessie-system-testing/${{ github.run_number }}
run: go test -v ./nessie --system-tests
- name: Check files in bucket
- name: Check files in S3 bucket
env:
AWS_ACCESS_KEY_ID: ${{ secrets.NESSIE_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }}
Expand All @@ -45,3 +45,19 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }}
run: |
docker-compose ps -q postgres && docker-compose exec postgres pg_dumpall --username=lakefs | gzip | aws s3 cp - s3://nessie-system-testing/${{ github.run_number }}/dump.gz
- name: Run lakeFS GS
itaiad200 marked this conversation as resolved.
Show resolved Hide resolved
env:
TAG: dev
LAKEFS_STATS_ENABLED: "false"
LAKEFS_BLOCKSTORE_TYPE: gs
LAKEFS_GATEWAYS_S3_DOMAIN_NAME: s3.local.lakefs.io:8000
AWS_ACCESS_KEY_ID: ""
AWS_SECRET_ACCESS_KEY: ""
LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON: ${{ secrets.LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON }}
run: |
docker-compose down
docker-compose up --quiet-pull --build -d
- name: Run Nessie GS
env:
NESSIE_STORAGE_NAMESPACE: gs://nessie-system-testing/${{ github.run_number }}
run: go test -v ./nessie --system-tests
4 changes: 4 additions & 0 deletions api/api_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,10 @@ func (c *Controller) CreateRepositoryHandler() repositories.CreateRepositoryHand

err = ensureStorageNamespaceRW(deps.BlockAdapter, swag.StringValue(params.Repository.StorageNamespace))
if err != nil {
c.deps.logger.
WithError(err).
WithField("storage_namespace", swag.StringValue(params.Repository.StorageNamespace)).
Warn("Cloud not access storage namespace")
nopcoder marked this conversation as resolved.
Show resolved Hide resolved
return repositories.NewCreateRepositoryBadRequest().
WithPayload(responseError("error creating repository: could not access storage namespace"))
}
Expand Down
2 changes: 1 addition & 1 deletion block/factory/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func buildGSAdapter(params params.GS) (*gs.Adapter, error) {
opts = append(opts, option.WithCredentialsFile(params.CredentialsFile))
} else if params.CredentialsJSON != "" {
ctx := context.Background()
cred, err := google.CredentialsFromJSON(ctx, []byte(params.CredentialsJSON))
cred, err := google.CredentialsFromJSON(ctx, []byte(params.CredentialsJSON), "https://www.googleapis.com/auth/cloud-platform")
nopcoder marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return nil, err
}
Expand Down
16 changes: 9 additions & 7 deletions docker-compose.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ services:
depends_on:
- "postgres"
environment:
LAKEFS_AUTH_ENCRYPT_SECRET_KEY: some random secret string
LAKEFS_DATABASE_CONNECTION_STRING: postgres://lakefs:lakefs@postgres/postgres?sslmode=disable
LAKEFS_BLOCKSTORE_TYPE: ${LAKEFS_BLOCKSTORE_TYPE}
LAKEFS_BLOCKSTORE_LOCAL_PATH: /home/lakefs
LAKEFS_GATEWAYS_S3_DOMAIN_NAME: s3.local.lakefs.io:8000
LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_SECRET_KEY: ${AWS_SECRET_ACCESS_KEY}
- LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string
- LAKEFS_DATABASE_CONNECTION_STRING=postgres://lakefs:lakefs@postgres/postgres?sslmode=disable
- LAKEFS_BLOCKSTORE_TYPE=${LAKEFS_BLOCKSTORE_TYPE}
- LAKEFS_BLOCKSTORE_LOCAL_PATH=/home/lakefs
- LAKEFS_GATEWAYS_S3_DOMAIN_NAME=s3.local.lakefs.io:8000
- LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
- LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_SECRET_KEY=${AWS_SECRET_ACCESS_KEY}
- LAKEFS_LOGGING_LEVEL=DEBUG
itaiad200 marked this conversation as resolved.
Show resolved Hide resolved
- LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON
itaiad200 marked this conversation as resolved.
Show resolved Hide resolved
entrypoint: ["/app/wait-for", "postgres:5432", "--", "/app/lakefs", "run"]
postgres:
image: "postgres:11"
Expand Down
2 changes: 1 addition & 1 deletion nessie/system_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func createRepository(ctx context.Context, t *testing.T, name string, repoStorag
ID: swag.String(name),
StorageNamespace: swag.String(repoStorage),
}), nil)
require.NoErrorf(t, err, "failed to create repository %s, storage %s", name, repoStorage)
require.NoErrorf(t, err, "failed to create repository '%s', storage '%s'", name, repoStorage)
}

func uploadFileRandomData(ctx context.Context, t *testing.T, repo, branch, objPath string) (checksum, content string) {
Expand Down