From 6dd286fc280b5f8ac9d52923de7a7c283464f0b6 Mon Sep 17 00:00:00 2001 From: Vasco Guita <33404234+vascoguita@users.noreply.github.com> Date: Tue, 10 Jan 2023 13:42:07 +0100 Subject: [PATCH] Migrate the litmusOcisSpacesDav test from Drone to GitHub Actions (#3566) --- .drone.star | 63 +--------------- .github/workflows/docker.yml | 42 +++++++++++ .github/workflows/lint.yml | 2 + .github/workflows/release.yml | 38 ++++------ .github/workflows/test.yml | 67 +++++++++-------- Makefile | 59 +++++++-------- README.md | 2 +- changelog/unreleased/enhancement-litmus.md | 3 + .../en/docs/getting-started/build-reva.md | 2 +- tests/litmus/docker-compose.yml | 72 ++++++++++++++++--- tests/litmus/frontend.toml | 2 +- tests/litmus/gateway.toml | 4 +- .../permissions.toml} | 0 tests/litmus/storage-home-ocis.toml | 6 +- tests/litmus/storage-users-ocis.toml | 42 +++++++++++ 15 files changed, 240 insertions(+), 164 deletions(-) create mode 100644 .github/workflows/docker.yml create mode 100644 changelog/unreleased/enhancement-litmus.md rename tests/{oc-integration-tests/drone/permissions-ocis-ci.toml => litmus/permissions.toml} (100%) create mode 100644 tests/litmus/storage-users-ocis.toml diff --git a/.drone.star b/.drone.star index 90bdc24b61..3724f410cd 100644 --- a/.drone.star +++ b/.drone.star @@ -2,8 +2,6 @@ OC_CI_GOLANG = "owncloudci/golang:1.19" OC_CI_ALPINE = "owncloudci/alpine:latest" OSIXIA_OPEN_LDAP = "osixia/openldap:1.3.0" OC_CI_PHP = "owncloudci/php:7.4" -OC_LITMUS = "owncloud/litmus:latest" -OC_CS3_API_VALIDATOR = "owncloud/cs3api-validator:0.2.0" OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier:latest" def makeStep(): @@ -11,7 +9,7 @@ def makeStep(): "name": "build", "image": OC_CI_GOLANG, "commands": [ - "make build", + "make build-revad", ], } @@ -69,7 +67,6 @@ def main(ctx): # implemented for: ocisIntegrationTests and s3ngIntegrationTests return [ checkStarlark(), - litmusOcisSpacesDav(), virtualViews(), ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12) @@ -132,64 +129,6 @@ def virtualViews(): ], } -def litmusOcisSpacesDav(): - return { - "kind": "pipeline", - "type": "docker", - "name": "litmus-owncloud-spaces-dav", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "trigger": { - "event": { - "include": [ - "pull_request", - "tag", - ], - }, - }, - "steps": [ - makeStep(), - { - "name": "revad-services", - "image": OC_CI_GOLANG, - "detach": True, - "commands": [ - "cd /drone/src/tests/oc-integration-tests/drone/", - "/drone/src/cmd/revad/revad -c frontend.toml &", - "/drone/src/cmd/revad/revad -c gateway.toml &", - "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", - "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", - "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &", - "/drone/src/cmd/revad/revad -c users.toml", - ], - }, - { - "name": "sleep-for-revad-start", - "image": OC_CI_GOLANG, - "commands": [ - "sleep 5", - ], - }, - { - "name": "litmus-owncloud-spaces-dav", - "image": OC_LITMUS, - "environment": { - "LITMUS_USERNAME": "einstein", - "LITMUS_PASSWORD": "relativity", - "TESTS": "basic http copymove props", - }, - "commands": [ - # The spaceid is randomly generated during the first login so we need this hack to construct the correct url. - "curl -s -k -u einstein:relativity -I http://revad-services:20080/remote.php/dav/files/einstein", - "export LITMUS_URL=http://revad-services:20080/remote.php/dav/spaces/123e4567-e89b-12d3-a456-426655440000!$(ls /drone/src/tmp/reva/data/spaces/personal/)", - "/usr/local/bin/litmus-wrapper", - ], - }, - ], - } - def ocisIntegrationTests(parallelRuns, skipExceptParts = []): pipelines = [] debugPartsEnabled = (len(skipExceptParts) != 0) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000..abe05d835d --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,42 @@ +name: Docker +on: + workflow_call: + inputs: + file: + required: true + type: string + tag: + type: string + load: + type: boolean + push: + type: boolean + +jobs: + docker: + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v3.1.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + if: ${{ inputs.push }} + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build ${{ (inputs.push && 'and push') || '' }} ${{ inputs.tag }} Docker image + uses: docker/build-push-action@v3 + with: + context: . + file: ${{ inputs.file }} + tags: ${{ (inputs.push && format('{0}/', secrets.DOCKERHUB_ORGANIZATION)) || '' }}${{ inputs.tag }} + load: ${{ inputs.load }} + push: ${{ inputs.push }} + - name: Upload ${{ inputs.tag }} Docker image to artifacts + uses: ishworkh/docker-image-artifact-upload@v1 + if: ${{ inputs.load }} + with: + image: ${{ inputs.tag }} + retention_days: '1' \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7a06e4b35f..81aa6a61e6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,6 +5,8 @@ on: - "go.mod" - "go.sum" push: + tags-ignore: + - "*" workflow_dispatch: jobs: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 484a01ccf6..bce6c412aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,29 +22,21 @@ jobs: files: "dist/*" body_path: "changelog/NOTE.md" docker: - runs-on: self-hosted strategy: + fail-fast: false matrix: include: - - tag: ${{ secrets.DOCKERHUB_ORGANIZATION }}/reva:${{ github.ref_name }} - file: Dockerfile.reva - - tag: ${{ secrets.DOCKERHUB_ORGANIZATION }}/revad:${{ github.ref_name }} - file: Dockerfile.revad - - tag: ${{ secrets.DOCKERHUB_ORGANIZATION }}/revad:${{ github.ref_name }}-eos - file: Dockerfile.revad-eos - - tag: ${{ secrets.DOCKERHUB_ORGANIZATION }}/revad:${{ github.ref_name }}-ceph - file: Dockerfile.revad-ceph - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push ${{ matrix.tag }} - uses: docker/build-push-action@v3 - with: - push: true - file: docker/${{ matrix.file }} - tags: ${{ matrix.tag }} + - image: reva + tag: reva:${{ github.ref_name }} + - image: revad + tag: revad:${{ github.ref_name }} + - image: revad-eos + tag: revad:${{ github.ref_name }}-eos + - image: revad-ceph + tag: revad:${{ github.ref_name }}-ceph + uses: ./.github/workflows/docker.yml + secrets: inherit + with: + file: docker/Dockerfile.${{ matrix.image }} + tag: ${{ matrix.tag }} + push: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f6e62f1c9..4d9838136c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,11 +1,13 @@ name: Test -on: [pull_request, push, workflow_dispatch] - -env: - TEST_IMAGE: 'revad:test' +on: + pull_request: + push: + tags-ignore: + - "*" + workflow_dispatch: jobs: - test: + integration-test: runs-on: self-hosted steps: - name: Checkout @@ -14,8 +16,6 @@ jobs: uses: actions/setup-go@v3.3.0 with: go-version-file: go.mod - - name: Test - run: make test - name: Test integration run: make test-integration env: @@ -23,41 +23,48 @@ jobs: services: redis: image: registry.cern.ch/docker.io/webhippie/redis - docker: + go-test: runs-on: self-hosted - strategy: - matrix: - image: [reva, revad, revad-eos, revad-ceph] steps: - name: Checkout - uses: actions/checkout@v3.1.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build Docker image from ${{ matrix.image }} - uses: docker/build-push-action@v3 - with: - context: . - file: docker/Dockerfile.${{ matrix.image }} - tags: ${{ env.TEST_IMAGE }} - load: ${{ matrix.image == 'revad' }} - - name: Upload ${{ env.TEST_IMAGE }} to artifacts - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ matrix.image == 'revad' }} + uses: actions/checkout@v3 + - name: Setup Go environment + uses: actions/setup-go@v3.3.0 with: - image: ${{ env.TEST_IMAGE }} - retention_days: '1' + go-version-file: go.mod + - name: Test + run: make test + docker: + strategy: + fail-fast: false + matrix: + image: [reva, revad-eos, revad-ceph] + uses: ./.github/workflows/docker.yml + with: + file: docker/Dockerfile.${{ matrix.image }} + docker-revad: + name: docker (revad) + uses: ./.github/workflows/docker.yml + with: + file: docker/Dockerfile.revad + tag: revad:test + load: true litmus: - needs: docker + needs: docker-revad runs-on: self-hosted strategy: + fail-fast: false matrix: - test: [litmus-1-only, litmus-2-only] + profile: [1, 2, 3] steps: - name: Checkout uses: actions/checkout@v3 - name: Download image uses: ishworkh/docker-image-artifact-download@v1 with: - image: ${{ env.TEST_IMAGE }} + image: revad:test - name: Test - run: make ${{ matrix.test }} \ No newline at end of file + run: make litmus-only + env: + PROFILE: ${{ matrix.profile }} + IMAGE: revad:test \ No newline at end of file diff --git a/Makefile b/Makefile index 59b5cce878..07d4ee5cae 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,32 @@ -BUILD_DATE=`date +%FT%T%z` -GIT_COMMIT ?= `git rev-parse --short HEAD` -GIT_DIRTY=`git diff-index --quiet HEAD -- || echo "dirty-"` -VERSION ?= `git describe --always` -GO_VERSION ?= `go version | awk '{print $$3}'` -BUILD_FLAGS="-X main.gitCommit=${GIT_COMMIT} -X main.version=${VERSION} -X main.goVersion=${GO_VERSION} -X main.buildDate=${BUILD_DATE}" +BUILD_DATE = `date +%FT%T%z` +GIT_COMMIT ?= `git rev-parse --short HEAD` +GIT_DIRTY = `git diff-index --quiet HEAD -- || echo "dirty-"` +VERSION ?= `git describe --always` +GO_VERSION ?= `go version | awk '{print $$3}'` +BUILD_FLAGS = "-X main.gitCommit=${GIT_COMMIT} -X main.version=${VERSION} -X main.goVersion=${GO_VERSION} -X main.buildDate=${BUILD_DATE}" -TEST_IMAGE ?= revad:test +.PHONY: all +all: build-revad build-reva test lint gen-doc + +IMAGE ?= revad:test .PHONY: test-image test-image: - docker build -t $(TEST_IMAGE) -f docker/Dockerfile.revad . - -LITMUS ?= $(CURDIR)/tests/litmus -TIMEOUT ?= 3600 -URL_PATH ?= /remote.php/webdav + docker build -t $(IMAGE) -f docker/Dockerfile.revad . -.PHONY: litmus-1-only -litmus-1-only: - @cd $(LITMUS) && URL_PATH=$(URL_PATH) TEST_IMAGE=$(TEST_IMAGE) docker-compose up --remove-orphans --force-recreate --exit-code-from litmus --abort-on-container-exit --timeout $(TIMEOUT) +LITMUS ?= $(CURDIR)/tests/litmus +TIMEOUT ?= 3600 -.PHONY: litmus-1 -litmus-1: test-image litmus-1-only - -.PHONY: litmus-2-only -litmus-2-only: URL_PATH=/remote.php/dav/files/4c510ada-c86b-4815-8820-42cdf82c3d51 -litmus-2-only: litmus-1-only +.PHONY: litmus-only +litmus-only: +ifndef PROFILE + $(error PROFILE is not defined) +else + @cd $(LITMUS) && IMAGE=$(IMAGE) docker-compose --profile $(PROFILE) up --remove-orphans --exit-code-from litmus-$(PROFILE) --abort-on-container-exit --timeout $(TIMEOUT) +endif -.PHONY: litmus-2 -litmus-2: test-image litmus-2-only +.PHONY: litmus +litmus: test-image litmus-only TOOLCHAIN ?= $(CURDIR)/toolchain GOLANGCI_LINT ?= $(TOOLCHAIN)/golangci-lint @@ -47,7 +46,7 @@ $(GOLANGCI_LINT): .PHONY: check-changelog lint: $(GOLANGCI_LINT) - $(GOLANGCI_LINT) run + @$(GOLANGCI_LINT) run || (echo "Tip: many lint errors can be automatically fixed with \"make lint-fix\""; exit 1) .PHONY: lint-fix lint-fix: $(GOLANGCI_LINT) @@ -86,9 +85,6 @@ off: imports: off $(GOIMPORTS) $(GOIMPORTS) -w tools pkg internal cmd -.PHONY: build -build: build-revad build-reva - .PHONY: build-cephfs build-cephfs: build-revad-cephfs build-reva @@ -123,11 +119,11 @@ build-reva-docker: off .PHONY: test test: off - go test -coverprofile coverage.out -race $$(go list ./... | grep -v /tests/integration) + go test -race $$(go list ./... | grep -v /tests/integration) .PHONY: test-integration -test-integration: build - cd tests/integration && go test -race ./... +test-integration: build-revad + go test -race ./tests/integration/... .PHONY: contrib contrib: @@ -141,9 +137,6 @@ gen-doc: clean: toolchain-clean rm -rf dist -.PHONY: all -all: build test lint gen-doc - # create local build versions dist: gen-doc go run tools/create-artifacts/main.go -version ${VERSION} -commit ${GIT_COMMIT} -goversion ${GO_VERSION} diff --git a/README.md b/README.md index 230b4f4006..6af17147b6 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ You need to have [Go](https://golang.org/doc/install) (version 1.16 or higher), ``` $ git clone https://github.com/cs3org/reva $ cd reva -$ make build +$ make $ mkdir -p /etc/revad $ cp examples/storage-references/users.demo.json /etc/revad/users.json $ cp examples/storage-references/groups.demo.json /etc/revad/groups.json diff --git a/changelog/unreleased/enhancement-litmus.md b/changelog/unreleased/enhancement-litmus.md new file mode 100644 index 0000000000..9a5a344ca9 --- /dev/null +++ b/changelog/unreleased/enhancement-litmus.md @@ -0,0 +1,3 @@ +Enhancement: Migrate the litmusOcisSpacesDav test from Drone to GitHub Actions + +https://github.com/cs3org/reva/pull/3566 \ No newline at end of file diff --git a/docs/content/en/docs/getting-started/build-reva.md b/docs/content/en/docs/getting-started/build-reva.md index 3f98fa28f8..509c6019a2 100644 --- a/docs/content/en/docs/getting-started/build-reva.md +++ b/docs/content/en/docs/getting-started/build-reva.md @@ -12,5 +12,5 @@ the Go programming language that installs the Go compiler. ``` git clone https://github.com/cs3org/reva cd reva -make build +make ``` diff --git a/tests/litmus/docker-compose.yml b/tests/litmus/docker-compose.yml index e65608bc39..5a05d20f5a 100644 --- a/tests/litmus/docker-compose.yml +++ b/tests/litmus/docker-compose.yml @@ -1,36 +1,92 @@ version: "3.3" services: frontend: - image: ${TEST_IMAGE} + image: ${IMAGE} command: -c /litmus/frontend.toml volumes: - .:/litmus working_dir: /litmus gateway: - image: ${TEST_IMAGE} + image: ${IMAGE} command: -c /litmus/gateway.toml volumes: - .:/litmus storage-home-ocis: - image: ${TEST_IMAGE} + image: ${IMAGE} command: -c /litmus/storage-home-ocis.toml volumes: - .:/litmus + - shared-volume:/var/tmp users: - image: ${TEST_IMAGE} + image: ${IMAGE} command: -c /litmus/users.toml volumes: - .:/litmus working_dir: /litmus - litmus: + permissions: + profiles: ["3"] + image: ${IMAGE} + command: -c /litmus/permissions.toml + volumes: + - .:/litmus + storage-users-ocis: + profiles: ["3"] + image: ${IMAGE} + command: -c /litmus/storage-users-ocis.toml + volumes: + - .:/litmus + - shared-volume:/var/tmp + litmus-1: + profiles: ["1"] + image: registry.cern.ch/docker.io/owncloud/litmus:latest + environment: + LITMUS_URL: http://frontend:20080/remote.php/webdav + LITMUS_USERNAME: einstein + LITMUS_PASSWORD: relativity + TESTS: basic http copymove props + depends_on: + - frontend + - gateway + - users + - storage-home-ocis + litmus-2: + profiles: ["2"] image: registry.cern.ch/docker.io/owncloud/litmus:latest environment: - LITMUS_URL: http://frontend:20080${URL_PATH} + LITMUS_URL: http://frontend:20080/remote.php/dav/files/4c510ada-c86b-4815-8820-42cdf82c3d51 LITMUS_USERNAME: einstein - LITMUS_PASSWORD: relativity + LITMUS_PASSWORD: relativity TESTS: basic http copymove props depends_on: - frontend - gateway + - users + - storage-home-ocis + litmus-3: + profiles: ["3"] + image: registry.cern.ch/docker.io/owncloud/litmus:latest + environment: + LITMUS_USERNAME: einstein + LITMUS_PASSWORD: relativity + TESTS: basic http copymove props + entrypoint: + - /bin/sh + command: + - -c + - | + curl -s -k -u einstein:relativity -I http://frontend:20080/remote.php/dav/files/einstein + export LITMUS_URL=http://frontend:20080/remote.php/dav/spaces/123e4567-e89b-12d3-a456-426655440000!$$(ls /var/tmp/reva/data/spaces/personal/) + /usr/local/bin/litmus-wrapper + volumes: + - .:/litmus + - shared-volume:/var/tmp + depends_on: + - frontend + - gateway + - users - storage-home-ocis - - users \ No newline at end of file + - permissions + - storage-users-ocis + +volumes: + shared-volume: \ No newline at end of file diff --git a/tests/litmus/frontend.toml b/tests/litmus/frontend.toml index a0de367e70..9a8433eff3 100644 --- a/tests/litmus/frontend.toml +++ b/tests/litmus/frontend.toml @@ -29,7 +29,7 @@ allow_credentials = true [http.services.ocdav] # serve ocdav on the root path prefix = "" -chunk_folder = "/tmp/reva/chunks" +chunk_folder = "/var/tmp/reva/chunks" # for user lookups # prefix the path of requests to /dav/files with this namespace # While owncloud has only listed usernames at this endpoint CERN has diff --git a/tests/litmus/gateway.toml b/tests/litmus/gateway.toml index 021b790bf2..182f3eb9e1 100644 --- a/tests/litmus/gateway.toml +++ b/tests/litmus/gateway.toml @@ -31,7 +31,7 @@ ocmshareprovidersvc = "localhost:14000" ocminvitemanagersvc = "localhost:14000" ocmproviderauthorizersvc = "localhost:14000" # permissions -permissionssvc = "localhost:10000" +permissionssvc = "permissions:10000" # other commit_share_to_storage_grant = true commit_share_to_storage_ref = true @@ -40,7 +40,7 @@ datagateway = "http://localhost:19001/data" transfer_shared_secret = "replace-me-with-a-transfer-secret" # for direct uploads transfer_expires = 6 # give it a moment #disable_home_creation_on_login = true -link_grants_file = "/tmp/reva/link_grants_file.json" +link_grants_file = "/var/tmp/reva/link_grants_file.json" [grpc.services.authregistry] driver = "static" diff --git a/tests/oc-integration-tests/drone/permissions-ocis-ci.toml b/tests/litmus/permissions.toml similarity index 100% rename from tests/oc-integration-tests/drone/permissions-ocis-ci.toml rename to tests/litmus/permissions.toml diff --git a/tests/litmus/storage-home-ocis.toml b/tests/litmus/storage-home-ocis.toml index ceb255b6ba..4dfd693ecb 100644 --- a/tests/litmus/storage-home-ocis.toml +++ b/tests/litmus/storage-home-ocis.toml @@ -26,7 +26,7 @@ enable_home_creation = true gateway_addr = "gateway:19000" [grpc.services.storageprovider.drivers.ocis] -root = "/tmp/reva/data" +root = "/var/tmp/reva/data" enable_home = true treetime_accounting = true treesize_accounting = true @@ -38,10 +38,10 @@ address = "0.0.0.0:12001" [http.services.dataprovider] driver = "ocis" -temp_folder = "/tmp/reva/tmp" +temp_folder = "/var/tmp/reva/tmp" [http.services.dataprovider.drivers.ocis] -root = "/tmp/reva/data" +root = "/var/tmp/reva/data" enable_home = true treetime_accounting = true treesize_accounting = true diff --git a/tests/litmus/storage-users-ocis.toml b/tests/litmus/storage-users-ocis.toml new file mode 100644 index 0000000000..d7455393cb --- /dev/null +++ b/tests/litmus/storage-users-ocis.toml @@ -0,0 +1,42 @@ +# This config file will start a reva service that: +# - uses the ocis driver to serve users (/users) +# - serves the storage provider on grpc port 11000 +# - serves http dataprovider for this storage on port 11001 +# - /data - dataprovider: file up and download + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "gateway:19000" + +[grpc] +address = "0.0.0.0:11000" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/users" +mount_id = "123e4567-e89b-12d3-a456-426655440000" +expose_data_server = true +data_server_url = "http://storage-users-ocis:11001/data" +gateway_addr = "gateway:19000" + +[grpc.services.storageprovider.drivers.ocis] +root = "/var/tmp/reva/data" +treetime_accounting = true +treesize_accounting = true +userprovidersvc = "users:18000" +gateway_addr = "gateway:19000" + +# we have a locally running dataprovider +[http] +address = "0.0.0.0:11001" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/var/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/var/tmp/reva/data" +treetime_accounting = true +treesize_accounting = true