From cc14925803c7692ec69a3b8b0cd320fe68bfeae0 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 27 Aug 2019 15:19:49 -0700 Subject: [PATCH] reduce memory pressure by reducing golangci-lint parallelism in ci seperate 'lint' and 'test' CI steps 'go test' and 'go build' should use files from the 'vendor' folder pact tests should use go modules use vendor in 'go vet' run golangci-lint on each subdir in pkg individually in ci skip running golangci-lint on integration tests go-bindata and go-modules do not get along --- .circleci/config.yml | 35 +++++++++++++++++++++++++- Makefile | 49 +++++++++++++++++++++++------------- contracts/Dockerfile.testing | 1 + go.mod | 2 +- go.sum | 2 ++ hack/get_build_deps.sh | 7 +----- hack/get_lint_deps.sh | 6 +++++ 7 files changed, 77 insertions(+), 25 deletions(-) create mode 100755 hack/get_lint_deps.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 7848decd2..0e453cf2b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -183,7 +183,7 @@ jobs: command: | mkdir -p $GOCACHE - make build-deps citest + make citest if [ -n "$CC_TEST_REPORTER_ID" ]; then make ci-upload-coverage fi @@ -192,6 +192,29 @@ jobs: - /tmp/go/cache key: ship-unit-test-build-cache-{{ epoch }} + + lint: + docker: + - image: circleci/golang:1.12 + environment: + GOCACHE: "/tmp/go/cache" + working_directory: /go/src/github.com/replicatedhq/ship + steps: + - checkout + - restore_cache: + keys: + - ship-lint-build-cache + - run: + name: make test + command: | + mkdir -p $GOCACHE + + make cilint + - save_cache: + paths: + - /tmp/go/cache + key: ship-lint-build-cache-{{ epoch }} + pacts: docker: - image: circleci/golang:1.12 @@ -635,6 +658,7 @@ workflows: - e2e_setup - test + - lint - pacts - windows_build_test - docs @@ -671,6 +695,7 @@ workflows: - e2e_setup - e2e_init - test + - lint - pacts - windows_build_test - integration_base @@ -740,6 +765,13 @@ workflows: branches: ignore: /.*/ + - lint: + filters: + tags: + only: /^v[0-9]+(\.[0-9]+)*(-.*)*/ + branches: + ignore: /.*/ + - pacts: filters: tags: @@ -817,6 +849,7 @@ workflows: - e2e_setup - e2e_init - test + - lint - pacts - windows_build_test - integration_base diff --git a/Makefile b/Makefile index ea787432f..caae17cb6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .NOTPARALLEL: -.PHONY: build-deps dep-deps docker shell githooks dep e2e run citest ci-upload-coverage goreleaser integration-test build_ship_integration_test build-ui build-ui-dev mark-ui-gitignored fmt lint vet test build embed-ui clean-ship clean clean-integration +.PHONY: build-deps docker shell githooks dep e2e run citest ci-upload-coverage goreleaser integration-test build_ship_integration_test build-ui build-ui-dev mark-ui-gitignored fmt lint vet test build embed-ui clean-ship clean clean-integration export GO111MODULE=on @@ -47,14 +47,18 @@ define LDFLAGS endef .state/build-deps: hack/get_build_deps.sh - ./hack/get_build_deps.sh + time ./hack/get_build_deps.sh @mkdir -p .state/ @touch .state/build-deps build-deps: .state/build-deps -dep-deps: - go get -u github.com/golang/dep/cmd/dep +.state/lint-deps: hack/get_lint_deps.sh + time ./hack/get_lint_deps.sh + @mkdir -p .state/ + @touch .state/lint-deps + +lint-deps: .state/lint-deps docker: docker build -t ship . @@ -76,14 +80,14 @@ githooks: .PHONY: pacts pacts: - go test -v ./contracts/... + go test -v -mod vendor ./contracts/... .PHONY: pacts-ci pacts-ci: docker build -t ship-contract-tests -f contracts/Dockerfile.testing . docker run --rm --name ship-contract-tests \ ship-contract-tests \ - bash -c 'go test -v ./contracts/...' + bash -c 'go test -v -mod vendor ./contracts/...' .PHONY: pacts-ci-publish pacts-ci-publish: @@ -91,9 +95,9 @@ pacts-ci-publish: docker run --rm --name ship-contract-tests \ -e PACT_BROKER_USERNAME -e PACT_BROKER_PASSWORD -e VERSION=$$CIRCLE_TAG \ ship-contract-tests \ - bash -c 'go test -v ./contracts/... && ./contracts/publish.sh' + bash -c 'go test -v -mod vendor ./contracts/... && ./contracts/publish.sh' -_mockgen: +_mockgen: build-deps rm -rf pkg/test-mocks mkdir -p pkg/test-mocks/ui mkdir -p pkg/test-mocks/config @@ -244,18 +248,24 @@ deps: @touch .state/fmt -fmt: .state/build-deps .state/fmt +fmt: .state/lint-deps .state/fmt .state/vet: $(SRC) - go vet ./pkg/... - go vet ./cmd/... - go vet ./integration/... + go vet -mod vendor ./pkg/... + go vet -mod vendor ./cmd/... + go vet -mod vendor ./integration/... @mkdir -p .state @touch .state/vet vet: .state/vet -.state/golangci-lint: .state/build-deps $(SRC) +.state/golangci-lint-ci: .state/lint-deps $(SRC) + golangci-lint run -j 1 ./cmd/... + for D in ./pkg/*; do echo $$D; golangci-lint run -j 1 $$D/...; done + @mkdir -p .state + @touch .state/golangci-lint-ci + +.state/golangci-lint: .state/lint-deps $(SRC) golangci-lint run ./pkg/... golangci-lint run ./cmd/... golangci-lint run ./integration/... @@ -273,14 +283,14 @@ golangci-lint: .state/golangci-lint lint: vet golangci-lint .state/lint .state/test: $(SRC) - go test ./pkg/... ./integration | grep -v '?' + go test -mod vendor ./pkg/... ./integration | grep -v '?' @mkdir -p .state @touch .state/test test: lint .state/test .state/race: $(SRC) - go test --race ./pkg/... + go test --race -mod vendor ./pkg/... @mkdir -p .state @touch .state/race @@ -289,9 +299,12 @@ race: lint .state/race .state/coverage.out: $(SRC) @mkdir -p .state/ #the reduced parallelism here is to avoid hitting the memory limits - we consistently did so with two threads on a 4gb instance - go test -parallel 1 -p 1 -coverprofile=.state/coverage.out ./pkg/... ./integration + go test -parallel 1 -p 1 -coverprofile=.state/coverage.out -mod vendor ./pkg/... ./integration + +citest: .state/coverage.out -citest: .state/vet .state/golangci-lint .state/lint .state/coverage.out +.PHONY: cilint +cilint: .state/vet .state/golangci-lint-ci .state/lint .state/cc-test-reporter: @mkdir -p .state/ @@ -312,6 +325,7 @@ build-minimal: build-ui pkg/lifecycle/daemon/ui.bindatafs.go bin/ship bin/ship: $(FULLSRC) go build \ + -mod vendor \ ${LDFLAGS} \ -i \ -o bin/ship \ @@ -320,6 +334,7 @@ bin/ship: $(FULLSRC) bin/ship.exe: $(SRC) GOOS=windows go build \ + -mod vendor \ ${LDFLAGS} \ -i \ -o bin/ship.exe \ diff --git a/contracts/Dockerfile.testing b/contracts/Dockerfile.testing index 61d8fe4d5..c577e42f7 100644 --- a/contracts/Dockerfile.testing +++ b/contracts/Dockerfile.testing @@ -2,6 +2,7 @@ FROM golang:1.12 RUN cd /opt && curl -fsSL https://raw.githubusercontent.com/pact-foundation/pact-ruby-standalone/v1.66.0/install.sh | bash ENV PATH="/opt/pact/bin:${PATH}" +ENV GO111MODULE=on WORKDIR /go/src/github.com/replicatedhq/ship diff --git a/go.mod b/go.mod index cd1b20096..995d3e022 100644 --- a/go.mod +++ b/go.mod @@ -117,7 +117,7 @@ require ( golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 // indirect golang.org/x/text v0.3.2 // indirect - golang.org/x/tools v0.0.0-20190827152308-062dbaebb618 // indirect + golang.org/x/tools v0.0.0-20190827205025-b29f5f60c37a // indirect google.golang.org/appengine v1.3.0 // indirect google.golang.org/grpc v1.21.0 gopkg.in/go-playground/assert.v1 v1.2.1 // indirect diff --git a/go.sum b/go.sum index 6aa40e830..439dff945 100644 --- a/go.sum +++ b/go.sum @@ -725,6 +725,8 @@ golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c h1:KfpJVdWhuRqNk4XVXzjXf2K golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190827152308-062dbaebb618 h1:WtF22n/HcPWMhvZm4KWiQ0FcC1m8kk5ILpXYtY+qN7s= golang.org/x/tools v0.0.0-20190827152308-062dbaebb618/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190827205025-b29f5f60c37a h1:0JEq5ZQ3TgsRlFmz4BcD+E6U6cOk4pOImCQSyIG59ZM= +golang.org/x/tools v0.0.0-20190827205025-b29f5f60c37a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20171005000305-7a7376eff6a5/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/appengine v0.0.0-20150527042145-b667a5000b08/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/hack/get_build_deps.sh b/hack/get_build_deps.sh index eda481384..ca8a544b6 100755 --- a/hack/get_build_deps.sh +++ b/hack/get_build_deps.sh @@ -1,12 +1,7 @@ #!/bin/sh set -v -go get -u golang.org/x/tools/cmd/goimports -go get -u golang.org/x/lint/golint go get github.com/golang/mock/gomock go install github.com/golang/mock/mockgen go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs -go get -u github.com/jteeuwen/go-bindata/go-bindata -go get -u github.com/gordonklaus/ineffassign -go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.17.1 - +GO111MODULE=auto go get -u github.com/jteeuwen/go-bindata/go-bindata diff --git a/hack/get_lint_deps.sh b/hack/get_lint_deps.sh new file mode 100755 index 000000000..5db35833a --- /dev/null +++ b/hack/get_lint_deps.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -v + +go get -u golang.org/x/tools/cmd/goimports +go get -u golang.org/x/lint/golint +go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.17.1