reduce memory pressure by reducing golangci-lint parallelism in ci

seperate 'lint' and 'test' CI steps

'go test' and 'go build' should use files from the 'vendor' folder

pact tests should use go modules

use vendor in 'go vet'

run golangci-lint on each subdir in pkg individually in ci

skip running golangci-lint on integration tests

go-bindata and go-modules do not get along

.circleci/config.yml

@@ -183,7 +183,7 @@ jobs:
           command: |
             mkdir -p $GOCACHE
 
-            make build-deps citest
+            make citest
             if [ -n "$CC_TEST_REPORTER_ID" ]; then
               make ci-upload-coverage
             fi
@@ -192,6 +192,29 @@ jobs:
             - /tmp/go/cache
           key: ship-unit-test-build-cache-{{ epoch }}
 
+
+  lint:
+    docker:
+    - image: circleci/golang:1.12
+    environment:
+      GOCACHE: "/tmp/go/cache"
+    working_directory: /go/src/github.com/replicatedhq/ship
+    steps:
+    - checkout
+    - restore_cache:
+        keys:
+        - ship-lint-build-cache
+    - run:
+        name: make test
+        command: |
+          mkdir -p $GOCACHE
+
+          make cilint
+    - save_cache:
+        paths:
+        - /tmp/go/cache
+        key: ship-lint-build-cache-{{ epoch }}
+
   pacts:
     docker:
       - image: circleci/golang:1.12
@@ -635,6 +658,7 @@ workflows:
             - e2e_setup
 
       - test
+      - lint
       - pacts
       - windows_build_test
       - docs
@@ -671,6 +695,7 @@ workflows:
             - e2e_setup
             - e2e_init
             - test
+            - lint
             - pacts
             - windows_build_test
             - integration_base
@@ -740,6 +765,13 @@ workflows:
             branches:
               ignore: /.*/
 
+      - lint:
+          filters:
+            tags:
+              only: /^v[0-9]+(\.[0-9]+)*(-.*)*/
+            branches:
+              ignore: /.*/
+
       - pacts:
           filters:
             tags:
@@ -817,6 +849,7 @@ workflows:
             - e2e_setup
             - e2e_init
             - test
+            - lint
             - pacts
             - windows_build_test
             - integration_base

Makefile

@@ -1,6 +1,6 @@
 .NOTPARALLEL:
 
-.PHONY: build-deps dep-deps docker shell githooks dep e2e run citest ci-upload-coverage goreleaser integration-test build_ship_integration_test build-ui build-ui-dev mark-ui-gitignored fmt lint vet test build embed-ui clean-ship clean clean-integration
+.PHONY: build-deps docker shell githooks dep e2e run citest ci-upload-coverage goreleaser integration-test build_ship_integration_test build-ui build-ui-dev mark-ui-gitignored fmt lint vet test build embed-ui clean-ship clean clean-integration
 
 export GO111MODULE=on
 
@@ -47,14 +47,18 @@ define LDFLAGS
 endef
 
 .state/build-deps: hack/get_build_deps.sh
-	./hack/get_build_deps.sh
+	time ./hack/get_build_deps.sh
 	@mkdir -p .state/
 	@touch .state/build-deps
 
 build-deps: .state/build-deps
 
-dep-deps:
-	go get -u github.com/golang/dep/cmd/dep
+.state/lint-deps: hack/get_lint_deps.sh
+	time ./hack/get_lint_deps.sh
+	@mkdir -p .state/
+	@touch .state/lint-deps
+
+lint-deps: .state/lint-deps
 
 docker:
 	docker build -t ship .
@@ -76,24 +80,24 @@ githooks:
 
 .PHONY: pacts
 pacts:
-	go test -v ./contracts/...
+	go test -v -mod vendor ./contracts/...
 
 .PHONY: pacts-ci
 pacts-ci:
 	docker build -t ship-contract-tests -f contracts/Dockerfile.testing .
 	docker run --rm --name ship-contract-tests \
 		ship-contract-tests \
-		bash -c 'go test -v ./contracts/...'
+		bash -c 'go test -v -mod vendor ./contracts/...'
 
 .PHONY: pacts-ci-publish
 pacts-ci-publish:
 	docker build -t ship-contract-tests -f contracts/Dockerfile.testing .
 	docker run --rm --name ship-contract-tests \
 		-e PACT_BROKER_USERNAME -e PACT_BROKER_PASSWORD -e VERSION=$$CIRCLE_TAG \
 		ship-contract-tests \
-		bash -c 'go test -v ./contracts/... && ./contracts/publish.sh'
+		bash -c 'go test -v -mod vendor ./contracts/... && ./contracts/publish.sh'
 
-_mockgen:
+_mockgen: build-deps
 	rm -rf pkg/test-mocks
 	mkdir -p pkg/test-mocks/ui
 	mkdir -p pkg/test-mocks/config
@@ -244,18 +248,24 @@ deps:
 	@touch .state/fmt
 
 
-fmt: .state/build-deps .state/fmt
+fmt: .state/lint-deps .state/fmt
 
 .state/vet: $(SRC)
-	go vet ./pkg/...
-	go vet ./cmd/...
-	go vet ./integration/...
+	go vet -mod vendor ./pkg/...
+	go vet -mod vendor ./cmd/...
+	go vet -mod vendor ./integration/...
 	@mkdir -p .state
 	@touch .state/vet
 
 vet: .state/vet
 
-.state/golangci-lint: .state/build-deps $(SRC)
+.state/golangci-lint-ci: .state/lint-deps $(SRC)
+	golangci-lint run -j 1 ./cmd/...
+	for D in ./pkg/*; do echo $$D; golangci-lint run -j 1 $$D/...; done
+	@mkdir -p .state
+	@touch .state/golangci-lint-ci
+
+.state/golangci-lint: .state/lint-deps $(SRC)
 	golangci-lint run ./pkg/...
 	golangci-lint run ./cmd/...
 	golangci-lint run ./integration/...
@@ -273,14 +283,14 @@ golangci-lint: .state/golangci-lint
 lint: vet golangci-lint .state/lint
 
 .state/test: $(SRC)
-	go test ./pkg/... ./integration | grep -v '?'
+	go test -mod vendor ./pkg/... ./integration | grep -v '?'
 	@mkdir -p .state
 	@touch .state/test
 
 test: lint .state/test
 
 .state/race: $(SRC)
-	go test --race ./pkg/...
+	go test --race -mod vendor ./pkg/...
 	@mkdir -p .state
 	@touch .state/race
 
@@ -289,9 +299,12 @@ race: lint .state/race
 .state/coverage.out: $(SRC)
 	@mkdir -p .state/
 	#the reduced parallelism here is to avoid hitting the memory limits - we consistently did so with two threads on a 4gb instance
-	go test -parallel 1 -p 1 -coverprofile=.state/coverage.out ./pkg/... ./integration
+	go test -parallel 1 -p 1 -coverprofile=.state/coverage.out -mod vendor ./pkg/... ./integration
+
+citest: .state/coverage.out
 
-citest: .state/vet .state/golangci-lint .state/lint .state/coverage.out
+.PHONY: cilint
+cilint: .state/vet .state/golangci-lint-ci .state/lint
 
 .state/cc-test-reporter:
 	@mkdir -p .state/
@@ -312,6 +325,7 @@ build-minimal: build-ui pkg/lifecycle/daemon/ui.bindatafs.go bin/ship
 
 bin/ship: $(FULLSRC)
 	go build \
+		-mod vendor \
 		${LDFLAGS} \
 		-i \
 		-o bin/ship \
@@ -320,6 +334,7 @@ bin/ship: $(FULLSRC)
 
 bin/ship.exe: $(SRC)
 	GOOS=windows go build \
+		-mod vendor \
 		${LDFLAGS} \
 		-i \
 		-o bin/ship.exe \

contracts/Dockerfile.testing

@@ -2,6 +2,7 @@ FROM golang:1.12
 
 RUN cd /opt && curl -fsSL https://raw.githubusercontent.com/pact-foundation/pact-ruby-standalone/v1.66.0/install.sh | bash
 ENV PATH="/opt/pact/bin:${PATH}"
+ENV GO111MODULE=on
 
 WORKDIR /go/src/github.com/replicatedhq/ship
 

go.mod

@@ -117,7 +117,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba
 	golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 // indirect
 	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/tools v0.0.0-20190827152308-062dbaebb618 // indirect
+	golang.org/x/tools v0.0.0-20190827205025-b29f5f60c37a // indirect
 	google.golang.org/appengine v1.3.0 // indirect
 	google.golang.org/grpc v1.21.0
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect

go.sum

@@ -725,6 +725,8 @@ golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c h1:KfpJVdWhuRqNk4XVXzjXf2K
 golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190827152308-062dbaebb618 h1:WtF22n/HcPWMhvZm4KWiQ0FcC1m8kk5ILpXYtY+qN7s=
 golang.org/x/tools v0.0.0-20190827152308-062dbaebb618/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190827205025-b29f5f60c37a h1:0JEq5ZQ3TgsRlFmz4BcD+E6U6cOk4pOImCQSyIG59ZM=
+golang.org/x/tools v0.0.0-20190827205025-b29f5f60c37a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.0.0-20171005000305-7a7376eff6a5/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/appengine v0.0.0-20150527042145-b667a5000b08/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

hack/get_build_deps.sh

@@ -1,12 +1,7 @@
 #!/bin/sh
 set -v
 
-go get -u golang.org/x/tools/cmd/goimports
-go get -u golang.org/x/lint/golint
 go get github.com/golang/mock/gomock
 go install github.com/golang/mock/mockgen
 go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs
-go get -u github.com/jteeuwen/go-bindata/go-bindata
-go get -u github.com/gordonklaus/ineffassign
-go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.17.1
-
+GO111MODULE=auto go get -u github.com/jteeuwen/go-bindata/go-bindata

hack/get_lint_deps.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -v
+
+go get -u golang.org/x/tools/cmd/goimports
+go get -u golang.org/x/lint/golint
+go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.17.1