From c4fff05f19773aafd5e447b4ca65a5bedd6b891d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20P=C5=82otka?= Date: Mon, 2 Sep 2019 15:37:51 +0100 Subject: [PATCH] Baked required deps in docker image for CircleCI. (#1484) This should fix https://github.com/thanos-io/thanos/issues/1483 Also it should massively improve our CI time. Signed-off-by: Bartek Plotka --- .circleci/config.yml | 11 ++++------ .gitignore | 2 ++ Dockerfile.thanos-ci | 6 ++++++ Makefile | 49 ++++++++++++++++++++++++++++++++++---------- 4 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 Dockerfile.thanos-ci diff --git a/.circleci/config.yml b/.circleci/config.yml index 47ee70c398..b1d34a15c2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,12 +3,11 @@ version: 2 jobs: test: docker: - # Available from https://hub.docker.com/r/circleci/golang/ - - image: circleci/golang:1.12.5 + # Build by Thanos make docker-ci + - image: quay.io/thanos/thanos-ci:v0.1.0 working_directory: /go/src/github.com/thanos-io/thanos environment: GO111MODULE: 'on' - GOBIN: "/go/bin" # Run garbage collection more aggresively to avoid getting OOMed during the lint phase (4GB limit). GOGC: "20" # By default Go uses GOMAXPROCS but a Circle CI executor has many @@ -60,11 +59,9 @@ jobs: publish_master: docker: - # Available from https://hub.docker.com/r/circleci/golang/ - - image: circleci/golang:1.12.5 + # Build by Thanos make docker-ci + - image: quay.io/thanos/thanos-ci:v0.1.0 working_directory: /go/src/github.com/thanos-io/thanos - environment: - GOBIN: "/go/bin" steps: - checkout - setup_remote_docker: diff --git a/.gitignore b/.gitignore index 6ba1763b5b..4e999979a2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ data/ website/public/ website/docs-pre-processed/ !website/data + +tmp/bin diff --git a/Dockerfile.thanos-ci b/Dockerfile.thanos-ci new file mode 100644 index 0000000000..cb8ce2a985 --- /dev/null +++ b/Dockerfile.thanos-ci @@ -0,0 +1,6 @@ +# Available from https://hub.docker.com/r/circleci/golang/ +FROM circleci/golang:1.12.5 + +ENV GOBIN=/go/bin + +COPY tmp/bin ${GOBIN} diff --git a/Makefile b/Makefile index 2de40f37dd..62913763a8 100644 --- a/Makefile +++ b/Makefile @@ -46,9 +46,13 @@ ME ?= $(shell whoami) # Limited prom version, because testing was not possible. This should fix it: https://github.com/thanos-io/thanos/issues/758 PROM_VERSIONS ?= v2.4.3 v2.5.0 v2.8.1 v2.9.2 +PROMS ?= $(GOBIN)/prometheus-v2.4.3 $(GOBIN)/prometheus-v2.5.0 $(GOBIN)/prometheus-v2.8.1 $(GOBIN)/prometheus-v2.9.2 ALERTMANAGER_VERSION ?= v0.15.2 +ALERTMANAGER ?= $(GOBIN)/alertmanager-$(ALERTMANAGER_VERSION) + MINIO_SERVER_VERSION ?= RELEASE.2018-10-06T00-15-16Z +MINIO_SERVER ?=$(GOBIN)/minio-$(MINIO_SERVER_VERSION) # fetch_go_bin_version downloads (go gets) the binary from specific version and installs it in $(GOBIN)/- # arguments: @@ -178,11 +182,17 @@ tarballs-release: $(PROMU) # test runs all Thanos golang tests against each supported version of Prometheus. .PHONY: test test: export GOCACHE= $(TMP_GOPATH)/gocache -test: check-git test-deps +test: export THANOS_TEST_MINIO_PATH= $(MINIO_SERVER) +test: export THANOS_TEST_PROMETHEUS_VERSIONS= $(PROM_VERSIONS) +test: export THANOS_TEST_ALERTMANAGER_PATH= $(ALERTMANAGER) +test: check-git install-deps + @echo ">> install thanos GOOPTS=${GOOPTS}" + # Thanos binary is required by e2e tests. + @go install github.com/thanos-io/thanos/cmd/thanos # Be careful on GOCACHE. Those tests are sometimes using built Thanos/Prometheus binaries directly. Don't cache those. @rm -rf ${GOCACHE} @echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets" - THANOS_TEST_PROMETHEUS_VERSIONS="$(PROM_VERSIONS)" THANOS_TEST_ALERTMANAGER_PATH="alertmanager-$(ALERTMANAGER_VERSION)" go test $(shell go list ./... | grep -v /vendor/); + @go test $(shell go list ./... | grep -v /vendor/); .PHONY: test-only-gcs test-only-gcs: export THANOS_SKIP_S3_AWS_TESTS = true @@ -202,15 +212,23 @@ test-local: @echo ">> Skipping GCE tests" $(MAKE) test-only-gcs -# test-deps installs dependency for e2e tets. -# It installs current Thanos, supported versions of Prometheus and alertmanager to test against in e2e. -.PHONY: test-deps -test-deps: - @echo ">> install thanos GOOPTS=${GOOPTS}" - @go install github.com/thanos-io/thanos/cmd/thanos - $(foreach ver,$(PROM_VERSIONS),$(call fetch_go_bin_version,github.com/prometheus/prometheus/cmd/prometheus,$(ver))) - $(call fetch_go_bin_version,github.com/prometheus/alertmanager/cmd/alertmanager,$(ALERTMANAGER_VERSION)) - $(call fetch_go_bin_version,github.com/minio/minio,$(MINIO_SERVER_VERSION)) +# install-deps installs dependencies for e2e tetss. +# It installs supported versions of Prometheus and alertmanager to test against in e2e. +.PHONY: install-deps +install-deps: $(ALERTMANAGER) $(MINIO_SERVER) $(PROMS) + @echo ">>GOBIN=$(GOBIN)" + +.PHONY: docker-ci +# To be run by Thanos maintainer. +docker-ci: install-deps + # Copy all to tmp local dir as this is required by docker. + @rm -rf ./tmp/bin + @mkdir -p ./tmp/bin + @cp -r $(GOBIN)/* ./tmp/bin + @docker build -t thanos-ci -f Dockerfile.thanos-ci . + @echo ">> pushing thanos-ci image" + @docker tag "thanos-ci" "quay.io/thanos/thanos-ci:v0.1.0" + @docker push "quay.io/thanos/thanos-ci:v0.1.0" # tooling deps. TODO(bwplotka): Pin them all to certain version! .PHONY: check-git @@ -277,6 +295,15 @@ $(GOLANGCILINT): $(MISSPELL): $(call fetch_go_bin_version,github.com/client9/misspell/cmd/misspell,$(MISSPELL_VERSION)) +$(ALERTMANAGER): + $(call fetch_go_bin_version,github.com/prometheus/alertmanager/cmd/alertmanager,$(ALERTMANAGER_VERSION)) + +$(MINIO_SERVER): + $(call fetch_go_bin_version,github.com/minio/minio,$(MINIO_SERVER_VERSION)) + +$(PROMS): + $(foreach ver,$(PROM_VERSIONS),$(call fetch_go_bin_version,github.com/prometheus/prometheus/cmd/prometheus,$(ver))) + $(PROTOC): @mkdir -p $(TMP_GOPATH) @echo ">> fetching protoc@${PROTOC_VERSION}"