From 5cfdda95bd70f090a5865b795060c15b80fe1725 Mon Sep 17 00:00:00 2001 From: Bogdan Drutu Date: Tue, 10 May 2022 10:23:11 -0700 Subject: [PATCH] Fix github action cache schema * There were multiple workflows with same cache key, but no dependency between them, and also caching different paths. * When the tests were changed to run on different go version, cache was not updated to include the version. * Race condition between go1.17-test/go1.18-test/lint to update the same cache key, split into 3 caches. Signed-off-by: Bogdan Drutu --- .github/workflows/build-and-test-windows.yml | 2 +- .github/workflows/build-and-test.yml | 119 +++++++------------ .github/workflows/load-tests.yml | 26 +--- 3 files changed, 49 insertions(+), 98 deletions(-) diff --git a/.github/workflows/build-and-test-windows.yml b/.github/workflows/build-and-test-windows.yml index 17c07cf12b6d..21e973dffcf4 100644 --- a/.github/workflows/build-and-test-windows.yml +++ b/.github/workflows/build-and-test-windows.yml @@ -33,7 +33,7 @@ jobs: path: | \Users\runneradmin\go\pkg\mod %LocalAppData%\go-build - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} + key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: Run Unit tests # use default Go build tags from Makefile.common run: go test --tags containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper ./... diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index c7fb47dbcf35..f18b90ab4145 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -23,23 +23,18 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - name: Cache Go - id: module-cache + id: go-cache uses: actions/cache@v3 with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} - - name: Cache Tools - id: tool-cache - uses: actions/cache@v3 - with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: Install dependencies - if: steps.module-cache.outputs.hit != 'true' + if: steps.go-cache.outputs.hit != 'true' run: make -j2 gomoddownload - name: Install Tools - if: steps.tool-cache.outputs.cache-hit != 'true' + if: steps.go-cache.outputs.cache-hit != 'true' run: make install-tools check-collector-module-version: runs-on: ubuntu-latest @@ -64,23 +59,18 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - name: Cache Go - id: module-cache + id: go-cache uses: actions/cache@v3 with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} - - name: Cache Tools - id: tool-cache - uses: actions/cache@v3 - with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} - - name: Cache Build + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Cache Lint Build uses: actions/cache@v3 with: path: /home/runner/.cache/go-build - key: go-build-lint-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} + key: go-lint-build-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: Lint run: make -j2 golint checks: @@ -98,23 +88,13 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - name: Cache Go - id: module-cache + id: go-cache uses: actions/cache@v3 with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} - - name: Cache Tools - id: tool-cache - uses: actions/cache@v3 - with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} - - name: Cache Build - uses: actions/cache@v3 - with: - path: /home/runner/.cache/go-build - key: go-build-lint-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: CheckDoc run: make checkdoc - name: Porto @@ -149,23 +129,18 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - name: Cache Go - id: module-cache + id: go-cache uses: actions/cache@v3 with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} - - name: Cache Tools - id: tool-cache - uses: actions/cache@v3 - with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} - - name: Cache Build + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Cache Test Build uses: actions/cache@v3 with: path: /home/runner/.cache/go-build - key: go-build-unittest-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} + key: go-test-build-${{ runner.os }}-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} - name: Run Unit Tests run: make gotest integration-tests: @@ -183,18 +158,13 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - name: Cache Go - id: module-cache + id: go-cache uses: actions/cache@v3 with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} - - name: Cache Tools - id: tool-cache - uses: actions/cache@v3 - with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: Run Integration Tests run: make integration-tests-with-cover @@ -213,18 +183,13 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - name: Cache Go - id: module-cache + id: go-cache uses: actions/cache@v3 with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} - - name: Cache Tools - id: tool-cache - uses: actions/cache@v3 - with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: Correctness run: make -C testbed run-correctness-traces-tests correctness-metrics: @@ -242,18 +207,13 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - name: Cache Go - id: module-cache + id: go-cache uses: actions/cache@v3 with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} - - name: Cache Tools - id: tool-cache - uses: actions/cache@v3 - with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: Correctness run: make -C testbed run-correctness-metrics-tests @@ -289,12 +249,13 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - name: Cache Go - id: module-cache + id: go-cache uses: actions/cache@v3 with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: Build Collector ${{ matrix.binary }} run: make otelcontribcol-${{ matrix.binary }} - name: Upload Collector Binaries @@ -416,12 +377,14 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH mkdir bin/ dist/ - - name: Cache Tools - id: tool-cache + - name: Cache Go + id: go-cache uses: actions/cache@v3 with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} + path: | + /home/runner/go/bin + /home/runner/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: Download Binaries uses: actions/download-artifact@v3 with: @@ -475,12 +438,14 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH mkdir bin/ dist/ - - name: Cache Tools - id: tool-cache + - name: Cache Go + id: go-cache uses: actions/cache@v3 with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} + path: | + /home/runner/go/bin + /home/runner/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - name: Set Release Tag id: github_tag run: ./.github/workflows/scripts/set_release_tag.sh diff --git a/.github/workflows/load-tests.yml b/.github/workflows/load-tests.yml index 4b2214664d7b..ac88465de9ec 100644 --- a/.github/workflows/load-tests.yml +++ b/.github/workflows/load-tests.yml @@ -29,22 +29,15 @@ jobs: cache-name: cache-go-modules with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod /home/runner/.cache/go-build - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} - - name: Cache Tools - id: tool-cache - uses: actions/cache@v3 - env: - cache-name: cache-tool-binaries - with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} - - name: Install dependencies + key: loadtest-go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Install Dependencies if: steps.module-cache.outputs.hit != 'true' run: make -j2 gomoddownload - name: Install Tools - if: steps.tool-cache.outputs.cache-hit != 'true' + if: steps.module-cache.outputs.hit != 'true' run: make install-tools - run: make otelcontribcol-testbed - name: Upload Collector Binaries @@ -65,14 +58,6 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v3 - - name: Cache Tools - id: tool-cache - uses: actions/cache@v3 - env: - cache-name: cache-tool-binaries - with: - path: /home/runner/go/bin - key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }} - run: sudo chmod 0777 -R /opt - name: Fluentbit Cache id: fluentbit-cache @@ -103,9 +88,10 @@ jobs: cache-name: cache-go-modules with: path: | + /home/runner/go/bin /home/runner/go/pkg/mod /home/runner/.cache/go-build - key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }} + key: loadtest-go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - run: mkdir -p results && touch results/TESTRESULTS.md - name: Download Collector Binaries uses: actions/download-artifact@v3