CI.yml

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.

name: C/I
on:
  push:
    branches:
      - main
      - release/v*
      - dev
      - dev/*
    paths-ignore:
      - '.github/**'
      - '!.github/workflows/CI.yml'
      - '**.md'

  # from collector and contrib repo
  repository_dispatch:
    types: [dependency-build, workflow-run]

env:
  IMAGE_NAME: aws-otel-collector
  PACKAGING_ROOT: build/packages
  ECR_REPO: aws-otel-test/adot-collector-integration-test
  TF_VAR_aws_access_key_id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
  TF_VAR_aws_secret_access_key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
  TF_VAR_aoc_vpc_name: aoc-vpc-large
  TF_VAR_aoc_vpc_security_group: aoc-vpc-security-group-large
  # TF_VAR_patch: 'true'
  PKG_SIGN_PRIVATE_KEY_NAME: aoc-linux-pkg-signing-gpg-key
  WIN_UNSIGNED_PKG_BUCKET: aoc-aws-signer-unsigned-artifact-src
  WIN_SIGNED_PKG_BUCKET: aoc-aws-signer-signed-artifact-dest
  WIN_UNSIGNED_PKG_FOLDER: OTelCollectorAuthenticode/AuthenticodeSigner-SHA256-RSA
  WIN_SIGNED_PKG_FOLDER: OTelCollectorAuthenticode/AuthenticodeSigner-SHA256-RSA

concurrency:
  group: ci-${{ github.ref_name }}
  cancel-in-progress: true

jobs:
  build-aotutil:
    runs-on: ubuntu-latest
    steps:
      - name: Check out testing framework
        uses: actions/checkout@v2
        with:
          repository: 'aws-observability/aws-otel-test-framework'
          path: testing-framework
      - name: Set up Go 1.x
        uses: actions/setup-go@v2
        with:
          go-version: 1.17
      - name: Install Go tools
        run: cd /tmp && go get -u golang.org/x/tools/cmd/goimports
      - name: Build aotutil
        run: cd testing-framework/cmd/aotutil && make build
      - name: Cache aotutil
        uses: actions/cache@v2
        with:
          key: "aotutil_${{ github.run_id }}"
          path: testing-framework/cmd/aotutil/aotutil

  build:
    needs:
      - build-aotutil
    runs-on: ubuntu-latest

    steps:
    # Set up building environment, patch the dev repo code on dispatch events.
    - name: Set up Go 1.x
      uses: actions/setup-go@v2
      with:
        go-version: 1.17

    - uses: actions/checkout@v2

    - name: Checkout dev opentelemetry-collector-contrib
      if: github.event_name == 'repository_dispatch' && github.event.action == 'dependency-build'
      uses: actions/checkout@v2
      with:
        repository: ${{ github.repository_owner }}/opentelemetry-collector-contrib
        ref: main
        path: pkg/opentelemetry-collector-contrib

    - name: Checkout dev opentelemetry-collector
      if: github.event_name == 'repository_dispatch' && github.event.action == 'dependency-build'
      uses: actions/checkout@v2
      with:
        repository: ${{ github.repository_owner }}/opentelemetry-collector
        ref: main
        path: pkg/opentelemetry-collector

    - name: append replace statement to go.mod to build with dev repo
      if: github.event_name == 'repository_dispatch' && github.event.action == 'dependency-build'
      run: |
        echo "replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => ./pkg/opentelemetry-collector-contrib/exporter/awsxrayexporter" >> go.mod
        echo "replace go.opentelemetry.io/collector => ./pkg/opentelemetry-collector" >> go.mod
        cat go.mod
        ls pkg

    #Cache go build and dependencies before making unit testing and build
    #Samples codes for different OS: https://github.com/actions/cache/blob/main/examples.md#go---modules
    #Since we are using Linux, the go packages are in /go/pkg/mod and build are in /.cache/go-build
    #Also speed up unit testing since go test uses the go build cache and also speed up the go build.
    - name: Cache go
      id: cached_go
      uses: actions/cache@v2
      env:
        cache-name: cached_go_modules
      with:
        path: |
          ~/go/pkg/mod
          ~/.cache/go-build
        key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }}

    - name: Cache binaries
      id: cached_binaries
      uses: actions/cache@v2
      with:
        key: "cached_binaries_${{ github.run_id }}"
        path: build

    # Unit Test and attach test coverage badge
    - name: Unit Test
      if: steps.cached_binaries.outputs.cache-hit != 'true'
      run: make test

    - name: Upload Coverage report to CodeCov
      if: steps.cached_binaries.outputs.cache-hit != 'true'
      uses: codecov/codecov-action@v2
      with:
        file: ./coverage.txt

    # Build and archive binaries into cache.
    - name: Build Binaries
      if: steps.cached_binaries.outputs.cache-hit != 'true'
      run: make build

    # upload the binaries to artifact as well because cache@v2 hasn't support windows
    - name: Upload
      uses: actions/upload-artifact@v2
      with:
        name: binary_artifacts
        path: build

  packaging-msi:
    needs: build
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v2

      - name: Download built artifacts
        uses: actions/download-artifact@v2
        with:
          name: binary_artifacts
          path: build

      - name: Display structure of downloaded files
        run: ls -R

      - name: Create msi file using candle and light
        run: .\tools\packaging\windows\create_msi.ps1

      - name: Install AWS Cli v2
        run: |
          Invoke-WebRequest -Uri "https://awscli.amazonaws.com/AWSCLIV2.msi" -OutFile "AWSCLIV2.msi"
          msiexec.exe /i AWSCLIV2.msi /passive
          [System.Environment]::SetEnvironmentVariable('Path',$Env:Path + ";C:\\Program Files\\Amazon\\AWSCLIV2",'User')

      - name: Sign windows artifacts
        run: |
          $pkgfile = "build\packages\windows\amd64\aws-otel-collector.msi"
          $hashobj = Get-FileHash -Algorithm sha256 $pkgfile
          $hash = $hashobj.Hash
          $create_date = Get-Date -format s
          aws s3api put-object "--body" $pkgfile "--bucket" ${{ env.WIN_UNSIGNED_PKG_BUCKET }} "--key" ${{ env.WIN_UNSIGNED_PKG_FOLDER }}/aws-otel-collector-$hash.msi
          $objkey = ""
          for ($num = 1 ; $num -le 60 ; $num++) { # 60 * 10 = 600s = 10min timeout
             Start-Sleep -s 10
             Write-Output "Poll number $num"
             $objkey = aws s3api list-objects "--bucket" ${{ env.WIN_SIGNED_PKG_BUCKET }} "--prefix" ${{ env.WIN_SIGNED_PKG_FOLDER }}/aws-otel-collector-$hash.msi "--output" text "--query" "Contents[?LastModified>'$create_date'].Key|[0]"
             if ($objkey -ne "None") {
               Write-Output "Found: $objkey"
               break
             } else {
               Write-Output "$objkey returned, obj not available yet, try again later"
             }
          }
          if ($objkey -eq "None") {
            Throw "Could not find the signed artifact"
          }
          aws s3api get-object "--bucket" ${{ env.WIN_SIGNED_PKG_BUCKET }} "--key" $objkey $pkgfile
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.SIGN_PKG_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SIGN_PKG_AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-west-2

      - name: Verify package signature
        run: |
          $pkgfile = "build\packages\windows\amd64\aws-otel-collector.msi"
          $sig = Get-AuthenticodeSignature $pkgfile
          $status = $sig.Status
          if ($status -ne "Valid") {
            Throw "Invalid signature found: $status"
          }
          Write-Output "Valid signature found from the package"

      - name: Upload the msi
        uses: actions/upload-artifact@v2
        with:
          name: msi_artifacts
          path: build/packages

  packaging-rpm:
    runs-on: ubuntu-latest
    needs: build
    steps:
      # Build and archive rpms into cache.
      - uses: actions/checkout@v2

      - name: Cache rpms
        id: cached_rpms
        uses: actions/cache@v2
        with:
          key: "cached_rpms_${{ github.run_id }}"
          path: build/packages

      - name: restore cached binaries
        if: steps.cached_rpms.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "cached_binaries_${{ github.run_id }}"
          path: build

      - name: Display structure of downloaded files
        run: ls -R

      - name: Build RPM
        if: steps.cached_rpms.outputs.cache-hit != 'true'
        run: |
          ARCH=x86_64 DEST=build/packages/linux/amd64 tools/packaging/linux/create_rpm.sh
          ARCH=aarch64 DEST=build/packages/linux/arm64 tools/packaging/linux/create_rpm.sh

      - name: Download Package Signing GPG key
        if: steps.cached_rpms.outputs.cache-hit != 'true'
        run: |
          aws secretsmanager get-secret-value --region us-west-2 --secret-id "$PKG_SIGN_PRIVATE_KEY_NAME" | jq -r ".SecretString" > pkg_sign_private.key
          md5sum pkg_sign_private.key
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.SIGN_PKG_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SIGN_PKG_AWS_SECRET_ACCESS_KEY }}

      - name: Import Package Signing GPG Key
        if: steps.cached_rpms.outputs.cache-hit != 'true'
        run: |
          gpg --import pkg_sign_private.key
          gpg --list-keys
          gpg --armor --export -a "aws-otel-collector@amazon.com" > pkg_sign_public.key
          rpm --import pkg_sign_public.key
          echo "%_gpg_name aws-otel-collector@amazon.com" > ~/.rpmmacros
          md5sum pkg_sign_public.key
          shred -fuvz pkg_sign_private.key

      - name: Sign RPM Pakcage
        if: steps.cached_rpms.outputs.cache-hit != 'true'
        run: |
          rpmsign --addsign build/packages/linux/*/*.rpm

      - name: Remove Package Signing GPG Key from local GPG Key Ring
        if: steps.cached_rpms.outputs.cache-hit != 'true'
        run: |
          gpg --fingerprint --with-colons aws-otel-collector@amazon.com grep "^fpr" | sed -n 's/^fpr:::::::::\([[:alnum:]]\+\):/\1/p' | xargs gpg --batch --yes --delete-secret-keys
          gpg --list-secret-keys

  packaging-deb:
    runs-on: ubuntu-latest
    needs: build
    steps:
      # Build and archive debs into cache.
      - uses: actions/checkout@v2

      - name: Cache Debs
        id: cached_debs
        uses: actions/cache@v2
        with:
          key: "cached_debs_${{ github.run_id }}"
          path: build/packages

      - name: restore cached binaries
        if: steps.cached_debs.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "cached_binaries_${{ github.run_id }}"
          path: build

      - name: Build Debs
        if: steps.cached_debs.outputs.cache-hit != 'true'
        run: |
          ARCH=amd64 TARGET_SUPPORTED_ARCH=x86_64 DEST=build/packages/debian/amd64 tools/packaging/debian/create_deb.sh
          ARCH=arm64 TARGET_SUPPORTED_ARCH=aarch64 DEST=build/packages/debian/arm64 tools/packaging/debian/create_deb.sh

      - name: Download Package Signing GPG key
        if: steps.cached_debs.outputs.cache-hit != 'true'
        run: |
          aws secretsmanager get-secret-value --region us-west-2 --secret-id "$PKG_SIGN_PRIVATE_KEY_NAME" | jq -r ".SecretString" > pkg_sign_private.key
          md5sum pkg_sign_private.key
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.SIGN_PKG_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.SIGN_PKG_AWS_SECRET_ACCESS_KEY }}

      - name: Import Package Signing GPG Key
        if: steps.cached_debs.outputs.cache-hit != 'true'
        run: |
          gpg --import pkg_sign_private.key
          gpg --list-secret-keys
          shred -fuvz pkg_sign_private.key

      - name: Sign DEB Pakcage
        if: steps.cached_debs.outputs.cache-hit != 'true'
        run: |
          sudo apt install -y dpkg-sig
          dpkg-sig --sign origin -k "aws-otel-collector@amazon.com" build/packages/debian/*/*.deb

      - name: Remove Package Signing GPG Key from local GPG Key Ring
        if: steps.cached_debs.outputs.cache-hit != 'true'
        run: |
          gpg --fingerprint --with-colons aws-otel-collector@amazon.com grep "^fpr" | sed -n 's/^fpr:::::::::\([[:alnum:]]\+\):/\1/p' | xargs gpg --batch --yes --delete-secret-keys
          gpg --list-secret-keys

  packaging-image:
    runs-on: ubuntu-latest
    needs: build
    steps:
      # Build and archive image into cache
      - uses: actions/checkout@v2

      - name: Cache Image
        id: cached_image
        uses: actions/cache@v2
        with:
          key: "cached_image_${{ github.run_id }}"
          path: "${{ env.PACKAGING_ROOT }}"

      - name: restore cached binaries
        if: steps.cached_image.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "cached_binaries_${{ github.run_id }}"
          path: build

      - name: Build Image
        if: steps.cached_image.outputs.cache-hit != 'true'
        run: docker build -t $IMAGE_NAME -f cmd/awscollector/Dockerfile .

      - name: Extract the Image file
        if: steps.cached_image.outputs.cache-hit != 'true'
        run: |
          mkdir -p $PACKAGING_ROOT
          docker save --output $PACKAGING_ROOT/$IMAGE_NAME.tar $IMAGE_NAME

  packaging-ssm:
    runs-on: ubuntu-latest
    needs: [packaging-rpm, packaging-deb, packaging-msi]
    steps:
      # Build and archive SSM package into cache.
      - uses: actions/checkout@v2

      - name: Cache SSM files
        id: cached_ssm
        uses: actions/cache@v2
        with:
          key: "cached_ssm_${{ github.run_id }}"
          path: build/packages/ssm

      - name: Restore cached rpms
        if: steps.cached_ssm.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "cached_rpms_${{ github.run_id }}"
          path: build/packages

      - name: Restore cached debs
        if: steps.cached_ssm.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "cached_debs_${{ github.run_id }}"
          path: build/packages

      - name: Download built artifacts
        if: steps.cached_ssm.outputs.cache-hit != 'true'
        uses: actions/download-artifact@v2
        with:
          name: msi_artifacts
          path: build/packages

      - run: ls -R

      - name: Create zip file and manifest for SSM package
        if: steps.cached_ssm.outputs.cache-hit != 'true'
        run: |
          # use a version with github run id as same as e2etest-preparation
          rm -rf build/packages/ssm
          python3 tools/ssm/ssm_manifest.py "`cat VERSION`-${{ github.run_id }}"

  e2etest-preparation:
    runs-on: ubuntu-latest
    needs: [packaging-rpm, packaging-deb, packaging-msi, packaging-image, packaging-ssm]
    outputs:
      version: ${{ steps.versioning.outputs.version }}
    steps:
      # Archive all the packages into one, and build a unique version number for e2etesting
      - uses: actions/checkout@v2

      - name: Cache the packages
        id: cached_packages
        uses: actions/cache@v2
        with:
          key: "cached_packages_${{ github.run_id }}"
          path: build/packages

      - name: Restore cached rpms
        if: steps.cached_packages.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "cached_rpms_${{ github.run_id }}"
          path: build/packages

      - name: Restore cached debs
        if: steps.cached_packages.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "cached_debs_${{ github.run_id }}"
          path: build/packages

      - name: Restore cached image
        if: steps.cached_packages.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "cached_image_${{ github.run_id }}"
          path: build/packages

      - name: Download built artifacts
        if: steps.cached_packages.outputs.cache-hit != 'true'
        uses: actions/download-artifact@v2
        with:
          name: msi_artifacts
          path: build/packages

      - name: Restore cached ssm
        if: steps.cached_packages.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "cached_ssm_${{ github.run_id }}"
          path: build/packages/ssm

      - run: ls -R

      - name: Versioning for testing
        id: versioning
        run: |
          # build a version with github run id so that we can distingush each build for integ-test
          Version="`cat VERSION`-$GITHUB_RUN_ID"
          echo $Version > build/packages/VERSION
          cat build/packages/VERSION
          echo "::set-output name=version::$Version"

      - name: prepare CI&CD stack
        run: |
          cp .aoc-stack-test.yml build/packages/
          cp .aoc-stack-release.yml build/packages/

  e2etest-release:
    runs-on: ubuntu-latest
    needs: [e2etest-preparation]
    steps:
      - uses: actions/checkout@v2

      - name: Cache if success
        id: e2etest-release
        uses: actions/cache@v2
        with:
          path: |
            VERSION
          key: e2etest-release-${{ github.run_id }}

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: restore cached rpms
        uses: actions/cache@v2
        with:
          key: "cached_packages_${{ github.run_id }}"
          path: build/packages

      - name: Create SSM package
        run: |
          ssm_pkg_version=`cat build/packages/VERSION`
          (aws ssm describe-document --name "testAWSDistroOTel-Collector" --version-name ${ssm_pkg_version} >/dev/null 2>&1) || {
            pip3 install boto3
            python3 tools/ssm/ssm_manifest.py ${ssm_pkg_version}
            aws s3 cp build/packages/ssm s3://aws-otel-collector-test/ssm/${ssm_pkg_version} --recursive
            python3 tools/ssm/ssm_create.py --no-default testAWSDistroOTel-Collector ${ssm_pkg_version} aws-otel-collector-test/ssm/${ssm_pkg_version} us-west-2
          }

      - name: Upload to S3 in the testing stack
        if: steps.e2etest-release.outputs.cache-hit != 'true'
        run: s3_bucket_name=aws-otel-collector-test upload_to_latest=0 bash tools/release/image-binary-release/s3-release.sh

      - name: Login to Public ECR
        if: steps.e2etest-release.outputs.cache-hit != 'true'
        id: login-ecr
        uses: docker/login-action@v1
        with:
          registry: public.ecr.aws
          username: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          password: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
        env:
          AWS_REGION: us-east-1

      - name: upload to ECR
        if: steps.e2etest-release.outputs.cache-hit != 'true'
        run: |
          docker load < build/packages/$IMAGE_NAME.tar
          docker tag $IMAGE_NAME public.ecr.aws/$ECR_REPO:${{ needs.e2etest-preparation.outputs.version }}
          docker push public.ecr.aws/$ECR_REPO:${{ needs.e2etest-preparation.outputs.version }}


  get-testing-suites:
    runs-on: ubuntu-latest
    outputs:
      eks-matrix: ${{ steps.set-matrix.outputs.eks-matrix }}
      eks-fargate-matrix: ${{ steps.set-matrix.outputs.eks-fargate-matrix }}
      eks-adot-operator-matrix: ${{ steps.set-matrix.outputs.eks-adot-operator-matrix }}
      ecs-matrix: ${{ steps.set-matrix.outputs.ecs-matrix }}
      ec2-matrix-1: ${{ steps.set-matrix.outputs.ec2-matrix-1 }}
      ec2-matrix-2: ${{ steps.set-matrix.outputs.ec2-matrix-2 }}
      ec2-matrix-3: ${{ steps.set-matrix.outputs.ec2-matrix-3 }}
    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Setup Python
        uses: actions/setup-python@v2.3.1

      - name: Get all the testing suites
        id: set-matrix
        run: |
          ec2_matrix_1=$(python e2etest/get-testcases.py ec2_matrix_1)
          ec2_matrix_2=$(python e2etest/get-testcases.py ec2_matrix_2)
          ec2_matrix_3=$(python e2etest/get-testcases.py ec2_matrix_3)
          ecs_matrix=$(python e2etest/get-testcases.py ecs_matrix)
          eks_matrix=$(python e2etest/get-testcases.py eks_matrix)
          eks_fargate_matrix=$(python e2etest/get-testcases.py eks_fargate_matrix)
          eks_adot_operator_matrix=$(python e2etest/get-testcases.py eks_adot_operator_matrix)
          echo "::set-output name=eks-matrix::$eks_matrix"
          echo "::set-output name=eks-fargate-matrix::$eks_fargate_matrix"
          echo "::set-output name=eks-adot-operator-matrix::$eks_adot_operator_matrix"
          echo "::set-output name=ecs-matrix::$ecs_matrix"
          echo "::set-output name=ec2-matrix-1::$ec2_matrix_1"
          echo "::set-output name=ec2-matrix-2::$ec2_matrix_2"
          echo "::set-output name=ec2-matrix-3::$ec2_matrix_3"
      - name: List testing suites
        run: |
          echo ${{ steps.set-matrix.outputs.eks-matrix }}
          echo ${{ steps.set-matrix.outputs.eks-fargate-matrix }}
          echo ${{ steps.set-matrix.outputs.eks-adot-operator-matrix }}
          echo ${{ steps.set-matrix.outputs.ecs-matrix }}
          echo ${{ steps.set-matrix.outputs.ec2-matrix-1 }}
          echo ${{ steps.set-matrix.outputs.ec2-matrix-2 }}
          echo ${{ steps.set-matrix.outputs.ec2-matrix-3 }}

  e2etest-ec2-1:
    runs-on: ubuntu-latest
    needs: [get-testing-suites, e2etest-release, e2etest-preparation]
    strategy:
      fail-fast: false
      max-parallel: 10
      matrix: ${{ fromJson(needs.get-testing-suites.outputs.ec2-matrix-1) }}

    steps:
      - uses: actions/checkout@v2

      - name: Cache if success
        id: e2etest-ec2-1
        uses: actions/cache@v2
        with:
          path: |
            VERSION
          key: e2etest-ec2-1-${{ github.run_id }}-${{ matrix.testcase }}-${{ matrix.testing_ami }}

      - name: Configure AWS Credentials
        if: steps.e2etest-ec2-1.outputs.cache-hit != 'true'
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: Set up JDK 11
        if: steps.e2etest-ec2-1.outputs.cache-hit != 'true'
        uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up terraform
        if: steps.e2etest-ec2-1.outputs.cache-hit != 'true'
        uses: hashicorp/setup-terraform@v1

      - name: Check out testing framework
        if: steps.e2etest-ec2-1.outputs.cache-hit != 'true'
        uses: actions/checkout@v2
        with:
          repository: 'aws-observability/aws-otel-collector-test-framework'
          path: testing-framework

      - name: Restore aoutil
        if: steps.e2etest-ec2-1.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "aotutil_${{ github.run_id }}"
          path: testing-framework/cmd/aotutil/aotutil

      - name: Run testing suite on ec2
        if: steps.e2etest-ec2-1.outputs.cache-hit != 'true'
        uses: nick-invision/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 120
          command: |
            if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
            cd testing-framework/terraform/ec2
            terraform init
            if terraform apply -auto-approve -lock=false $opts -var="testing_ami=${{ matrix.testing_ami }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve ; else terraform destroy -auto-approve && exit 1 ; fi

      #This is here just in case workflow cancel
      - name: Destroy resources
        if: ${{ cancelled() && steps.e2etest-ec2-1.outputs.cache-hit != 'true' }}
        run: |
          cd testing-framework/terraform/ec2 && terraform destroy -auto-approve

  e2etest-ec2-2:
    runs-on: ubuntu-latest
    needs: [get-testing-suites, e2etest-release, e2etest-preparation]
    strategy:
      fail-fast: false
      max-parallel: 10
      matrix: ${{ fromJson(needs.get-testing-suites.outputs.ec2-matrix-2) }}

    steps:
      - uses: actions/checkout@v2

      - name: Cache if success
        id: e2etest-ec2-2
        uses: actions/cache@v2
        with:
          path: |
            VERSION
          key: e2etest-ec2-2-${{ github.run_id }}-${{ matrix.testcase }}-${{ matrix.testing_ami }}

      - name: Configure AWS Credentials
        if: steps.e2etest-ec2-2.outputs.cache-hit != 'true'
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: Set up JDK 11
        if: steps.e2etest-ec2-2.outputs.cache-hit != 'true'
        uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up terraform
        if: steps.e2etest-ec2-2.outputs.cache-hit != 'true'
        uses: hashicorp/setup-terraform@v1

      - name: Check out testing framework
        if: steps.e2etest-ec2-2.outputs.cache-hit != 'true'
        uses: actions/checkout@v2
        with:
          repository: 'aws-observability/aws-otel-collector-test-framework'
          path: testing-framework

      - name: Restore aoutil
        if: steps.e2etest-ec2-2.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "aotutil_${{ github.run_id }}"
          path: testing-framework/cmd/aotutil/aotutil

      - name: Run testing suite on ec2
        if: steps.e2etest-ec2-2.outputs.cache-hit != 'true'
        uses: nick-invision/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 120
          command: |
            if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
            cd testing-framework/terraform/ec2
            terraform init
            if terraform apply -auto-approve -lock=false $opts -var="testing_ami=${{ matrix.testing_ami }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve ; else terraform destroy -auto-approve && exit 1 ; fi

      #This is here just in case workflow cancel
      - name: Destroy resources
        if: ${{ cancelled() && steps.e2etest-ec2-2.outputs.cache-hit != 'true' }}
        run: |
          cd testing-framework/terraform/ec2 && terraform destroy -auto-approve

  e2etest-ec2-3:
    runs-on: ubuntu-latest
    needs: [get-testing-suites, e2etest-release, e2etest-preparation]
    strategy:
      fail-fast: false
      max-parallel: 10
      matrix: ${{ fromJson(needs.get-testing-suites.outputs.ec2-matrix-3) }}

    steps:
      - uses: actions/checkout@v2

      - name: Cache if success
        id: e2etest-ec2-3
        uses: actions/cache@v2
        with:
          path: |
            VERSION
          key: e2etest-ec2-3-${{ github.run_id }}-${{ matrix.testcase }}-${{ matrix.testing_ami }}

      - name: Configure AWS Credentials
        if: steps.e2etest-ec2-3.outputs.cache-hit != 'true'
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: Set up JDK 11
        if: steps.e2etest-ec2-3.outputs.cache-hit != 'true'
        uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up terraform
        if: steps.e2etest-ec2-3.outputs.cache-hit != 'true'
        uses: hashicorp/setup-terraform@v1

      - name: Check out testing framework
        if: steps.e2etest-ec2-3.outputs.cache-hit != 'true'
        uses: actions/checkout@v2
        with:
          repository: 'aws-observability/aws-otel-collector-test-framework'
          path: testing-framework

      - name: Restore aoutil
        if: steps.e2etest-ec2-3.outputs.cache-hit != 'true'
        uses: actions/cache@v2
        with:
          key: "aotutil_${{ github.run_id }}"
          path: testing-framework/cmd/aotutil/aotutil

      - name: Run testing suite on ec2
        if: steps.e2etest-ec2-3.outputs.cache-hit != 'true'
        uses: nick-invision/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 120
          command: |
            if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
            cd testing-framework/terraform/ec2
            terraform init
            if terraform apply -auto-approve -lock=false $opts -var="testing_ami=${{ matrix.testing_ami }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve ; else terraform destroy -auto-approve && exit 1 ; fi

      #This is here just in case workflow cancel
      - name: Destroy resources
        if: ${{ cancelled() && steps.e2etest-ec2-3.outputs.cache-hit != 'true' }}
        run: |
          cd testing-framework/terraform/ec2 && terraform destroy -auto-approve

  e2etest-ecs:
    runs-on: ubuntu-latest
    needs: [get-testing-suites, e2etest-release, e2etest-preparation]
    strategy:
      fail-fast: false
      max-parallel: 10
      matrix: ${{ fromJson(needs.get-testing-suites.outputs.ecs-matrix) }}

    steps:
      - uses: actions/checkout@v2

      - name: Cache if success
        id: e2etest-ecs
        uses: actions/cache@v2
        with:
          path: |
            VERSION
          key: e2etest-ecs-${{ github.run_id }}-${{ matrix.testcase }}-${{ matrix.launch_type }}

      - name: Configure AWS Credentials
        if: steps.e2etest-ecs.outputs.cache-hit != 'true'
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: Set up JDK 11
        if: steps.e2etest-ecs.outputs.cache-hit != 'true'
        uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up terraform
        if: steps.e2etest-ecs.outputs.cache-hit != 'true'
        uses: hashicorp/setup-terraform@v1

      - name: Check out testing framework
        if: steps.e2etest-ecs.outputs.cache-hit != 'true'
        uses: actions/checkout@v2
        with:
          repository: 'aws-observability/aws-otel-collector-test-framework'
          path: testing-framework

      - name: Run testing suite on ecs
        if: steps.e2etest-ecs.outputs.cache-hit != 'true'
        uses: nick-invision/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 120
          command: |
            if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
            cd testing-framework/terraform/ecs
            terraform init
            if terraform apply -auto-approve -lock=false $opts -var="ecs_launch_type=${{ matrix.launch_type }}" -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve ; else terraform destroy -auto-approve && exit 1 ; fi

      #This is here just in case workflow cancel
      - name: Destroy resources
        if: ${{ cancelled() && steps.e2etest-ecs.outputs.cache-hit != 'true' }}
        run: |
          cd testing-framework/terraform/ecs && terraform destroy -auto-approve

  e2etest-eks:
    runs-on: ubuntu-latest
    needs: [get-testing-suites, e2etest-release, e2etest-preparation]
    strategy:
      fail-fast: false
      max-parallel: 5
      matrix: ${{ fromJson(needs.get-testing-suites.outputs.eks-matrix) }}

    steps:
      - uses: actions/checkout@v2

      - name: Cache if success
        id: e2etest-eks
        uses: actions/cache@v2
        with:
          path: |
            VERSION
          key: e2etest-eks-${{ github.run_id }}-${{ matrix.testcase }}

      - name: Configure AWS Credentials
        if: steps.e2etest-eks.outputs.cache-hit != 'true'
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: Set up JDK 11
        if: steps.e2etest-eks.outputs.cache-hit != 'true'
        uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up terraform
        if: steps.e2etest-eks.outputs.cache-hit != 'true'
        uses: hashicorp/setup-terraform@v1

      - name: Check out testing framework
        if: steps.e2etest-eks.outputs.cache-hit != 'true'
        uses: actions/checkout@v2
        with:
          repository: 'aws-observability/aws-otel-collector-test-framework'
          path: testing-framework

      - name: Run testing suite on eks
        if: steps.e2etest-eks.outputs.cache-hit != 'true'
        run: |
          if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
          cd testing-framework/terraform/eks && terraform init && terraform apply -auto-approve -lock=false $opts -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}"

      - name: Destroy resources
        if: ${{ always() && steps.e2etest-eks.outputs.cache-hit != 'true' }}
        run: |
          cd testing-framework/terraform/eks && terraform destroy -auto-approve

  e2etest-eks-fargate:
    runs-on: ubuntu-latest
    needs: [get-testing-suites, e2etest-release, e2etest-preparation]
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix: ${{ fromJson(needs.get-testing-suites.outputs.eks-fargate-matrix) }}

    steps:
      - uses: actions/checkout@v2

      - name: Cache if success
        id: e2etest-eks-fargate
        uses: actions/cache@v2
        with:
          path: |
            VERSION
          key: e2etest-eks-fargate-${{ github.run_id }}-${{ matrix.testcase }}

      - name: Configure AWS Credentials
        if: steps.e2etest-eks-fargate.outputs.cache-hit != 'true'
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: Set up JDK 11
        if: steps.e2etest-eks-fargate.outputs.cache-hit != 'true'
        uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up terraform
        if: steps.e2etest-eks-fargate.outputs.cache-hit != 'true'
        uses: hashicorp/setup-terraform@v1

      - name: Check out testing framework
        if: steps.e2etest-eks-fargate.outputs.cache-hit != 'true'
        uses: actions/checkout@v2
        with:
          repository: 'aws-observability/aws-otel-collector-test-framework'
          path: testing-framework

      - name: Run testing suite on eks
        if: steps.e2etest-eks-fargate.outputs.cache-hit != 'true'
        uses: nick-invision/retry@v2
        with:
          max_attempts: 3
          timeout_minutes: 120
          command: |
            if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
            cd testing-framework/terraform/eks
            terraform init
            if terraform apply -auto-approve -lock=false $opts -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}" ; then terraform destroy -auto-approve -var-file="../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else terraform destroy -auto-approve -var-file="../testcases/${{ matrix.testcase }}/parameters.tfvars" && exit 1 ; fi

      #This is here just in case workflow cancel
      - name: Destroy resources
        if: ${{ cancelled() && steps.e2etest-eks-fargate.outputs.cache-hit != 'true' }}
        run: |
          cd testing-framework/terraform/eks && terraform destroy -auto-approve

  e2etest-eks-adot-operator:
    runs-on: ubuntu-latest
    needs: [get-testing-suites, e2etest-release, e2etest-preparation]
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix: ${{ fromJson(needs.get-testing-suites.outputs.eks-adot-operator-matrix) }}

    steps:
      - uses: actions/checkout@v2

      - name: Cache if success
        id: e2etest-eks-adot-operator
        uses: actions/cache@v2
        with:
          path: |
            VERSION
          key: e2etest-eks-adot-operator-${{ github.run_id }}-${{ matrix.testcase }}

      - name: Configure AWS Credentials
        if: steps.e2etest-eks-adot-operator.outputs.cache-hit != 'true'
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: Set up JDK 11
        if: steps.e2etest-eks-adot-operator.outputs.cache-hit != 'true'
        uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up terraform
        if: steps.e2etest-eks-adot-operator.outputs.cache-hit != 'true'
        uses: hashicorp/setup-terraform@v1

      - name: Check out testing framework
        if: steps.e2etest-eks-adot-operator.outputs.cache-hit != 'true'
        uses: actions/checkout@v2
        with:
          repository: 'aws-observability/aws-otel-collector-test-framework'
          path: testing-framework

      - name: Run ADOT Operator testing suite on eks
        if: steps.e2etest-eks-adot-operator.outputs.cache-hit != 'true'
        run: |
          if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
          cd testing-framework/terraform/eks && terraform init && terraform apply -auto-approve $opts -var="aoc_version=${{ needs.e2etest-preparation.outputs.version }}" -var="testcase=../testcases/${{ matrix.testcase }}"

      - name: Destroy resources
        if: ${{ always() && steps.e2etest-eks-adot-operator.outputs.cache-hit != 'true' }}
        run: |
          cd testing-framework/terraform/eks && terraform destroy -auto-approve -var-file="../testcases/${{ matrix.testcase }}/parameters.tfvars"

  release-candidate:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' # only create the artifact when there's a push, not for dispatch.
    needs: [e2etest-eks, e2etest-eks-adot-operator, e2etest-eks-fargate, e2etest-ecs, e2etest-ec2-1, e2etest-ec2-2, e2etest-ec2-3]
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: restore cached packages
        uses: actions/cache@v2
        with:
          path: build/packages
          key: "cached_packages_${{ github.run_id }}"

      - name: prepare production version
        run: |
          TESTING_VERSION=`cat build/packages/VERSION`
          VERSION=`echo $TESTING_VERSION | awk -F "-" '{print $1}'`
          echo $VERSION > build/packages/VERSION
          echo $GITHUB_SHA > build/packages/GITHUB_SHA
          echo $TESTING_VERSION > build/packages/TESTING_VERSION

      - name: upload packages as release candidate on s3
        run: |
          tar czvf $GITHUB_SHA.tar.gz build
          aws s3 cp $GITHUB_SHA.tar.gz s3://aws-otel-collector-release-candidate/$GITHUB_SHA.tar.gz

      - name: Trigger performance test
        uses: peter-evans/repository-dispatch@v1.1.3
        with:
          token: "${{ secrets.REPO_WRITE_ACCESS_TOKEN }}"
          event-type: trigger-perf
          client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'

  clean:
    runs-on: ubuntu-latest
    if: ${{ always() }}
    needs: [e2etest-eks, e2etest-eks-adot-operator, e2etest-eks-fargate, e2etest-ecs, e2etest-ec2-1, e2etest-ec2-2, e2etest-ec2-3]
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }}
          aws-region: us-west-2

      - name: restore cached packages
        uses: actions/cache@v2
        with:
          path: build/packages
          key: "cached_packages_${{ github.run_id }}"

      - name: clean up SSM test package
        run: |
          ssm_pkg_version=`cat build/packages/VERSION`
          aws ssm delete-document --name "testAWSDistroOTel-Collector" --version-name ${ssm_pkg_version}