From 73d78f8d6a7d031ec7a0e06365e729e75f1b677d Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 15 Oct 2020 08:15:53 -0700 Subject: [PATCH] ci: use multiple stage (#13557) Signed-off-by: Lizan Zhou --- .azure-pipelines/pipelines.yml | 688 +++++++++++++++++---------------- 1 file changed, 349 insertions(+), 339 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index f8c5ce972174..151736a8f335 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -10,370 +10,380 @@ trigger: # PR build config is manually overridden in Azure pipelines UI with different secrets pr: none -jobs: - - job: format - dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. - pool: - vmImage: "ubuntu-18.04" - steps: - - task: Cache@2 - inputs: - key: "format | ./WORKSPACE | **/*.bzl" - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true +stages: + - stage: precheck + jobs: + - job: format + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + pool: + vmImage: "ubuntu-18.04" + steps: + - task: Cache@2 + inputs: + key: "format | ./WORKSPACE | **/*.bzl" + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true - - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Run check format scripts" + - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Run check format scripts" - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff" - artifactName: format - condition: failed() + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff" + artifactName: format + condition: failed() - - job: docs - dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. - pool: - vmImage: "ubuntu-18.04" - steps: - - task: Cache@2 - inputs: - key: "docs | ./WORKSPACE | **/*.bzl" - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true + - job: docs + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + pool: + vmImage: "ubuntu-18.04" + steps: + - task: Cache@2 + inputs: + key: "docs | ./WORKSPACE | **/*.bzl" + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Generate docs" + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Generate docs" - - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs' - displayName: "Upload Docs to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) + - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs' + displayName: "Upload Docs to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.SourcesDirectory)/generated/docs" - artifactName: docs - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.SourcesDirectory)/generated/docs" + artifactName: docs + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(DocsPublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(DocsPrivateKey)" - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DocsPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DocsPrivateKey)" + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - - script: docs/publish.sh - displayName: "Publish to GitHub" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) + - script: docs/publish.sh + displayName: "Publish to GitHub" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true)) - - job: release - displayName: "Linux-x64 release" - dependsOn: ["format"] - # For master builds, continue even if format fails - condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) - timeoutInMinutes: 360 - pool: - vmImage: "ubuntu-18.04" - steps: - - template: bazel.yml - parameters: - ciTarget: bazel.release - - - job: release_arm64 - displayName: "Linux-arm64 release" - dependsOn: ["format"] - # For master builds, continue even if format fails - condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) - timeoutInMinutes: 360 - pool: "arm-large" - steps: - - template: bazel.yml - parameters: - managedAgent: false - ciTarget: bazel.release - rbe: false - artifactSuffix: ".arm64" - bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" - - - job: filter_example - displayName: "filter-example sync" - dependsOn: [] - pool: - vmImage: "ubuntu-18.04" + - stage: sync condition: and(succeeded(), eq(variables['PostSubmit'], true)) - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(FilterExamplePublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(FilterExamplePrivateKey)" + dependsOn: [] + jobs: + - job: filter_example + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(FilterExamplePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(FilterExamplePrivateKey)" - - bash: ci/filter_example_mirror.sh - displayName: "Sync envoy-filter-example" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) + - bash: ci/filter_example_mirror.sh + displayName: "Sync envoy-filter-example" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) - - job: api - displayName: "data-plane-api sync" - dependsOn: [] - condition: and(succeeded(), eq(variables['PostSubmit'], true)) - pool: - vmImage: "ubuntu-18.04" - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(DataPlaneApiPublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(DataPlaneApiPrivateKey)" + - job: data_plane_api + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DataPlaneApiPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DataPlaneApiPrivateKey)" - - bash: ci/api_mirror.sh - displayName: "Sync data-plane-api" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) + - bash: ci/api_mirror.sh + displayName: "Sync data-plane-api" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) - - job: go_control_plane - displayName: "go-control-plane sync" - dependsOn: [] - condition: and(succeeded(), eq(variables['PostSubmit'], true)) - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(GoControlPlanePublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(GoControlPlanePrivateKey)" + - job: go_control_plane + dependsOn: [] + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(GoControlPlanePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(GoControlPlanePrivateKey)" - - bash: | - cp -a ~/.ssh $(Build.StagingDirectory)/ - ci/run_envoy_docker.sh 'ci/go_mirror.sh' - displayName: "Sync go-control-plane" - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - AZP_BRANCH: $(Build.SourceBranch) + - bash: | + cp -a ~/.ssh $(Build.StagingDirectory)/ + ci/run_envoy_docker.sh 'ci/go_mirror.sh' + displayName: "Sync go-control-plane" + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + AZP_BRANCH: $(Build.SourceBranch) - - job: bazel - displayName: "Linux-x64" - dependsOn: ["release"] - # For master builds, continue even if format fails + - stage: linux_x64 + dependsOn: ["precheck"] + # For master builds, continue even if precheck fails condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) - strategy: - maxParallel: 3 - matrix: - api: - CI_TARGET: "bazel.api" - gcc: - CI_TARGET: "bazel.gcc" - clang_tidy: - CI_TARGET: "bazel.clang_tidy" - asan: - CI_TARGET: "bazel.asan" - tsan: - CI_TARGET: "bazel.tsan" - compile_time_options: - CI_TARGET: "bazel.compile_time_options" - timeoutInMinutes: 360 - pool: - vmImage: "ubuntu-18.04" - steps: - - template: bazel.yml - parameters: - ciTarget: $(CI_TARGET) + jobs: + - job: release + # For master builds, continue even if format fails + timeoutInMinutes: 360 + pool: + vmImage: "ubuntu-18.04" + steps: + - template: bazel.yml + parameters: + ciTarget: bazel.release + + - stage: linux_arm64 + dependsOn: ["precheck"] + # For master builds, continue even if precheck fails + condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))) + jobs: + - job: release + timeoutInMinutes: 360 + pool: "arm-large" + steps: + - template: bazel.yml + parameters: + managedAgent: false + ciTarget: bazel.release + rbe: false + artifactSuffix: ".arm64" + bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" + + - stage: check + dependsOn: ["linux_x64"] + jobs: + - job: bazel + displayName: "linux_x64" + dependsOn: [] + strategy: + maxParallel: 3 + matrix: + api: + CI_TARGET: "bazel.api" + gcc: + CI_TARGET: "bazel.gcc" + clang_tidy: + CI_TARGET: "bazel.clang_tidy" + asan: + CI_TARGET: "bazel.asan" + tsan: + CI_TARGET: "bazel.tsan" + compile_time_options: + CI_TARGET: "bazel.compile_time_options" + timeoutInMinutes: 360 + pool: + vmImage: "ubuntu-18.04" + steps: + - template: bazel.yml + parameters: + ciTarget: $(CI_TARGET) - - job: coverage - displayName: "Linux-x64" - dependsOn: ["release"] - timeoutInMinutes: 360 - pool: "x64-large" - strategy: - maxParallel: 2 - matrix: - coverage: - CI_TARGET: "coverage" - fuzz_coverage: - CI_TARGET: "fuzz_coverage" - steps: - - template: bazel.yml - parameters: - managedAgent: false - ciTarget: bazel.$(CI_TARGET) - rbe: false - # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces - bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" + - job: coverage + displayName: "linux_x64" + dependsOn: [] + timeoutInMinutes: 360 + pool: "x64-large" + strategy: + maxParallel: 2 + matrix: + coverage: + CI_TARGET: "coverage" + fuzz_coverage: + CI_TARGET: "fuzz_coverage" + steps: + - template: bazel.yml + parameters: + managedAgent: false + ciTarget: bazel.$(CI_TARGET) + rbe: false + # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces + bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" - - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)' - displayName: "Upload $(CI_TARGET) Report to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) - condition: always() + - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)' + displayName: "Upload $(CI_TARGET) Report to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) + condition: always() - - job: docker - displayName: "Linux multi-arch docker" - dependsOn: ["release", "release_arm64"] - pool: - vmImage: "ubuntu-18.04" - steps: - - bash: .azure-pipelines/cleanup.sh - displayName: "Removing tools from agent" - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "bazel.release" - itemPattern: "bazel.release/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "bazel.release.arm64" - itemPattern: "bazel.release.arm64/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: | - set -e - mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64 - mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64 - ci/docker_ci.sh - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: $(DockerUsername) - DOCKERHUB_PASSWORD: $(DockerPassword) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/build_images" - artifactName: docker - condition: always() + - stage: docker + dependsOn: ["linux_x64", "linux_arm64"] + jobs: + - job: docker + displayName: "linux multiarch" + pool: + vmImage: "ubuntu-18.04" + steps: + - bash: .azure-pipelines/cleanup.sh + displayName: "Removing tools from agent" + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release" + itemPattern: "bazel.release/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release.arm64" + itemPattern: "bazel.release.arm64/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: | + set -e + mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64 + mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64 + ci/docker_ci.sh + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + DOCKERHUB_USERNAME: $(DockerUsername) + DOCKERHUB_PASSWORD: $(DockerPassword) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/build_images" + artifactName: docker + condition: always() - - job: examples + - stage: verify dependsOn: ["docker"] - displayName: "Verify examples run as documented" - pool: - vmImage: "ubuntu-18.04" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "docker" - itemPattern: "docker/envoy-docker-images.tar.xz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: ./ci/do_ci.sh verify_examples - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - NO_BUILD_SETUP: 1 + jobs: + - job: examples + pool: + vmImage: "ubuntu-18.04" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "docker" + itemPattern: "docker/envoy-docker-images.tar.xz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: ./ci/do_ci.sh verify_examples + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + NO_BUILD_SETUP: 1 - - job: macOS - dependsOn: ["format"] - timeoutInMinutes: 360 - pool: - vmImage: "macos-latest" - steps: - - script: ./ci/mac_ci_setup.sh - displayName: "Install dependencies" + - stage: macos + dependsOn: ["precheck"] + jobs: + - job: test + timeoutInMinutes: 360 + pool: + vmImage: "macos-latest" + steps: + - script: ./ci/mac_ci_setup.sh + displayName: "Install dependencies" - - script: ./ci/mac_ci_steps.sh - displayName: "Run Mac CI" - env: - BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + - script: ./ci/mac_ci_steps.sh + displayName: "Run Mac CI" + env: + BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/bazel-testlogs/**/test.xml" - testRunTitle: "macOS" - condition: always() + - task: PublishTestResults@2 + inputs: + testResultsFiles: "**/bazel-testlogs/**/test.xml" + testRunTitle: "macOS" + condition: always() - - script: ./ci/flaky_test/run_process_xml.sh - displayName: "Process Test Results" - env: - TEST_TMPDIR: $(Build.SourcesDirectory) - SLACK_TOKEN: $(SLACK_TOKEN) - CI_TARGET: "MacOS" - REPO_URI: $(Build.Repository.Uri) - BUILD_URI: $(Build.BuildUri) + - script: ./ci/flaky_test/run_process_xml.sh + displayName: "Process Test Results" + env: + TEST_TMPDIR: $(Build.SourcesDirectory) + SLACK_TOKEN: $(SLACK_TOKEN) + CI_TARGET: "MacOS" + REPO_URI: $(Build.Repository.Uri) + BUILD_URI: $(Build.BuildUri) - - job: Windows - dependsOn: ["format"] - timeoutInMinutes: 360 - pool: - vmImage: "windows-latest" - steps: - - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh - displayName: "Run Windows CI" - env: - ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)" - ENVOY_RBE: "true" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/envoy" - artifactName: windows.release - condition: always() + - stage: windows + dependsOn: ["precheck"] + jobs: + - job: release + timeoutInMinutes: 360 + pool: + vmImage: "windows-latest" + steps: + - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh + displayName: "Run Windows CI" + env: + ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)" + ENVOY_RBE: "true" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/envoy" + artifactName: windows.release + condition: always() - - job: docker_windows - displayName: "Windows docker" - dependsOn: ["Windows"] - timeoutInMinutes: 360 - pool: - vmImage: "windows-latest" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "windows.release" - itemPattern: "windows.release/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: | - set -e - # Convert to Unix-style path so tar doesn't think drive letter is a hostname - STAGING_DIR="/$(echo '$(Build.StagingDirectory)' | tr -d ':' | tr '\\' '/')" - mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/windows.release/envoy_binary.tar.gz" -C ./windows/amd64 - ci/docker_ci.sh - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: $(DockerUsername) - DOCKERHUB_PASSWORD: $(DockerPassword) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/build_images" - artifactName: docker_windows - condition: always() + - job: docker + dependsOn: ["release"] + timeoutInMinutes: 360 + pool: + vmImage: "windows-latest" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "windows.release" + itemPattern: "windows.release/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: | + set -e + # Convert to Unix-style path so tar doesn't think drive letter is a hostname + STAGING_DIR="/$(echo '$(Build.StagingDirectory)' | tr -d ':' | tr '\\' '/')" + mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/windows.release/envoy_binary.tar.gz" -C ./windows/amd64 + ci/docker_ci.sh + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + DOCKERHUB_USERNAME: $(DockerUsername) + DOCKERHUB_PASSWORD: $(DockerPassword) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/build_images" + artifactName: docker_windows + condition: always()