ci: use azp to generate docs (#13481)

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

.azure-pipelines/pipelines.yml

@@ -37,6 +37,49 @@ jobs:
           artifactName: format
         condition: failed()
 
+  - job: docs
+    dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
+    pool:
+      vmImage: "ubuntu-18.04"
+    steps:
+      - task: Cache@2
+        inputs:
+          key: "docs | ./WORKSPACE | **/*.bzl"
+          path: $(Build.StagingDirectory)/repository_cache
+        continueOnError: true
+
+      - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs'
+        workingDirectory: $(Build.SourcesDirectory)
+        env:
+          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+          BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+          BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+          GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+        displayName: "Generate docs"
+
+      - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs'
+        displayName: "Upload Docs to GCS"
+        env:
+          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+          GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+          GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
+
+      - task: InstallSSHKey@0
+        inputs:
+          hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
+          sshPublicKey: "$(DocsPublicKey)"
+          sshPassphrase: "$(SshDeployKeyPassphrase)"
+          sshKeySecureFile: "$(DocsPrivateKey)"
+        condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true))
+
+      - script: docs/publish.sh
+        displayName: "Publish to GitHub"
+        workingDirectory: $(Build.SourcesDirectory)
+        env:
+          AZP_BRANCH: $(Build.SourceBranch)
+          AZP_SHA1: $(Build.SourceVersion)
+        condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true))
+
   - job: release
     displayName: "Linux-x64 release"
     dependsOn: ["format"]

.circleci/config.yml

@@ -45,27 +45,10 @@ jobs:
              - "f6:f9:df:90:9c:4b:5f:9c:f4:69:fd:42:94:ff:88:24"
        - run: ci/filter_example_mirror.sh
 
-   docs:
-     executor: ubuntu-build
-     steps:
-       - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
-       - checkout
-       - run: ci/do_circle_ci.sh docs
-       - add_ssh_keys:
-           fingerprints:
-             - "44:c7:a1:9e:f4:9e:a5:33:11:f1:0e:79:e1:55:c9:04"
-       - run: docs/publish.sh
-       - store_artifacts:
-           path: generated/docs
-
 workflows:
   version: 2
   all:
     jobs:
       - api
       - go_control_plane_mirror
       - filter_example_mirror
-      - docs:
-          filters:
-            tags:
-              only: /^v.*/

ci/run_envoy_docker.sh

@@ -64,6 +64,7 @@ docker run --rm \
        "${ENVOY_DOCKER_OPTIONS[@]}" \
        -v "${ENVOY_DOCKER_BUILD_DIR}":"${BUILD_DIR_MOUNT_DEST}" \
        -v "${SOURCE_DIR}":"${SOURCE_DIR_MOUNT_DEST}" \
+       -e AZP_BRANCH \
        -e HTTP_PROXY \
        -e HTTPS_PROXY \
        -e NO_PROXY \

docs/build.sh

@@ -7,25 +7,30 @@
 
 set -e
 
+RELEASE_TAG_REGEX="^refs/tags/v.*"
+
+if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
+  DOCS_TAG="${AZP_BRANCH/refs\/tags\//}"
+fi
+
 # We need to set ENVOY_DOCS_VERSION_STRING and ENVOY_DOCS_RELEASE_LEVEL for Sphinx.
 # We also validate that the tag and version match at this point if needed.
-if [ -n "$CIRCLE_TAG" ]
-then
+if [[ -n "${DOCS_TAG}" ]]; then
   # Check the git tag matches the version number in the VERSION file.
   VERSION_NUMBER=$(cat VERSION)
-  if [ "v${VERSION_NUMBER}" != "${CIRCLE_TAG}" ]; then
+  if [[ "v${VERSION_NUMBER}" != "${DOCS_TAG}" ]]; then
     echo "Given git tag does not match the VERSION file content:"
-    echo "${CIRCLE_TAG} vs $(cat VERSION)"
+    echo "${DOCS_TAG} vs $(cat VERSION)"
     exit 1
   fi
   # Check the version_history.rst contains current release version.
   grep --fixed-strings "$VERSION_NUMBER" docs/root/version_history/current.rst \
     || (echo "Git tag not found in version_history/current.rst" && exit 1)
 
   # Now that we know there is a match, we can use the tag.
-  export ENVOY_DOCS_VERSION_STRING="tag-$CIRCLE_TAG"
+  export ENVOY_DOCS_VERSION_STRING="tag-${DOCS_TAG}"
   export ENVOY_DOCS_RELEASE_LEVEL=tagged
-  export ENVOY_BLOB_SHA="$CIRCLE_TAG"
+  export ENVOY_BLOB_SHA="${DOCS_TAG}"
 else
   BUILD_SHA=$(git rev-parse HEAD)
   VERSION_NUM=$(cat VERSION)

docs/publish.sh

@@ -10,35 +10,36 @@
 set -e
 
 DOCS_DIR=generated/docs
-CHECKOUT_DIR=../envoy-docs
+CHECKOUT_DIR=envoy-docs
 BUILD_SHA=$(git rev-parse HEAD)
 
-if [ -n "$CIRCLE_TAG" ]
-then
-  PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy/"$CIRCLE_TAG"
-elif [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ]
-then
-  PUBLISH_DIR="$CHECKOUT_DIR"/docs/envoy/latest
+MAIN_BRANCH="refs/heads/master"
+RELEASE_TAG_REGEX="^refs/tags/v.*"
+
+if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
+  PUBLISH_DIR="${CHECKOUT_DIR}"/docs/envoy/"${AZP_BRANCH/refs\/tags\//}"
+elif [[ "$AZP_BRANCH" == "${MAIN_BRANCH}" ]]; then
+  PUBLISH_DIR="${CHECKOUT_DIR}"/docs/envoy/latest
 else
   echo "Ignoring docs push"
   exit 0
 fi
 
+DOCS_MAIN_BRANCH="master"
+
 echo 'cloning'
-git clone git@github.com:envoyproxy/envoyproxy.github.io "$CHECKOUT_DIR"
+git clone git@github.com:envoyproxy/envoyproxy.github.io "${CHECKOUT_DIR}" -b "${DOCS_MAIN_BRANCH}" --depth 1
 
-git -C "$CHECKOUT_DIR" fetch
-git -C "$CHECKOUT_DIR" checkout -B master origin/master
 rm -fr "$PUBLISH_DIR"
 mkdir -p "$PUBLISH_DIR"
 cp -r "$DOCS_DIR"/* "$PUBLISH_DIR"
-cd "$CHECKOUT_DIR"
+cd "${CHECKOUT_DIR}"
 
-git config user.name "envoy-docs(travis)"
+git config user.name "envoy-docs(Azure Pipelines)"
 git config user.email envoy-docs@users.noreply.github.com
-echo 'add'
+
+set -x
+
 git add .
-echo 'commit'
 git commit -m "docs envoy@$BUILD_SHA"
-echo 'push'
-git push origin master
+git push origin "${DOCS_MAIN_BRANCH}"