From f32840303a26bf74de66a5bd3893413f5311e06f Mon Sep 17 00:00:00 2001 From: Nathan Melehan Date: Fri, 16 Feb 2024 15:02:07 -0500 Subject: [PATCH] CI/CD SFTP Updates for production deploy workflow (#6843) * Workflow for testing with test kitchen server * Update staging and testing workflows for SFTP * Add cicd-sftp-update to trigger branches for testing * Update image cache workflow * Testing SFTP connections * Un-do quick SFTP test changes * Test curl connections * Test curl connections, 2 * Test curl connections, 3 * Test curl connections, 4 * Test curl connections, 5 * Test curl connections, 6 * Test curl connections, 7 * Test curl connections, 8 * Test curl connections, 9 * Test curl connections, 10 * Test curl connections, 11 * Test curl connections, 12 * Test curl connections, 13 * Test curl connections, 14 * Test curl connections, 15 * Test curl connections, 16 * Test curl connections, 17 * Test curl connections, 18 * Testing curl cacert * Testing curl cacert, 2 * Testing curl cacert, 3 * Test SFTP to new prod boxes, 1 * Test SFTP to new prod boxes, 2 * Test SFTP to new prod boxes, 3 * Test SFTP to new prod boxes, 4 * Test SFTP to new prod boxes, 5 * Testconnections to new prod boxes * Test connections to new prod boxes, 2 * Test connections to new prod boxes, 3 * Test connections to new prod boxes, 4 * Test connections to new prod boxes, 5 * Test connections to new prod boxes, 6 * Test connections to new prod boxes, 7 * Update production deploy workflow, 3 * Update production deploy workflow, 4 * Update production deploy workflow, 5 * Update production deploy workflow, 6 * Update production deploy workflow, 7 * Update production deploy workflow, 8 --- .../deploy-to-feature-testing-server.yaml | 3 +- .../deploy-to-main-staging-server.yaml | 3 +- .github/workflows/deploy-to-production.yaml | 349 ++++++++++++++---- 3 files changed, 282 insertions(+), 73 deletions(-) diff --git a/.github/workflows/deploy-to-feature-testing-server.yaml b/.github/workflows/deploy-to-feature-testing-server.yaml index 14d8b713ff3..7ff451716e2 100644 --- a/.github/workflows/deploy-to-feature-testing-server.yaml +++ b/.github/workflows/deploy-to-feature-testing-server.yaml @@ -4,7 +4,6 @@ on: push: branches: - 'feature/testing-linodedocs' - - 'cicd-sftp-update' # This workflow builds the site with Hugo, uploads it to a testing web # server, and updates the sandbox Algolia app to reflect the content in the @@ -370,8 +369,8 @@ jobs: echo "Commit hash of workflow run is $GITHUB_SHA, but commit hash reported by server is $CURRENT_GIT_COMMIT_HASH. Deployment to server has failed. Please inspect workflow logs and server logs." exit 1 fi - else + rm cacert.pem echo "No gitcommithash.txt exists on server after deployment. Deployment to server has failed. Please inspect workflow logs and server logs." exit 1 fi \ No newline at end of file diff --git a/.github/workflows/deploy-to-main-staging-server.yaml b/.github/workflows/deploy-to-main-staging-server.yaml index 332985b51d5..e7bbefd88d2 100644 --- a/.github/workflows/deploy-to-main-staging-server.yaml +++ b/.github/workflows/deploy-to-main-staging-server.yaml @@ -4,7 +4,6 @@ on: push: branches: - 'main' - - 'cicd-sftp-update' # This workflow builds the site with Hugo, uploads it to a staging web # server, and updates the sandbox Algolia app to reflect the content in the @@ -370,8 +369,8 @@ jobs: echo "Commit hash of workflow run is $GITHUB_SHA, but commit hash reported by server is $CURRENT_GIT_COMMIT_HASH. Deployment to server has failed. Please inspect workflow logs and server logs." exit 1 fi - else + rm cacert.pem echo "No gitcommithash.txt exists on server after deployment. Deployment to server has failed. Please inspect workflow logs and server logs." exit 1 fi \ No newline at end of file diff --git a/.github/workflows/deploy-to-production.yaml b/.github/workflows/deploy-to-production.yaml index 1ef3bce2150..34389b54f2f 100644 --- a/.github/workflows/deploy-to-production.yaml +++ b/.github/workflows/deploy-to-production.yaml @@ -4,10 +4,6 @@ on: release: types: [published] -env: - hugo_image_cache_name: hugo-generated-images - hugo_image_cache_path: /home/runner/work/docs/docs/docs-repo/resources/_gen/images/ - # This workflow builds the site with Hugo, syncs it over to the production web # servers, and updates the production Algolia app to reflect the content in the # new version of the site. @@ -46,6 +42,7 @@ env: jobs: deploy-to-production: + if: github.repository_owner == 'linode' runs-on: ubuntu-latest environment: @@ -53,25 +50,35 @@ jobs: url: ${{ vars.DOCS_WEBSITE_URL }} steps: - - name: Set up Hugo + - name: 1. Set up Hugo uses: peaceiris/actions-hugo@v2 with: - hugo-version: '0.116.1' + hugo-version: ${{ vars.HUGO_VERSION }} - - name: Checkout docs repo + - name: 2. Checkout docs repo uses: actions/checkout@v3 with: path: 'docs-repo' + # ref: 'main' # Debugging step so we can check if the right version # of the docs repo is checked out - - name: Print current docs repo branch/ref/commit + - name: (Debugging info) Print current docs repo branch/ref/commit working-directory: ./docs-repo run: | git status git log -1 - - name: Get linode-docs-theme version + - name: 3. Set up SSH agent (linode-docs-theme repo deploy key) + uses: webfactory/ssh-agent@v0.7.0 + with: + ssh-private-key: ${{ secrets.LINODE_DOCS_THEME_DEPLOY_KEY_FOR_DOCS_DEPLOY_GHA }} + + - name: 4. Clone docs theme repo + run: | + git clone git@github.com:$GITHUB_REPOSITORY_OWNER/linode-docs-theme linode-docs-theme-repo + + - name: 5. Get linode-docs-theme commit hash from docs go.mod id: get-theme-version working-directory: ./docs-repo run: | @@ -84,23 +91,14 @@ jobs: LINODE_DOCS_THEME_VERSION=$(hugo mod graph | grep linode-docs-theme | cut -d '+' -f 1 | grep -o '[^-]*$') echo "VERSION=$LINODE_DOCS_THEME_VERSION" >> $GITHUB_OUTPUT - - name: Set up SSH agent (linode-docs-theme repo deploy key) - uses: webfactory/ssh-agent@v0.7.0 - with: - ssh-private-key: ${{ secrets.LINODE_DOCS_THEME_DEPLOY_KEY_FOR_DOCS_DEPLOY_GHA }} - - - name: Checkout docs theme repo - run: | - git clone git@github.com:linode/linode-docs-theme linode-docs-theme-repo - - - name: Check out theme repo commit from docs repo go.mod + - name: 6. Check out linode-docs-theme commit hash from docs go.mod working-directory: ./linode-docs-theme-repo run: | git checkout ${{ steps.get-theme-version.outputs.VERSION }} # Debugging step so we can check if the right version # of the Algolia admin tool is checked out - - name: Print Linode Algolia admin tool version + - name: (Debugging info) Print Linode Algolia admin tool version working-directory: ./linode-docs-theme-repo run: | cd scripts/linode_algolia_admin/ @@ -108,34 +106,34 @@ jobs: # Debugging step that lists the Hugo-generated images # directory *before* the imache cache restore step - - name: List contents of images dir + - name: (Debugging info) List contents of images dir continue-on-error: true - run: ls -al ${{ env.hugo_image_cache_path }} + run: ls -al ${{ vars.HUGO_IMAGE_CACHE_PATH }} - - name: Restore Hugo generated images cache + - name: 7. Restore Hugo generated images cache uses: ylemkimon/cache-restore@v2 with: - path: ${{ env.hugo_image_cache_path }} - key: ${{ env.hugo_image_cache_name }} - restore-keys: ${{ env.hugo_image_cache_name }} + path: ${{ vars.HUGO_IMAGE_CACHE_PATH }} + key: ${{ vars.HUGO_IMAGE_CACHE_NAME }} + restore-keys: ${{ vars.HUGO_IMAGE_CACHE_NAME }} # Debugging step that lists the Hugo-generated images # directory *after* the imache cache restore step, # to make sure that the restore happened successfully - - name: List contents of images dir + - name: (Debugging info) List contents of images dir continue-on-error: true - run: ls -al ${{ env.hugo_image_cache_path }} + run: ls -al ${{ vars.HUGO_IMAGE_CACHE_PATH }} - - name: Use Node.js + - name: 8. Install Node.js uses: actions/setup-node@v3 with: node-version: 14 - - name: Install dependencies (Node) + - name: 9. Install dependencies (Node) working-directory: ./docs-repo run: npm ci - - name: Build Hugo + - name: 10. Build Hugo env: HUGOxPARAMSxSEARCH_CONFIG2xAPP_ID: ${{ vars.ALGOLIA_APP_ID }} HUGOxPARAMSxSEARCH_CONFIG2xAPI_KEY: ${{ vars.ALGOLIA_SEARCH_KEY }} @@ -146,7 +144,7 @@ jobs: # Debugging step that lists the Hugo-rendered files # to make sure the site was built - - name: List rendered files + - name: (Debugging info) List rendered files working-directory: ./docs-repo run: | sudo apt install -y tree @@ -155,7 +153,7 @@ jobs: # When Hugo builds, it renders search index data to a collection of # data files under public/. This step uploads that data to the # Algolia search backend - - name: Update Algolia + - name: 11. Update Algolia working-directory: ./linode-docs-theme-repo env: ALGOLIA_APP_ID: ${{ vars.ALGOLIA_APP_ID }} @@ -191,7 +189,7 @@ jobs: # - Build Hugo again so that the navigation UI can be rendered with that # updated info from Algolia # It's a little redundant, but solves the chicken-or-egg problem - - name: Build Hugo (second time) + - name: 12. Build Hugo (second time) env: HUGOxPARAMSxSEARCH_CONFIG2xAPP_ID: ${{ vars.ALGOLIA_APP_ID }} HUGOxPARAMSxSEARCH_CONFIG2xAPI_KEY: ${{ vars.ALGOLIA_SEARCH_KEY }} @@ -200,43 +198,256 @@ jobs: hugo config hugo -b "${{ vars.DOCS_WEBSITE_URL }}" --gc --minify -d public - - name: Set up SSH agent (docs webserver key) + # The gitcommithash.txt file is used in the last workflow step to verify that the deployment was successful + - name: 13. Add gitcommithash.txt to rendered public/ folder + working-directory: ./docs-repo + run: | + echo $GITHUB_SHA > public/gitcommithash.txt + + - name: 14. Set up SSH agent (docs webserver key) uses: webfactory/ssh-agent@v0.7.0 with: - ssh-private-key: ${{ secrets.DOCS_WEBSITE_RSYNC_PRIVATE_KEY }} - - - name: Rsync site to webserver - run: | - echo "${{ secrets.DOCS_WEBSITE_RSYNC_HOST_VERIFICATION_1 }}" > rsync_known_hosts - echo "${{ secrets.DOCS_WEBSITE_RSYNC_HOST_VERIFICATION_2 }}" >> rsync_known_hosts - - rsync_to_destination() { - # Sync these assets without the --delete flag. These - # assets' filenames are fingerprinted, so they will not - # conflict with old versions that remain on the server. - rsync -r -e "ssh -o UserKnownHostsFile=rsync_known_hosts" \ - docs-repo/public/css \ - docs-repo/public/images \ - docs-repo/public/js \ - docs-repo/public/jslibs \ - docs-repo/public/linode \ - ${{ secrets.DOCS_WEBSITE_USER }}@$1:${{ secrets.DOCS_WEBSITE_RSYNC_DIR }} - - # Sync everything else, and delete files in the destination - # rsync dir that do not exist in the source docs-repo/public/ - # folder - rsync -r -e "ssh -o UserKnownHostsFile=rsync_known_hosts" \ - --exclude="/css/" \ - --exclude="/images/" \ - --exclude="/js/" \ - --exclude="/jslibs/" \ - --exclude="/linode/" \ - --delete \ - docs-repo/public/ \ - ${{ secrets.DOCS_WEBSITE_USER }}@$1:${{ secrets.DOCS_WEBSITE_RSYNC_DIR }} + ssh-private-key: ${{ secrets.DOCS_WEBSITE_SFTP_PRIVATE_KEY }} + + # Make a tarball of the site, because it will upload much, much quicker + # than the uncompressed rendered site. The commit for this workflow run + # is encoded in the name of the tarball. + - name: 15. Create tarball of docs website + run: | + tar --owner=${{ secrets.DOCS_WEBSITE_USER }} --group=${{ secrets.DOCS_WEBSITE_USER }} -czf docs-$GITHUB_SHA.tar.gz -C docs-repo/public/ . + + # Upload the tarball to the website document root. Before the upload, the + # document root looks like this: + # {{ document_root }}/ + # docs -> docs-PREVIOUS_GITHUB_SHA + # docs-PREVIOUS_GITHUB_SHA/ + # + # Where PREVIOUS_GITHUB_SHA is the commit for the last time the deploy + # workflow ran, and the docs-PREVIOUS_GITHUB_SHA/ folder contains the + # rendered site from that last workflow run. `docs` is a symlink to that + # folder. + # + # After the upload, it looks like: + # {{ document_root }}/ + # docs -> docs-PREVIOUS_GITHUB_SHA + # docs-PREVIOUS_GITHUB_SHA/ + # docs-CURRENT_GITHUB_SHA.tar.gz + # + # Note that the tarball is temporarily uploaded to + # {{ document_root }}/new-tarball-uploads/, and then moved into + # {{ document_root }}/. If it was uploaded directly to {{ document_root }}/, then + # the unzip script (see next workflow step for description) could try + # to unzip a half-uploaded tarball. + - name: 16. SFTP upload website tarball to webserver + run: | + echo "${{ secrets.DOCS_WEBSITE_SFTP_DESTINATION_1 }} ${{ secrets.DOCS_WEBSITE_SFTP_HOST_VERIFICATION_FINGERPRINT_1 }}" > sftp_known_hosts + echo "${{ secrets.DOCS_WEBSITE_SFTP_DESTINATION_2 }} ${{ secrets.DOCS_WEBSITE_SFTP_HOST_VERIFICATION_FINGERPRINT_2 }}" >> sftp_known_hosts + + upload_to_destination() { + echo "" + echo "" + echo "Uploading tarball to server $1" + + sftp -o "UserKnownHostsFile=sftp_known_hosts" ${{ secrets.DOCS_WEBSITE_USER }}@$2 < docs-PREVIOUS_GITHUB_SHA + # docs-PREVIOUS_GITHUB_SHA/ + # docs-CURRENT_GITHUB_SHA/ + - name: 17. Wait for server to unzip tarball + shell: bash + env: + CACERT_BASE64: ${{ secrets.DOCS_WEBSITE_NO_CDN_CA_CERT_BASE64 }} + run: | + set +e + + check_for_unzipped_directory() { + echo -n $CACERT_BASE64 | base64 --decode > cacert.pem + + echo "" + echo "" + echo "Checking for unzipped tarball on server $1" + + CHECK_URL="$2/docs-$GITHUB_SHA/" + echo "Waiting for server to unzip uploaded docs site tarball:" + echo "- Every one second, checking $CHECK_URL for an HTTP 200 OK response" + echo "- Waiting up to ${{ vars.DOCS_WEBSERVER_UNZIP_TIMEOUT }} seconds..." + + # Wait for server to unzip the tarball of the docs site + for i in {1..${{ vars.DOCS_WEBSERVER_UNZIP_TIMEOUT }}} + do + echo "Checking server $1..." + curl --cacert cacert.pem -ILs $CHECK_URL | grep 'HTTP' | grep '200 OK' + if [ 0 -eq $? ]; then + rm cacert.pem + echo 0 + return 0 + fi + + sleep 1 + done + + echo "Server did not unzip docs tarball before ${{ vars.DOCS_WEBSERVER_UNZIP_TIMEOUT }} seconds" + echo "This is often because there is not enough space on server. An administrator should review the available disk space on the server." + + rm cacert.pem + echo 1 + return 1 + } + + if check_for_unzipped_directory "1" "${{ secrets.DOCS_WEBSITE_URL_NO_CDN_1 }}" && check_for_unzipped_directory "2" "${{ secrets.DOCS_WEBSITE_URL_NO_CDN_2 }}"; then + exit 0 + else + exit 1 + fi + + - name: 18. Get commit hash of previous deploy + id: get-previous-deploy-commit + shell: bash + env: + CACERT_BASE64: ${{ secrets.DOCS_WEBSITE_NO_CDN_CA_CERT_BASE64 }} + run: | + set +e + + get_current_commit_hash_on_server() { + echo -n $CACERT_BASE64 | base64 --decode > cacert.pem + + echo "" + echo "" + echo "Fetching commit hash for currently deployed website on server $1" + + GIT_COMMIT_HASH_URL="$2/docs/gitcommithash.txt" + + curl --cacert cacert.pem -ILs $GIT_COMMIT_HASH_URL | grep 'HTTP' | grep '200 OK' + if [ 0 -eq $? ]; then + GIT_COMMIT_HASH=$(curl --cacert cacert.pem -s $GIT_COMMIT_HASH_URL) + rm cacert.pem + + echo "Hash is $GIT_COMMIT_HASH" + echo "$2=$GIT_COMMIT_HASH" >> $GITHUB_OUTPUT + else + rm cacert.pem + echo "No gitcommithash.txt exists; this is probably because the webserver is currently displaying the initial setup instructions." + fi + } + + get_current_commit_hash_on_server "1" "${{ secrets.DOCS_WEBSITE_URL_NO_CDN_1 }}" "HASH_1" + get_current_commit_hash_on_server "2" "${{ secrets.DOCS_WEBSITE_URL_NO_CDN_2 }}" "HASH_2" + + # Update the `docs` symlink in the website document root to point to the + # new folder created for the unzipped tarball. + # + # The website document root will look like this after the symlink is + # updated: + # {{ document_root }}/ + # docs -> docs-CURRENT_GITHUB_SHA + # docs-CURRENT_GITHUB_SHA/ + # docs-PREVIOUS_GITHUB_SHA + # + # Note: a garbage collection script is periodically run on the server + # to remove the previous docs deployment folders + - name: 19. SFTP symlink docs folder on webserver + run: | + echo "${{ secrets.DOCS_WEBSITE_SFTP_DESTINATION_1 }} ${{ secrets.DOCS_WEBSITE_SFTP_HOST_VERIFICATION_FINGERPRINT_1 }}" > sftp_known_hosts + echo "${{ secrets.DOCS_WEBSITE_SFTP_DESTINATION_2 }} ${{ secrets.DOCS_WEBSITE_SFTP_HOST_VERIFICATION_FINGERPRINT_2 }}" >> sftp_known_hosts + + symlink_docs_on_destination() { + echo "" + echo "" + echo "Updating docs symlink on server $1" + + sftp -o "UserKnownHostsFile=sftp_known_hosts" ${{ secrets.DOCS_WEBSITE_USER }}@$2 < cacert.pem + + echo "" + echo "" + echo "Verifying commit hash on server $1 matches new deployment" + + GIT_COMMIT_HASH_URL="$2/docs/gitcommithash.txt" + + curl --cacert cacert.pem -ILs $GIT_COMMIT_HASH_URL | grep 'HTTP' | grep '200 OK' + if [ 0 -eq $? ]; then + CURRENT_GIT_COMMIT_HASH=$(curl --cacert cacert.pem -s $GIT_COMMIT_HASH_URL) + rm cacert.pem + + echo "Hash reported by server is $CURRENT_GIT_COMMIT_HASH" + echo "Hash for current deployment is $GITHUB_SHA" + if [ $CURRENT_GIT_COMMIT_HASH == $GITHUB_SHA ]; then + echo "Hashes match!" + echo 0 + return 0 + else + echo "Commit hash of workflow run is $GITHUB_SHA, but commit hash reported by server is $CURRENT_GIT_COMMIT_HASH. Deployment to server has failed. Please inspect workflow logs and server logs." + echo 1 + return 1 + fi + + else + rm cacert.pem + echo "No gitcommithash.txt exists on server after deployment. Deployment to server has failed. Please inspect workflow logs and server logs." + echo 1 + return 1 + fi + } - rm rsync_known_hosts \ No newline at end of file + if verify_hash_updated "1" "${{ secrets.DOCS_WEBSITE_URL_NO_CDN_1 }}" && verify_hash_updated "2" "${{ secrets.DOCS_WEBSITE_URL_NO_CDN_2 }}"; then + exit 0 + else + exit 1 + fi \ No newline at end of file