[CI] Try to prevent npm/yarn problems from causing an agent to fail a…

…ll future jobs (#122358)

.buildkite/scripts/common/setup_node.sh

@@ -61,7 +61,14 @@ YARN_VERSION=$(node -e "console.log(String(require('./package.json').engines.yar
 export YARN_VERSION
 
 if [[ ! $(which yarn) || $(yarn --version) != "$YARN_VERSION" ]]; then
-  npm install -g "yarn@^${YARN_VERSION}"
+  rm -rf "$(npm root -g)/yarn" # in case the directory is in a bad state
+  if [[ ! $(npm install -g "yarn@^${YARN_VERSION}") ]]; then
+    # If this command is terminated early, e.g. because the build was cancelled in buildkite,
+    # a yarn directory is left behind in a bad state that can cause all subsequent installs to fail
+    rm -rf "$(npm root -g)/yarn"
+    echo "Trying again to install yarn..."
+    npm install -g "yarn@^${YARN_VERSION}"
+  fi
 fi
 
 yarn config set yarn-offline-mirror "$YARN_OFFLINE_CACHE"

.buildkite/scripts/lifecycle/pre_command.sh

@@ -9,8 +9,22 @@ export BUILDKITE_TOKEN
 
 echo '--- Install buildkite dependencies'
 cd '.buildkite'
-retry 5 15 yarn install --production --pure-lockfile
-cd -
+
+# If this yarn install is terminated early, e.g. if the build is cancelled in buildkite,
+# A node module could end up in a bad state that can cause all future builds to fail
+# So, let's cache clean and try again to make sure that's not what caused the error
+install_deps() {
+  yarn install --production --pure-lockfile
+  EXIT=$?
+  if [[ "$EXIT" != "0" ]]; then
+    yarn cache clean
+  fi
+  return $EXIT
+}
+
+retry 5 15 install_deps
+
+cd ..
 
 node .buildkite/scripts/lifecycle/print_agent_links.js || true