From b69df7bd5cc413dc98fad78fa9e10a6d625d761d Mon Sep 17 00:00:00 2001 From: Jiaxin Shan Date: Fri, 20 Nov 2020 22:35:24 -0800 Subject: [PATCH 1/2] Add resuable scripts to kubeflow/testing --- hack/cherry-picks.md | 32 +++++ hack/cherry_pick_pull.sh | 253 ++++++++++++++++++++++++++++++++++++ scripts/update-changelog.sh | 44 +++++++ 3 files changed, 329 insertions(+) create mode 100644 hack/cherry-picks.md create mode 100755 hack/cherry_pick_pull.sh create mode 100755 scripts/update-changelog.sh diff --git a/hack/cherry-picks.md b/hack/cherry-picks.md new file mode 100644 index 000000000..b3e2af3ba --- /dev/null +++ b/hack/cherry-picks.md @@ -0,0 +1,32 @@ +# Automatically Creating Cherry Picks to Release Branches + +This is a script that automates creation of cherry-pick PRs to release branches, courtesy of +the Kubernetes team. + +## Prerequisites + +* A pull request is already merged against the master branch. +* A release branch (e.g. `v0.3-branch`) already exists. +* Hub is installed (see the following section) + +## Installing Hub + +Download and install hub from its [release page](https://github.com/github/hub/releases): +``` +HUB_VER=hub-linux-amd64-2.5.1 +wget https://github.com/github/hub/releases/download/v2.5.1/$HUB_VER.tgz +tar -xvf $HUB_VER.tgz +PATH=$PATH:$(pwd)/$HUB_VER/bin +``` + +## Running the Script +Suppose that you want to cherry-pick PR #1234 to the `v0.3-branch` release: +``` +GITHUB_USER=[your github id] +./cherry_pick_pull.sh upstream/v0.3-branch 1234 +``` +Follow the commandline prompt to enter authentication credentials. Assuming that there are no +conflicts, a PR should be created with the title `Automated cherry pick of #1234...`. PRs +with conflicts need to be resolved manually. + +The PR should be assigned to the current release czar for approval. diff --git a/hack/cherry_pick_pull.sh b/hack/cherry_pick_pull.sh new file mode 100755 index 000000000..7ebd81412 --- /dev/null +++ b/hack/cherry_pick_pull.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash + +# Copyright 2020 The Kubeflow Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Checkout a PR from GitHub. (Yes, this is sitting in a Git tree. How +# meta.) Assumes you care about pulls from remote "upstream" and +# checks them out to a branch named: +# automated-cherry-pick-of--- + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +declare -r REPO_ROOT +cd "${REPO_ROOT}" + +STARTINGBRANCH=$(git symbolic-ref --short HEAD) +declare -r STARTINGBRANCH +declare -r REBASEMAGIC="${REPO_ROOT}/.git/rebase-apply" +DRY_RUN=${DRY_RUN:-""} +REGENERATE_DOCS=${REGENERATE_DOCS:-""} +UPSTREAM_REMOTE=${UPSTREAM_REMOTE:-upstream} +FORK_REMOTE=${FORK_REMOTE:-origin} +MAIN_REPO_ORG=${MAIN_REPO_ORG:-$(git remote get-url "$UPSTREAM_REMOTE" | awk '{gsub(/http[s]:\/\/|git@/,"")}1' | awk -F'[@:./]' 'NR==1{print $3}')} +MAIN_REPO_NAME=${MAIN_REPO_NAME:-$(git remote get-url "$UPSTREAM_REMOTE" | awk '{gsub(/http[s]:\/\/|git@/,"")}1' | awk -F'[@:./]' 'NR==1{print $4}')} + +if [[ -z ${GITHUB_USER:-} ]]; then + echo "Please export GITHUB_USER= (or GH organization, if that's where your fork lives)" + exit 1 +fi + +if ! which hub > /dev/null; then + echo "Can't find 'hub' tool in PATH, please install from https://github.com/github/hub" + exit 1 +fi + +if [[ "$#" -lt 2 ]]; then + echo "${0} ...: cherry pick one or more onto and leave instructions for proposing pull request" + echo + echo " Checks out and handles the cherry-pick of (possibly multiple) for you." + echo " Examples:" + echo " $0 upstream/release-3.14 12345 # Cherry-picks PR 12345 onto upstream/release-3.14 and proposes that as a PR." + echo " $0 upstream/release-3.14 12345 56789 # Cherry-picks PR 12345, then 56789 and proposes the combination as a single PR." + echo + echo " Set the DRY_RUN environment var to skip git push and creating PR." + echo " This is useful for creating patches to a release branch without making a PR." + echo " When DRY_RUN is set the script will leave you in a branch containing the commits you cherry-picked." + echo + echo " Set the REGENERATE_DOCS environment var to regenerate documentation for the target branch after picking the specified commits." + echo " This is useful when picking commits containing changes to API documentation." + echo + echo " Set UPSTREAM_REMOTE (default: upstream) and FORK_REMOTE (default: origin)" + echo " To override the default remote names to what you have locally." + exit 2 +fi + +if git_status=$(git status --porcelain --untracked=no 2>/dev/null) && [[ -n "${git_status}" ]]; then + echo "!!! Dirty tree. Clean up and try again." + exit 1 +fi + +if [[ -e "${REBASEMAGIC}" ]]; then + echo "!!! 'git rebase' or 'git am' in progress. Clean up and try again." + exit 1 +fi + +declare -r BRANCH="$1" +shift 1 +declare -r PULLS=("$@") + +join() { + local IFS="$1" + shift + echo "$*" +} + +declare -r PULLDASH=$(join - "${PULLS[@]/#/#}") # Generates something like "#12345-#56789" +declare -r PULLSUBJ=$(join " " "${PULLS[@]/#/#}") # Generates something like "#12345 #56789" + +echo "+++ Updating remotes..." +git remote update "${UPSTREAM_REMOTE}" "${FORK_REMOTE}" + +if ! git log -n1 --format=%H "${BRANCH}" >/dev/null 2>&1; then + echo "!!! '${BRANCH}' not found. The second argument should be something like ${UPSTREAM_REMOTE}/release-0.21." + echo " (In particular, it needs to be a valid, existing remote branch that I can 'git checkout'.)" + exit 1 +fi + +declare -r NEWBRANCHREQ="automated-cherry-pick-of-${PULLDASH}" # "Required" portion for tools. +declare -r NEWBRANCH="$(echo "${NEWBRANCHREQ}-${BRANCH}" | sed 's/\//-/g')" +declare -r NEWBRANCHUNIQ="${NEWBRANCH}-$(date +%s)" +echo "+++ Creating local branch ${NEWBRANCHUNIQ}" + +cleanbranch="" +prtext="" +gitamcleanup=false + +return_to_kansas() { + if [[ "${gitamcleanup}" == "true" ]]; then + echo + echo "+++ Aborting in-progress git am." + git am --abort >/dev/null 2>&1 || true + fi + + # return to the starting branch and delete the PR text file + if [[ -z "${DRY_RUN}" ]]; then + echo + echo "+++ Returning you to the ${STARTINGBRANCH} branch and cleaning up." + git checkout -f "${STARTINGBRANCH}" >/dev/null 2>&1 || true + if [[ -n "${cleanbranch}" ]]; then + git branch -D "${cleanbranch}" >/dev/null 2>&1 || true + fi + if [[ -n "${prtext}" ]]; then + rm "${prtext}" + fi + fi +} + +trap return_to_kansas EXIT + +SUBJECTS=() + +make-a-pr() { + local rel="$(basename "${BRANCH}")" + echo + echo "+++ Creating a pull request on GitHub at ${GITHUB_USER}:${NEWBRANCH}" + + # This looks like an unnecessary use of a tmpfile, but it avoids + # https://github.com/github/hub/issues/976 Otherwise stdin is stolen + # when we shove the heredoc at hub directly, tickling the ioctl + # crash. + prtext="$(mktemp -t prtext.XXXX)" # cleaned in return_to_kansas + local numandtitle=$(printf '%s\n' "${SUBJECTS[@]}") + cat >"${prtext}" <&2 + exit 1 + fi + done + + if [[ "${conflicts}" != "true" ]]; then + echo "!!! git am failed, likely because of an in-progress 'git am' or 'git rebase'" + exit 1 + fi + } + + # set the subject + subject=$(grep -m 1 "^Subject" "/tmp/${pull}.patch" | sed -e 's/Subject: \[PATCH//g' | sed 's/.*] //') + SUBJECTS+=("#${pull}: ${subject}") + + # remove the patch file from /tmp + rm -f "/tmp/${pull}.patch" +done +gitamcleanup=false + +# Re-generate docs (if needed) +if [[ -n "${REGENERATE_DOCS}" ]]; then + echo + echo "Regenerating docs..." + if ! hack/generate-docs.sh; then + echo + echo "hack/generate-docs.sh FAILED to complete." + exit 1 + fi +fi + +if [[ -n "${DRY_RUN}" ]]; then + echo "!!! Skipping git push and PR creation because you set DRY_RUN." + echo "To return to the branch you were in when you invoked this script:" + echo + echo " git checkout ${STARTINGBRANCH}" + echo + echo "To delete this branch:" + echo + echo " git branch -D ${NEWBRANCHUNIQ}" + exit 0 +fi + +if git remote -v | grep ^${FORK_REMOTE} | grep ${MAIN_REPO_ORG}/${MAIN_REPO_NAME}.git; then + echo "!!! You have ${FORK_REMOTE} configured as your ${MAIN_REPO_ORG}/${MAIN_REPO_NAME}.git" + echo "This isn't normal. Leaving you with push instructions:" + echo + echo "+++ First manually push the branch this script created:" + echo + echo " git push REMOTE ${NEWBRANCHUNIQ}:${NEWBRANCH}" + echo + echo "where REMOTE is your personal fork (maybe ${UPSTREAM_REMOTE}? Consider swapping those.)." + echo "OR consider setting UPSTREAM_REMOTE and FORK_REMOTE to different values." + echo + make-a-pr + cleanbranch="" + exit 0 +fi + +echo +echo "+++ I'm about to do the following to push to GitHub (and I'm assuming ${FORK_REMOTE} is your personal fork):" +echo +echo " git push ${FORK_REMOTE} ${NEWBRANCHUNIQ}:${NEWBRANCH}" +echo +read -p "+++ Proceed (anything but 'y' aborts the cherry-pick)? [y/n] " -r +if ! [[ "${REPLY}" =~ ^[yY]$ ]]; then + echo "Aborting." >&2 + exit 1 +fi + +git push "${FORK_REMOTE}" -f "${NEWBRANCHUNIQ}:${NEWBRANCH}" +make-a-pr diff --git a/scripts/update-changelog.sh b/scripts/update-changelog.sh new file mode 100755 index 000000000..cb7829947 --- /dev/null +++ b/scripts/update-changelog.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Copyright 2020 The Kubeflow Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Update CHANGELOG.md using github_changelog_generator. +# +# The script will compute changes between release tags. So make sure there is +# a release tag corresponding to the release you want to compute the changes +# for. +set -o errexit +set -o nounset +set -o pipefail +set -x + +GITHUB_TOKEN=${GITHUB_TOKEN:-"NO"} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." > /dev/null && pwd)" + +cd ${ROOT} + +if [ "${GITHUB_TOKEN}" == "NO" ]; then + echo "Environment variable GITHUB_TOKEN is not set." + exit 1 +fi + +github_changelog_generator -t ${GITHUB_TOKEN} -u kubeflow -p kubeflow \ + --exclude-labels community/discussion,community/question,duplicate,question,invalid,wontfix \ + --bug-labels kind/bug,problems/bug \ + --enhancement-labels improvement/optimization,kind/enhancement,improvement/enhancement,addition/feature,kind/feature \ + --enhancement-label "**Features and improvements:**" + +cd - > /dev/null From e6716a9fde17a214bfd5b512ca5aaab2de6a1e8d Mon Sep 17 00:00:00 2001 From: Jiaxin Shan Date: Sun, 22 Nov 2020 22:02:08 -0800 Subject: [PATCH 2/2] Remove github_changelog_generator This scripts will encounter rate limit issue in large repo. We should find better ways to generate changlog --- scripts/update-changelog.sh | 44 ------------------------------------- 1 file changed, 44 deletions(-) delete mode 100755 scripts/update-changelog.sh diff --git a/scripts/update-changelog.sh b/scripts/update-changelog.sh deleted file mode 100755 index cb7829947..000000000 --- a/scripts/update-changelog.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2020 The Kubeflow Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Update CHANGELOG.md using github_changelog_generator. -# -# The script will compute changes between release tags. So make sure there is -# a release tag corresponding to the release you want to compute the changes -# for. -set -o errexit -set -o nounset -set -o pipefail -set -x - -GITHUB_TOKEN=${GITHUB_TOKEN:-"NO"} - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." > /dev/null && pwd)" - -cd ${ROOT} - -if [ "${GITHUB_TOKEN}" == "NO" ]; then - echo "Environment variable GITHUB_TOKEN is not set." - exit 1 -fi - -github_changelog_generator -t ${GITHUB_TOKEN} -u kubeflow -p kubeflow \ - --exclude-labels community/discussion,community/question,duplicate,question,invalid,wontfix \ - --bug-labels kind/bug,problems/bug \ - --enhancement-labels improvement/optimization,kind/enhancement,improvement/enhancement,addition/feature,kind/feature \ - --enhancement-label "**Features and improvements:**" - -cd - > /dev/null