From c250cf2589e5f996046ef2d67a54ce74fc52845f Mon Sep 17 00:00:00 2001 From: Nathan Cutler Date: Thu, 24 Oct 2019 13:26:50 +0200 Subject: [PATCH 1/3] ceph-backport.sh: implement interactive setup routine and new options This commit implements several new features: * a --cherry-pick-only option * a --force option * an --existing-pr option * an interactive setup routine The --cherry-pick-only option can be used to test whether a backport cherry-picks cleanly, for example. This is the same as the --prepare functionality that was provided by an earlier version of the script, and --prepare is re-introduced as a synonym for --cherry-pick-only. The --force option can be used to make the script less careful (less "cowardly"). For example, if the script refuses to do a backport because the backport tracker issue is assigned to someone else, the script will "cowardly" refuse to continue. Use --force to override. Be aware that --force will also blow away an existing wip branch - the script asks for user confirmation in this case. The new --existing-pr option can be used to specify the number (ID) of an existing backport PR that addresses the backport tracker issue given via the positional argument. The new "interactive setup routine" should make the setup process much simpler for the user. If there is a setup issue, the script produces a report and starts the interactive setup routine, which prompts the user for the needed information. Also, the script no longer requires the user to explicitly provide values for github_user and redmine_user_id. Instead, it divines the correct values from the GitHub token and the Redmine key, respectively. Finally, the existing ~/bin/backport_common.sh file is deprecated in favor of two files: ~/.github_token ~/.redmine_key (The latter is already used by Sage's build-integration-branch tool and it didn't make sense to have two different configuration files for a single purpose.) Signed-off-by: Nathan Cutler --- src/script/ceph-backport.sh | 1395 ++++++++++++++++++++++++----------- 1 file changed, 984 insertions(+), 411 deletions(-) diff --git a/src/script/ceph-backport.sh b/src/script/ceph-backport.sh index aab85741b90..39cb37c08d8 100755 --- a/src/script/ceph-backport.sh +++ b/src/script/ceph-backport.sh @@ -8,19 +8,40 @@ # This script automates the process of staging a backport starting from a # Backport tracker issue. # -# Setup, usage and troubleshooting: +# Setup: +# +# ceph-backport.sh --setup +# +# Usage and troubleshooting: # # ceph-backport.sh --help -# ceph-backport.sh --setup-advice -# ceph-backport.sh --usage-advice -# ceph-backport.sh --troubleshooting-advice -# +# ceph-backport.sh --usage | less +# ceph-backport.sh --troubleshooting | less # -SCRIPT_VERSION="15.0.0.6270" full_path="$0" + +SCRIPT_VERSION="15.0.0.6612" +active_milestones="" +backport_pr_number="" +backport_pr_url="" +deprecated_backport_common="$HOME/bin/backport_common.sh" +github_token="" +github_token_file="$HOME/.github_token" +github_token_ok="" +github_user="" +non_interactive="" +original_issue="" +original_issue_url="" +original_pr="" +original_pr_url="" +redmine_key="" +redmine_key_file="$HOME/.redmine_key" +redmine_key_ok="" +redmine_login="" +redmine_user_id="" +setup_ok="" this_script=$(basename "$full_path") -how_to_get_setup_advice="For setup advice, run: \"${this_script} --setup-advice | less\"" if [[ $* == *--debug* ]]; then set -x @@ -29,6 +50,7 @@ fi # associative array keyed on "component" strings from PR titles, mapping them to # GitHub PR labels that make sense in backports declare -A comp_hash=( +["auth"]="core" ["bluestore"]="bluestore" ["build/ops"]="build/ops" ["ceph.spec"]="build/ops" @@ -68,25 +90,39 @@ declare -A comp_hash=( declare -A flagged_pr_hash=() +function abort_due_to_setup_problem { + error "problem detected in your setup" + info "Run \"${this_script} --setup\" to fix" + false +} + +function assert_fail { + local message="$1" + error "(internal error) $message" + info "This could be reported as a bug!" + false +} + function bail_out_github_api { local api_said="$1" + local hint="$2" info "GitHub API said:" log bare "$api_said" - info "For setup report, run: ${this_script} --setup" - info "For setup advice, run: ${this_script} --setup-advice" - info "(hint) Check the value of github_token" - info "(hint) Run the script with --debug" - false + if [ "$hint" ] ; then + info "(hint) $hint" + fi + abort_due_to_setup_problem } function blindly_set_pr_metadata { local pr_number="$1" local json_blob="$2" - curl --silent --data-binary "$json_blob" 'https://api.github.com/repos/ceph/ceph/issues/'$pr_number'?access_token='$github_token >/dev/null 2>&1 || true + curl --silent --data-binary "$json_blob" "https://api.github.com/repos/ceph/ceph/issues/${pr_number}?access_token=${github_token}" >/dev/null 2>&1 || true } function check_milestones { - local milestones_to_check="$(echo "$1" | tr '\n' ' ' | xargs)" + local milestones_to_check + milestones_to_check="$(echo "$1" | tr '\n' ' ' | xargs)" info "Active milestones: $milestones_to_check" for m in $milestones_to_check ; do info "Examining all PRs targeting base branch \"$m\"" @@ -98,6 +134,7 @@ function check_milestones { function check_tracker_status { local -a ok_statuses=("new" "need more info") local ts="$1" + local error_msg local tslc="${ts,,}" local tslc_is_ok= for oks in "${ok_statuses[@]}"; do @@ -111,41 +148,48 @@ function check_tracker_status { true else if [ "$tslc" = "in progress" ] ; then - error "Backport $redmine_url is already in progress" - false + error_msg="backport $redmine_url is already in progress" else - error "Backport $redmine_url is closed (status: ${ts})" - false + error_msg="backport $redmine_url is closed (status: ${ts})" + fi + if [ "$FORCE" ] || [ "$EXISTING_PR" ] ; then + warning "$error_msg" + else + error "$error_msg" fi fi echo "$tslc_is_ok" } function cherry_pick_phase { - local base_branch= - local merged= - local number_of_commits= - local offset=0 - local singular_or_plural_commit= + local base_branch + local default_val + local i + local merged + local number_of_commits + local offset + local sha1_to_cherry_pick + local singular_or_plural_commit + local yes_or_no_answer populate_original_issue if [ -z "$original_issue" ] ; then error "Could not find original issue" info "Does ${redmine_url} have a \"Copied from\" relation?" false fi - info "Parent issue: ${redmine_endpoint}/issues/${original_issue}" + info "Parent issue: ${original_issue_url}" populate_original_pr if [ -z "$original_pr" ]; then error "Could not find original PR" - info "Is the \"Pull request ID\" field of ${redmine_endpoint}/issues/${original_issue} populated?" + info "Is the \"Pull request ID\" field of ${original_issue_url} populated?" false fi info "Parent issue ostensibly fixed by: ${original_pr_url}" verbose "Examining ${original_pr_url}" - remote_api_output=$(curl --silent https://api.github.com/repos/ceph/ceph/pulls/${original_pr}?access_token=${github_token}) - base_branch=$(echo ${remote_api_output} | jq -r .base.label) + remote_api_output=$(curl --silent "https://api.github.com/repos/ceph/ceph/pulls/${original_pr}?access_token=${github_token}") + base_branch=$(echo "${remote_api_output}" | jq -r '.base.label') if [ "$base_branch" = "ceph:master" ] ; then true else @@ -154,14 +198,15 @@ function cherry_pick_phase { info "You can still use the script to stage the backport, though. Just prepare the local branch \"${local_branch}\" manually and re-run the script." false fi - merged=$(echo ${remote_api_output} | jq -r .merged) + merged=$(echo "${remote_api_output}" | jq -r '.merged') if [ "$merged" = "true" ] ; then true else - error "${original_pr_url} is not merged yet: cowardly refusing to perform automated cherry-pick" + error "${original_pr_url} is not merged yet" + info "Cowardly refusing to perform automated cherry-pick" false fi - number_of_commits=$(echo ${remote_api_output} | jq .commits) + number_of_commits=$(echo "${remote_api_output}" | jq '.commits') if [ "$number_of_commits" -eq "$number_of_commits" ] 2>/dev/null ; then # \$number_of_commits is set, and is an integer if [ "$number_of_commits" -eq "1" ] ; then @@ -175,34 +220,60 @@ function cherry_pick_phase { fi info "Found $number_of_commits $singular_or_plural_commit in $original_pr_url" - debug "Fetching latest commits from $upstream_remote" - git fetch $upstream_remote + set -x + git fetch "$upstream_remote" - debug "Initializing local branch $local_branch to $milestone" - if git show-ref --verify --quiet refs/heads/$local_branch ; then - error "Cannot initialize $local_branch - local branch already exists" - false + if git show-ref --verify --quiet "refs/heads/$local_branch" ; then + if [ "$FORCE" ] ; then + if [ "$non_interactive" ] ; then + git checkout "$local_branch" + git reset --hard "${upstream_remote}/${milestone}" + else + echo + echo "A local branch $local_branch already exists and the --force option was given." + echo "If you continue, any local changes in $local_branch will be lost!" + echo + default_val="y" + echo -n "Do you really want to overwrite ${local_branch}? (default: ${default_val}) " + yes_or_no_answer="$(get_user_input "$default_val")" + [ "$yes_or_no_answer" ] && yes_or_no_answer="${yes_or_no_answer:0:1}" + if [ "$yes_or_no_answer" = "y" ] ; then + git checkout "$local_branch" + git reset --hard "${upstream_remote}/${milestone}" + else + info "OK, bailing out!" + false + fi + fi + else + set +x + error "Cannot initialize $local_branch - local branch already exists" + false + fi else - git checkout $upstream_remote/$milestone -b $local_branch + git checkout "${upstream_remote}/${milestone}" -b "$local_branch" fi - debug "Fetching latest commits from ${original_pr_url}" - git fetch $upstream_remote pull/$original_pr/head:pr-$original_pr + git fetch "$upstream_remote" "pull/$original_pr/head:pr-$original_pr" + set +x info "Attempting to cherry pick $number_of_commits commits from ${original_pr_url} into local branch $local_branch" - let offset=${number_of_commits}-1 || true # don't fail on set -e when result is 0 - for ((i=$offset; i>=0; i--)) ; do - debug "Cherry-picking commit $(git log --oneline --max-count=1 --no-decorate pr-$original_pr~$i)" - if git cherry-pick -x "pr-$original_pr~$i" ; then - true + offset="$((number_of_commits - 1))" || true + for ((i=offset; i>=0; i--)) ; do + info "Running \"git cherry-pick -x\" on $(git log --oneline --max-count=1 --no-decorate "pr-${original_pr}~${i}")" + sha1_to_cherry_pick=$(git rev-parse --verify "pr-${original_pr}~${i}") + set -x + if git cherry-pick -x "$sha1_to_cherry_pick" ; then + set +x else + set +x [ "$VERBOSE" ] && git status error "Cherry pick failed" info "Next, manually fix conflicts and complete the current cherry-pick" if [ "$i" -gt "0" ] >/dev/null 2>&1 ; then info "Then, cherry-pick the remaining commits from ${original_pr_url}, i.e.:" - for ((j=$i-1; j>=0; j--)) ; do - info "-> missing commit: $(git log --oneline --max-count=1 --no-decorate pr-$original_pr~$j)" + for ((j=i-1; j>=0; j--)) ; do + info "-> missing commit: $(git log --oneline --max-count=1 --no-decorate "pr-${original_pr}~${j}")" done info "Finally, re-run the script" else @@ -222,29 +293,6 @@ function debug { log debug "$@" } -function deduce_remote { - local remote_type="$1" - local remote="" - local url_component="" - if [ "$remote_type" = "upstream" ] ; then - url_component="ceph" - elif [ "$remote_type" = "fork" ] ; then - url_component="$github_user" - else - error "Internal error in deduce_remote" - false - fi - remote=$(git remote -v | egrep --ignore-case '(://|@)github.com(/|:)'$url_component'/ceph(\s|\.|\/)' | head -n1 | cut -f 1) - if [ "$remote" ] ; then - true - else - error "Cannot auto-determine ${remote_type}_remote" - info "There is something wrong with your remotes - to start with, check 'git remote -v'" - false - fi - echo "$remote" -} - function display_version_message_and_exit { echo "$this_script: Ceph backporting script, version $SCRIPT_VERSION" exit 0 @@ -259,7 +307,7 @@ function dump_flagged_prs { warning "Some backport PRs had problematic milestone settings" log bare "===========" log bare "Flagged PRs" - log bare "===========" + log bare "-----------" for url in "${!flagged_pr_hash[@]}" ; do log bare "$url - ${flagged_pr_hash[$url]}" done @@ -268,7 +316,7 @@ function dump_flagged_prs { } function eol { - log mtt=$1 + local mtt="$1" error "$mtt is EOL" false } @@ -277,10 +325,83 @@ function error { log error "$@" } +function existing_pr_routine { + local base_branch + local clipped_pr_body + local new_pr_body + local new_pr_title + local pr_body + local pr_json_tempfile + local pr_title + local remote_api_output + remote_api_output="$(curl --silent "https://api.github.com/repos/ceph/ceph/pulls/${backport_pr_number}?access_token=${github_token}")" + pr_title="$(echo "$remote_api_output" | jq -r '.title')" + if [ "$pr_title" = "null" ] ; then + error "could not get PR title of existing PR ${backport_pr_number}" + bail_out_github_api "$remote_api_output" + fi + pr_body="$(echo "$remote_api_output" | jq -r '.body')" + if [ "$pr_body" = "null" ] ; then + error "could not get PR body of existing PR ${backport_pr_number}" + bail_out_github_api "$remote_api_output" + fi + base_branch=$(echo "${remote_api_output}" | jq -r '.base.label') + base_branch="${base_branch#ceph:}" + if [ -z "$(is_active_milestone "$base_branch")" ] ; then + error "existing PR $backport_pr_url is targeting $base_branch which is not an active milestone" + info "Cowardly refusing to work on a backport to $base_branch" + false + fi + pr_json_tempfile=$(mktemp) + echo "$pr_body" | sed -n '/