From e20cbbc16c5e4041912c342e03d83885aa0f211d Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sun, 22 Sep 2024 02:19:16 +0800 Subject: [PATCH] [ZEPPELIN-6089][INFRA] Improve the merge PR script ### What is this PR for? Zeppelin has `dev/merge_zeppelin_pr.py` that was borrowed from Spark, I would recommend committers use this script over the GitHub button to merge PR, which has some benefits: 1. Simplify the backport process the tools will ask you to backport the commit to lower maintained branches after you merge a PR to master, if there are no conflicts, all things you need to do are just type a "branch name" that you want to backport. 2. Automatically update JIRA information the script uses the python jira client to update JIRA ticket, for example, automatically closes the JIRA ticket after PR is merged, fills in the fixed versions, which is important to users to know the features/bug fixes applied to versions. 3. Better PR title, body, and "Signed-off-by" info Before image After image This PR syncs the change from the Spark upstream (around 4.0.0-preview2), and has several improvements recently, e.g. support using tokens instead of passwords for GitHub and JIRA authentication. Additionally, this PR switches to the GitHub open API jongyoul suggested to merge the PR, which fixed the merged PR status from "Closed" to "Merged" ### What type of PR is it? Improvement ### Todos * [x] - verify this script by merging at least 3 PRs ### What is the Jira issue? ZEPPELIN-6089 ### How should this be tested? Manually test. Currently not work due to permission issues. ``` $ dev/merge_zeppelin_pr.py git rev-parse --abbrev-ref HEAD Which pull request would you like to merge? (e.g. 34): 4837 === Pull Request #4837 === title [MINOR] Remove duplicate entry in .gitignore source MyLanPangzi/patch-2 target master url https://api.github.com/repos/apache/zeppelin/pulls/4837 Proceed with merging pull request #4837? (y/N): y git config --get user.name git config --get user.email git fetch apache master remote: Enumerating objects: 5, done. remote: Counting objects: 100% (5/5), done. remote: Compressing objects: 100% (3/3), done. remote: Total 3 (delta 2), reused 0 (delta 0), pack-reused 0 (from 0) Unpacking objects: 100% (3/3), 1.53 KiB | 260.00 KiB/s, done. From github.com:apache/zeppelin * branch master -> FETCH_HEAD ad79848a9..35e129912 master -> apache/master Pull request #4837 merged! Merge hash: 35e12991 Would you like to pick 35e12991 into another branch? (y/N): y Enter a branch name [branch-0.9]: branch-0.11 git fetch apache branch-0.11:PR_TOOL_PICK_PR_4837_BRANCH-0.11 From github.com:apache/zeppelin * [new branch] branch-0.11 -> PR_TOOL_PICK_PR_4837_BRANCH-0.11 git checkout PR_TOOL_PICK_PR_4837_BRANCH-0.11 Switched to branch 'PR_TOOL_PICK_PR_4837_BRANCH-0.11' git cherry-pick -sx 35e12991 Pick complete (local ref PR_TOOL_PICK_PR_4837_BRANCH-0.11). Push to apache? (y/N): y git push apache PR_TOOL_PICK_PR_4837_BRANCH-0.11:branch-0.11 Enumerating objects: 5, done. Counting objects: 100% (5/5), done. Delta compression using up to 10 threads Compressing objects: 100% (3/3), done. Writing objects: 100% (3/3), 1.60 KiB | 1.60 MiB/s, done. Total 3 (delta 2), reused 0 (delta 0), pack-reused 0 remote: Resolving deltas: 100% (2/2), completed with 2 local objects. remote: remote: GitHub found 199 vulnerabilities on apache/zeppelin's default branch (19 critical, 70 high, 87 moderate, 23 low). To find out more, visit: remote: https://github.com/apache/zeppelin/security/dependabot remote: To github.com:apache/zeppelin.git 7128f7da4..a04da2e09 PR_TOOL_PICK_PR_4837_BRANCH-0.11 -> branch-0.11 git rev-parse PR_TOOL_PICK_PR_4837_BRANCH-0.11 Restoring head pointer to ZEPPELIN-6089 git checkout ZEPPELIN-6089 Switched to branch 'ZEPPELIN-6089' git branch Deleting local branch PR_TOOL_PICK_PR_4837_BRANCH-0.11 git branch -D PR_TOOL_PICK_PR_4837_BRANCH-0.11 Pull request #4837 picked into branch-0.11! Pick hash: a04da2e0 Would you like to pick 35e12991 into another branch? (y/N): n Would you like to update an associated JIRA? (y/N): n Okay, exiting Restoring head pointer to ZEPPELIN-6089 git checkout ZEPPELIN-6089 Already on 'ZEPPELIN-6089' git branch Restoring head pointer to ZEPPELIN-6089 git checkout ZEPPELIN-6089 Already on 'ZEPPELIN-6089' git branch ``` ### Screenshots (if appropriate) ### Questions: * Does the license files need to update? No. * Is there breaking changes for older versions? No. * Does this needs documentation? No. Closes #4831 from pan3793/ZEPPELIN-6089. Signed-off-by: Cheng Pan --- dev/merge_zeppelin_pr.py | 669 +++++++++++++++++++++++++++------------ 1 file changed, 473 insertions(+), 196 deletions(-) diff --git a/dev/merge_zeppelin_pr.py b/dev/merge_zeppelin_pr.py index a31eb28eaeb..87b2e1cbdfa 100755 --- a/dev/merge_zeppelin_pr.py +++ b/dev/merge_zeppelin_pr.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,24 +17,22 @@ # limitations under the License. # -# Utility for creating well-formed pull request merges and pushing them to Apache. -# usage: ./merge-zeppelin-pr.py (see config env vars below) +# Utility for creating well-formed pull request merges and pushing them to Apache +# Zeppelin. +# usage: ./merge_zeppelin_pr.py (see config env vars below) # -# This utility assumes you already have local a Zeppelin git folder and that you -# have added remotes corresponding to both (i) the github apache Zeppelin -# mirror and (ii) the apache git repo. +# This utility assumes you already have a local Zeppelin git folder and that you +# have added remotes corresponding to the git@github.com:apache/zeppelin.git import json import os import re import subprocess import sys - -if sys.version_info < (3,0,0): - print(__file__ + ' requires Python 3, while Python ' + str(sys.version[0] + ' was detected. Terminating. ')) - sys.exit(1) - -import urllib.request +import traceback +from urllib.request import urlopen +from urllib.request import Request +from urllib.error import HTTPError try: import jira.client @@ -45,13 +43,24 @@ # Location of your Zeppelin git development area ZEPPELIN_HOME = os.environ.get("ZEPPELIN_HOME", os.getcwd()) # Remote name which points to the Github site -PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache-github") +PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache") # Remote name which points to Apache git PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache") # ASF JIRA username -JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "moon") +JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "") # ASF JIRA password -JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD", "00000") +JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD", "") +# ASF JIRA access token +# If it is configured, username and password are dismissed +# Go to https://issues.apache.org/jira/secure/ViewProfile.jspa -> Personal Access Tokens for +# your own token management. +JIRA_ACCESS_TOKEN = os.environ.get("JIRA_ACCESS_TOKEN") +# OAuth key used for issuing requests against the GitHub API. If this is not defined, then requests +# will be unauthenticated. You should only need to configure this if you find yourself regularly +# exceeding your IP's unauthenticated request rate limit. You can create an OAuth key at +# https://github.com/settings/tokens. This script only requires the "public_repo" scope. +GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY") + GITHUB_BASE = "https://github.com/apache/zeppelin/pull" GITHUB_API_BASE = "https://api.github.com/repos/apache/zeppelin" @@ -60,19 +69,51 @@ # Prefix added to temporary branches BRANCH_PREFIX = "PR_TOOL" -os.chdir(ZEPPELIN_HOME) + +def print_error(msg): + print("\033[91m%s\033[0m" % msg) -def get_json(url): +def bold_input(prompt) -> str: + return input("\033[1m%s\033[0m" % prompt) + + +def http_req_and_return_json(req): try: - return json.load(urllib.request.urlopen(url)) - except urllib.error.HTTPError as e: - print("Unable to fetch URL, exiting: %s" % url) - sys.exit(-1) + if GITHUB_OAUTH_KEY: + req.add_header("Authorization", "token %s" % GITHUB_OAUTH_KEY) + return json.load(urlopen(req)) + except HTTPError as e: + if "X-RateLimit-Remaining" in e.headers and e.headers["X-RateLimit-Remaining"] == "0": + print_error( + "Exceeded the GitHub API rate limit; see the instructions in " + + "dev/merge_zeppelin_pr.py to configure an OAuth token for making authenticated " + + "GitHub requests." + ) + sys.exit(-1) + elif e.code == 401: + print_error( + "GITHUB_OAUTH_KEY is invalid or expired. Please regenerate a new one with " + + "at least the 'public_repo' scope on https://github.com/settings/tokens and " + + "update your local settings before you try again." + ) + sys.exit(-1) + else: + raise e + + +def http_put(url, data): + req = Request(url, data=json.dumps(data).encode('utf-8'), method="PUT") + return http_req_and_return_json(req) + + +def http_get(url): + req = Request(url) + return http_req_and_return_json(req) def fail(msg): - print(msg) + print_error(msg) clean_up() sys.exit(-1) @@ -80,105 +121,67 @@ def fail(msg): def run_cmd(cmd): print(cmd) if isinstance(cmd, list): - return subprocess.check_output(cmd).decode('utf-8') + return subprocess.check_output(cmd).decode("utf-8") else: - return subprocess.check_output(cmd.split(" ")).decode('utf-8') + return subprocess.check_output(cmd.split(" ")).decode("utf-8") -def continue_maybe(prompt): - result = input("\n%s (y/n): " % prompt) +def continue_maybe(prompt, cherry=False): + result = bold_input("%s (y/N): " % prompt) if result.lower() != "y": + if cherry: + try: + run_cmd("git cherry-pick --abort") + except Exception: + print_error("Unable to abort and get back to the state before cherry-pick") fail("Okay, exiting") -original_head = run_cmd("git rev-parse HEAD")[:8] - - def clean_up(): - print("Restoring head pointer to %s" % original_head) - run_cmd("git checkout %s" % original_head) + if "original_head" in globals(): + print("Restoring head pointer to %s" % original_head) + run_cmd("git checkout %s" % original_head) - branches = run_cmd("git branch").replace(" ", "").split("\n") + branches = run_cmd("git branch").replace(" ", "").split("\n") - for branch in filter(lambda x: x.startswith(BRANCH_PREFIX), branches): - print("Deleting local branch %s" % branch) - run_cmd("git branch -D %s" % branch) + for branch in list(filter(lambda x: x.startswith(BRANCH_PREFIX), branches)): + print("Deleting local branch %s" % branch) + run_cmd("git branch -D %s" % branch) # merge the requested PR and return the merge hash -def merge_pr(pr_num, target_ref): - pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num) - target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper()) - run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, pr_branch_name)) - run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, target_branch_name)) - run_cmd("git checkout %s" % target_branch_name) - - had_conflicts = False - try: - run_cmd(['git', 'merge', pr_branch_name, '--squash']) - except Exception as e: - msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e - continue_maybe(msg) - msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?" - continue_maybe(msg) - had_conflicts = True - - commit_authors = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, - '--pretty=format:%an <%ae>']).split("\n") - commit_date = run_cmd(['git', 'log', '%s' % pr_branch_name, '-1', - '--pretty=format:%ad']) - distinct_authors = sorted(set(commit_authors), - key=lambda x: commit_authors.count(x), reverse=True) - primary_author = distinct_authors[0] - commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, - '--pretty=format:%h [%an] %s']).split("\n\n") +def merge_pr(pr_num, target_ref, title, body, pr_repo_desc): + # We replace @ symbols with from the body to avoid triggering e-mails + # to people every time someone creates a public fork of Zeppelin. + message = body.replace("@", "") - merge_message_flags = [] - - merge_message_flags += ["-m", title] - if body is not None: - # We remove @ symbols from the body to avoid triggering e-mails - # to people every time someone creates a public fork of Zeppelin. - merge_message_flags += ["-m", body.replace("@", "")] - - authors = "\n".join(["Author: %s" % a for a in distinct_authors]) - - merge_message_flags += ["-m", authors] - - if had_conflicts: - committer_name = run_cmd("git config --get user.name").strip() - committer_email = run_cmd("git config --get user.email").strip() - message = "This patch had conflicts when merged, resolved by\nCommitter: %s <%s>" % ( - committer_name, committer_email) - merge_message_flags += ["-m", message] + committer_name = run_cmd("git config --get user.name").strip() + committer_email = run_cmd("git config --get user.email").strip() # The string "Closes #%s" string is required for GitHub to correctly close the PR - merge_message_flags += [ - "-m", - "Closes #%s from %s and squashes the following commits:" % (pr_num, pr_repo_desc)] - for c in commits: - merge_message_flags += ["-m", c] - - run_cmd(['git', 'commit', '--author="%s"' % primary_author, '--date="%s"' % commit_date] + merge_message_flags) - - continue_maybe("Merge complete (local ref %s). Push to %s?" % ( - target_branch_name, PUSH_REMOTE_NAME)) + message = "%s\n\nCloses #%s from %s." % (message, pr_num, pr_repo_desc) + message = "%s\n\nSigned-off-by: %s <%s>" % (message, committer_name, committer_email) + merge_pr_resp = None try: - run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name, target_ref)) - except Exception as e: - clean_up() - fail("Exception while pushing: %s" % e) - - merge_hash = run_cmd("git rev-parse %s" % target_branch_name)[:8] - clean_up() + merge_pr_resp = http_put( + "%s/pulls/%s/merge" % (GITHUB_API_BASE, pr_num), + {"commit_title": title, "commit_message": message, "merge_method": "squash"}) + except HTTPError as e: + if e.code == 405: + fail("Merge pull request #%s is not allowed." % pr_num) + + merge_hash = merge_pr_resp["sha"][:8] print("Pull request #%s merged!" % pr_num) print("Merge hash: %s" % merge_hash) + + # we must do a git fetch to make the merged commit visible in local + run_cmd("git fetch %s %s" % (PUSH_REMOTE_NAME, target_ref)) return merge_hash def cherry_pick(pr_num, merge_hash, default_branch): - pick_ref = input("Enter a branch name [%s]: " % default_branch) + pick_ref = bold_input("Enter a branch name [%s]: " % default_branch) if pick_ref == "": pick_ref = default_branch @@ -191,15 +194,16 @@ def cherry_pick(pr_num, merge_hash, default_branch): run_cmd("git cherry-pick -sx %s" % merge_hash) except Exception as e: msg = "Error cherry-picking: %s\nWould you like to manually fix-up this merge?" % e - continue_maybe(msg) + continue_maybe(msg, True) msg = "Okay, please fix any conflicts and finish the cherry-pick. Finished?" - continue_maybe(msg) + continue_maybe(msg, True) - continue_maybe("Pick complete (local ref %s). Push to %s?" % ( - pick_branch_name, PUSH_REMOTE_NAME)) + continue_maybe( + "Pick complete (local ref %s). Push to %s?" % (pick_branch_name, PUSH_REMOTE_NAME) + ) try: - run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref)) + run_cmd("git push %s %s:%s" % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref)) except Exception as e: clean_up() fail("Exception while pushing: %s" % e) @@ -212,49 +216,85 @@ def cherry_pick(pr_num, merge_hash, default_branch): return pick_ref -def fix_version_from_branch(branch, versions): - # Note: Assumes this is a sorted (newest->oldest) list of un-released versions - if branch == "master": - return versions[0] - else: - branch_ver = branch.replace("branch-", "") - return list(filter(lambda x: x.name.startswith(branch_ver), versions))[-1] - - -def resolve_jira_issue(merge_branches, comment, default_jira_id=""): - asf_jira = jira.client.JIRA({'server': JIRA_API_BASE}, - basic_auth=(JIRA_USERNAME, JIRA_PASSWORD)) - - jira_id = input("Enter a JIRA id [%s]: " % default_jira_id) +def print_jira_issue_summary(issue): + summary = "Summary\t\t%s\n" % issue.fields.summary + assignee = issue.fields.assignee + if assignee is not None: + assignee = assignee.displayName + assignee = "Assignee\t%s\n" % assignee + status = "Status\t\t%s\n" % issue.fields.status.name + url = "Url\t\t%s/%s\n" % (JIRA_BASE, issue.key) + target_versions = "Affected\t%s\n" % [x.name for x in issue.fields.versions] + fix_versions = "" + if len(issue.fields.fixVersions) > 0: + fix_versions = "Fixed\t\t%s\n" % [x.name for x in issue.fields.fixVersions] + print("=== JIRA %s ===" % issue.key) + print("%s%s%s%s%s%s" % (summary, assignee, status, url, target_versions, fix_versions)) + + +def get_jira_issue(prompt, default_jira_id=""): + jira_id = bold_input("%s [%s]: " % (prompt, default_jira_id)) if jira_id == "": jira_id = default_jira_id - + if jira_id == "": + print("JIRA ID not found, skipping.") + return None try: issue = asf_jira.issue(jira_id) + print_jira_issue_summary(issue) + status = issue.fields.status.name + if status == "Resolved" or status == "Closed": + print("JIRA issue %s already has status '%s'" % (jira_id, status)) + return None + if bold_input("Check if the JIRA information is as expected (y/N): ").lower() == "y": + return issue + else: + return get_jira_issue("Enter the revised JIRA ID again or leave blank to skip") except Exception as e: - fail("ASF JIRA could not find %s\n%s" % (jira_id, e)) + print_error("ASF JIRA could not find %s: %s" % (jira_id, e)) + return get_jira_issue("Enter the revised JIRA ID again or leave blank to skip") - cur_status = issue.fields.status.name - cur_summary = issue.fields.summary - cur_assignee = issue.fields.assignee - if cur_assignee is None: - cur_assignee = "NOT ASSIGNED!!!" - else: - cur_assignee = cur_assignee.displayName - if cur_status == "Resolved" or cur_status == "Closed": - fail("JIRA issue %s already has status '%s'" % (jira_id, cur_status)) - print ("=== JIRA %s ===" % jira_id) - print ("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n" % ( - cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id)) +def resolve_jira_issue(merge_branches, comment, default_jira_id=""): + issue = get_jira_issue("Enter a JIRA id", default_jira_id) + if issue is None: + return + + if issue.fields.assignee is None: + choose_jira_assignee(issue) versions = asf_jira.project_versions("ZEPPELIN") + # Consider only x.y.z, unreleased, unarchived versions + versions = [ + x + for x in versions + if not x.raw["released"] and not x.raw["archived"] and re.match(r"\d+\.\d+\.\d+", x.name) + ] versions = sorted(versions, key=lambda x: x.name, reverse=True) - versions = list(filter(lambda x: x.raw['released'] is False, versions)) - # Consider only x.y.z versions - versions = list(filter(lambda x: re.match('\d+\.\d+\.\d+', x.name), versions)) - default_fix_versions = set(map(lambda x: fix_version_from_branch(x, versions).name, merge_branches)) + default_fix_versions = [] + for b in merge_branches: + if b == "master": + default_fix_versions.append(versions[0].name) + else: + found = False + found_versions = [] + for v in versions: + if v.name.startswith(b.replace("branch-", "")): + found_versions.append(v.name) + found = True + if found: + # There might be several unreleased versions for specific branches + # For example, assuming + # versions = ['4.0.0', '3.5.1', '3.5.0', '3.4.2', '3.3.4', '3.3.3'] + # we've found two candidates for branch-3.5, we pick the last/smallest one + default_fix_versions.append(found_versions[-1]) + else: + print_error( + "Target version for %s is not found on JIRA, it may be archived or " + "not created. Skipping it." % b + ) + for v in default_fix_versions: # Handles the case where we have forked a release branch but not yet made the release. # In this case, if the PR is committed to the master branch and the release branch, we @@ -267,25 +307,129 @@ def resolve_jira_issue(merge_branches, comment, default_jira_id=""): default_fix_versions = list(filter(lambda x: x != v, default_fix_versions)) default_fix_versions = ",".join(default_fix_versions) - fix_versions = input("Enter comma-separated fix version(s) [%s]: " % default_fix_versions) - if fix_versions == "": - fix_versions = default_fix_versions - fix_versions = fix_versions.replace(" ", "").split(",") + available_versions = set(list(map(lambda v: v.name, versions))) + while True: + try: + fix_versions = bold_input( + "Enter comma-separated fix version(s) [%s]: " % default_fix_versions + ) + if fix_versions == "": + fix_versions = default_fix_versions + fix_versions = fix_versions.replace(" ", "").split(",") + if set(fix_versions).issubset(available_versions): + break + else: + print( + "Specified version(s) [%s] not found in the available versions, try " + "again (or leave blank and fix manually)." % (", ".join(fix_versions)) + ) + except KeyboardInterrupt: + raise + except BaseException: + traceback.print_exc() + print("Error setting fix version(s), try again (or leave blank and fix manually)") def get_version_json(version_str): return list(filter(lambda v: v.name == version_str, versions))[0].raw jira_fix_versions = list(map(lambda v: get_version_json(v), fix_versions)) - resolve = list(filter(lambda a: a['name'] == "Resolve Issue", asf_jira.transitions(jira_id)))[0] + resolve = list(filter(lambda a: a["name"] == "Resolve Issue", asf_jira.transitions(issue.key)))[ + 0 + ] + resolution = list(filter(lambda r: r.raw["name"] == "Fixed", asf_jira.resolutions()))[0] asf_jira.transition_issue( - jira_id, resolve["id"], fixVersions=jira_fix_versions, comment=comment) + issue.key, + resolve["id"], + fixVersions=jira_fix_versions, + comment=comment, + resolution={"id": resolution.raw["id"]}, + ) - print("Succesfully resolved %s with fixVersions=%s!" % (jira_id, fix_versions)) + try: + print_jira_issue_summary(asf_jira.issue(issue.key)) + except Exception: + print("Unable to fetch JIRA issue %s after resolving" % issue.key) + print("Successfully resolved %s with fixVersions=%s!" % (issue.key, fix_versions)) + + +def choose_jira_assignee(issue): + """ + Prompt the user to choose who to assign the issue to in jira, given a list of candidates, + including the original reporter and all commentators + """ + while True: + try: + reporter = issue.fields.reporter + commentators = list(map(lambda x: x.author, issue.fields.comment.comments)) + candidates = set(commentators) + candidates.add(reporter) + candidates = list(candidates) + print("JIRA is unassigned, choose assignee") + for idx, author in enumerate(candidates): + if author.key == "apachezeppelin": + continue + annotations = ["Reporter"] if author == reporter else [] + if author in commentators: + annotations.append("Commentator") + print("[%d] %s (%s)" % (idx, author.displayName, ",".join(annotations))) + raw_assignee = bold_input( + "Enter number of user, or userid, to assign to (blank to leave unassigned):" + ) + if raw_assignee == "": + return None + else: + try: + id = int(raw_assignee) + assignee = candidates[id] + except BaseException: + # assume it's a user id, and try to assign (might fail, we just prompt again) + assignee = asf_jira.user(raw_assignee) + try: + assign_issue(issue.key, assignee.name) + except Exception as e: + if ( + e.__class__.__name__ == "JIRAError" + and ("'%s' cannot be assigned" % assignee.name) + in getattr(e, "response").text + ): + continue_maybe( + "User '%s' cannot be assigned, add to contributors role and try again?" + % assignee.name + ) + grant_contributor_role(assignee.name) + assign_issue(issue.key, assignee.name) + else: + raise e + return assignee + except KeyboardInterrupt: + raise + except BaseException: + traceback.print_exc() + print("Error assigning JIRA, try again (or leave blank and fix manually)") + + +def grant_contributor_role(user: str): + role = asf_jira.project_role("ZEPPELIN", 10010) + role.add_user(user) + print("Successfully added user '%s' to contributors role" % user) + + +def assign_issue(issue: int, assignee: str) -> bool: + """ + Assign an issue to a user, which is a shorthand for jira.client.JIRA.assign_issue. + The original one has an issue that it will search users again and only choose the assignee + from 20 candidates. If it's unmatched, it picks the head blindly. In our case, the assignee + is already resolved. + """ + url = getattr(asf_jira, "_get_latest_url")(f"issue/{issue}/assignee") + payload = {"name": assignee} + getattr(asf_jira, "_session").put(url, data=json.dumps(payload)) + return True def resolve_jira_issues(title, merge_branches, comment): - jira_ids = re.findall("ZEPPELIN-[0-9]{3,5}", title) + jira_ids = re.findall("ZEPPELIN-[0-9]{3,6}", title) if len(jira_ids) == 0: resolve_jira_issue(merge_branches, comment) @@ -293,69 +437,202 @@ def resolve_jira_issues(title, merge_branches, comment): resolve_jira_issue(merge_branches, comment, jira_id) -#branches = get_json("%s/branches" % GITHUB_API_BASE) -#branch_names = filter(lambda x: x.startswith("branch-"), [x['name'] for x in branches]) -# Assumes branch names can be sorted lexicographically -#latest_branch = sorted(branch_names, reverse=True)[0] -latest_branch = "master" - -pr_num = input("Which pull request would you like to merge? (e.g. 34): ") -pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num)) -pr_events = get_json("%s/issues/%s/events" % (GITHUB_API_BASE, pr_num)) - -url = pr["url"] -title = pr["title"] -body = pr["body"] -target_ref = pr["base"]["ref"] -user_login = pr["user"]["login"] -base_ref = pr["head"]["ref"] -pr_repo_desc = "%s/%s" % (user_login, base_ref) - -# Merged pull requests don't appear as merged in the GitHub API; -# Instead, they're closed by asfgit. -merge_commits = \ - [e for e in pr_events if e["actor"]["login"] == "asfgit" and e["event"] == "closed"] - -if merge_commits: - merge_hash = merge_commits[0]["commit_id"] - message = get_json("%s/commits/%s" % (GITHUB_API_BASE, merge_hash))["commit"]["message"] +def standardize_jira_ref(text): + """ + Standardize the [ZEPPELIN-XXXX][MODULE] prefix + Convert + "[ZEPPELIN-XXXX][spark] Issue" or + "[Spark] ZEPPELIN-XXXX. Issue" or + "ZEPPELIN XXXX [SPARK]: Issue" + to + "[ZEPPELIN-XXXX][SPARK] Issue" + """ + jira_refs = [] + components = [] + + # If this is a Revert PR, no need to process any further + if text.startswith('Revert "') and text.endswith('"'): + return text + + # If the string is compliant, no need to process any further + if re.search(r"^\[ZEPPELIN-[0-9]{3,6}\](\[[A-Z0-9_\s,]+\] )+\S+", text): + return text + + # Extract JIRA ref(s): + pattern = re.compile(r"(ZEPPELIN[-\s]*[0-9]{3,6})+", re.IGNORECASE) + for ref in pattern.findall(text): + # Add brackets, replace spaces with a dash, & convert to uppercase + jira_refs.append("[" + re.sub(r"\s+", "-", ref.upper()) + "]") + text = text.replace(ref, "") + + # Extract zeppelin component(s): + # Look for alphanumeric chars, spaces, dashes, periods, and/or commas + pattern = re.compile(r"(\[[\w\s,.-]+\])", re.IGNORECASE) + for component in pattern.findall(text): + components.append(component.upper()) + text = text.replace(component, "") + + # Cleanup any remaining symbols: + pattern = re.compile(r"^\W+(.*)", re.IGNORECASE) + if pattern.search(text) is not None: + text = pattern.search(text).groups()[0] + + # Assemble full text (JIRA ref(s), module(s), remaining text) + clean_text = "".join(jira_refs).strip() + "".join(components).strip() + " " + text.strip() + + # Replace multiple spaces with a single space, e.g. if no jira refs and/or components were + # included + clean_text = re.sub(r"\s+", " ", clean_text.strip()) + + return clean_text + + +def get_current_ref(): + ref = run_cmd("git rev-parse --abbrev-ref HEAD").strip() + if ref == "HEAD": + # The current ref is a detached HEAD, so grab its SHA. + return run_cmd("git rev-parse HEAD").strip() + else: + return ref + + +def initialize_jira(): + global asf_jira + jira_server = {"server": JIRA_API_BASE} + + if not JIRA_IMPORTED: + print_error("ERROR finding jira library. Run 'pip3 install jira' to install.") + continue_maybe("Continue without jira?") + elif JIRA_ACCESS_TOKEN: + client = jira.client.JIRA(jira_server, token_auth=JIRA_ACCESS_TOKEN) + try: + # Eagerly check if the token is valid to align with the behavior of username/password + # authn + client.current_user() + asf_jira = client + except Exception as e: + if e.__class__.__name__ == "JIRAError" and getattr(e, "status_code", None) == 401: + msg = ( + "ASF JIRA could not authenticate with the invalid or expired token '%s'" + % JIRA_ACCESS_TOKEN + ) + fail(msg) + else: + raise e + elif JIRA_USERNAME and JIRA_PASSWORD: + print("You can use JIRA_ACCESS_TOKEN instead of JIRA_USERNAME/JIRA_PASSWORD.") + print("Visit https://issues.apache.org/jira/secure/ViewProfile.jspa ") + print("and click 'Personal Access Tokens' menu to manage your own tokens.") + asf_jira = jira.client.JIRA(jira_server, basic_auth=(JIRA_USERNAME, JIRA_PASSWORD)) + else: + print("Neither JIRA_ACCESS_TOKEN nor JIRA_USERNAME/JIRA_PASSWORD are set.") + continue_maybe("Continue without jira?") - print("Pull request %s has already been merged, assuming you want to backport" % pr_num) - commit_is_downloaded = run_cmd(['git', 'rev-parse', '--quiet', '--verify', - "%s^{commit}" % merge_hash]).strip() != "" - if not commit_is_downloaded: - fail("Couldn't find any merge commit for #%s, you may need to update HEAD." % pr_num) - print("Found commit %s:\n%s" % (merge_hash, message)) - cherry_pick(pr_num, merge_hash, latest_branch) - sys.exit(0) +def main(): + initialize_jira() + global original_head -if not bool(pr["mergeable"]): - msg = "Pull request %s is not mergeable in its current form.\n" % pr_num + \ - "Continue? (experts only!)" - continue_maybe(msg) + os.chdir(ZEPPELIN_HOME) + original_head = get_current_ref() -print ("\n=== Pull Request #%s ===" % pr_num) -print ("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" % ( - title, pr_repo_desc, target_ref, url)) -continue_maybe("Proceed with merging pull request #%s?" % pr_num) + branches = http_get("%s/branches" % GITHUB_API_BASE) + branch_names = list(filter(lambda x: x.startswith("branch-"), [x["name"] for x in branches])) + # Assumes branch names can be sorted lexicographically + branch_names = sorted(branch_names, reverse=True) + branch_iter = iter(branch_names) -merged_refs = [target_ref] + pr_num = bold_input("Which pull request would you like to merge? (e.g. 34): ") + pr = http_get("%s/pulls/%s" % (GITHUB_API_BASE, pr_num)) + pr_events = http_get("%s/issues/%s/events" % (GITHUB_API_BASE, pr_num)) -merge_hash = merge_pr(pr_num, target_ref) + url = pr["url"] -pick_prompt = "Would you like to pick %s into another branch?" % merge_hash -while input("\n%s (y/n): " % pick_prompt).lower() == "y": - merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)] + # Warn if the PR is WIP + if "[WIP]" in pr["title"]: + msg = "The PR title has `[WIP]`:\n%s\nContinue?" % pr["title"] + continue_maybe(msg) -if JIRA_IMPORTED: - if JIRA_USERNAME and JIRA_PASSWORD: + # Decide whether to use the modified title or not + modified_title = standardize_jira_ref(pr["title"]).rstrip(".") + if modified_title != pr["title"]: + print("I've re-written the title as follows to match the standard format:") + print("Original: %s" % pr["title"]) + print("Modified: %s" % modified_title) + result = bold_input("Would you like to use the modified title? (y/N): ") + if result.lower() == "y": + title = modified_title + print("Using modified title:") + else: + title = pr["title"] + print("Using original title:") + print(title) + else: + title = pr["title"] + + body = pr["body"] + if body is None: + body = "" + modified_body = re.sub(re.compile(r"\n?", re.DOTALL), "", body).lstrip() + if modified_body != body: + print("=" * 80) + print(modified_body) + print("=" * 80) + print("I've removed the comments from PR template like the above:") + result = bold_input("Would you like to use the modified body? (y/N): ") + if result.lower() == "y": + body = modified_body + print("Using modified body:") + else: + print("Using original body:") + print("=" * 80) + print(body) + print("=" * 80) + target_ref = pr["base"]["ref"] + user_login = pr["user"]["login"] + base_ref = pr["head"]["ref"] + pr_repo_desc = "%s/%s" % (user_login, base_ref) + + if not bool(pr["mergeable"]): + fail("Pull request %s is not mergeable in its current form." % pr_num) + + if asf_jira is not None: + jira_ids = re.findall("ZEPPELIN-[0-9]{3,6}", title) + for jira_id in jira_ids: + try: + print_jira_issue_summary(asf_jira.issue(jira_id)) + except Exception: + print_error("Unable to fetch summary of %s" % jira_id) + + print("\n=== Pull Request #%s ===" % pr_num) + print("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" % (title, pr_repo_desc, target_ref, url)) + continue_maybe("Proceed with merging pull request #%s?" % pr_num) + + merged_refs = [target_ref] + + merge_hash = merge_pr(pr_num, target_ref, title, body, pr_repo_desc) + + pick_prompt = "Would you like to pick %s into another branch?" % merge_hash + while bold_input("\n%s (y/N): " % pick_prompt).lower() == "y": + merged_refs = merged_refs + [ + cherry_pick(pr_num, merge_hash, next(branch_iter, branch_names[0])) + ] + + if asf_jira is not None: continue_maybe("Would you like to update an associated JIRA?") - jira_comment = "Issue resolved by pull request %s\n[%s/%s]" % (pr_num, GITHUB_BASE, pr_num) + jira_comment = "Issue resolved by pull request %s\n[%s/%s]" % ( + pr_num, + GITHUB_BASE, + pr_num, + ) resolve_jira_issues(title, merged_refs, jira_comment) else: - print("JIRA_USERNAME and JIRA_PASSWORD not set") print("Exiting without trying to close the associated JIRA.") -else: - print("Could not find jira library. Run 'sudo pip install jira' to install.") - print("Exiting without trying to close the associated JIRA.") + + +if __name__ == "__main__": + try: + main() + except BaseException: + clean_up() + raise