diff --git a/.githubmap b/.githubmap index c53bc6a33ff1b..c31be2abac536 100644 --- a/.githubmap +++ b/.githubmap @@ -43,3 +43,4 @@ xiexingguo xie xingguo yangdongsheng Dongsheng Yang yuyuyu101 Haomai Wang jtlayton Jeff Layton +yuriw Yuri Weinstein diff --git a/src/script/ptl-tool.py b/src/script/ptl-tool.py old mode 100644 new mode 100755 index 1284956ef2d72..cb754101e3fde --- a/src/script/ptl-tool.py +++ b/src/script/ptl-tool.py @@ -1,27 +1,89 @@ +#!/usr/bin/env python2 + +# README: +# +# This tool's purpose is to make it easier to merge PRs into Ceph. +# +# Because developers often have custom names for the ceph upstream remote +# (https://github.com/ceph/ceph.git), You will probably want to export the +# PTL_TOOL_BASE_PATH environment variable in your shell rc files before using +# this script: +# +# export PTL_TOOL_BASE_PATH=refs/remotes// +# +# and PTL_TOOL_BASE_REMOTE as the name of your Ceph upstream remote (default: "upstream"): +# +# export PTL_TOOL_BASE_REMOTE= +# +# +# ** Here are some basic exmples to get started: ** +# +# Merging PR #1234567 and #2345678 into a new test branch with a testing label added to the PR: +# +# $ env PTL_TOOL_BASE_PATH=refs/remotes/upstream/ src/script/ptl-tool.py --base master 1234567 2345678 +# Detaching HEAD onto base: master +# Merging PR #1234567 +# Labeled PR #1234567 wip-pdonnell-testing +# Merging PR #2345678 +# Labeled PR #2345678 wip-pdonnell-testing +# Deleted old test branch wip-pdonnell-testing-20170928 +# Created branch wip-pdonnell-testing-20170928 +# Created tag testing/wip-pdonnell-testing-20170928_03 +# +# +# Merging PR #1234567 into master leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label: +# +# $ env PTL_TOOL_BASE_PATH=refs/remotes/upstream/ src/script/ptl-tool.py --base master --branch HEAD --merge-branch-name --label - master 1234567 +# Detaching HEAD onto base: master +# Merging PR #1234567 +# Leaving HEAD detached; no branch anchors your commits +# +# Now push to master: +# $ git push upstream HEAD:master +# +# +# Merging PR #12345678 into luminous leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label: +# $ env PTL_TOOL_BASE_PATH=refs/remotes/upstream/ src/script/ptl-tool.py --base luminous --branch HEAD --merge-branch-name luminous --label - 12345678 +# Detaching HEAD onto base: luminous +# Merging PR #12345678 +# Leaving HEAD detached; no branch anchors your commits +# +# Now push to luminous: +# $ git push upstream HEAD:luminous + + # TODO +# Fetch PRs by label. # Look for check failures? # redmine issue update: http://www.redmine.org/projects/redmine/wiki/Rest_Issues +import argparse import datetime import getpass import git +import itertools import json import logging +import os import re import requests import sys + from os.path import expanduser log = logging.getLogger(__name__) log.addHandler(logging.StreamHandler()) log.setLevel(logging.INFO) -BASE = "refs/remotes/upstream/heads/%s" +BASE_REMOTE = os.getenv("PTL_TOOL_BASE_REMOTE", "upstream") +BASE_PATH = os.getenv("PTL_TOOL_BASE_PATH", "refs/remotes/upstream/heads/") +GITDIR = os.getenv("PTL_TOOL_GITDIR", ".") USER = getpass.getuser() with open(expanduser("~/.github.key")) as f: PASSWORD = f.read().strip() BRANCH_PREFIX = "wip-%s-testing-" % USER -TESTING_LABEL = ["wip-%s-testing" % USER] +TESTING_LABEL = "wip-%s-testing" % USER +TESTING_BRANCH_NAME = BRANCH_PREFIX + datetime.datetime.now().strftime("%Y%m%d") SPECIAL_BRANCHES = ('master', 'luminous', 'jewel', 'HEAD') @@ -42,33 +104,56 @@ m = patt.match(line) CONTRIBUTORS[m.group(1)] = m.group(2) -def build_branch(branch_name, pull_requests): - repo = git.Repo(".") +def build_branch(args): + base = args.base + branch = args.branch + label = args.label + + if label and label != '-': + #Check the label format + if re.search(r'\bwip-(.*?)-testing\b', label) is None: + log.error("Unknown Label '{lblname}'. Label Format: wip--testing".format(lblname=label)) + sys.exit(1) + + #Check if the Label exist in the repo + res = requests.get("https://api.github.com/repos/ceph/ceph/labels/{lblname}".format(lblname=label), auth=(USER, PASSWORD)) + if res.status_code != 200: + log.error("Label '{lblname}' not found in the repo".format(lblname=label)) + sys.exit(1) + + G = git.Repo(args.git) - repo.remotes.upstream.fetch() + # First get the latest base branch and PRs from BASE_REMOTE + remote = getattr(G.remotes, BASE_REMOTE) + remote.fetch() - # First get the latest base branch from upstream - if branch_name == 'HEAD': + if base == 'HEAD': log.info("Branch base is HEAD; not checking out!") else: - if branch_name in SPECIAL_BRANCHES: - base = BASE % branch_name - else: - base = BASE % "master" - log.info("Branch base on {}".format(base)) - base = filter(lambda r: r.path == base, repo.refs)[0] + log.info("Detaching HEAD onto base: {}".format(base)) + try: + base_path = args.base_path + base + base = filter(lambda r: r.path == base_path, G.refs)[0] + except IndexError: + log.error("Branch " + base + " does not exist!") + sys.exit(1) # So we know that we're not on an old test branch, detach HEAD onto ref: base.checkout() - for pr in pull_requests: - log.info("Merging PR {pr}".format(pr=pr)) + for pr in args.prs: + log.info("Merging PR #{pr}".format(pr=pr)) pr = int(pr) - r = filter(lambda r: r.path == "refs/remotes/upstream/pull/%d/head" % pr, repo.refs)[0] + remote_ref = "refs/pull/{pr}/head".format(pr=pr) + fi = remote.fetch(remote_ref) + if len(fi) != 1: + log.error("PR {pr} does not exist?".format(pr=pr)) + sys.exit(1) + tip = fi[0].ref.commit - message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, branch_name, r.path) + message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, args.merge_branch_name, remote_ref) - for commit in repo.iter_commits(rev="HEAD.."+r.path): + for commit in G.iter_commits(rev="HEAD.."+str(tip)): message = message + ("\t%s\n" % commit.message.split('\n', 1)[0]) message = message + "\n" @@ -76,17 +161,17 @@ def build_branch(branch_name, pull_requests): comments = requests.get("https://api.github.com/repos/ceph/ceph/issues/{pr}/comments".format(pr=pr), auth=(USER, PASSWORD)) if comments.status_code != 200: log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments)) - return + sys.exit(1) reviews = requests.get("https://api.github.com/repos/ceph/ceph/pulls/{pr}/reviews".format(pr=pr), auth=(USER, PASSWORD)) if reviews.status_code != 200: log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments)) - return + sys.exit(1) review_comments = requests.get("https://api.github.com/repos/ceph/ceph/pulls/{pr}/comments".format(pr=pr), auth=(USER, PASSWORD)) if review_comments.status_code != 200: log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments)) - return + sys.exit(1) indications = set() for comment in comments.json()+review_comments.json(): @@ -120,49 +205,81 @@ def build_branch(branch_name, pull_requests): if new_new_contributors: # Check out the PR, add a commit adding to .githubmap - HEAD = repo.head.commit + HEAD = G.head.commit log.info("adding new contributors to githubmap on top of PR #%s" % pr) - r.checkout() + G.head.reset(commit=tip, index=True, working_tree=True) with open(".githubmap", "a") as f: for c in new_new_contributors: f.write("%s %s\n" % (c, new_new_contributors[c])) - repo.index.add([".githubmap"]) - repo.git.commit("-s", "-m", "githubmap: update contributors") - c = repo.head.commit - repo.head.reset(HEAD, index=True, working_tree=True) + G.index.add([".githubmap"]) + G.git.commit("-s", "-m", "githubmap: update contributors") + c = G.head.commit + G.head.reset(HEAD, index=True, working_tree=True) else: - c = r.commit + c = tip - repo.git.merge(c.hexsha, '--no-ff', m=message) + G.git.merge(c.hexsha, '--no-ff', m=message) - if branch_name not in SPECIAL_BRANCHES: - req = requests.post("https://api.github.com/repos/ceph/ceph/issues/{pr}/labels".format(pr=pr), data=json.dumps(TESTING_LABEL), auth=(USER, PASSWORD)) + if label and label != '-': + req = requests.post("https://api.github.com/repos/ceph/ceph/issues/{pr}/labels".format(pr=pr), data=json.dumps([label]), auth=(USER, PASSWORD)) if req.status_code != 200: - log.error("PR #%d could not be labeled %s: %s" % (pr, wip, req)) - return + log.error("PR #%d could not be labeled %s: %s" % (pr, label, req)) + sys.exit(1) + log.info("Labeled PR #{pr} {label}".format(pr=pr, label=label)) - # If the branch is master, leave HEAD detached (but use "master" for commit message) - if branch_name not in SPECIAL_BRANCHES: + # If the branch is 'HEAD', leave HEAD detached (but use "master" for commit message) + if branch == 'HEAD': + log.info("Leaving HEAD detached; no branch anchors your commits") + else: # Delete test branch if it already existed try: - getattr(repo.branches, branch_name).delete( - repo, getattr(repo.branches, branch_name), force=True) - log.info("Deleted old test branch %s" % branch_name) + getattr(G.branches, branch).delete( + G, getattr(G.branches, branch), force=True) + log.info("Deleted old test branch %s" % branch) except AttributeError: pass - log.info("Creating branch {branch_name}".format(branch_name=branch_name)) - repo.create_head(branch_name) + G.create_head(branch) + log.info("Created branch {branch}".format(branch=branch)) + # tag it for future reference. - name = "testing/%s" % branch_name - log.info("Creating tag %s" % name) - git.refs.tag.Tag.create(repo, name, force=True) + for i in range(0, 100): + if i == 0: + name = "testing/%s" % branch + else: + name = "testing/%s_%02d" % (branch, i) + try: + git.refs.tag.Tag.create(G, name) + log.info("Created tag %s" % name) + break + except: + pass + if i == 99: + raise RuntimeException("ran out of numbers") -if __name__ == "__main__": - if sys.argv[1] in SPECIAL_BRANCHES: - branch_name = sys.argv[1] - pull_requests = sys.argv[2:] +def main(): + parser = argparse.ArgumentParser(description="Ceph PTL tool") + default_base = 'master' + default_branch = TESTING_BRANCH_NAME + default_label = TESTING_LABEL + if len(sys.argv) > 1 and sys.argv[1] in SPECIAL_BRANCHES: + argv = sys.argv[2:] + default_branch = 'HEAD' # Leave HEAD deatched + default_base = default_branch + default_label = False else: - branch_name = BRANCH_PREFIX + datetime.datetime.now().strftime("%Y%m%d") - pull_requests = sys.argv[1:] - build_branch(branch_name, pull_requests) + argv = sys.argv[1:] + parser.add_argument('--branch', dest='branch', action='store', default=default_branch, help='branch to create ("HEAD" leaves HEAD detached; i.e. no branch is made)') + parser.add_argument('--merge-branch-name', dest='merge_branch_name', action='store', help='name of the branch for merge messages') + parser.add_argument('--base', dest='base', action='store', default=default_base, help='base for branch') + parser.add_argument('--base-path', dest='base_path', action='store', default=BASE_PATH, help='base for branch') + parser.add_argument('--git-dir', dest='git', action='store', default=GITDIR, help='git directory') + parser.add_argument('--label', dest='label', action='store', default=default_label, help='label PRs for testing') + parser.add_argument('prs', metavar="PR", type=int, nargs='+', help='Pull Requests to merge') + args = parser.parse_args(argv) + if getattr(args, 'merge_branch_name') is None: + setattr(args, 'merge_branch_name', args.branch) + return build_branch(args) + +if __name__ == "__main__": + main()