Merge PR ceph#18032 into master

* refs/pull/18032/head: Merge commit 'refs/pull/4/head' of github.com:batrick/ceph into ptl-tool-2 script/ptl-tool.py: Label Validation Merge commit 'refs/pull/2/head' of github.com:batrick/ceph into ptl-tool-2 Merge commit 'refs/pull/1/head' of github.com:batrick/ceph into ptl-tool-2 script/ptl-tool.py: Fix the Comment example script/ptl-tool.py: Fixed error message ptl-tool: make more configurable and usable Reviewed-by: Yuri Weinstein <[email protected]> Reviewed-by: Jos Collin <[email protected]>
beihuaihai · Oct 6, 2017 · 723cb35 · 723cb35
2 parents 42ab0e0 + 69254f2
commit 723cb35
Show file tree

Hide file tree

Showing 2 changed files with 168 additions and 50 deletions.
diff --git a/.githubmap b/.githubmap
@@ -43,3 +43,4 @@ xiexingguo xie xingguo <[email protected]>
 yangdongsheng Dongsheng Yang <[email protected]>
 yuyuyu101 Haomai Wang <[email protected]>
 jtlayton Jeff Layton <[email protected]>
+yuriw Yuri Weinstein <[email protected]>
diff --git a/src/script/ptl-tool.py b/src/script/ptl-tool.py
@@ -1,27 +1,89 @@
+#!/usr/bin/env python2
+
+# README:
+#
+# This tool's purpose is to make it easier to merge PRs into Ceph.
+#
+# Because developers often have custom names for the ceph upstream remote
+# (https://github.com/ceph/ceph.git), You will probably want to export the
+# PTL_TOOL_BASE_PATH environment variable in your shell rc files before using
+# this script:
+#
+#     export PTL_TOOL_BASE_PATH=refs/remotes/<remotename>/
+#
+# and PTL_TOOL_BASE_REMOTE as the name of your Ceph upstream remote (default: "upstream"):
+#
+#     export PTL_TOOL_BASE_REMOTE=<remotename>
+#
+#
+# ** Here are some basic exmples to get started: **
+#
+# Merging PR #1234567 and #2345678 into a new test branch with a testing label added to the PR:
+#
+# $ env PTL_TOOL_BASE_PATH=refs/remotes/upstream/ src/script/ptl-tool.py --base master 1234567 2345678
+# Detaching HEAD onto base: master
+# Merging PR #1234567
+# Labeled PR #1234567 wip-pdonnell-testing
+# Merging PR #2345678
+# Labeled PR #2345678 wip-pdonnell-testing
+# Deleted old test branch wip-pdonnell-testing-20170928
+# Created branch wip-pdonnell-testing-20170928
+# Created tag testing/wip-pdonnell-testing-20170928_03
+#
+#
+# Merging PR #1234567 into master leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
+#
+# $ env PTL_TOOL_BASE_PATH=refs/remotes/upstream/ src/script/ptl-tool.py --base master --branch HEAD --merge-branch-name --label - master 1234567
+# Detaching HEAD onto base: master
+# Merging PR #1234567
+# Leaving HEAD detached; no branch anchors your commits
+#
+# Now push to master:
+# $ git push upstream HEAD:master
+#
+#
+# Merging PR #12345678 into luminous leaving a detached HEAD (i.e. do not update your repo's master branch) and do not label:
+# $ env PTL_TOOL_BASE_PATH=refs/remotes/upstream/ src/script/ptl-tool.py --base luminous --branch HEAD --merge-branch-name luminous --label - 12345678
+# Detaching HEAD onto base: luminous
+# Merging PR #12345678
+# Leaving HEAD detached; no branch anchors your commits
+#
+# Now push to luminous:
+# $ git push upstream HEAD:luminous
+
+
 # TODO
+# Fetch PRs by label.
 # Look for check failures?
 # redmine issue update: http://www.redmine.org/projects/redmine/wiki/Rest_Issues
 
+import argparse
 import datetime
 import getpass
 import git
+import itertools
 import json
 import logging
+import os
 import re
 import requests
 import sys
+
 from os.path import expanduser
 
 log = logging.getLogger(__name__)
 log.addHandler(logging.StreamHandler())
 log.setLevel(logging.INFO)
 
-BASE = "refs/remotes/upstream/heads/%s"
+BASE_REMOTE = os.getenv("PTL_TOOL_BASE_REMOTE", "upstream")
+BASE_PATH = os.getenv("PTL_TOOL_BASE_PATH", "refs/remotes/upstream/heads/")
+GITDIR = os.getenv("PTL_TOOL_GITDIR", ".")
 USER = getpass.getuser()
 with open(expanduser("~/.github.key")) as f:
     PASSWORD = f.read().strip()
 BRANCH_PREFIX = "wip-%s-testing-" % USER
-TESTING_LABEL = ["wip-%s-testing" % USER]
+TESTING_LABEL = "wip-%s-testing" % USER
+TESTING_BRANCH_NAME = BRANCH_PREFIX + datetime.datetime.now().strftime("%Y%m%d")
 
 SPECIAL_BRANCHES = ('master', 'luminous', 'jewel', 'HEAD')
 
@@ -42,51 +104,74 @@
         m = patt.match(line)
         CONTRIBUTORS[m.group(1)] = m.group(2)
 
-def build_branch(branch_name, pull_requests):
-    repo = git.Repo(".")
+def build_branch(args):
+    base = args.base
+    branch = args.branch
+    label = args.label
+
+    if label and label != '-':
+        #Check the label format
+        if re.search(r'\bwip-(.*?)-testing\b', label) is None:
+            log.error("Unknown Label '{lblname}'. Label Format: wip-<name>-testing".format(lblname=label))
+            sys.exit(1)
+
+        #Check if the Label exist in the repo
+        res = requests.get("https://api.github.com/repos/ceph/ceph/labels/{lblname}".format(lblname=label), auth=(USER, PASSWORD))
+        if res.status_code != 200:
+            log.error("Label '{lblname}' not found in the repo".format(lblname=label))
+            sys.exit(1)
+
+    G = git.Repo(args.git)
 
-    repo.remotes.upstream.fetch()
+    # First get the latest base branch and PRs from BASE_REMOTE
+    remote = getattr(G.remotes, BASE_REMOTE)
+    remote.fetch()
 
-    # First get the latest base branch from upstream
-    if branch_name == 'HEAD':
+    if base == 'HEAD':
         log.info("Branch base is HEAD; not checking out!")
     else:
-        if branch_name in SPECIAL_BRANCHES:
-            base = BASE % branch_name
-        else:
-            base = BASE % "master"
-        log.info("Branch base on {}".format(base))
-        base = filter(lambda r: r.path == base, repo.refs)[0]
+        log.info("Detaching HEAD onto base: {}".format(base))
+        try:
+            base_path = args.base_path + base
+            base = filter(lambda r: r.path == base_path, G.refs)[0]
+        except IndexError:
+            log.error("Branch " + base + " does not exist!")
+            sys.exit(1)
 
         # So we know that we're not on an old test branch, detach HEAD onto ref:
         base.checkout()
 
-    for pr in pull_requests:
-        log.info("Merging PR {pr}".format(pr=pr))
+    for pr in args.prs:
+        log.info("Merging PR #{pr}".format(pr=pr))
         pr = int(pr)
-        r = filter(lambda r: r.path == "refs/remotes/upstream/pull/%d/head" % pr, repo.refs)[0]
+        remote_ref = "refs/pull/{pr}/head".format(pr=pr)
+        fi = remote.fetch(remote_ref)
+        if len(fi) != 1:
+            log.error("PR {pr} does not exist?".format(pr=pr))
+            sys.exit(1)
+        tip = fi[0].ref.commit
 
-        message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, branch_name, r.path)
+        message = "Merge PR #%d into %s\n\n* %s:\n" % (pr, args.merge_branch_name, remote_ref)
 
-        for commit in repo.iter_commits(rev="HEAD.."+r.path):
+        for commit in G.iter_commits(rev="HEAD.."+str(tip)):
             message = message + ("\t%s\n" % commit.message.split('\n', 1)[0])
 
         message = message + "\n"
 
         comments = requests.get("https://api.github.com/repos/ceph/ceph/issues/{pr}/comments".format(pr=pr), auth=(USER, PASSWORD))
         if comments.status_code != 200:
             log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
-            return
+            sys.exit(1)
 
         reviews = requests.get("https://api.github.com/repos/ceph/ceph/pulls/{pr}/reviews".format(pr=pr), auth=(USER, PASSWORD))
         if reviews.status_code != 200:
             log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
-            return
+            sys.exit(1)
 
         review_comments = requests.get("https://api.github.com/repos/ceph/ceph/pulls/{pr}/comments".format(pr=pr), auth=(USER, PASSWORD))
         if review_comments.status_code != 200:
             log.error("PR '{pr}' not found: {c}".format(pr=pr,c=comments))
-            return
+            sys.exit(1)
 
         indications = set()
         for comment in comments.json()+review_comments.json():
@@ -120,49 +205,81 @@ def build_branch(branch_name, pull_requests):
 
         if new_new_contributors:
             # Check out the PR, add a commit adding to .githubmap
-            HEAD = repo.head.commit
+            HEAD = G.head.commit
             log.info("adding new contributors to githubmap on top of PR #%s" % pr)
-            r.checkout()
+            G.head.reset(commit=tip, index=True, working_tree=True)
             with open(".githubmap", "a") as f:
                 for c in new_new_contributors:
                     f.write("%s %s\n" % (c, new_new_contributors[c]))
-            repo.index.add([".githubmap"])
-            repo.git.commit("-s", "-m", "githubmap: update contributors")
-            c = repo.head.commit
-            repo.head.reset(HEAD, index=True, working_tree=True)
+            G.index.add([".githubmap"])
+            G.git.commit("-s", "-m", "githubmap: update contributors")
+            c = G.head.commit
+            G.head.reset(HEAD, index=True, working_tree=True)
         else:
-            c = r.commit
+            c = tip
 
-        repo.git.merge(c.hexsha, '--no-ff', m=message)
+        G.git.merge(c.hexsha, '--no-ff', m=message)
 
-        if branch_name not in SPECIAL_BRANCHES:
-            req = requests.post("https://api.github.com/repos/ceph/ceph/issues/{pr}/labels".format(pr=pr), data=json.dumps(TESTING_LABEL), auth=(USER, PASSWORD))
+        if label and label != '-':
+            req = requests.post("https://api.github.com/repos/ceph/ceph/issues/{pr}/labels".format(pr=pr), data=json.dumps([label]), auth=(USER, PASSWORD))
             if req.status_code != 200:
-                log.error("PR #%d could not be labeled %s: %s" % (pr, wip, req))
-                return
+                log.error("PR #%d could not be labeled %s: %s" % (pr, label, req))
+                sys.exit(1)
+            log.info("Labeled PR #{pr} {label}".format(pr=pr, label=label))
 
-    # If the branch is master, leave HEAD detached (but use "master" for commit message)
-    if branch_name not in SPECIAL_BRANCHES:
+    # If the branch is 'HEAD', leave HEAD detached (but use "master" for commit message)
+    if branch == 'HEAD':
+        log.info("Leaving HEAD detached; no branch anchors your commits")
+    else:
         # Delete test branch if it already existed
         try:
-            getattr(repo.branches, branch_name).delete(
-                    repo, getattr(repo.branches, branch_name), force=True)
-            log.info("Deleted old test branch %s" % branch_name)
+            getattr(G.branches, branch).delete(
+                    G, getattr(G.branches, branch), force=True)
+            log.info("Deleted old test branch %s" % branch)
         except AttributeError:
             pass
 
-        log.info("Creating branch {branch_name}".format(branch_name=branch_name))
-        repo.create_head(branch_name)
+        G.create_head(branch)
+        log.info("Created branch {branch}".format(branch=branch))
+
         # tag it for future reference.
-        name = "testing/%s" % branch_name
-        log.info("Creating tag %s" % name)
-        git.refs.tag.Tag.create(repo, name, force=True)
+        for i in range(0, 100):
+            if i == 0:
+                name = "testing/%s" % branch
+            else:
+                name = "testing/%s_%02d" % (branch, i)
+            try:
+                git.refs.tag.Tag.create(G, name)
+                log.info("Created tag %s" % name)
+                break
+            except:
+                pass
+            if i == 99:
+                raise RuntimeException("ran out of numbers")
 
-if __name__ == "__main__":
-    if sys.argv[1] in SPECIAL_BRANCHES:
-        branch_name = sys.argv[1]
-        pull_requests = sys.argv[2:]
+def main():
+    parser = argparse.ArgumentParser(description="Ceph PTL tool")
+    default_base = 'master'
+    default_branch = TESTING_BRANCH_NAME
+    default_label = TESTING_LABEL
+    if len(sys.argv) > 1 and sys.argv[1] in SPECIAL_BRANCHES:
+        argv = sys.argv[2:]
+        default_branch = 'HEAD' # Leave HEAD deatched
+        default_base = default_branch
+        default_label = False
     else:
-        branch_name = BRANCH_PREFIX + datetime.datetime.now().strftime("%Y%m%d")
-        pull_requests = sys.argv[1:]
-    build_branch(branch_name, pull_requests)
+        argv = sys.argv[1:]
+    parser.add_argument('--branch', dest='branch', action='store', default=default_branch, help='branch to create ("HEAD" leaves HEAD detached; i.e. no branch is made)')
+    parser.add_argument('--merge-branch-name', dest='merge_branch_name', action='store', help='name of the branch for merge messages')
+    parser.add_argument('--base', dest='base', action='store', default=default_base, help='base for branch')
+    parser.add_argument('--base-path', dest='base_path', action='store', default=BASE_PATH, help='base for branch')
+    parser.add_argument('--git-dir', dest='git', action='store', default=GITDIR, help='git directory')
+    parser.add_argument('--label', dest='label', action='store', default=default_label, help='label PRs for testing')
+    parser.add_argument('prs', metavar="PR", type=int, nargs='+', help='Pull Requests to merge')
+    args = parser.parse_args(argv)
+    if getattr(args, 'merge_branch_name') is None:
+        setattr(args, 'merge_branch_name', args.branch)
+    return build_branch(args)
+
+if __name__ == "__main__":
+    main()