[AIRFLOW-1091] Add script that can compare jira target against merges

When working towards a release it is convenient to
be able to compare
what has been merged into the current branch and
what has not.

Closes apache#2231 from bolkedebruin/AIRFLOW-1091

dev/airflow-jira

@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Utility for creating well-formed pull request merges and pushing them to
+# Apache.
+#
+#   usage: ./airflow-jira    (see config env vars below)
+#
+# This utility assumes you already have a local Airflow git folder and that you
+# have added remotes corresponding to both (i) the github apache Airflow
+# mirror and (ii) the apache git repo.
+
+# This tool is based on the Spark merge_spark_pr script:
+# https://github.com/apache/spark/blob/master/dev/merge_spark_pr.py
+
+from __future__ import print_function
+
+import jira
+import re
+import sys
+
+TMP_CREDENTIALS = {}
+PROJECT = "AIRFLOW"
+
+# Python 3 compatibility
+try:
+    import urllib2 as urllib
+except ImportError:
+    import urllib.request as urllib
+if sys.version_info[0] == 3:
+    raw_input = input
+
+try:
+    import click
+except ImportError:
+    print("Could not find the click library. Run 'sudo pip install click' to install.")
+    sys.exit(-1)
+
+try:
+    import keyring
+except ImportError:
+    print("Could not find the keyring library. Run 'sudo pip install keyring' to install.")
+    sys.exit(-1)
+
+try:
+    import git
+except ImportError:
+    print("Could not import git. Run 'sudo pip install gitpython' to install")
+    sys.exit(-1)
+
+JIRA_BASE = "https://issues.apache.org/jira/browse"
+JIRA_API_BASE = "https://issues.apache.org/jira"
+
+GIT_COMMIT_FIELDS = ['id', 'author_name', 'author_email', 'date', 'subject', 'body']
+GIT_LOG_FORMAT = ['%H', '%an', '%ae', '%ad', '%s', '%b']
+GIT_LOG_FORMAT = '%x1f'.join(GIT_LOG_FORMAT) + '%x1e'
+
+
+def get_jiras_for_version(version):
+    asf_jira = jira.client.JIRA(
+            {'server': JIRA_API_BASE})
+
+    issues = asf_jira.search_issues('PROJECT={} and fixVersion={}'.format(PROJECT, version))
+    return issues
+
+
+def get_merged_issues(version):
+    repo = git.Repo(".", search_parent_directories=True)
+    log = repo.git.log('--format={}'.format(GIT_LOG_FORMAT))
+    log = log.strip('\n\x1e').split("\x1e")
+    log = [row.strip().split("\x1f") for row in log]
+    log = [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]
+
+    issue_re = re.compile(".*(AIRFLOW-[0-9]{1,6})(\]|\s|:)")
+    pr_re = re.compile("(.*)Closes (#[0-9]{1,6})", flags=re.DOTALL)
+
+    merges = {}
+    for log_item in log:
+        issue_id = None
+
+        match = issue_re.match(log_item['subject'])
+        if match:
+            issue_id = match.group(1)
+        if log_item.has_key('body'):
+            match = pr_re.match(log_item['body'])
+            if match:
+                log_item['pull_request'] = match.group(2)
+            else:
+                log_item['pull_request'] = '#na'
+        else:
+            log_item['pull_request'] = '#na'
+
+        if issue_id:
+            merges[issue_id] = log_item
+
+    return merges
+
+@click.group()
+def cli():
+    r"""
+    This tool should be used by Airflow Release Manager to verify what Jira's
+     were merged in the current working branch.
+
+        airflow-jira compare <target_version>
+    """
+
+
+@cli.command(short_help='Compare a jira target version against git merges')
+@click.argument('target_version', default=None)
+def compare(target_version):
+    merges = get_merged_issues(target_version)
+    issues = get_jiras_for_version(target_version)
+
+    print("{:<18}|{:<12}||{:<10}||{:<10}|{:<50}|{:<6}|{:<6}|{:<40}"
+          .format("ISSUE ID", "TYPE", "PRIORITY",
+                  "STATUS", "DESCRIPTION", "MERGED",
+                  "PR", "COMMIT"))
+
+    for issue in issues:
+        is_merged = issue.key in merges
+        print("{:<18}|{:<12}||{:<10}||{:<10}|{:<50}|{:<6}|{:<6}|{:<40}"
+              .format(issue.key,
+                      issue.fields.issuetype,
+                      issue.fields.priority,
+                      issue.fields.status,
+                      issue.fields.summary[:50],
+                      is_merged,
+                      merges[issue.key]['pull_request'] if is_merged else "-",
+                      merges[issue.key]['id'] if is_merged else "-"))
+
+
+if __name__ == "__main__":
+    import doctest
+    (failure_count, test_count) = doctest.testmod()
+    if failure_count:
+        exit(-1)
+    try:
+        cli()
+    except:
+        raise
+
+

dev/requirements.txt

@@ -1,3 +1,5 @@
 click>=6.6,<7
 jira>=1.0.7,<2
 keyring==10.1
+gitpython
+