|
| 1 | +# This file is part of Buildbot. Buildbot is free software: you can |
| 2 | +# redistribute it and/or modify it under the terms of the GNU General Public |
| 3 | +# License as published by the Free Software Foundation, version 2. |
| 4 | +# |
| 5 | +# This program is distributed in the hope that it will be useful, but WITHOUT |
| 6 | +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
| 7 | +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more |
| 8 | +# details. |
| 9 | +# |
| 10 | +# You should have received a copy of the GNU General Public License along with |
| 11 | +# this program; if not, write to the Free Software Foundation, Inc., 51 |
| 12 | +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 13 | +# |
| 14 | +# Copyright Buildbot Team Members |
| 15 | + |
| 16 | + |
| 17 | +# Grabbed from https://github.com/buildbot/buildbot/pull/2585, mid-review. The |
| 18 | +# only changes here a few bug fixes that are pending there. |
| 19 | + |
| 20 | +from __future__ import absolute_import |
| 21 | +from __future__ import print_function |
| 22 | + |
| 23 | +from datetime import datetime |
| 24 | + |
| 25 | +from twisted.internet import defer |
| 26 | + |
| 27 | +from buildbot import config |
| 28 | +from buildbot.changes import base |
| 29 | +from buildbot.util import ascii2unicode |
| 30 | +from buildbot.util import datetime2epoch |
| 31 | +from buildbot.util import httpclientservice |
| 32 | +from buildbot.util.logger import Logger |
| 33 | +from buildbot.util.state import StateMixin |
| 34 | + |
| 35 | +log = Logger() |
| 36 | + |
| 37 | +HOSTED_BASE_URL = "https://api.github.com" |
| 38 | +link_urls = { |
| 39 | + "https": "clone_url", |
| 40 | + "svn": "svn_url", |
| 41 | + "git": "git_url", |
| 42 | + "ssh": "ssh_url" |
| 43 | +} |
| 44 | + |
| 45 | + |
| 46 | +class GitHubPullrequestPoller(base.ReconfigurablePollingChangeSource, |
| 47 | + StateMixin): |
| 48 | + compare_attrs = ("owner", "repo", "token", "branches", "pollInterval", |
| 49 | + "category", "project", "pollAtLaunch", "name") |
| 50 | + db_class_name = 'GitHubPullrequestPoller' |
| 51 | + |
| 52 | + def __init__(self, owner, repo, **kwargs): |
| 53 | + name = kwargs.get("name") |
| 54 | + if not name: |
| 55 | + kwargs["name"] = "GitHubPullrequestPoller:" + owner + "/" + repo |
| 56 | + super(GitHubPullrequestPoller, self).__init__(owner, repo, **kwargs) |
| 57 | + |
| 58 | + def checkConfig(self, |
| 59 | + owner, |
| 60 | + repo, |
| 61 | + branches=None, |
| 62 | + category=None, |
| 63 | + baseURL=None, |
| 64 | + project='', |
| 65 | + pullrequest_filter=True, |
| 66 | + token=None, |
| 67 | + repository_link="https", |
| 68 | + **kwargs): |
| 69 | + if repository_link not in ["https", "svn", "git", "ssh"]: |
| 70 | + config.error( |
| 71 | + "repository_link must be one of {https, svn, git, ssh}") |
| 72 | + base.ReconfigurablePollingChangeSource.checkConfig( |
| 73 | + self, name=self.name, **kwargs) |
| 74 | + |
| 75 | + @defer.inlineCallbacks |
| 76 | + def reconfigService(self, |
| 77 | + owner, |
| 78 | + repo, |
| 79 | + branches=None, |
| 80 | + pollInterval=10 * 60, |
| 81 | + category=None, |
| 82 | + baseURL=None, |
| 83 | + project='', |
| 84 | + pullrequest_filter=True, |
| 85 | + token=None, |
| 86 | + pollAtLaunch=False, |
| 87 | + repository_link="https", |
| 88 | + **kwargs): |
| 89 | + yield base.ReconfigurablePollingChangeSource.reconfigService( |
| 90 | + self, name=self.name, **kwargs) |
| 91 | + |
| 92 | + if baseURL is None: |
| 93 | + baseURL = HOSTED_BASE_URL |
| 94 | + if baseURL.endswith('/'): |
| 95 | + baseURL = baseURL[:-1] |
| 96 | + |
| 97 | + http_headers = {'User-Agent': 'Buildbot'} |
| 98 | + if token is not None: |
| 99 | + http_headers.update({'Authorization': 'token ' + token}) |
| 100 | + |
| 101 | + self._http = yield httpclientservice.HTTPClientService.getService( |
| 102 | + self.master, baseURL, headers=http_headers) |
| 103 | + |
| 104 | + if not branches: |
| 105 | + branches = ['master'] |
| 106 | + |
| 107 | + self.token = token |
| 108 | + self.owner = owner |
| 109 | + self.repo = repo |
| 110 | + self.branches = branches |
| 111 | + self.project = project |
| 112 | + self.pollInterval = pollInterval |
| 113 | + self.repository_link = link_urls[repository_link] |
| 114 | + |
| 115 | + if callable(pullrequest_filter): |
| 116 | + self.pullrequest_filter = pullrequest_filter |
| 117 | + else: |
| 118 | + self.pullrequest_filter = (lambda _: pullrequest_filter) |
| 119 | + |
| 120 | + self.category = category if callable(category) else ascii2unicode( |
| 121 | + category) |
| 122 | + self.project = ascii2unicode(project) |
| 123 | + |
| 124 | + def describe(self): |
| 125 | + return "GitHubPullrequestPoller watching the "\ |
| 126 | + "GitHub repository %s/%s" % ( |
| 127 | + self.owner, self.repo) |
| 128 | + |
| 129 | + @defer.inlineCallbacks |
| 130 | + def _getPulls(self): |
| 131 | + log.debug("GitHubPullrequestPoller: polling " |
| 132 | + "GitHub repository %s/%s, branches: %s" % |
| 133 | + (self.owner, self.repo, self.branches)) |
| 134 | + result = yield self._http.get('/'.join( |
| 135 | + ['/repos', self.owner, self.repo, 'pulls'])) |
| 136 | + my_json = yield result.json() |
| 137 | + defer.returnValue(my_json) |
| 138 | + |
| 139 | + @defer.inlineCallbacks |
| 140 | + def _getEmail(self, user): |
| 141 | + result = yield self._http.get("/".join(['/users', user])) |
| 142 | + my_json = yield result.json() |
| 143 | + defer.returnValue(my_json["email"]) |
| 144 | + |
| 145 | + @defer.inlineCallbacks |
| 146 | + def _getFiles(self, prnumber): |
| 147 | + result = yield self._http.get("/".join([ |
| 148 | + '/repos', self.owner, self.repo, 'pulls', str(prnumber), 'files' |
| 149 | + ])) |
| 150 | + my_json = yield result.json() |
| 151 | + |
| 152 | + defer.returnValue([f["filename"] for f in my_json]) |
| 153 | + |
| 154 | + @defer.inlineCallbacks |
| 155 | + def _getCurrentRev(self, prnumber): |
| 156 | + # Get currently assigned revision of PR number |
| 157 | + |
| 158 | + result = yield self._getStateObjectId() |
| 159 | + rev = yield self.master.db.state.getState(result, 'pull_request%d' % |
| 160 | + prnumber, None) |
| 161 | + defer.returnValue(rev) |
| 162 | + |
| 163 | + @defer.inlineCallbacks |
| 164 | + def _setCurrentRev(self, prnumber, rev): |
| 165 | + # Set the updated revision for PR number. |
| 166 | + |
| 167 | + result = yield self._getStateObjectId() |
| 168 | + yield self.master.db.state.setState(result, |
| 169 | + 'pull_request%d' % prnumber, rev) |
| 170 | + |
| 171 | + @defer.inlineCallbacks |
| 172 | + def _getStateObjectId(self): |
| 173 | + # Return a deferred for object id in state db. |
| 174 | + result = yield self.master.db.state.getObjectId( |
| 175 | + '%s/%s' % (self.owner, self.repo), self.db_class_name) |
| 176 | + defer.returnValue(result) |
| 177 | + |
| 178 | + @defer.inlineCallbacks |
| 179 | + def _processChanges(self, github_result): |
| 180 | + for pr in github_result: |
| 181 | + # Track PRs for specified branches |
| 182 | + base_branch = pr['base']['ref'] |
| 183 | + prnumber = pr['number'] |
| 184 | + revision = pr['head']['sha'] |
| 185 | + |
| 186 | + # Check to see if the branch is set or matches |
| 187 | + if base_branch not in self.branches: |
| 188 | + continue |
| 189 | + if (self.pullrequest_filter is not None and |
| 190 | + not self.pullrequest_filter(pr)): |
| 191 | + continue |
| 192 | + current = yield self._getCurrentRev(prnumber) |
| 193 | + if not current or current[0:12] != revision[0:12]: |
| 194 | + # Access title, repo, html link, and comments |
| 195 | + branch = pr['head']['ref'] |
| 196 | + title = pr['title'] |
| 197 | + repo = pr['head']['repo'][self.repository_link] |
| 198 | + revlink = pr['html_url'] |
| 199 | + comments = pr['body'] |
| 200 | + updated = datetime.strptime(pr['updated_at'], |
| 201 | + '%Y-%m-%dT%H:%M:%SZ') |
| 202 | + # update database |
| 203 | + yield self._setCurrentRev(prnumber, revision) |
| 204 | + |
| 205 | + author = pr['user']['login'] |
| 206 | + |
| 207 | + dl = defer.DeferredList( |
| 208 | + [self._getFiles(prnumber), self._getEmail(author)], |
| 209 | + consumeErrors=True) |
| 210 | + |
| 211 | + results = yield dl |
| 212 | + failures = [r[1] for r in results if not r[0]] |
| 213 | + if failures: |
| 214 | + for failure in failures: |
| 215 | + log.error("while processing changes for " |
| 216 | + "Pullrequest {} revision {}".format( |
| 217 | + prnumber, revision)) |
| 218 | + # Fail on the first error! |
| 219 | + failures[0].raiseException() |
| 220 | + [files, email] = [r[1] for r in results] |
| 221 | + |
| 222 | + if email is not None or email is not "null": |
| 223 | + author += " <" + str(email) + ">" |
| 224 | + |
| 225 | + # emit the change |
| 226 | + yield self.master.data.updates.addChange( |
| 227 | + author=ascii2unicode(author), |
| 228 | + revision=ascii2unicode(revision), |
| 229 | + revlink=ascii2unicode(revlink), |
| 230 | + comments=u'pull-request #%d: %s\n%s\n%s' % |
| 231 | + (prnumber, title, revlink, comments), |
| 232 | + when_timestamp=datetime2epoch(updated), |
| 233 | + branch=branch, |
| 234 | + category=self.category, |
| 235 | + project=self.project, |
| 236 | + repository=ascii2unicode(repo), |
| 237 | + files=files, |
| 238 | + src=u'git') |
| 239 | + |
| 240 | + @defer.inlineCallbacks |
| 241 | + def poll(self): |
| 242 | + result = yield self._getPulls() |
| 243 | + yield self._processChanges(result) |
0 commit comments