plugins/marge.py

   1 """
   2 Margebot: A Errbot Plugin for Gitlab MR reminders
   3 """
   4 from datetime import datetime, timezone
   5 from time import sleep
   6 from dateutil import parser
   7 from dateutil.tz import tzutc
   8 from dateutil.relativedelta import relativedelta
   9 from errbot import BotPlugin, arg_botcmd, botcmd, re_botcmd, webhook
  10 from errbot.templating import tenv
  11 from errcron.bot import CrontabMixin
  12 import gitlab
  13 import requests
  14
  15
  16 class MargeGitlab(gitlab.Gitlab):
  17     """
  18     Subclass gitlab.Gitlab so extra_data args can be added
  19     to the addprojecthook() and editprojecthook() methods
  20     """
  21
  22     def __init__(self, host, token="", oauth_token="", verify_ssl=True):
  23         super().__init__(host, token, oauth_token, verify_ssl)
  24
  25     def addprojecthook_extra(self, project_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
  26         """
  27         A copy parent addprojecthook with an extra_data field
  28         """
  29         data = {"id": project_id, "url": url}
  30         if extra_data:
  31             for ed_key, ed_value in extra_data.items():
  32                 data[ed_key] = ed_value
  33         data['push_events'] = int(bool(push))
  34         data['issues_events'] = int(bool(issues))
  35         data['merge_requests_events'] = int(bool(merge_requests))
  36         data['tag_push_events'] = int(bool(tag_push))
  37         request = requests.post("{0}/{1}/hooks".format(self.projects_url, project_id),
  38                                 headers=self.headers, data=data, verify=self.verify_ssl)
  39         if request.status_code == 201:
  40             return request.json()
  41         return False
  42
  43     def editprojecthook_extra(self, project_id, hook_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
  44         """
  45         A copy of the parent editprojecthook with an extra_data field
  46         """
  47         data = {"id": project_id, "hook_id": hook_id, "url": url}
  48         if extra_data:
  49             for ed_key, ed_value in extra_data.items():
  50                 data[ed_key] = ed_value
  51         data['push_events'] = int(bool(push))
  52         data['issues_events'] = int(bool(issues))
  53         data['merge_requests_events'] = int(bool(merge_requests))
  54         data['tag_push_events'] = int(bool(tag_push))
  55         request = requests.put("{0}/{1}/hooks/{2}".format(self.projects_url, project_id, hook_id),
  56                                headers=self.headers, data=data, verify=self.verify_ssl)
  57         return request.status_code == 200
  58
  59
  60 def deltastr(any_delta):
  61     """
  62     Output a datetime delta in the format "x days, y hours, z minutes ago"
  63     """
  64     l_delta = []
  65     days = any_delta.days
  66     hours = any_delta.seconds // 3600
  67     mins = (any_delta.seconds // 60) % 60
  68
  69     for (key, val) in [("day", days), ("hour", hours), ("minute", mins)]:
  70         if val == 1:
  71             l_delta.append("1 " + key)
  72         elif val > 1:
  73             l_delta.append("{} {}s".format(val, key))
  74
  75     if l_delta == []:
  76         retval = "now"
  77     else:
  78         retval = ", ".join(l_delta) + " ago"
  79     return retval
  80
  81
  82 class Marge(BotPlugin, CrontabMixin):
  83     """
  84     I remind you about merge requests
  85
  86     Use:
  87        In gitlab:
  88           Add a merge request webook of the form
  89              'https://webookserver/margeboot/<rooms>'
  90           to the projects you want tracked.  <rooms> should be a
  91           comma-separated list of short room names (anything before the '@')
  92           that you want notified.
  93        In errbot:
  94           Add <roomname>@domain to the CHATROOM_PRESENCE list in config.py for
  95           rooms margebot should join
  96     """
  97
  98     CRONTAB = [
  99         # Set in config now: '0 11,17 * * * .crontab_hook'    # 7:00AM and 1:00PM EST warnings
 100     ]
 101
 102     def __init__(self, *args, **kwargs):
 103         self.git_host = None
 104         self.chatroom_host = None
 105         self.gitlab = None
 106         self.soak_delta = None
 107         self.webhook_url = None
 108         super().__init__(*args, **kwargs)
 109
 110     def get_configuration_template(self):
 111         """
 112         GITLAB_HOST:      Host name of your gitlab server
 113         GITLAB_ADMIN_TOKEN: PAT from an admin's https://${GIT_HOST}/profile/personal_access_tokens.
 114         CHATROOM_HOST: Chatroom host.  Usually 'chatroom' + FQDN of Jabber server
 115         CRONTAB: Schedule of automated merge request checks in '%M %H %d %m %w' format
 116         VERIFY_SSL : True, False, or path to CA cert to verify cert
 117         CRONTAB_SOAK_HOURS : Don't send out reminders about MRs opened less than this many hours
 118         WEBHOOK_URL :  URL to use for defining MR integration in gitlab
 119         """
 120         return {'GITLAB_HOST': 'gitlab.example.com',
 121                 'GITLAB_ADMIN_TOKEN': 'gitlab-admin-user-private-token',
 122                 'CHATROOM_HOST': 'conference.jabber.example.com',
 123                 'CRONTAB': '0 11,17 * * *',
 124                 'VERIFY_SSL': True,
 125                 'CRONTAB_SOAK_HOURS': 1,
 126                 'WEBHOOK_URL': 'https://webhooks.example.com:3142/margebot/'}
 127
 128     def check_configuration(self, configuration):
 129         """
 130         Check that the plugin has been configured properly
 131         """
 132         super().check_configuration(configuration)
 133
 134     def activate(self):
 135         """
 136         Initialization done when the plugin is activated
 137         """
 138         if not self.config:
 139             self.log.info('Margebot is not configured. Forbid activation')
 140             return
 141         self.git_host = self.config['GITLAB_HOST']
 142         self.chatroom_host = self.config['CHATROOM_HOST']
 143         Marge.CRONTAB = ['{} .crontab_hook'.format(self.config['CRONTAB'])]
 144         gitlab_auth_token = self.config['GITLAB_ADMIN_TOKEN']
 145         verify_ssl = self.config['VERIFY_SSL']
 146         # self.gitlab = gitlab.Gitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl)
 147         self.gitlab = MargeGitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl)
 148         self.activate_crontab()
 149
 150         self.soak_delta = relativedelta(hours=self.config['CRONTAB_SOAK_HOURS'])
 151         self.webhook_url = self.config['WEBHOOK_URL']
 152         if self.webhook_url[-1] != '/':
 153             self.webhook_url += '/'
 154         super().activate()
 155
 156     def deactivate(self):
 157         """
 158         Anything that needs to be tore down when the plugin is deactivated goes here.
 159         """
 160         super().deactivate()
 161
 162     @webhook('/margebot/<rooms>/')
 163     def gitlab_hook(self, request, rooms):
 164         """
 165         Webhook that listens on http://<server>:<port>/gitlab
 166         """
 167
 168         self.log.info("webhook: request: {}, rooms: {}".format(request, rooms))
 169         self.log.info("state: {}".format(request['object_attributes']['state']))
 170
 171         # verify it's a merge request
 172         if request['object_kind'] != 'merge_request':
 173             self.log.error('unexpecting object_kind: {}'.format(request['object_kind']))
 174         elif 'opened' in request['object_attributes']['state']:
 175
 176             if request['object_attributes']['work_in_progress']:
 177                 wip = "WIP "
 178             else:
 179                 wip = ""
 180             url = request['project']['homepage']
 181             title = request['object_attributes']['title']
 182
 183             author_id = request['object_attributes']['author_id']  # map this to user name ...
 184             author = self.gitlab.getuser(author_id)
 185             if author:
 186                 author_name = author['username']
 187             else:
 188                 self.log.info("unexpected author_id {}".format(author_id))
 189                 author_name = author_id
 190
 191             target_project_id = request['object_attributes']['target_project_id']
 192             iid = request['object_attributes']['iid']
 193             mr_id = request['object_attributes']['id']
 194
 195             msg_template = "Hi there ! {} has opened a new {}MR: \"{}\"\n{}/merge_requests/{}"
 196             msg = msg_template.format(author_name, wip, title, url, iid)
 197
 198             if 'OPEN_MRS' not in self.keys():
 199                 empty_dict = {}
 200                 self['OPEN_MRS'] = empty_dict
 201
 202             open_mrs = self['OPEN_MRS']
 203
 204             if (target_project_id, mr_id, rooms) not in open_mrs:
 205                 for a_room in rooms.split(','):
 206                     if self.config:
 207                         self.send(self.build_identifier(a_room + '@' + self.chatroom_host), msg)
 208
 209                 self.log.info("webhook: Saving ({}, {}, {})".format(target_project_id, mr_id, rooms))
 210                 open_mrs[(target_project_id, mr_id, rooms)] = True
 211                 self['OPEN_MRS'] = open_mrs
 212         return "OK"
 213
 214     def mr_status_msg(self, a_mr, author=None):
 215         """
 216         Create the merge request status message
 217         """
 218         self.log.info("mr_status_msg: a_mr: {}".format(a_mr))
 219
 220         # Only weed out MRs less than the soak time for the crontab output (where author==None)
 221         now = datetime.now(timezone.utc)
 222         creation_time = parser.parse(a_mr['created_at'], tzinfos=tzutc)
 223         if not author:
 224             self.log.info("times: {}, {}, {}".format(creation_time, self.soak_delta, now))
 225             if creation_time + self.soak_delta > now:
 226                 project_id = a_mr['target_project_id']
 227                 mr_id = a_mr['id']
 228                 soak_hours = self.config['CRONTAB_SOAK_HOURS']
 229                 info_template = "skipping: MR <{},{}> was opened less than {} hours ago"
 230                 info_msg = info_template.format(project_id, mr_id, soak_hours)
 231                 self.log.info(info_msg)
 232                 return None
 233
 234         str_open_since = deltastr(now - creation_time)
 235
 236         warning = ""
 237         if a_mr['work_in_progress']:
 238             warning = "still WIP"
 239         elif a_mr['merge_status'] != 'can_be_merged':
 240             warning = "there are merge conflicts"
 241
 242         if author:
 243             authored = (a_mr['author']['id'] == author)
 244         else:
 245             authored = False
 246
 247         # getapprovals is only available in GitLab 8.9 EE or greater
 248         # (not the open source CE version)
 249         # approvals = self.gitlab.getapprovals(a_mr['id'])
 250         # also_approved = ""
 251         # for approved in approvals['approved_by']:
 252         #    also_approved += "," + approved['user']['name']
 253
 254         upvotes = a_mr['upvotes']
 255         msg = "{} (opened {})".format(a_mr['web_url'], str_open_since)
 256         if upvotes >= 2:
 257             msg += ": Has 2+ upvotes and could be merged in now"
 258             if warning != "":
 259                 msg += " except {}.".format(warning)
 260             else:
 261                 msg += "."
 262
 263         elif upvotes == 1:
 264             if authored:
 265                 msg += ": Your MR is waiting for another upvote"
 266             else:
 267                 msg += ": Waiting for another upvote"
 268             if warning != "":
 269                 msg += " but {}.".format(warning)
 270             else:
 271                 msg += "."
 272
 273         else:
 274             if authored:
 275                 msg += ": Your MR has no upvotes"
 276             else:
 277                 msg += ": No upvotes, please review"
 278             if warning != "":
 279                 msg += " but {}.".format(warning)
 280             else:
 281                 msg += "."
 282
 283         return {'creation_time': creation_time, 'msg': msg}
 284
 285     def crontab_hook(self, polled_time):
 286         """
 287         Send a scheduled message to the rooms margebot is watching
 288         about open MRs the room cares about.
 289         """
 290
 291         self.log.info("crontab_hook triggered at {}".format(polled_time))
 292
 293         reminder_msg = {}  # Map of reminder_msg['roomname@domain'] = [(created_at,msg)]
 294
 295         # initialize the reminders
 296         rooms = self.rooms()
 297         for a_room in rooms:
 298             reminder_msg[a_room.node] = []
 299
 300         still_open_mrs = {}
 301
 302         # Let's walk through the MRs we've seen already:
 303         open_mrs = self['OPEN_MRS']
 304
 305         for (project_id, mr_id, notify_rooms) in open_mrs:
 306
 307             # Lookup the MR from the project/id
 308             a_mr = self.gitlab.getmergerequest(project_id, mr_id)
 309             if not a_mr:
 310                 self.log.debug("Couldn't find project: {}, id: {}".format(project_id, mr_id))
 311                 continue
 312
 313             # If the MR is tagged 'never-close' ignore it
 314             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 315                 continue
 316
 317             self.log.info("a_mr: {} {} {} {}".format(project_id, mr_id, notify_rooms, a_mr['state']))
 318
 319             # If the MR is no longer open, skip to the next MR,
 320             # and don't include this MR in the next check
 321             if 'opened' not in a_mr['state']:
 322                 continue
 323             else:
 324                 still_open_mrs[(project_id, mr_id, notify_rooms)] = True
 325
 326             msg_dict = self.mr_status_msg(a_mr)
 327             if msg_dict is None:
 328                 continue
 329
 330             for a_room in notify_rooms.split(','):
 331                 reminder_msg[a_room].append(msg_dict)
 332
 333         # Remind each of the rooms about open MRs
 334         for a_room, room_msg_list in reminder_msg.items():
 335             if room_msg_list != []:
 336                 if self.config:
 337                     to_room = a_room + '@' + self.config['CHATROOM_HOST']
 338                     msg = tenv().get_template('reviews.md').render(msg_list=room_msg_list)
 339                     self.send(self.build_identifier(to_room), msg)
 340
 341         self['OPEN_MRS'] = still_open_mrs
 342
 343     @botcmd(template="reviews")
 344     def reviews(self, msg, args):
 345         """
 346         Returns a list of MRs that are waiting for some luv.
 347         Also returns a list of MRs that have had enough luv but aren't merged in yet.
 348         """
 349         # Sending directly to Margbot:  sender in the form sender@....
 350         # Sending to a chatroom:  snder in the form room@rooms/sender
 351
 352         log_template = 'reviews: msg: {}, args: {}, \nmsg.frm: {}, \nmsg.msg.frm: {}, chatroom_host: {}'
 353         self.log.info(log_template.format(msg, args, msg.frm.__dict__, dir(msg.frm), self.config['CHATROOM_HOST']))
 354         self.log.info('reviews: bot mode: {}'.format(self._bot.mode))
 355
 356         if self._bot.mode == "xmpp":
 357             if msg.frm.domain == self.config['CHATROOM_HOST']:
 358                 sender = msg.frm.resource
 359             else:
 360                 sender = msg.frm.node
 361         else:
 362             sender = str(msg.frm).split('@')[0]
 363
 364         keys = self.keys()
 365         if 'OPEN_MRS' not in keys:
 366             self.log.error('OPEN_MRS not in {}'.format(keys))
 367             return "No MRs to review"
 368
 369         sender_gitlab_id = None
 370         sender_users = self.gitlab.getusers(search=(('username', sender)))
 371         if not sender_users:
 372             self.log.error('problem mapping {} to a gitlab user'.format(sender))
 373             sender_gitlab_id = None
 374         else:
 375             sender_gitlab_id = sender_users[0]['id']
 376
 377         # Walk through the MRs we've seen already:
 378         msg_list = []
 379         msg = ""
 380         still_open_mrs = {}
 381         open_mrs = self['OPEN_MRS']
 382         for (project, mr_id, notify_rooms) in open_mrs:
 383
 384             # Lookup the MR from the project/id
 385             a_mr = self.gitlab.getmergerequest(project, mr_id)
 386             if not a_mr:
 387                 self.log.debug("Couldn't find project: {}, id: {}".format(project, id))
 388                 continue
 389
 390             self.log.info('project: {}, id: {}, a_mr: {}'.format(project, id, a_mr))
 391
 392             # If the MR is tagged 'never-close' ignore it
 393             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 394                 continue
 395
 396             # If the MR is no longer open, skip to the next MR,
 397             # and don't include this MR in the next check
 398             if 'opened' not in a_mr['state']:
 399                 continue
 400             else:
 401                 still_open_mrs[(project, mr_id, notify_rooms)] = True
 402
 403             msg_dict = self.mr_status_msg(a_mr, author=sender_gitlab_id)
 404             if msg_dict is None:
 405                 continue
 406
 407             msg_list.append(msg_dict)
 408
 409         self['OPEN_MRS'] = still_open_mrs
 410
 411         return {'sender': sender, 'msg_list': msg_list}
 412
 413     # -----------------------------------------------------------
 414     # webhook maintenance commands
 415
 416     @arg_botcmd('rooms', type=str)
 417     @arg_botcmd('repo', type=str)
 418     def watchrepo(self, msg, repo, rooms):
 419         """
 420         Add the margebot webhook to a repo, and prepopulate any open MRs in the repo with margebot
 421         args: repo: gitlab repo name in the 'NAMESPACE/PROJECT_NAME' format
 422               rooms:  comma separates list of rooms to notify when the webhook triggers
 423         """
 424         self.log.info("msg={}".format(msg))
 425         self.log.info("repo={}".format(repo))
 426         self.log.info("rooms={}".format(rooms))
 427
 428         # get the group/repo repo, error out if it doesn't exist
 429         project = self.gitlab.getproject(repo)
 430         if not project:
 431             msg = "Couldn't find repo {}".format(repo)
 432             self.log.info("watchrepo: {}".format(msg))
 433             return msg
 434
 435         self.log.info('project: {}'.format(project))
 436
 437         target_project_id = project['id']
 438
 439         # Check is the project already includes the margebot hook
 440         # If no hooks, will it return False or [] ?
 441         marge_hook = None
 442         hooks = self.gitlab.getprojecthooks(target_project_id)
 443         if not hooks:
 444             msg = "Couldn't find {} hooks".format(repo)
 445             self.log.error("watchrepo: {}".format(msg))
 446             return msg
 447         else:
 448             for a_hook in hooks:
 449                 self.log.info('a_hook: {}'.format(a_hook))
 450                 if a_hook['merge_requests_events'] and a_hook['url'].startswith(self.webhook_url):
 451                     marge_hook = a_hook
 452                     break
 453
 454         # If so replace it (or error out ?)
 455         url = "{}{}".format(self.webhook_url, rooms)  # webhooks_url will end in '/'
 456         if marge_hook:
 457
 458             old_rooms = marge_hook['url'].split(self.webhook_url, 1)[1]
 459             if old_rooms == rooms:
 460                 msg = "Already reporting {} MRs to the {} room(s)".format(repo, rooms)
 461                 self.log.info('watchrepo: {}'.format(msg))
 462                 return msg
 463             else:
 464                 hook_updated = self.gitlab.editprojecthook_extra(target_project_id, marge_hook['id'], url, merge_requests=True, extra_data={'enable_ssl_verification': False})
 465                 s_watch_msg = "Updating room list for {} MRs from {} to {}".format(repo, old_rooms, rooms)
 466                 s_action = "update"
 467         else:
 468             hook_updated = self.gitlab.addprojecthook_extra(target_project_id, url, merge_requests=True, extra_data={'enable_ssl_verification': False})
 469             s_watch_msg = "Now watching for new MRs in the {} repo to the {} roomi(s)".format(repo, rooms)
 470             s_action = "add"
 471
 472         if not hook_updated:
 473             msg = "Couldn't {} hook: {}".format(s_action, repo)
 474             self.log.error("watchrepo: {}".format(msg))
 475             return msg
 476
 477         open_mrs = self['OPEN_MRS']
 478         mr_count = 0
 479         # get the open MRs in the repo
 480
 481         # If updating the room list, walk through the existing MR list
 482         if s_action == "update":
 483             new_open_mrs = {}
 484             for (project_id, mr_id, old_rooms) in open_mrs:
 485                 # pragma pylint: disable=simplifiable-if-statement
 486                 if project_id == target_project_id:
 487                     new_open_mrs[(project_id, mr_id, rooms)] = True
 488                 else:
 489                     new_open_mrs[(project_id, mr_id, old_rooms)] = True
 490                 # pragma pylint: enable=simplifiable-if-statement
 491             open_mrs = new_open_mrs
 492
 493         # If adding a new repo, check for existing opened/reopened MRs in the repo.
 494         else:
 495             for state in ['opened', 'reopened']:
 496                 page = 1
 497                 mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100, state=state)
 498                 while not mr_list and mr_list != []:
 499                     for an_mr in mr_list:
 500                         mr_count += 1
 501                         self.log.info('watchrepo: an_mr WATS THE ID\n{}'.format(an_mr))
 502                         mr_id = an_mr['id']
 503                         open_mrs[(target_project_id, mr_id, rooms)] = True
 504                     # Get the next page of MRs
 505                     page += 1
 506                     mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100)
 507
 508         self['OPEN_MRS'] = open_mrs
 509
 510         if mr_count == 0:
 511             mr_msg = "No open MRs were found in the repo."
 512         elif mr_count == 1:
 513             mr_msg = "1 open MR was found in the repo.  Run !reviews to see the updated MR list."
 514         else:
 515             mr_msg = "{} open MRs were found in the repo.  Run !reviews to see the updated MR list."
 516
 517         return "{}\n{}".format(s_watch_msg, mr_msg)
 518
 519     # pragma pylint: disable=unused-argument
 520     @botcmd()
 521     def hello(self, msg, args):
 522         """
 523         A simple command to check if the bot is responding
 524         """
 525         return "Hi there"
 526
 527     @botcmd()
 528     def xyzzy(self, msg, args):
 529         """
 530         Don't call this command...
 531         """
 532         yield "/me whispers \"All open MRs have been merged into master.\""
 533         sleep(5)
 534         yield "(just kidding)"
 535
 536     @re_botcmd(pattern=r"I blame [Mm]arge([Bb]ot)?")
 537     def dont_blame_margebot(self, msg, match):
 538         """
 539         margebot is innocent.
 540         """
 541         yield "(」ﾟﾛﾟ)｣NOOOooooo say it ain't so."
 542
 543     @re_botcmd(pattern=r"good bot")
 544     def best_bot(self, msg, match):
 545         """
 546         margebot is the best.
 547         """
 548         yield "Best bot"
 549
 550     # pragma pylint: enable=unused-argument