plugins/marge.py

   1 """
   2 Margebot: A Errbot Plugin for Gitlab MR reminders
   3 """
   4 import re
   5 from datetime import datetime, timezone
   6 from time import sleep
   7 from dateutil import parser
   8 from dateutil.tz import tzutc
   9 from dateutil.relativedelta import relativedelta
  10 from errbot import BotPlugin, arg_botcmd, botcmd, re_botcmd, webhook
  11 from errbot.templating import tenv
  12 from errcron.bot import CrontabMixin
  13 import gitlab
  14 import requests
  15
  16 class MargeGitlab(gitlab.Gitlab):
  17     """
  18     Subclass gitlab.Gitlab so extra_data args can be added
  19     to the addprojecthook() and editprojecthook() methods
  20     """
  21
  22     def __init__(self, host, token="", oauth_token="", verify_ssl=True):
  23         super().__init__(host, token, oauth_token, verify_ssl)
  24
  25     def addprojecthook_extra(self, project_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
  26         """
  27         A copy parent addprojecthook with an extra_data field
  28         """
  29         data = {"id": project_id, "url": url}
  30         if extra_data:
  31             for ed_key, ed_value in extra_data.items():
  32                 data[ed_key] = ed_value
  33         data['push_events'] = int(bool(push))
  34         data['issues_events'] = int(bool(issues))
  35         data['merge_requests_events'] = int(bool(merge_requests))
  36         data['tag_push_events'] = int(bool(tag_push))
  37         request = requests.post("{0}/{1}/hooks".format(self.projects_url, project_id),
  38                                 headers=self.headers, data=data, verify=self.verify_ssl)
  39         if request.status_code == 201:
  40             return request.json()
  41         return False
  42
  43     def editprojecthook_extra(self, project_id, hook_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
  44         """
  45         A copy of the parent editprojecthook with an extra_data field
  46         """
  47         data = {"id": project_id, "hook_id": hook_id, "url": url}
  48         if extra_data:
  49             for ed_key, ed_value in extra_data.items():
  50                 data[ed_key] = ed_value
  51         data['push_events'] = int(bool(push))
  52         data['issues_events'] = int(bool(issues))
  53         data['merge_requests_events'] = int(bool(merge_requests))
  54         data['tag_push_events'] = int(bool(tag_push))
  55         request = requests.put("{0}/{1}/hooks/{2}".format(self.projects_url, project_id, hook_id),
  56                                headers=self.headers, data=data, verify=self.verify_ssl)
  57         return request.status_code == 200
  58
  59
  60 def deltastr(any_delta):
  61     """
  62     Output a datetime delta in the format "x days, y hours, z minutes ago"
  63     """
  64     l_delta = []
  65     days = any_delta.days
  66     hours = any_delta.seconds // 3600
  67     mins = (any_delta.seconds // 60) % 60
  68
  69     for (key, val) in [("day", days), ("hour", hours), ("minute", mins)]:
  70         if val == 1:
  71             l_delta.append("1 " + key)
  72         elif val > 1:
  73             l_delta.append("{} {}s".format(val, key))
  74
  75     if l_delta == []:
  76         retval = "now"
  77     else:
  78         retval = ", ".join(l_delta) + " ago"
  79     return retval
  80
  81
  82 class Marge(BotPlugin, CrontabMixin):
  83     """
  84     I remind you about merge requests
  85
  86     Use:
  87        In gitlab:
  88           Add a merge request webook of the form
  89              'https://webookserver/margeboot/<rooms>'
  90           to the projects you want tracked.  <rooms> should be a
  91           comma-separated list of short room names (anything before the '@')
  92           that you want notified.
  93        In errbot:
  94           Add <roomname>@domain to the CHATROOM_PRESENCE list in config.py for
  95           rooms margebot should join
  96     """
  97
  98     CRONTAB = [
  99         # Set in config now: '0 11,17 * * * .crontab_hook'    # 7:00AM and 1:00PM EST warnings
 100     ]
 101
 102     def __init__(self, *args, **kwargs):
 103         self.git_host = None
 104         self.chatroom_host = None
 105         self.gitlab = None
 106         self.soak_delta = None
 107         self.webhook_url = None
 108         super().__init__(*args, **kwargs)
 109
 110     def get_configuration_template(self):
 111         """
 112         GITLAB_HOST:      Host name of your gitlab server
 113         GITLAB_ADMIN_TOKEN: PAT from an admin's https://${GIT_HOST}/profile/personal_access_tokens.
 114         CHATROOM_HOST: Chatroom host.  Usually 'chatroom' + FQDN of Jabber server
 115         CRONTAB: Schedule of automated merge request checks in '%M %H %d %m %w' format
 116         VERIFY_SSL : True, False, or path to CA cert to verify cert
 117         CRONTAB_SOAK_HOURS : Don't send out reminders about MRs opened less than this many hours
 118         WEBHOOK_URL :  URL to use for defining MR integration in gitlab
 119         """
 120         return {'GITLAB_HOST': 'gitlab.example.com',
 121                 'GITLAB_ADMIN_TOKEN': 'gitlab-admin-user-private-token',
 122                 'CHATROOM_HOST': 'conference.jabber.example.com',
 123                 'CRONTAB': '0 11,17 * * *',
 124                 'VERIFY_SSL': True,
 125                 'CRONTAB_SOAK_HOURS': 1,
 126                 'WEBHOOK_URL': 'https://webhooks.example.com:3142/margebot/'}
 127
 128     def check_configuration(self, configuration):
 129         """
 130         Check that the plugin has been configured properly
 131         """
 132         super().check_configuration(configuration)
 133
 134     def activate(self):
 135         """
 136         Initialization done when the plugin is activated
 137         """
 138         if not self.config:
 139             self.log.info('Margebot is not configured. Forbid activation')
 140             return
 141         self.git_host = self.config['GITLAB_HOST']
 142         self.chatroom_host = self.config['CHATROOM_HOST']
 143         Marge.CRONTAB = ['{} .crontab_hook'.format(self.config['CRONTAB'])]
 144         gitlab_auth_token = self.config['GITLAB_ADMIN_TOKEN']
 145         verify_ssl = self.config['VERIFY_SSL']
 146         self.gitlab = MargeGitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl)
 147         self.activate_crontab()
 148
 149         self.soak_delta = relativedelta(hours=self.config['CRONTAB_SOAK_HOURS'])
 150         self.webhook_url = self.config['WEBHOOK_URL']
 151         if self.webhook_url[-1] != '/':
 152             self.webhook_url += '/'
 153         super().activate()
 154
 155     def deactivate(self):
 156         """
 157         Anything that needs to be tore down when the plugin is deactivated goes here.
 158         """
 159         super().deactivate()
 160
 161     @webhook('/margebot/<rooms>/')
 162     def gitlab_hook(self, request, rooms):
 163         """
 164         Webhook that listens on http://<server>:<port>/gitlab
 165         """
 166
 167         self.log.info("webhook: request: {}, rooms: {}".format(request, rooms))
 168         # self.log.info("state: {}".format(request['object_attributes']['state']))
 169
 170         # verify it's a merge request
 171         if request['object_kind'] != 'merge_request':
 172             self.log.error('unexpecting object_kind: {}'.format(request['object_kind']))
 173         elif 'opened' in request['object_attributes']['state']:
 174
 175             if request['object_attributes']['work_in_progress']:
 176                 wip = "WIP "
 177             else:
 178                 wip = ""
 179             url = request['project']['homepage']
 180             title = request['object_attributes']['title']
 181
 182             author_id = request['object_attributes']['author_id']  # map this to user name ...
 183             author = self.gitlab.getuser(author_id)
 184             if author:
 185                 author_name = author['username']
 186             else:
 187                 self.log.info("unexpected author_id {}".format(author_id))
 188                 author_name = author_id
 189
 190             target_project_id = request['object_attributes']['target_project_id']
 191             iid = request['object_attributes']['iid']
 192             mr_id = request['object_attributes']['id']
 193
 194             msg_template = "Hi there ! {} has opened a new {}MR: \"{}\"\n{}/merge_requests/{}"
 195             msg = msg_template.format(author_name, wip, title, url, iid)
 196
 197             if 'OPEN_MRS' not in self.keys():
 198                 empty_dict = {}
 199                 self['OPEN_MRS'] = empty_dict
 200
 201             open_mrs = self['OPEN_MRS']
 202
 203             if (target_project_id, mr_id, rooms) not in open_mrs:
 204                 for a_room in rooms.split(','):
 205                     if self.config:
 206                         self.send(self.build_identifier(a_room + '@' + self.chatroom_host), msg)
 207
 208                 self.log.info("webhook: Saving ({}, {}, {})".format(target_project_id, mr_id, rooms))
 209                 open_mrs[(target_project_id, mr_id, rooms)] = True
 210                 self['OPEN_MRS'] = open_mrs
 211         return "OK"
 212
 213     def mr_status_msg(self, a_mr, author=None):
 214         """
 215         Create the merge request status message
 216         """
 217         self.log.info("mr_status_msg: a_mr: {}".format(a_mr))
 218
 219         # Only weed out MRs less than the soak time for the crontab output (where author==None)
 220         now = datetime.now(timezone.utc)
 221         creation_time = parser.parse(a_mr['created_at'], tzinfos=tzutc)
 222         if not author:
 223             self.log.info("times: {}, {}, {}".format(creation_time, self.soak_delta, now))
 224             if creation_time + self.soak_delta > now:
 225                 project_id = a_mr['target_project_id']
 226                 mr_id = a_mr['id']
 227                 soak_hours = self.config['CRONTAB_SOAK_HOURS']
 228                 info_template = "skipping: MR <{},{}> was opened less than {} hours ago"
 229                 info_msg = info_template.format(project_id, mr_id, soak_hours)
 230                 self.log.info(info_msg)
 231                 return None
 232
 233         str_open_since = deltastr(now - creation_time)
 234
 235         warning = ""
 236         if a_mr['work_in_progress']:
 237             warning = "still WIP"
 238         elif a_mr['merge_status'] != 'can_be_merged':
 239             warning = "there are merge conflicts"
 240
 241         if author:
 242             authored = (a_mr['author']['id'] == author)
 243         else:
 244             authored = False
 245
 246         # getapprovals is only available in GitLab 8.9 EE or greater
 247         # (not the open source CE version)
 248         # approvals = self.gitlab.getapprovals(a_mr['id'])
 249         # also_approved = ""
 250         # for approved in approvals['approved_by']:
 251         #    also_approved += "," + approved['user']['name']
 252
 253         upvotes = a_mr['upvotes']
 254         msg = "{} (opened {})".format(a_mr['web_url'], str_open_since)
 255         if upvotes >= 2:
 256             msg += ": Has 2+ upvotes and could be merged in now"
 257             if warning != "":
 258                 msg += " except {}.".format(warning)
 259             else:
 260                 msg += "."
 261
 262         elif upvotes == 1:
 263             if authored:
 264                 msg += ": Your MR is waiting for another upvote"
 265             else:
 266                 msg += ": Waiting for another upvote"
 267             if warning != "":
 268                 msg += " but {}.".format(warning)
 269             else:
 270                 msg += "."
 271
 272         else:
 273             if authored:
 274                 msg += ": Your MR has no upvotes"
 275             else:
 276                 msg += ": No upvotes, please review"
 277             if warning != "":
 278                 msg += " but {}.".format(warning)
 279             else:
 280                 msg += "."
 281
 282         return {'creation_time': creation_time, 'msg': msg}
 283
 284     def crontab_hook(self, polled_time):
 285         """
 286         Send a scheduled message to the rooms margebot is watching
 287         about open MRs the room cares about.
 288         """
 289
 290         self.log.info("crontab_hook triggered at {}".format(polled_time))
 291
 292         reminder_msg = {}  # Map of reminder_msg['roomname@domain'] = [(created_at,msg)]
 293
 294         # initialize the reminders
 295         rooms = self.rooms()
 296         for a_room in rooms:
 297             reminder_msg[a_room.node] = []
 298
 299         still_open_mrs = {}
 300
 301         # Let's walk through the MRs we've seen already:
 302         open_mrs = self['OPEN_MRS']
 303
 304         for (project_id, mr_id, notify_rooms) in open_mrs:
 305
 306             # Lookup the MR from the project/id
 307             a_mr = self.gitlab.getmergerequest(project_id, mr_id)
 308             if not a_mr:
 309                 self.log.debug("Couldn't find project: {}, id: {}".format(project_id, mr_id))
 310                 continue
 311
 312             # If the MR is tagged 'never-close' ignore it
 313             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 314                 continue
 315
 316             self.log.info("a_mr: {} {} {} {}".format(project_id, mr_id, notify_rooms, a_mr['state']))
 317
 318             # If the MR is no longer open, skip to the next MR,
 319             # and don't include this MR in the next check
 320             if 'opened' not in a_mr['state']:
 321                 continue
 322             else:
 323                 still_open_mrs[(project_id, mr_id, notify_rooms)] = True
 324
 325             msg_dict = self.mr_status_msg(a_mr)
 326             if msg_dict is None:
 327                 continue
 328
 329             for a_room in notify_rooms.split(','):
 330                 reminder_msg[a_room].append(msg_dict)
 331
 332         # Remind each of the rooms about open MRs
 333         for a_room, room_msg_list in reminder_msg.items():
 334             if room_msg_list != []:
 335                 if self.config:
 336                     to_room = a_room + '@' + self.config['CHATROOM_HOST']
 337                     msg = tenv().get_template('reviews.md').render(msg_list=room_msg_list)
 338                     self.send(self.build_identifier(to_room), msg)
 339
 340         self['OPEN_MRS'] = still_open_mrs
 341
 342     @botcmd(template="reviews")
 343     def reviews(self, msg, args):
 344         """
 345         Returns a list of MRs that are waiting for some luv.
 346         Also returns a list of MRs that have had enough luv but aren't merged in yet.
 347         """
 348         # Sending directly to Margbot:  sender in the form sender@....
 349         # Sending to a chatroom:  snder in the form room@rooms/sender
 350
 351         log_template = 'reviews: msg: {}, args: {}, \nmsg.frm: {}, \nmsg.msg.frm: {}, chatroom_host: {}'
 352         self.log.info(log_template.format(msg, args, msg.frm.__dict__, dir(msg.frm), self.config['CHATROOM_HOST']))
 353         self.log.info('reviews: bot mode: {}'.format(self._bot.mode))
 354
 355         if self._bot.mode == "xmpp":
 356             if msg.frm.domain == self.config['CHATROOM_HOST']:
 357                 sender = msg.frm.resource
 358             else:
 359                 sender = msg.frm.node
 360         else:
 361             sender = str(msg.frm).split('@')[0]
 362
 363         keys = self.keys()
 364         if 'OPEN_MRS' not in keys:
 365             self.log.error('OPEN_MRS not in {}'.format(keys))
 366             return "No MRs to review"
 367
 368         sender_gitlab_id = None
 369         sender_users = self.gitlab.getusers(search=(('username', sender)))
 370         if not sender_users:
 371             self.log.error('problem mapping {} to a gitlab user'.format(sender))
 372             sender_gitlab_id = None
 373         else:
 374             sender_gitlab_id = sender_users[0]['id']
 375
 376         # Walk through the MRs we've seen already:
 377         msg_list = []
 378         msg = ""
 379         still_open_mrs = {}
 380         open_mrs = self['OPEN_MRS']
 381         for (project, mr_id, notify_rooms) in open_mrs:
 382
 383             # Lookup the MR from the project/id
 384             a_mr = self.gitlab.getmergerequest(project, mr_id)
 385             if not a_mr:
 386                 self.log.debug("Couldn't find project: {}, id: {}".format(project, id))
 387                 continue
 388
 389             self.log.info('project: {}, id: {}, a_mr: {}'.format(project, id, a_mr))
 390
 391             # If the MR is tagged 'never-close' ignore it
 392             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 393                 continue
 394
 395             # If the MR is no longer open, skip to the next MR,
 396             # and don't include this MR in the next check
 397             if 'opened' not in a_mr['state']:
 398                 continue
 399             else:
 400                 still_open_mrs[(project, mr_id, notify_rooms)] = True
 401
 402             msg_dict = self.mr_status_msg(a_mr, author=sender_gitlab_id)
 403             if msg_dict is None:
 404                 continue
 405
 406             msg_list.append(msg_dict)
 407
 408         self['OPEN_MRS'] = still_open_mrs
 409
 410         return {'sender': sender, 'msg_list': msg_list}
 411
 412     @arg_botcmd('rooms', type=str, help="Comma-separated room list without @conference-room suffix")
 413     @arg_botcmd('repo', type=str, help="repo to start watching for MRs in NAMESPACE/PROJECT_NAME format")
 414     def watchrepo(self, msg, repo, rooms):
 415         """
 416         Add the margebot webhook to a repo, and prepopulate any open MRs in the repo with margebot
 417         """
 418         self.log.info("msg={}".format(msg))
 419         self.log.info("repo={}".format(repo))
 420         self.log.info("rooms={}".format(rooms))
 421
 422         # get the group/repo repo, error out if it doesn't exist
 423         project = self.gitlab.getproject(repo)
 424         if not project:
 425             msg = "Couldn't find repo {}".format(repo)
 426             self.log.info("watchrepo: {}".format(msg))
 427             return msg
 428
 429         self.log.info('project: {}'.format(project))
 430
 431         target_project_id = project['id']
 432
 433         # Check is the project already includes the margebot hook
 434         # If no hooks, will it return False or [] ?
 435         marge_hook = None
 436         hooks = self.gitlab.getprojecthooks(target_project_id)
 437         self.log.error("hooks = {}".format(hooks))
 438         if hooks is False:
 439             msg = "Couldn't find {} hooks".format(repo)
 440             self.log.error("watchrepo: {}".format(msg))
 441             return msg
 442         else:
 443             for a_hook in hooks:
 444                 self.log.info('a_hook: {}'.format(a_hook))
 445                 if a_hook['merge_requests_events'] and a_hook['url'].startswith(self.webhook_url):
 446                     marge_hook = a_hook
 447                     break
 448
 449         # If so replace it (or error out ?)
 450         url = "{}{}".format(self.webhook_url, rooms)  # webhooks_url will end in '/'
 451         if marge_hook:
 452
 453             old_rooms = marge_hook['url'].split(self.webhook_url, 1)[1]
 454             if old_rooms == rooms:
 455                 msg = "Already reporting {} MRs to the {} room(s)".format(repo, rooms)
 456                 self.log.info('watchrepo: {}'.format(msg))
 457                 return msg
 458             else:
 459                 hook_updated = self.gitlab.editprojecthook_extra(target_project_id, marge_hook['id'], url, merge_requests=True, extra_data={'enable_ssl_verification': False})
 460                 s_watch_msg = "Updating room list for {} MRs from {} to {}".format(repo, old_rooms, rooms)
 461                 s_action = "update"
 462         else:
 463             hook_updated = self.gitlab.addprojecthook_extra(target_project_id, url, merge_requests=True, extra_data={'enable_ssl_verification': False})
 464             s_watch_msg = "Now watching for new MRs in the {} repo to the {} roomi(s)".format(repo, rooms)
 465             s_action = "add"
 466
 467         if not hook_updated:
 468             msg = "Couldn't {} hook: {}".format(s_action, repo)
 469             self.log.error("watchrepo: {}".format(msg))
 470             return msg
 471
 472         open_mrs = self['OPEN_MRS']
 473         mr_count = 0
 474         # get the open MRs in the repo
 475
 476         # If updating the room list, walk through the existing MR list
 477         if s_action == "update":
 478             new_open_mrs = {}
 479             for (project_id, mr_id, old_rooms) in open_mrs:
 480                 # pragma pylint: disable=simplifiable-if-statement
 481                 if project_id == target_project_id:
 482                     new_open_mrs[(project_id, mr_id, rooms)] = True
 483                 else:
 484                     new_open_mrs[(project_id, mr_id, old_rooms)] = True
 485                 # pragma pylint: enable=simplifiable-if-statement
 486             open_mrs = new_open_mrs
 487
 488         # If adding a new repo, check for existing opened/reopened MRs in the repo.
 489         else:
 490             for state in ['opened', 'reopened']:
 491                 page = 1
 492                 mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100, state=state)
 493                 while (mr_list is not False) and (mr_list != []):
 494                     for an_mr in mr_list:
 495                         mr_count += 1
 496                         self.log.info('watchrepo: an_mr WATS THE ID\n{}'.format(an_mr))
 497                         mr_id = an_mr['id']
 498                         open_mrs[(target_project_id, mr_id, rooms)] = True
 499                     # Get the next page of MRs
 500                     page += 1
 501                     mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100)
 502
 503         self['OPEN_MRS'] = open_mrs
 504
 505         if mr_count == 0:
 506             mr_msg = "No open MRs were found in the repo."
 507         elif mr_count == 1:
 508             mr_msg = "1 open MR was found in the repo.  Run !reviews to see the updated MR list."
 509         else:
 510             mr_msg = "{} open MRs were found in the repo.  Run !reviews to see the updated MR list."
 511
 512         return "{}\n{}".format(s_watch_msg, mr_msg)
 513
 514     # pragma pylint: disable=unused-argument
 515
 516     @botcmd()
 517     def xyzzy(self, msg, args):
 518         """
 519         Don't call this command...
 520         """
 521         yield "/me whispers \"All open MRs have been merged into master.\""
 522         sleep(5)
 523         yield "(just kidding)"
 524
 525     @re_botcmd(pattern=r"I blame marge(bot)?", prefixed=False, flags=re.IGNORECASE)
 526     def dont_blame_margebot(self, msg, match):
 527         """
 528         margebot is innocent.
 529         """
 530         yield u"(\u300D\uFF9F\uFF9B\uFF9F)\uFF63NOOOooooo say it ain't so."
 531
 532     @re_botcmd(pattern=r"good bot", prefixed=False, flags=re.IGNORECASE)
 533     def best_bot(self, msg, match):
 534         """
 535         margebot is the best.
 536         """
 537         yield "Best bot"
 538
 539     @re_botcmd(pattern=r"magfest", prefixed=False, flags=re.IGNORECASE)
 540     def margefest(self, msg, args):
 541         """
 542         margefest4ever
 543         """
 544         return "More like MargeFest, amirite ?"
 545
 546     @re_botcmd(pattern=r"margebot sucks", prefixed=False, flags=re.IGNORECASE)
 547     def bring_it_up_with_the_steering_committee(self, msg, args):
 548         """
 549         Bring it up with the committee
 550         """
 551         return "Bring it up with the Margebot steering committee."
 552
 553     @re_botcmd(pattern=r".*", prefixed=True)
 554     def catchall(self, msg,args):
 555        """
 556        Don't have the bot complain about unknown commands if the first word in a msg is its name
 557        """
 558        return
 559
 560
 561     # pragma pylint: enable=unused-argument