plugins/marge.py

   1 """
   2 Margebot: A Errbot Plugin for Gitlab MR reminders
   3 """
   4 from datetime import datetime, timezone
   5 from dateutil import parser
   6 from dateutil.tz import tzutc
   7 from dateutil.relativedelta import relativedelta
   8 from errbot import BotPlugin, botcmd, arg_botcmd, re_botcmd, webhook
   9 from errbot.templating import tenv
  10 from errcron.bot import CrontabMixin
  11 import gitlab
  12
  13
  14 def deltastr(any_delta):
  15     """
  16     Output a datetime delta in the format "x days, y hours, z minutes ago"
  17     """
  18     l_delta = []
  19     days = any_delta.days
  20     hours = any_delta.seconds // 3600
  21     mins = (any_delta.seconds // 60) % 60
  22
  23     for (key, val) in [("day", days), ("hour", hours), ("minute", mins)]:
  24         if val == 1:
  25             l_delta.append("1 " + key)
  26         elif val > 1:
  27             l_delta.append("{} {}s".format(val, key))
  28
  29     if l_delta == []:
  30         retval = "now"
  31     else:
  32         retval = ", ".join(l_delta) + " ago"
  33     return retval
  34
  35
  36 class Marge(BotPlugin, CrontabMixin):
  37     """
  38     I remind you about merge requests
  39
  40     Use:
  41        In gitlab:
  42           Add a merge request webook of the form
  43              'https://webookserver/margeboot/<rooms>'
  44           to the projects you want tracked.  <rooms> should be a
  45           comma-separated list of short room names (anything before the '@')
  46           that you want notified.
  47        In errbot:
  48           Add <roomname>@domain to the CHATROOM_PRESENCE list in config.py for
  49           rooms margebot should join
  50     """
  51
  52     CRONTAB = [
  53         # Set in config now: '0 11,17 * * * .crontab_hook'    # 7:00AM and 1:00PM EST warnings
  54     ]
  55
  56     def __init__(self, *args, **kwargs):
  57         self.git_host = None
  58         self.chatroom_host = None
  59         self.gitlab = None
  60         self.soak_delta = None
  61         self.webhook_url = None
  62         super().__init__(*args, **kwargs)
  63
  64     def get_configuration_template(self):
  65         """
  66         GITLAB_HOST:      Host name of your gitlab server
  67         GITLAB_ADMIN_TOKEN: PAT from an admin's https://${GIT_HOST}/profile/personal_access_tokens.
  68         CHATROOM_HOST: Chatroom host.  Usually 'chatroom' + FQDN of Jabber server
  69         CRONTAB: Schedule of automated merge request checks in '%M %H %d %m %w' format
  70         VERIFY_SSL : True, False, or path to CA cert to verify cert
  71         CRONTAB_SOAK_HOURS : Don't send out reminders about MRs opened less than this many hours
  72         WEBHOOK_URL :  URL to use for defining MR integration in gitlab
  73         """
  74         return {'GITLAB_HOST': 'gitlab.example.com',
  75                 'GITLAB_ADMIN_TOKEN': 'gitlab-admin-user-private-token',
  76                 'CHATROOM_HOST': 'conference.jabber.example.com',
  77                 'CRONTAB': '0 11,17 * * *',
  78                 'VERIFY_SSL': True,
  79                 'CRONTAB_SOAK_HOURS': 1,
  80                 'WEBHOOK_URL': 'https://webhooks.example.com:3142/margebot/'}
  81
  82     def check_configuration(self, configuration):
  83         """
  84         Check that the plugin has been configured properly
  85         """
  86         super().check_configuration(configuration)
  87
  88     def activate(self):
  89         """
  90         Initialization done when the plugin is activated
  91         """
  92         if not self.config:
  93             self.log.info('Margebot is not configured. Forbid activation')
  94             return
  95         self.git_host = self.config['GITLAB_HOST']
  96         self.chatroom_host = self.config['CHATROOM_HOST']
  97         Marge.CRONTAB = ['{} .crontab_hook'.format(self.config['CRONTAB'])]
  98         gitlab_auth_token = self.config['GITLAB_ADMIN_TOKEN']
  99         verify_ssl = self.config['VERIFY_SSL']
 100         self.gitlab = gitlab.Gitlab('https://' + self.git_host, gitlab_auth_token, ssl_verify=verify_ssl)
 101         self.activate_crontab()
 102
 103         self.soak_delta = relativedelta(hours=self.config['CRONTAB_SOAK_HOURS'])
 104         self.webhook_url = self.config['WEBHOOK_URL']
 105         if self.webhook_url[-1] != '/':
 106             self.webhook_url += '/'
 107         super().activate()
 108
 109     def deactivate(self):
 110         """
 111         Anything that needs to be tore down when the plugin is deactivated goes here.
 112         """
 113         super().deactivate()
 114
 115     @webhook('/margebot/<rooms>/')
 116     def gitlab_hook(self, request, rooms):
 117         """
 118         Webhook that listens on http://<server>:<port>/gitlab
 119         """
 120
 121         self.log.info("webhook: request: {}, rooms: {}".format(request, rooms))
 122         # self.log.info("state: {}".format(request['object_attributes']['state']))
 123
 124         # verify it's a merge request
 125         if request['event_type'] != 'merge_request':
 126             self.log.error('unexpecting event_type: {}'.format(request['event_type']))
 127         elif 'opened' in request['object_attributes']['state']:
 128
 129             if request['object_attributes']['work_in_progress']:
 130                 wip = "WIP "
 131             else:
 132                 wip = ""
 133
 134             url = request['project']['homepage']
 135             title = request['object_attributes']['title']
 136             author_id = request['object_attributes']['author_id']  # map this to user name ...
 137             try:
 138                 author = self.gitlab.users.get(author_id)
 139                 author_name = author.attributes['username']
 140             except Exception as exp:
 141                 self.log.info("unexpected author_id {}, exp={}".format(author_id, exp))
 142                 author_name = author_id
 143
 144             target_project_id = request['object_attributes']['target_project_id']
 145             iid = request['object_attributes']['iid']
 146
 147             # If the MR is tagged 'never-close' or 'abandoned' ignore it
 148             if 'labels' in request:
 149                 for a_label in request['labels']:
 150                     if a_label['title'] in ['never-close', 'abandoned']:
 151                         self.log.info("Skipping {} notice for {} MR".format(a_label['title'], url))
 152                         return "OK"
 153
 154             msg_template = "Hi there ! {} has opened a new {}MR: \"{}\"\n{}/merge_requests/{}"
 155             msg = msg_template.format(author_name, wip, title, url, iid)
 156
 157             if 'OPEN_MRS' not in self.keys():
 158                 empty_dict = {}
 159                 self['OPEN_MRS'] = empty_dict
 160
 161             open_mrs = self['OPEN_MRS']
 162
 163             if (target_project_id, iid, rooms) not in open_mrs:
 164                 for a_room in rooms.split(','):
 165                     if self.config:
 166                         self.send(self.build_identifier(a_room + '@' + self.chatroom_host), msg)
 167
 168                 self.log.info("webhook: Saving ({}, {}, {})".format(target_project_id, iid, rooms))
 169                 open_mrs[(target_project_id, iid, rooms)] = True
 170                 self['OPEN_MRS'] = open_mrs
 171
 172         # TODO:  Add check if an MR has toggled the WIP indicator
 173         # (trigger on updates (what's that look like in request['object_attributes']['state'])
 174         # Then check in request['changes']['title']['previous'] starts with 'WIP:'
 175         # but not request['changes']['title']['current'], and vice versa
 176         # See https://gitlab.com/gitlab-org/gitlab-ce/issues/53529
 177
 178         return "OK"
 179
 180     def mr_status_msg(self, a_mr, author=None):
 181         """
 182         Create the merge request status message
 183         """
 184         self.log.info("mr_status_msg: a_mr: {}".format(a_mr))
 185         mr_attrs = a_mr.attributes
 186
 187         # Only weed out MRs less than the soak time for the crontab output (where author==None)
 188         now = datetime.now(timezone.utc)
 189         creation_time = parser.parse(mr_attrs['created_at'], tzinfos=[tzutc()]).astimezone(timezone.utc)
 190         if not author:
 191             self.log.info("times: {}, {}, {}".format(creation_time, self.soak_delta, now))
 192             if creation_time + self.soak_delta > now:
 193                 project_id = mr_attrs['target_project_id']
 194                 mr_id = mr_attrs['id']
 195                 soak_hours = self.config['CRONTAB_SOAK_HOURS']
 196                 info_template = "skipping: MR <{},{}> was opened less than {} hours ago"
 197                 info_msg = info_template.format(project_id, mr_id, soak_hours)
 198                 self.log.info(info_msg)
 199                 return None
 200
 201         str_open_since = deltastr(now - creation_time)
 202
 203         warning = ""
 204         if mr_attrs['work_in_progress']:
 205             warning = "still WIP"
 206         elif mr_attrs['merge_status'] != 'can_be_merged':
 207             warning = "there are merge conflicts"
 208
 209         if author:
 210             authored = (mr_attrs['author']['id'] == author)
 211         else:
 212             authored = False
 213
 214         msg = "{} (opened by {} {})".format(mr_attrs['web_url'], mr_attrs['author']['username'], str_open_since)
 215         upvotes = mr_attrs['upvotes']
 216         if upvotes >= 2:
 217             msg += ": Has 2+ upvotes and could be merged in now"
 218             if warning != "":
 219                 msg += " except {}.".format(warning)
 220             else:
 221                 msg += "."
 222
 223         elif upvotes == 1:
 224             self.log.error("pre-award a_mr: {}".format(dir(a_mr.awardemojis)))
 225             award_emoji = a_mr.awardemojis.list()
 226             self.log.info("award_emoji: {}".format(award_emoji))
 227             already_approved = "someone"
 228             for an_emoji in award_emoji:
 229                 emoji_attr = an_emoji.attributes
 230                 if emoji_attr["name"] == "thumbsup":
 231                     already_approved = emoji_attr["user"]["username"]
 232
 233             if authored:
 234                 msg += ": Your MR has been approved by " + already_approved + " and is waiting for another upvote"
 235             else:
 236                 msg += ": Approved by " + already_approved + " and waiting for another upvote"
 237             if warning != "":
 238                 msg += " but {}.".format(warning)
 239             else:
 240                 msg += "."
 241
 242         else:
 243             if authored:
 244                 msg += ": Your MR has no upvotes"
 245             else:
 246                 msg += ": No upvotes, please review"
 247             if warning != "":
 248                 msg += " but {}.".format(warning)
 249             else:
 250                 msg += "."
 251
 252         return {'creation_time': creation_time, 'msg': msg}
 253
 254     def crontab_hook(self, polled_time):
 255         """
 256         Send a scheduled message to the rooms margebot is watching
 257         about open MRs the room cares about.
 258         """
 259
 260         self.log.info("crontab_hook triggered at {}".format(polled_time))
 261
 262         reminder_msg = {}  # Map of reminder_msg['roomname@domain'] = [(created_at,msg)]
 263
 264         # initialize the reminders
 265         rooms = self.rooms()
 266         for a_room in rooms:
 267             self.log.info("poller: a_room.node: {}".format(a_room.node))
 268             reminder_msg[a_room.node] = []
 269
 270         still_open_mrs = {}
 271
 272         # Let's walk through the MRs we've seen already:
 273         open_mrs = self['OPEN_MRS']
 274
 275         for (project_id, mr_id, notify_rooms) in open_mrs:
 276
 277             a_project = self.gitlab.projects.get(project_id)
 278             if not a_project:
 279                 self.log.debug("Couldn't find project: {}".format(project_id))
 280                 continue
 281
 282             # Lookup the MR from the project/id
 283             a_mr = a_project.mergerequests.get(mr_id)
 284             mr_attrs = a_mr.attributes
 285             if not a_mr:
 286                 self.log.debug("Couldn't find project: {}, id: {}".format(project_id, mr_id))
 287                 continue
 288
 289             # If the MR is tagged 'never-close' or 'abandoned', ignore it
 290             if 'labels' in mr_attrs and ('never-close' in mr_attrs['labels'] or 'abandoned' in mr_attrs['labels']):
 291                 continue
 292
 293             self.log.info("a_mr: {} {} {} {}".format(project_id, mr_id, notify_rooms, mr_attrs['state']))
 294
 295             # If the MR is no longer open, skip to the next MR,
 296             # and don't include this MR in the next check
 297             if 'opened' not in mr_attrs['state']:
 298                 continue
 299             else:
 300                 still_open_mrs[(project_id, mr_id, notify_rooms)] = True
 301
 302             msg_dict = self.mr_status_msg(a_mr)
 303             if msg_dict is None:
 304                 continue
 305
 306             for a_room in notify_rooms.split(','):
 307                 if a_room in reminder_msg:
 308                     reminder_msg[a_room].append(msg_dict)
 309                 else:
 310                     self.log.error("{} not in reminder_msg (project_id={}, mr_id={})".format(a_room, project_id, mr_id))
 311
 312         self['OPEN_MRS'] = open_mrs
 313
 314         # Remind each of the rooms about open MRs
 315         for a_room, room_msg_list in reminder_msg.items():
 316             if room_msg_list != []:
 317                 if self.config:
 318                     to_room = a_room + '@' + self.config['CHATROOM_HOST']
 319                     msg = tenv().get_template('reviews.md').render(msg_list=room_msg_list)
 320                     self.send(self.build_identifier(to_room), msg)
 321
 322         self['OPEN_MRS'] = still_open_mrs
 323
 324     @botcmd(template="reviews")
 325     def reviews(self, msg, args):
 326         """
 327         Returns a list of MRs that are waiting for some luv.
 328         Also returns a list of MRs that have had enough luv but aren't merged in yet.
 329         """
 330         # Sending directly to Margbot:  sender in the form sender@....
 331         # Sending to a chatroom:  snder in the form room@rooms/sender
 332
 333         log_template = 'reviews: msg: {}, args: {}, \nmsg.frm: {}, \nmsg.msg.frm: {}, chatroom_host: {}'
 334         self.log.info(log_template.format(msg, args, msg.frm.__dict__, dir(msg.frm), self.config['CHATROOM_HOST']))
 335         self.log.info('reviews: bot mode: {}'.format(self._bot.mode))
 336
 337         if self._bot.mode == "xmpp":
 338             if msg.frm.domain == self.config['CHATROOM_HOST']:
 339                 sender = msg.frm.resource
 340             else:
 341                 sender = msg.frm.node
 342         else:
 343             sender = str(msg.frm).split('@')[0]
 344
 345         keys = self.keys()
 346         if 'OPEN_MRS' not in keys:
 347             self.log.error('OPEN_MRS not in {}'.format(keys))
 348             return "No MRs to review"
 349
 350         sender_gitlab_id = None
 351         try:
 352             sender_users = self.gitlab.users.list(username=sender)
 353             sender_gitlab_id = sender_users[0].attributes['id']
 354         except Exception as exp:
 355             self.log.error('problem mapping {} to a gitlab user, exp={}'.format(sender, exp))
 356             sender_gitlab_id = None
 357
 358         # Walk through the MRs we've seen already:
 359         msg_list = []
 360         msg = ""
 361         still_open_mrs = {}
 362         open_mrs = self['OPEN_MRS']
 363         self.log.info('open_mrs: {}'.format(open_mrs))
 364         for (project, mr_id, notify_rooms) in open_mrs:
 365
 366             try:
 367                 a_project = self.gitlab.projects.get(project)
 368             except Exception as exp:
 369                 self.log.debug("Couldn't find project: {}, exp: {}".format(project, exp))
 370                 continue
 371
 372             # Lookup the MR from the project/id
 373             try:
 374                 a_mr = a_project.mergerequests.get(mr_id)
 375                 mr_attrs = a_mr.attributes
 376             except Exception as exp:
 377                 self.log.debug("Couldn't find project: {}, id: {}, exp: {}".format(project, mr_id, exp))
 378                 continue
 379
 380             self.log.info('project: {}, id: {}, a_mr: {}'.format(project, id, a_mr))
 381
 382             # If the MR is tagged 'never-close' or 'abandoned',  ignore it
 383             if 'labels' in mr_attrs and ('never-close' in mr_attrs['labels'] or 'abandoned' in mr_attrs['labels']):
 384                 continue
 385
 386             # If the MR is no longer open, skip to the next MR,
 387             # and don't include this MR in the next check
 388             if 'opened' not in mr_attrs['state']:
 389                 self.log.info('state not opened: {}'.format(mr_attrs['state']))
 390                 continue
 391             else:
 392                 still_open_mrs[(project, mr_id, notify_rooms)] = True
 393
 394             msg_dict = self.mr_status_msg(a_mr, author=sender_gitlab_id)
 395             if msg_dict is None:
 396                 continue
 397
 398             msg_list.append(msg_dict)
 399
 400         self['OPEN_MRS'] = still_open_mrs
 401
 402         return {'sender': sender, 'msg_list': msg_list}
 403
 404     @arg_botcmd('rooms', type=str, help="Comma-separated room list without @conference-room suffix")
 405     @arg_botcmd('repo', type=str, help="repo to start watching for MRs in NAMESPACE/PROJECT_NAME format")
 406     def watchrepo(self, msg, repo, rooms):
 407         """
 408         Add the margebot webhook to a repo, and prepopulate any open MRs in the repo with margebot
 409         """
 410         self.log.info("msg={}".format(msg))
 411         self.log.info("repo={}".format(repo))
 412         self.log.info("rooms={}".format(rooms))
 413
 414         # get the group/repo repo, error out if it doesn't exist
 415         project = self.gitlab.projects.get(repo)
 416         if not project:
 417             msg = "Couldn't find repo {}".format(repo)
 418             self.log.info("watchrepo: {}".format(msg))
 419             return msg
 420
 421         self.log.info('project: {}'.format(project))
 422
 423         target_project_id = project.id
 424
 425         # Check is the project already includes the margebot hook
 426         # If no hooks, will it return False or [] ?
 427         marge_hook = None
 428         hooks = project.hooks.list()
 429         self.log.error("hooks = {}".format(hooks))
 430         if hooks is False:
 431             msg = "Couldn't find {} hooks".format(repo)
 432             self.log.error("watchrepo: {}".format(msg))
 433             return msg
 434         for a_hook in hooks:
 435             self.log.info('a_hook: {}'.format(a_hook))
 436             hook_attributes = a_hook.attributes
 437             if hook_attributes['merge_requests_events'] and hook_attributes['url'].startswith(self.webhook_url):
 438                 marge_hook = a_hook
 439                 break
 440
 441         # If so replace it (or error out ?)
 442         url = "{}{}".format(self.webhook_url, rooms)  # webhooks_url will end in '/'
 443         hook_updated = True
 444         if marge_hook:
 445             old_rooms = marge_hook.attributes['url'].split(self.webhook_url, 1)[1]
 446             if old_rooms == rooms:
 447                 msg = "Already reporting {} MRs to the {} room(s)".format(repo, rooms)
 448                 self.log.info('watchrepo: {}'.format(msg))
 449                 return msg
 450             try:
 451                 s_action = "update"
 452                 marge_hook.attributes['url'] = url
 453                 marge_hook.save()
 454                 s_watch_msg = "Updating room list for {} MRs from {} to {}".format(repo, old_rooms, rooms)
 455             except Exception as exp:
 456                 hook_updated = False
 457                 self.log.error("watchrepo; update hook {}  raised exception {}".format(repo, exp))
 458         else:
 459             try:
 460                 s_action = "add"
 461                 project.hooks.create({'url': url, 'merge_requests_events': 1, 'enable_ssl_verification': True})
 462                 s_watch_msg = "Now watching for new MRs in the {} repo to the {} room(s)".format(repo, rooms)
 463             except Exception as exp:
 464                 hook_updated = False
 465                 self.log.error("watchrepo; create hook {}  raised exception {}".format(repo, exp))
 466
 467         if not hook_updated:
 468             msg = "Couldn't {} hook: {}".format(s_action, repo)
 469             self.log.error("watchrepo: {}".format(msg))
 470             return msg
 471
 472         open_mrs = self['OPEN_MRS']
 473         mr_count = 0
 474         # get the open MRs in the repo
 475
 476         # If updating the room list, walk through the existing MR list
 477         if s_action == "update":
 478             new_open_mrs = {}
 479             for (project_id, mr_id, old_rooms) in open_mrs:
 480                 # pragma pylint: disable=simplifiable-if-statement
 481                 if project_id == target_project_id:
 482                     new_open_mrs[(project_id, mr_id, rooms)] = True
 483                 else:
 484                     new_open_mrs[(project_id, mr_id, old_rooms)] = True
 485                 # pragma pylint: enable=simplifiable-if-statement
 486             open_mrs = new_open_mrs
 487
 488         # If adding a new repo, check for existing opened MRs in the repo.
 489         else:
 490             for state in ['opened']:
 491                 a_project = self.gitlab.projects.get(target_project_id)
 492                 mr_list = a_project.mergerequests.list(state=state)
 493                 for an_mr in mr_list:
 494                     mr_count += 1
 495                     mr_id = an_mr.attributes['iid']
 496                     open_mrs[(target_project_id, mr_id, rooms)] = True
 497
 498         self['OPEN_MRS'] = open_mrs
 499
 500         if mr_count == 0:
 501             mr_msg = "No open MRs were found in the repo."
 502         elif mr_count == 1:
 503             mr_msg = "1 open MR was found in the repo.  Run !reviews to see the updated MR list."
 504         else:
 505             mr_msg = "{} open MRs were found in the repo.  Run !reviews to see the updated MR list.".format(mr_count)
 506         return "{}\n{}".format(s_watch_msg, mr_msg)
 507
 508     # pragma pylint: disable=unused-argument
 509
 510     # Check Chucklebot for the chuckles
 511
 512     @re_botcmd(pattern=r".*", prefixed=True)
 513     def catchall(self, msg, args):
 514         """
 515         Don't have the bot complain about unknown commands if the first word in a msg is its name
 516         """
 517         return
 518
 519     # pragma pylint: enable=unused-argument