plugins/marge.py

   1 """
   2 Margebot: A Errbot Plugin for Gitlab MR reminders
   3 """
   4 import re
   5 from datetime import datetime, timezone
   6 from dateutil import parser
   7 from dateutil.tz import tzutc
   8 from dateutil.relativedelta import relativedelta
   9 from errbot import BotPlugin, arg_botcmd, botcmd, re_botcmd, webhook
  10 from errbot.templating import tenv
  11 from errcron.bot import CrontabMixin
  12 import gitlab
  13
  14
  15 def deltastr(any_delta):
  16     """
  17     Output a datetime delta in the format "x days, y hours, z minutes ago"
  18     """
  19     l_delta = []
  20     days = any_delta.days
  21     hours = any_delta.seconds // 3600
  22     mins = (any_delta.seconds // 60) % 60
  23
  24     for (key, val) in [("day", days), ("hour", hours), ("minute", mins)]:
  25         if val == 1:
  26             l_delta.append("1 " + key)
  27         elif val > 1:
  28             l_delta.append("{} {}s".format(val, key))
  29
  30     if l_delta == []:
  31         retval = "now"
  32     else:
  33         retval = ", ".join(l_delta) + " ago"
  34     return retval
  35
  36
  37 class Marge(BotPlugin, CrontabMixin):
  38     """
  39     I remind you about merge requests
  40
  41     Use:
  42        In gitlab:
  43           Add a merge request webook of the form
  44              'https://webookserver/margeboot/<rooms>'
  45           to the projects you want tracked.  <rooms> should be a
  46           comma-separated list of short room names (anything before the '@')
  47           that you want notified.
  48        In errbot:
  49           Add <roomname>@domain to the CHATROOM_PRESENCE list in config.py for
  50           rooms margebot should join
  51     """
  52
  53     CRONTAB = [
  54         # Set in config now: '0 11,17 * * * .crontab_hook'    # 7:00AM and 1:00PM EST warnings
  55     ]
  56
  57     def __init__(self, *args, **kwargs):
  58         self.git_host = None
  59         self.chatroom_host = None
  60         self.gitlab = None
  61         self.soak_delta = None
  62         self.webhook_url = None
  63         super().__init__(*args, **kwargs)
  64
  65     def get_configuration_template(self):
  66         """
  67         GITLAB_HOST:      Host name of your gitlab server
  68         GITLAB_ADMIN_TOKEN: PAT from an admin's https://${GIT_HOST}/profile/personal_access_tokens.
  69         CHATROOM_HOST: Chatroom host.  Usually 'chatroom' + FQDN of Jabber server
  70         CRONTAB: Schedule of automated merge request checks in '%M %H %d %m %w' format
  71         VERIFY_SSL : True, False, or path to CA cert to verify cert
  72         CRONTAB_SOAK_HOURS : Don't send out reminders about MRs opened less than this many hours
  73         WEBHOOK_URL :  URL to use for defining MR integration in gitlab
  74         """
  75         return {'GITLAB_HOST': 'gitlab.example.com',
  76                 'GITLAB_ADMIN_TOKEN': 'gitlab-admin-user-private-token',
  77                 'CHATROOM_HOST': 'conference.jabber.example.com',
  78                 'CRONTAB': '0 11,17 * * *',
  79                 'VERIFY_SSL': True,
  80                 'CRONTAB_SOAK_HOURS': 1,
  81                 'WEBHOOK_URL': 'https://webhooks.example.com:3142/margebot/'}
  82
  83     def check_configuration(self, configuration):
  84         """
  85         Check that the plugin has been configured properly
  86         """
  87         super().check_configuration(configuration)
  88
  89     def activate(self):
  90         """
  91         Initialization done when the plugin is activated
  92         """
  93         if not self.config:
  94             self.log.info('Margebot is not configured. Forbid activation')
  95             return
  96         self.git_host = self.config['GITLAB_HOST']
  97         self.chatroom_host = self.config['CHATROOM_HOST']
  98         Marge.CRONTAB = ['{} .crontab_hook'.format(self.config['CRONTAB'])]
  99         gitlab_auth_token = self.config['GITLAB_ADMIN_TOKEN']
 100         verify_ssl = self.config['VERIFY_SSL']
 101         self.gitlab = gitlab.Gitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl)
 102         self.activate_crontab()
 103
 104         self.soak_delta = relativedelta(hours=self.config['CRONTAB_SOAK_HOURS'])
 105         self.webhook_url = self.config['WEBHOOK_URL']
 106         if self.webhook_url[-1] != '/':
 107             self.webhook_url += '/'
 108         super().activate()
 109
 110     def deactivate(self):
 111         """
 112         Anything that needs to be tore down when the plugin is deactivated goes here.
 113         """
 114         super().deactivate()
 115
 116     @webhook('/margebot/<rooms>/')
 117     def gitlab_hook(self, request, rooms):
 118         """
 119         Webhook that listens on http://<server>:<port>/gitlab
 120         """
 121
 122         self.log.info("webhook: request: {}, rooms: {}".format(request, rooms))
 123         # self.log.info("state: {}".format(request['object_attributes']['state']))
 124
 125         # verify it's a merge request
 126         if request['object_kind'] != 'merge_request':
 127             self.log.error('unexpecting object_kind: {}'.format(request['object_kind']))
 128         elif 'opened' in request['object_attributes']['state']:
 129
 130             if request['object_attributes']['work_in_progress']:
 131                 wip = "WIP "
 132             else:
 133                 wip = ""
 134             url = request['project']['homepage']
 135             title = request['object_attributes']['title']
 136
 137             author_id = request['object_attributes']['author_id']  # map this to user name ...
 138             author = self.gitlab.getuser(author_id)
 139             if author:
 140                 author_name = author['username']
 141             else:
 142                 self.log.info("unexpected author_id {}".format(author_id))
 143                 author_name = author_id
 144
 145             target_project_id = request['object_attributes']['target_project_id']
 146             iid = request['object_attributes']['iid']
 147             mr_id = request['object_attributes']['id']
 148
 149             msg_template = "Hi there ! {} has opened a new {}MR: \"{}\"\n{}/merge_requests/{}"
 150             msg = msg_template.format(author_name, wip, title, url, iid)
 151
 152             if 'OPEN_MRS' not in self.keys():
 153                 empty_dict = {}
 154                 self['OPEN_MRS'] = empty_dict
 155
 156             open_mrs = self['OPEN_MRS']
 157
 158             if (target_project_id, mr_id, rooms) not in open_mrs:
 159                 for a_room in rooms.split(','):
 160                     if self.config:
 161                         self.send(self.build_identifier(a_room + '@' + self.chatroom_host), msg)
 162
 163                 self.log.info("webhook: Saving ({}, {}, {})".format(target_project_id, mr_id, rooms))
 164                 open_mrs[(target_project_id, mr_id, rooms)] = True
 165                 self['OPEN_MRS'] = open_mrs
 166         return "OK"
 167
 168     def mr_status_msg(self, a_mr, author=None):
 169         """
 170         Create the merge request status message
 171         """
 172         self.log.info("mr_status_msg: a_mr: {}".format(a_mr))
 173
 174         # Only weed out MRs less than the soak time for the crontab output (where author==None)
 175         now = datetime.now(timezone.utc)
 176         creation_time = parser.parse(a_mr['created_at'], tzinfos=tzutc)
 177         if not author:
 178             self.log.info("times: {}, {}, {}".format(creation_time, self.soak_delta, now))
 179             if creation_time + self.soak_delta > now:
 180                 project_id = a_mr['target_project_id']
 181                 mr_id = a_mr['id']
 182                 soak_hours = self.config['CRONTAB_SOAK_HOURS']
 183                 info_template = "skipping: MR <{},{}> was opened less than {} hours ago"
 184                 info_msg = info_template.format(project_id, mr_id, soak_hours)
 185                 self.log.info(info_msg)
 186                 return None
 187
 188         str_open_since = deltastr(now - creation_time)
 189
 190         warning = ""
 191         if a_mr['work_in_progress']:
 192             warning = "still WIP"
 193         elif a_mr['merge_status'] != 'can_be_merged':
 194             warning = "there are merge conflicts"
 195
 196         if author:
 197             authored = (a_mr['author']['id'] == author)
 198         else:
 199             authored = False
 200
 201         # getapprovals is only available in GitLab 8.9 EE or greater
 202         # (not the open source CE version)
 203         # approvals = self.gitlab.getapprovals(a_mr['id'])
 204         # also_approved = ""
 205         # for approved in approvals['approved_by']:
 206         #    also_approved += "," + approved['user']['name']
 207
 208         upvotes = a_mr['upvotes']
 209         msg = "{} (opened {})".format(a_mr['web_url'], str_open_since)
 210         if upvotes >= 2:
 211             msg += ": Has 2+ upvotes and could be merged in now"
 212             if warning != "":
 213                 msg += " except {}.".format(warning)
 214             else:
 215                 msg += "."
 216
 217         elif upvotes == 1:
 218             if authored:
 219                 msg += ": Your MR is waiting for another upvote"
 220             else:
 221                 msg += ": Waiting for another upvote"
 222             if warning != "":
 223                 msg += " but {}.".format(warning)
 224             else:
 225                 msg += "."
 226
 227         else:
 228             if authored:
 229                 msg += ": Your MR has no upvotes"
 230             else:
 231                 msg += ": No upvotes, please review"
 232             if warning != "":
 233                 msg += " but {}.".format(warning)
 234             else:
 235                 msg += "."
 236
 237         return {'creation_time': creation_time, 'msg': msg}
 238
 239     def crontab_hook(self, polled_time):
 240         """
 241         Send a scheduled message to the rooms margebot is watching
 242         about open MRs the room cares about.
 243         """
 244
 245         self.log.info("crontab_hook triggered at {}".format(polled_time))
 246
 247         reminder_msg = {}  # Map of reminder_msg['roomname@domain'] = [(created_at,msg)]
 248
 249         # initialize the reminders
 250         rooms = self.rooms()
 251         for a_room in rooms:
 252             reminder_msg[a_room.node] = []
 253
 254         still_open_mrs = {}
 255
 256         # Let's walk through the MRs we've seen already:
 257         open_mrs = self['OPEN_MRS']
 258
 259         for (project_id, mr_id, notify_rooms) in open_mrs:
 260
 261             # Lookup the MR from the project/id
 262             a_mr = self.gitlab.getmergerequest(project_id, mr_id)
 263             if not a_mr:
 264                 self.log.debug("Couldn't find project: {}, id: {}".format(project_id, mr_id))
 265                 continue
 266
 267             # If the MR is tagged 'never-close' ignore it
 268             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 269                 continue
 270
 271             self.log.info("a_mr: {} {} {} {}".format(project_id, mr_id, notify_rooms, a_mr['state']))
 272
 273             # If the MR is no longer open, skip to the next MR,
 274             # and don't include this MR in the next check
 275             if 'opened' not in a_mr['state']:
 276                 continue
 277             else:
 278                 still_open_mrs[(project_id, mr_id, notify_rooms)] = True
 279
 280             msg_dict = self.mr_status_msg(a_mr)
 281             if msg_dict is None:
 282                 continue
 283
 284             for a_room in notify_rooms.split(','):
 285                 reminder_msg[a_room].append(msg_dict)
 286
 287         # Remind each of the rooms about open MRs
 288         for a_room, room_msg_list in reminder_msg.items():
 289             if room_msg_list != []:
 290                 if self.config:
 291                     to_room = a_room + '@' + self.config['CHATROOM_HOST']
 292                     msg = tenv().get_template('reviews.md').render(msg_list=room_msg_list)
 293                     self.send(self.build_identifier(to_room), msg)
 294
 295         self['OPEN_MRS'] = still_open_mrs
 296
 297     @botcmd(template="reviews")
 298     def reviews(self, msg, args):
 299         """
 300         Returns a list of MRs that are waiting for some luv.
 301         Also returns a list of MRs that have had enough luv but aren't merged in yet.
 302         """
 303         # Sending directly to Margbot:  sender in the form sender@....
 304         # Sending to a chatroom:  snder in the form room@rooms/sender
 305
 306         log_template = 'reviews: msg: {}, args: {}, \nmsg.frm: {}, \nmsg.msg.frm: {}, chatroom_host: {}'
 307         self.log.info(log_template.format(msg, args, msg.frm.__dict__, dir(msg.frm), self.config['CHATROOM_HOST']))
 308         self.log.info('reviews: bot mode: {}'.format(self._bot.mode))
 309
 310         if self._bot.mode == "xmpp":
 311             if msg.frm.domain == self.config['CHATROOM_HOST']:
 312                 sender = msg.frm.resource
 313             else:
 314                 sender = msg.frm.node
 315         else:
 316             sender = str(msg.frm).split('@')[0]
 317
 318         keys = self.keys()
 319         if 'OPEN_MRS' not in keys:
 320             self.log.error('OPEN_MRS not in {}'.format(keys))
 321             return "No MRs to review"
 322
 323         sender_gitlab_id = None
 324         sender_users = self.gitlab.getusers(search=(('username', sender)))
 325         if not sender_users:
 326             self.log.error('problem mapping {} to a gitlab user'.format(sender))
 327             sender_gitlab_id = None
 328         else:
 329             sender_gitlab_id = sender_users[0]['id']
 330
 331         # Walk through the MRs we've seen already:
 332         msg_list = []
 333         msg = ""
 334         still_open_mrs = {}
 335         open_mrs = self['OPEN_MRS']
 336         for (project, mr_id, notify_rooms) in open_mrs:
 337
 338             # Lookup the MR from the project/id
 339             a_mr = self.gitlab.getmergerequest(project, mr_id)
 340             if not a_mr:
 341                 self.log.debug("Couldn't find project: {}, id: {}".format(project, id))
 342                 continue
 343
 344             self.log.info('project: {}, id: {}, a_mr: {}'.format(project, id, a_mr))
 345
 346             # If the MR is tagged 'never-close' ignore it
 347             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 348                 continue
 349
 350             # If the MR is no longer open, skip to the next MR,
 351             # and don't include this MR in the next check
 352             if 'opened' not in a_mr['state']:
 353                 continue
 354             else:
 355                 still_open_mrs[(project, mr_id, notify_rooms)] = True
 356
 357             msg_dict = self.mr_status_msg(a_mr, author=sender_gitlab_id)
 358             if msg_dict is None:
 359                 continue
 360
 361             msg_list.append(msg_dict)
 362
 363         self['OPEN_MRS'] = still_open_mrs
 364
 365         return {'sender': sender, 'msg_list': msg_list}
 366
 367     @arg_botcmd('rooms', type=str, help="Comma-separated room list without @conference-room suffix")
 368     @arg_botcmd('repo', type=str, help="repo to start watching for MRs in NAMESPACE/PROJECT_NAME format")
 369     def watchrepo(self, msg, repo, rooms):
 370         """
 371         Add the margebot webhook to a repo, and prepopulate any open MRs in the repo with margebot
 372         """
 373         self.log.info("msg={}".format(msg))
 374         self.log.info("repo={}".format(repo))
 375         self.log.info("rooms={}".format(rooms))
 376
 377         # get the group/repo repo, error out if it doesn't exist
 378         project = self.gitlab.getproject(repo)
 379         if not project:
 380             msg = "Couldn't find repo {}".format(repo)
 381             self.log.info("watchrepo: {}".format(msg))
 382             return msg
 383
 384         self.log.info('project: {}'.format(project))
 385
 386         target_project_id = project['id']
 387
 388         # Check is the project already includes the margebot hook
 389         # If no hooks, will it return False or [] ?
 390         marge_hook = None
 391         hooks = self.gitlab.getprojecthooks(target_project_id)
 392         self.log.error("hooks = {}".format(hooks))
 393         if hooks is False:
 394             msg = "Couldn't find {} hooks".format(repo)
 395             self.log.error("watchrepo: {}".format(msg))
 396             return msg
 397         else:
 398             for a_hook in hooks:
 399                 self.log.info('a_hook: {} {}'.format(a_hook, self.webhook_url))
 400                 if a_hook['merge_requests_events'] and a_hook['url'].startswith(self.webhook_url):
 401                     marge_hook = a_hook
 402                     break
 403
 404         # If so replace it (or error out ?)
 405         url = "{}{}".format(self.webhook_url, rooms)  # webhooks_url will end in '/'
 406         if marge_hook:
 407
 408             old_rooms = marge_hook['url'].split(self.webhook_url, 1)[1]
 409             if old_rooms == rooms:
 410                 msg = "Already reporting {} MRs to the {} room(s)".format(repo, rooms)
 411                 self.log.info('watchrepo: {}'.format(msg))
 412                 return msg
 413             else:
 414                 hook_updated = self.gitlab.editprojecthook(target_project_id, marge_hook['id'], url, merge_requests=True)
 415                 s_watch_msg = "Updating room list for {} MRs from {} to {}".format(repo, old_rooms, rooms)
 416                 s_action = "update"
 417         else:
 418             hook_updated = self.gitlab.addprojecthook(target_project_id, url, merge_requests=True)
 419             s_watch_msg = "Now watching for new MRs in the {} repo to the {} room(s)".format(repo, rooms)
 420             s_action = "add"
 421
 422         if not hook_updated:
 423             msg = "Couldn't {} hook: {}".format(s_action, repo)
 424             self.log.error("watchrepo: {}".format(msg))
 425             return msg
 426
 427         open_mrs = self['OPEN_MRS']
 428         mr_count = 0
 429         # get the open MRs in the repo
 430
 431         # If updating the room list, walk through the existing MR list
 432         if s_action == "update":
 433             new_open_mrs = {}
 434             for (project_id, mr_id, old_rooms) in open_mrs:
 435                 # pragma pylint: disable=simplifiable-if-statement
 436                 if project_id == target_project_id:
 437                     new_open_mrs[(project_id, mr_id, rooms)] = True
 438                 else:
 439                     new_open_mrs[(project_id, mr_id, old_rooms)] = True
 440                 # pragma pylint: enable=simplifiable-if-statement
 441             open_mrs = new_open_mrs
 442
 443         # If adding a new repo, check for existing opened/reopened MRs in the repo.
 444         else:
 445             for state in ['opened', 'reopened']:
 446                 page = 1
 447                 mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100, state=state)
 448                 while (mr_list is not False) and (mr_list != []):
 449                     for an_mr in mr_list:
 450                         mr_count += 1
 451                         self.log.info('watchrepo: an_mr WATS THE ID\n{}'.format(an_mr))
 452                         mr_id = an_mr['id']
 453                         open_mrs[(target_project_id, mr_id, rooms)] = True
 454                     # Get the next page of MRs
 455                     page += 1
 456                     mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100)
 457
 458         self['OPEN_MRS'] = open_mrs
 459
 460         if mr_count == 0:
 461             mr_msg = "No open MRs were found in the repo."
 462         elif mr_count == 1:
 463             mr_msg = "1 open MR was found in the repo.  Run !reviews to see the updated MR list."
 464         else:
 465             mr_msg = "{} open MRs were found in the repo.  Run !reviews to see the updated MR list."
 466
 467         return "{}\n{}".format(s_watch_msg, mr_msg)
 468
 469     # pragma pylint: disable=unused-argument
 470
 471     @re_botcmd(pattern=r"I blame marge(bot)?", prefixed=False, flags=re.IGNORECASE)
 472     def dont_blame_margebot(self, msg, match):
 473         """
 474         margebot is innocent.
 475         """
 476         yield u"(\u300D\uFF9F\uFF9B\uFF9F)\uFF63NOOOooooo say it ain't so."
 477
 478     @re_botcmd(pattern=r"good bot", prefixed=False, flags=re.IGNORECASE)
 479     def best_bot(self, msg, match):
 480         """
 481         margebot is the best.
 482         """
 483         yield "Best bot"
 484
 485     @re_botcmd(pattern=r"magfest", prefixed=False, flags=re.IGNORECASE)
 486     def margefest(self, msg, args):
 487         """
 488         margefest4ever
 489         """
 490         return "More like MargeFest, amirite ?"
 491
 492     @re_botcmd(pattern=r"margebot sucks", prefixed=False, flags=re.IGNORECASE)
 493     def margebot_sucks(self, msg, args):
 494         """
 495         Bring it up with the committee
 496         """
 497         return "Bring it up with the Margebot steering committee."
 498
 499     @re_botcmd(pattern=r".*", prefixed=True)
 500     def catchall(self, msg, args):
 501         """
 502         Don't have the bot complain about unknown commands if the first word in a msg is its name
 503         """
 504         return
 505
 506     # pragma pylint: enable=unused-argument