plugins/marge.py

   1 """
   2 Margebot: A Errbot Plugin for Gitlab MR reminders
   3 """
   4 import re
   5 from datetime import datetime, timezone
   6 from time import sleep
   7 from dateutil import parser
   8 from dateutil.tz import tzutc
   9 from dateutil.relativedelta import relativedelta
  10 from errbot import BotPlugin, botcmd, arg_botcmd, re_botcmd, webhook
  11 from errbot.templating import tenv
  12 from errcron.bot import CrontabMixin
  13 import gitlab
  14 import requests
  15
  16 class MargeGitlab(gitlab.Gitlab):
  17     """
  18     Subclass gitlab.Gitlab so extra_data args can be added
  19     to the addprojecthook() and editprojecthook() methods
  20     """
  21
  22     def __init__(self, host, token="", oauth_token="", verify_ssl=True):
  23         super().__init__(host, token, oauth_token, verify_ssl)
  24
  25     def addprojecthook_extra(self, project_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
  26         """
  27         A copy parent addprojecthook with an extra_data field
  28         """
  29         data = {"id": project_id, "url": url}
  30         if extra_data:
  31             for ed_key, ed_value in extra_data.items():
  32                 data[ed_key] = ed_value
  33         data['push_events'] = int(bool(push))
  34         data['issues_events'] = int(bool(issues))
  35         data['merge_requests_events'] = int(bool(merge_requests))
  36         data['tag_push_events'] = int(bool(tag_push))
  37         request = requests.post("{0}/{1}/hooks".format(self.projects_url, project_id),
  38                                 headers=self.headers, data=data, verify=self.verify_ssl)
  39         if request.status_code == 201:
  40             return request.json()
  41         return False
  42
  43     def editprojecthook_extra(self, project_id, hook_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
  44         """
  45         A copy of the parent editprojecthook with an extra_data field
  46         """
  47         data = {"id": project_id, "hook_id": hook_id, "url": url}
  48         if extra_data:
  49             for ed_key, ed_value in extra_data.items():
  50                 data[ed_key] = ed_value
  51         data['push_events'] = int(bool(push))
  52         data['issues_events'] = int(bool(issues))
  53         data['merge_requests_events'] = int(bool(merge_requests))
  54         data['tag_push_events'] = int(bool(tag_push))
  55         request = requests.put("{0}/{1}/hooks/{2}".format(self.projects_url, project_id, hook_id),
  56                                headers=self.headers, data=data, verify=self.verify_ssl)
  57         return request.status_code == 200
  58
  59
  60 def deltastr(any_delta):
  61     """
  62     Output a datetime delta in the format "x days, y hours, z minutes ago"
  63     """
  64     l_delta = []
  65     days = any_delta.days
  66     hours = any_delta.seconds // 3600
  67     mins = (any_delta.seconds // 60) % 60
  68
  69     for (key, val) in [("day", days), ("hour", hours), ("minute", mins)]:
  70         if val == 1:
  71             l_delta.append("1 " + key)
  72         elif val > 1:
  73             l_delta.append("{} {}s".format(val, key))
  74
  75     if l_delta == []:
  76         retval = "now"
  77     else:
  78         retval = ", ".join(l_delta) + " ago"
  79     return retval
  80
  81
  82 class Marge(BotPlugin, CrontabMixin):
  83     """
  84     I remind you about merge requests
  85
  86     Use:
  87        In gitlab:
  88           Add a merge request webook of the form
  89              'https://webookserver/margeboot/<rooms>'
  90           to the projects you want tracked.  <rooms> should be a
  91           comma-separated list of short room names (anything before the '@')
  92           that you want notified.
  93        In errbot:
  94           Add <roomname>@domain to the CHATROOM_PRESENCE list in config.py for
  95           rooms margebot should join
  96     """
  97
  98     CRONTAB = [
  99         # Set in config now: '0 11,17 * * * .crontab_hook'    # 7:00AM and 1:00PM EST warnings
 100     ]
 101
 102     def __init__(self, *args, **kwargs):
 103         self.git_host = None
 104         self.chatroom_host = None
 105         self.gitlab = None
 106         self.soak_delta = None
 107         super().__init__(*args, **kwargs)
 108
 109     def get_configuration_template(self):
 110         """
 111         GITLAB_HOST:      Host name of your gitlab server
 112         GITLAB_ADMIN_TOKEN: PAT from an admin's https://${GIT_HOST}/profile/personal_access_tokens.
 113         CHATROOM_HOST: Chatroom host.  Usually 'chatroom' + FQDN of Jabber server
 114         CRONTAB: Schedule of automated merge request checks in '%M %H %d %m %w' format
 115         VERIFY_SSL : True, False, or path to CA cert to verify cert
 116         CRONTAB_SOAK_HOURS : Don't send out reminders about MRs opened less than this many hours
 117         WEBHOOK_URL :  URL to use for defining MR integration in gitlab
 118         """
 119         return {'GITLAB_HOST': 'gitlab.example.com',
 120                 'GITLAB_ADMIN_TOKEN': 'gitlab-admin-user-private-token',
 121                 'CHATROOM_HOST': 'conference.jabber.example.com',
 122                 'CRONTAB': '0 11,17 * * *',
 123                 'VERIFY_SSL': True,
 124                 'CRONTAB_SOAK_HOURS': 1,
 125                 'WEBHOOK_URL': 'https://webhooks.example.com:3142/margebot/'}
 126
 127     def check_configuration(self, configuration):
 128         """
 129         Check that the plugin has been configured properly
 130         """
 131         super().check_configuration(configuration)
 132
 133     def activate(self):
 134         """
 135         Initialization done when the plugin is activated
 136         """
 137         if not self.config:
 138             self.log.info('Margebot is not configured. Forbid activation')
 139             return
 140         self.git_host = self.config['GITLAB_HOST']
 141         self.chatroom_host = self.config['CHATROOM_HOST']
 142         Marge.CRONTAB = ['{} .crontab_hook'.format(self.config['CRONTAB'])]
 143         gitlab_auth_token = self.config['GITLAB_ADMIN_TOKEN']
 144         verify_ssl = self.config['VERIFY_SSL']
 145         self.gitlab = MargeGitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl)
 146         self.activate_crontab()
 147
 148         self.soak_delta = relativedelta(hours=self.config['CRONTAB_SOAK_HOURS'])
 149         self.webhook_url = self.config['WEBHOOK_URL']
 150         if self.webhook_url[-1] != '/':
 151             self.webhook_url += '/'
 152         super().activate()
 153
 154     def deactivate(self):
 155         """
 156         Anything that needs to be tore down when the plugin is deactivated goes here.
 157         """
 158         super().deactivate()
 159
 160     @webhook('/margebot/<rooms>/')
 161     def gitlab_hook(self, request, rooms):
 162         """
 163         Webhook that listens on http://<server>:<port>/gitlab
 164         """
 165
 166         self.log.info("webhook: request: {}, rooms: {}".format(request, rooms))
 167         # self.log.info("state: {}".format(request['object_attributes']['state']))
 168
 169         # verify it's a merge request
 170         if request['object_kind'] != 'merge_request':
 171             self.log.error('unexpecting object_kind: {}'.format(request['object_kind']))
 172         elif 'opened' in request['object_attributes']['state']:
 173
 174             if request['object_attributes']['work_in_progress']:
 175                 wip = "WIP "
 176             else:
 177                 wip = ""
 178             url = request['project']['homepage']
 179             title = request['object_attributes']['title']
 180
 181             author_id = request['object_attributes']['author_id']  # map this to user name ...
 182             author = self.gitlab.getuser(author_id)
 183             if author:
 184                 author_name = author['username']
 185             else:
 186                 self.log.info("unexpected author_id {}".format(author_id))
 187                 author_name = author_id
 188
 189             target_project_id = request['object_attributes']['target_project_id']
 190             iid = request['object_attributes']['iid']
 191             mr_id = request['object_attributes']['id']
 192
 193             msg_template = "Hi there ! {} has opened a new {}MR: \"{}\"\n{}/merge_requests/{}"
 194             msg = msg_template.format(author_name, wip, title, url, iid)
 195
 196             if 'OPEN_MRS' not in self.keys():
 197                 empty_dict = {}
 198                 self['OPEN_MRS'] = empty_dict
 199
 200             open_mrs = self['OPEN_MRS']
 201
 202             if (target_project_id, mr_id, rooms) not in open_mrs:
 203                 for a_room in rooms.split(','):
 204                     if self.config:
 205                         self.send(self.build_identifier(a_room + '@' + self.chatroom_host), msg)
 206
 207                 self.log.info("webhook: Saving ({}, {}, {})".format(target_project_id, mr_id, rooms))
 208                 open_mrs[(target_project_id, mr_id, rooms)] = True
 209                 self['OPEN_MRS'] = open_mrs
 210         return "OK"
 211
 212     def mr_status_msg(self, a_mr, author=None):
 213         """
 214         Create the merge request status message
 215         """
 216         self.log.info("mr_status_msg: a_mr: {}".format(a_mr))
 217
 218         # Only weed out MRs less than the soak time for the crontab output (where author==None)
 219         now = datetime.now(timezone.utc)
 220         creation_time = parser.parse(a_mr['created_at'], tzinfos=tzutc)
 221         if not author:
 222             self.log.info("times: {}, {}, {}".format(creation_time, self.soak_delta, now))
 223             if creation_time + self.soak_delta > now:
 224                 project_id = a_mr['target_project_id']
 225                 mr_id = a_mr['id']
 226                 soak_hours = self.config['CRONTAB_SOAK_HOURS']
 227                 info_template = "skipping: MR <{},{}> was opened less than {} hours ago"
 228                 info_msg = info_template.format(project_id, mr_id, soak_hours)
 229                 self.log.info(info_msg)
 230                 return None
 231
 232         str_open_since = deltastr(now - creation_time)
 233
 234         warning = ""
 235         if a_mr['work_in_progress']:
 236             warning = "still WIP"
 237         elif a_mr['merge_status'] != 'can_be_merged':
 238             warning = "there are merge conflicts"
 239
 240         if author:
 241             authored = (a_mr['author']['id'] == author)
 242         else:
 243             authored = False
 244
 245         # getapprovals is only available in GitLab 8.9 EE or greater
 246         # (not the open source CE version)
 247         # approvals = self.gitlab.getapprovals(a_mr['id'])
 248         # also_approved = ""
 249         # for approved in approvals['approved_by']:
 250         #    also_approved += "," + approved['user']['name']
 251
 252         upvotes = a_mr['upvotes']
 253         msg = "{} (opened {})".format(a_mr['web_url'], str_open_since)
 254         if upvotes >= 2:
 255             msg += ": Has 2+ upvotes and could be merged in now"
 256             if warning != "":
 257                 msg += " except {}.".format(warning)
 258             else:
 259                 msg += "."
 260
 261         elif upvotes == 1:
 262             if authored:
 263                 msg += ": Your MR is waiting for another upvote"
 264             else:
 265                 msg += ": Waiting for another upvote"
 266             if warning != "":
 267                 msg += " but {}.".format(warning)
 268             else:
 269                 msg += "."
 270
 271         else:
 272             if authored:
 273                 msg += ": Your MR has no upvotes"
 274             else:
 275                 msg += ": No upvotes, please review"
 276             if warning != "":
 277                 msg += " but {}.".format(warning)
 278             else:
 279                 msg += "."
 280
 281         return {'creation_time': creation_time, 'msg': msg}
 282
 283     def crontab_hook(self, polled_time):
 284         """
 285         Send a scheduled message to the rooms margebot is watching
 286         about open MRs the room cares about.
 287         """
 288
 289         self.log.info("crontab_hook triggered at {}".format(polled_time))
 290
 291         reminder_msg = {}  # Map of reminder_msg['roomname@domain'] = [(created_at,msg)]
 292
 293         # initialize the reminders
 294         rooms = self.rooms()
 295         for a_room in rooms:
 296             reminder_msg[a_room.node] = []
 297
 298         still_open_mrs = {}
 299
 300         # Let's walk through the MRs we've seen already:
 301         open_mrs = self['OPEN_MRS']
 302
 303         for (project_id, mr_id, notify_rooms) in open_mrs:
 304
 305             # Lookup the MR from the project/id
 306             a_mr = self.gitlab.getmergerequest(project_id, mr_id)
 307             if not a_mr:
 308                 self.log.debug("Couldn't find project: {}, id: {}".format(project_id, mr_id))
 309                 continue
 310
 311             # If the MR is tagged 'never-close' ignore it
 312             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 313                 continue
 314
 315             self.log.info("a_mr: {} {} {} {}".format(project_id, mr_id, notify_rooms, a_mr['state']))
 316
 317             # If the MR is no longer open, skip to the next MR,
 318             # and don't include this MR in the next check
 319             if 'opened' not in a_mr['state']:
 320                 continue
 321             else:
 322                 still_open_mrs[(project_id, mr_id, notify_rooms)] = True
 323
 324             msg_dict = self.mr_status_msg(a_mr)
 325             if msg_dict is None:
 326                 continue
 327
 328             for a_room in notify_rooms.split(','):
 329                 reminder_msg[a_room].append(msg_dict)
 330
 331         self['OPEN_MRS'] = open_mrs
 332
 333         # Remind each of the rooms about open MRs
 334         for a_room, room_msg_list in reminder_msg.items():
 335             if room_msg_list != []:
 336                 if self.config:
 337                     to_room = a_room + '@' + self.config['CHATROOM_HOST']
 338                     msg = tenv().get_template('reviews.md').render(msg_list=room_msg_list)
 339                     self.send(self.build_identifier(to_room), msg)
 340
 341         self['OPEN_MRS'] = still_open_mrs
 342
 343     @botcmd(template="reviews")
 344     def reviews(self, msg, args):
 345         """
 346         Returns a list of MRs that are waiting for some luv.
 347         Also returns a list of MRs that have had enough luv but aren't merged in yet.
 348         """
 349         # Sending directly to Margbot:  sender in the form sender@....
 350         # Sending to a chatroom:  snder in the form room@rooms/sender
 351
 352         log_template = 'reviews: msg: {}, args: {}, \nmsg.frm: {}, \nmsg.msg.frm: {}, chatroom_host: {}'
 353         self.log.info(log_template.format(msg, args, msg.frm.__dict__, dir(msg.frm), self.config['CHATROOM_HOST']))
 354         self.log.info('reviews: bot mode: {}'.format(self._bot.mode))
 355
 356         if self._bot.mode == "xmpp":
 357             if msg.frm.domain == self.config['CHATROOM_HOST']:
 358                 sender = msg.frm.resource
 359             else:
 360                 sender = msg.frm.node
 361         else:
 362             sender = str(msg.frm).split('@')[0]
 363
 364         keys = self.keys()
 365         if 'OPEN_MRS' not in keys:
 366             self.log.error('OPEN_MRS not in {}'.format(keys))
 367             return "No MRs to review"
 368
 369         sender_gitlab_id = None
 370         sender_users = self.gitlab.getusers(search=(('username', sender)))
 371         if not sender_users:
 372             self.log.error('problem mapping {} to a gitlab user'.format(sender))
 373             sender_gitlab_id = None
 374         else:
 375             sender_gitlab_id = sender_users[0]['id']
 376
 377         # Walk through the MRs we've seen already:
 378         msg_list = []
 379         msg = ""
 380         still_open_mrs = {}
 381         open_mrs = self['OPEN_MRS']
 382         for (project, mr_id, notify_rooms) in open_mrs:
 383
 384             # Lookup the MR from the project/id
 385             a_mr = self.gitlab.getmergerequest(project, mr_id)
 386             if not a_mr:
 387                 self.log.debug("Couldn't find project: {}, id: {}".format(project, id))
 388                 continue
 389
 390             self.log.info('project: {}, id: {}, a_mr: {}'.format(project, id, a_mr))
 391
 392             # If the MR is tagged 'never-close' ignore it
 393             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 394                 continue
 395
 396             # If the MR is no longer open, skip to the next MR,
 397             # and don't include this MR in the next check
 398             if 'opened' not in a_mr['state']:
 399                 continue
 400             else:
 401                 still_open_mrs[(project, mr_id, notify_rooms)] = True
 402
 403             msg_dict = self.mr_status_msg(a_mr, author=sender_gitlab_id)
 404             if msg_dict is None:
 405                 continue
 406
 407             msg_list.append(msg_dict)
 408
 409         self['OPEN_MRS'] = still_open_mrs
 410
 411         return {'sender': sender, 'msg_list': msg_list}
 412
 413     @arg_botcmd('rooms', type=str, help="Comma-separated room list without @conference-room suffix")
 414     @arg_botcmd('repo', type=str, help="repo to start watching for MRs in NAMESPACE/PROJECT_NAME format")
 415     def watchrepo(self, msg, repo, rooms):
 416         """
 417         Add the margebot webhook to a repo, and prepopulate any open MRs in the repo with margebot
 418         """
 419         self.log.info("msg={}".format(msg))
 420         self.log.info("repo={}".format(repo))
 421         self.log.info("rooms={}".format(rooms))
 422
 423         # get the group/repo repo, error out if it doesn't exist
 424         project = self.gitlab.getproject(repo)
 425         if not project:
 426             msg = "Couldn't find repo {}".format(repo)
 427             self.log.info("watchrepo: {}".format(msg))
 428             return msg
 429
 430         self.log.info('project: {}'.format(project))
 431
 432         target_project_id = project['id']
 433
 434         # Check is the project already includes the margebot hook
 435         # If no hooks, will it return False or [] ?
 436         marge_hook = None
 437         hooks = self.gitlab.getprojecthooks(target_project_id)
 438         self.log.error("hooks = {}".format(hooks))
 439         if hooks is False:
 440             msg = "Couldn't find {} hooks".format(repo)
 441             self.log.error("watchrepo: {}".format(msg))
 442             return msg
 443         else:
 444             for a_hook in hooks:
 445                 self.log.info('a_hook: {}'.format(a_hook))
 446                 if a_hook['merge_requests_events'] and a_hook['url'].startswith(self.webhook_url):
 447                     marge_hook = a_hook
 448                     break
 449
 450         # If so replace it (or error out ?)
 451         url = "{}{}".format(self.webhook_url, rooms)  # webhooks_url will end in '/'
 452         if marge_hook:
 453
 454             old_rooms = marge_hook['url'].split(self.webhook_url, 1)[1]
 455             if old_rooms == rooms:
 456                 msg = "Already reporting {} MRs to the {} room(s)".format(repo, rooms)
 457                 self.log.info('watchrepo: {}'.format(msg))
 458                 return msg
 459             else:
 460                 hook_updated = self.gitlab.editprojecthook_extra(target_project_id, marge_hook['id'], url, merge_requests=True, extra_data={'enable_ssl_verification': False})
 461                 s_watch_msg = "Updating room list for {} MRs from {} to {}".format(repo, old_rooms, rooms)
 462                 s_action = "update"
 463         else:
 464             hook_updated = self.gitlab.addprojecthook_extra(target_project_id, url, merge_requests=True, extra_data={'enable_ssl_verification': False})
 465             s_watch_msg = "Now watching for new MRs in the {} repo to the {} roomi(s)".format(repo, rooms)
 466             s_action = "add"
 467
 468         if not hook_updated:
 469             msg = "Couldn't {} hook: {}".format(s_action, repo)
 470             self.log.error("watchrepo: {}".format(msg))
 471             return msg
 472
 473         open_mrs = self['OPEN_MRS']
 474         mr_count = 0
 475         # get the open MRs in the repo
 476
 477         # If updating the room list, walk through the existing MR list
 478         if s_action == "update":
 479             new_open_mrs = {}
 480             for (project_id, mr_id, old_rooms) in open_mrs:
 481                 # pragma pylint: disable=simplifiable-if-statement
 482                 if project_id == target_project_id:
 483                     new_open_mrs[(project_id, mr_id, rooms)] = True
 484                 else:
 485                     new_open_mrs[(project_id, mr_id, old_rooms)] = True
 486                 # pragma pylint: enable=simplifiable-if-statement
 487             open_mrs = new_open_mrs
 488
 489         # If adding a new repo, check for existing opened/reopened MRs in the repo.
 490         else:
 491             for state in ['opened', 'reopened']:
 492                 page = 1
 493                 mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100, state=state)
 494                 while (mr_list is not False) and (mr_list != []):
 495                     for an_mr in mr_list:
 496                         mr_count += 1
 497                         self.log.info('watchrepo: an_mr WATS THE ID\n{}'.format(an_mr))
 498                         mr_id = an_mr['id']
 499                         open_mrs[(target_project_id, mr_id, rooms)] = True
 500                     # Get the next page of MRs
 501                     page += 1
 502                     mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100)
 503
 504         self['OPEN_MRS'] = open_mrs
 505
 506         if mr_count == 0:
 507             mr_msg = "No open MRs were found in the repo."
 508         elif mr_count == 1:
 509             mr_msg = "1 open MR was found in the repo.  Run !reviews to see the updated MR list."
 510         else:
 511             mr_msg = "{} open MRs were found in the repo.  Run !reviews to see the updated MR list."
 512
 513         return "{}\n{}".format(s_watch_msg, mr_msg)
 514
 515     # pragma pylint: disable=unused-argument
 516
 517     @botcmd()
 518     def xyzzy(self, msg, args):
 519         """
 520         Don't call this command...
 521         """
 522         yield "/me whispers \"All open MRs have been merged into master.\""
 523         sleep(5)
 524         yield "(just kidding)"
 525
 526     @re_botcmd(pattern=r"I blame marge(bot)?", prefixed=False, flags=re.IGNORECASE)
 527     def dont_blame_margebot(self, msg, match):
 528         """
 529         margebot is innocent.
 530         """
 531         yield u"(\u300D\uFF9F\uFF9B\uFF9F)\uFF63NOOOooooo say it ain't so."
 532
 533     @re_botcmd(pattern=r"good bot", prefixed=False, flags=re.IGNORECASE)
 534     def best_bot(self, msg, match):
 535         """
 536         margebot is the best.
 537         """
 538         yield "Best bot"
 539
 540     @re_botcmd(pattern=r"magfest", prefixed=False, flags=re.IGNORECASE)
 541     def margefest(self, msg, args):
 542         """
 543         margefest4ever
 544         """
 545         return "More like MargeFest, amirite ?"
 546
 547     @re_botcmd(pattern=r"margebot sucks", prefixed=False, flags=re.IGNORECASE)
 548     def bring_it_up_with_the_steering_committee(self, msg, args):
 549         """
 550         Bring it up with the committee
 551         """
 552         return "Bring it up with the Margebot steering committee."
 553
 554     @re_botcmd(pattern=r".*", prefixed=True)
 555     def catchall(self, msg,args):
 556        """
 557        Don't have the bot complain about unknown commands if the first word in a msg is its name
 558        """
 559        return
 560
 561
 562     # pragma pylint: enable=unused-argument