plugins/marge.py

   1 """
   2 Margebot: A Errbot Plugin for Gitlab MR reminders
   3 """
   4 from datetime import datetime, timezone
   5 from dateutil import parser
   6 from dateutil.tz import tzutc
   7 from dateutil.relativedelta import relativedelta
   8 from errbot import BotPlugin, botcmd, arg_botcmd, re_botcmd, webhook
   9 from errbot.templating import tenv
  10 from errcron.bot import CrontabMixin
  11 import gitlab
  12 import requests
  13
  14
  15 def addprojecthook_extra(self, project_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
  16     """
  17     A copy parent addprojecthook with an extra_data field
  18     """
  19     data = {"id": project_id, "url": url}
  20     if extra_data:
  21         for ed_key, ed_value in extra_data.items():
  22             data[ed_key] = ed_value
  23     data['push_events'] = int(bool(push))
  24     data['issues_events'] = int(bool(issues))
  25     data['merge_requests_events'] = int(bool(merge_requests))
  26     data['tag_push_events'] = int(bool(tag_push))
  27     request = requests.post("{0}/{1}/hooks".format(self.projects_url, project_id),
  28                             headers=self.headers, data=data, verify=self.verify_ssl)
  29     if request.status_code == 201:
  30         return request.json()
  31     return False
  32 gitlab.Gitlab.addprojecthook_extra = addprojecthook_extra
  33
  34
  35 def editprojecthook_extra(self, project_id, hook_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
  36     """
  37     A copy of the parent editprojecthook with an extra_data field
  38     """
  39     data = {"id": project_id, "hook_id": hook_id, "url": url}
  40     if extra_data:
  41         for ed_key, ed_value in extra_data.items():
  42             data[ed_key] = ed_value
  43     data['push_events'] = int(bool(push))
  44     data['issues_events'] = int(bool(issues))
  45     data['merge_requests_events'] = int(bool(merge_requests))
  46     data['tag_push_events'] = int(bool(tag_push))
  47     request = requests.put("{0}/{1}/hooks/{2}".format(self.projects_url, project_id, hook_id),
  48                            headers=self.headers, data=data, verify=self.verify_ssl)
  49     return request.status_code == 200
  50 gitlab.Gitlab.editprojecthook_extra = editprojecthook_extra
  51
  52
  53 def getawardemoji(self, project_id, mr_iid):
  54     """
  55     Give me the thumbs
  56     """
  57     url = "{0}/{1}/merge_requests/{2}/award_emoji".format(self.projects_url, project_id, mr_iid)
  58     request = requests.get(url, headers=self.headers, verify=self.verify_ssl)
  59     if request.status_code == 200:
  60         return request.json()
  61     return False
  62 gitlab.Gitlab.getawardemoji = getawardemoji
  63
  64
  65 def deltastr(any_delta):
  66     """
  67     Output a datetime delta in the format "x days, y hours, z minutes ago"
  68     """
  69     l_delta = []
  70     days = any_delta.days
  71     hours = any_delta.seconds // 3600
  72     mins = (any_delta.seconds // 60) % 60
  73
  74     for (key, val) in [("day", days), ("hour", hours), ("minute", mins)]:
  75         if val == 1:
  76             l_delta.append("1 " + key)
  77         elif val > 1:
  78             l_delta.append("{} {}s".format(val, key))
  79
  80     if l_delta == []:
  81         retval = "now"
  82     else:
  83         retval = ", ".join(l_delta) + " ago"
  84     return retval
  85
  86
  87 class Marge(BotPlugin, CrontabMixin):
  88     """
  89     I remind you about merge requests
  90
  91     Use:
  92        In gitlab:
  93           Add a merge request webook of the form
  94              'https://webookserver/margeboot/<rooms>'
  95           to the projects you want tracked.  <rooms> should be a
  96           comma-separated list of short room names (anything before the '@')
  97           that you want notified.
  98        In errbot:
  99           Add <roomname>@domain to the CHATROOM_PRESENCE list in config.py for
 100           rooms margebot should join
 101     """
 102
 103     CRONTAB = [
 104         # Set in config now: '0 11,17 * * * .crontab_hook'    # 7:00AM and 1:00PM EST warnings
 105     ]
 106
 107     def __init__(self, *args, **kwargs):
 108         self.git_host = None
 109         self.chatroom_host = None
 110         self.gitlab = None
 111         self.soak_delta = None
 112         self.webhook_url = None
 113         super().__init__(*args, **kwargs)
 114
 115     def get_configuration_template(self):
 116         """
 117         GITLAB_HOST:      Host name of your gitlab server
 118         GITLAB_ADMIN_TOKEN: PAT from an admin's https://${GIT_HOST}/profile/personal_access_tokens.
 119         CHATROOM_HOST: Chatroom host.  Usually 'chatroom' + FQDN of Jabber server
 120         CRONTAB: Schedule of automated merge request checks in '%M %H %d %m %w' format
 121         VERIFY_SSL : True, False, or path to CA cert to verify cert
 122         CRONTAB_SOAK_HOURS : Don't send out reminders about MRs opened less than this many hours
 123         WEBHOOK_URL :  URL to use for defining MR integration in gitlab
 124         """
 125         return {'GITLAB_HOST': 'gitlab.example.com',
 126                 'GITLAB_ADMIN_TOKEN': 'gitlab-admin-user-private-token',
 127                 'CHATROOM_HOST': 'conference.jabber.example.com',
 128                 'CRONTAB': '0 11,17 * * *',
 129                 'VERIFY_SSL': True,
 130                 'CRONTAB_SOAK_HOURS': 1,
 131                 'WEBHOOK_URL': 'https://webhooks.example.com:3142/margebot/'}
 132
 133     def check_configuration(self, configuration):
 134         """
 135         Check that the plugin has been configured properly
 136         """
 137         super().check_configuration(configuration)
 138
 139     def activate(self):
 140         """
 141         Initialization done when the plugin is activated
 142         """
 143         if not self.config:
 144             self.log.info('Margebot is not configured. Forbid activation')
 145             return
 146         self.git_host = self.config['GITLAB_HOST']
 147         self.chatroom_host = self.config['CHATROOM_HOST']
 148         Marge.CRONTAB = ['{} .crontab_hook'.format(self.config['CRONTAB'])]
 149         gitlab_auth_token = self.config['GITLAB_ADMIN_TOKEN']
 150         verify_ssl = self.config['VERIFY_SSL']
 151         self.gitlab = gitlab.Gitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl)
 152         self.activate_crontab()
 153
 154         self.soak_delta = relativedelta(hours=self.config['CRONTAB_SOAK_HOURS'])
 155         self.webhook_url = self.config['WEBHOOK_URL']
 156         if self.webhook_url[-1] != '/':
 157             self.webhook_url += '/'
 158         super().activate()
 159
 160     def deactivate(self):
 161         """
 162         Anything that needs to be tore down when the plugin is deactivated goes here.
 163         """
 164         super().deactivate()
 165
 166     @webhook('/margebot/<rooms>/')
 167     def gitlab_hook(self, request, rooms):
 168         """
 169         Webhook that listens on http://<server>:<port>/gitlab
 170         """
 171
 172         self.log.info("webhook: request: {}, rooms: {}".format(request, rooms))
 173         # self.log.info("state: {}".format(request['object_attributes']['state']))
 174
 175         # verify it's a merge request
 176         if request['object_kind'] != 'merge_request':
 177             self.log.error('unexpecting object_kind: {}'.format(request['object_kind']))
 178         elif 'opened' in request['object_attributes']['state']:
 179
 180             if request['object_attributes']['work_in_progress']:
 181                 wip = "WIP "
 182             else:
 183                 wip = ""
 184             url = request['project']['homepage']
 185             title = request['object_attributes']['title']
 186
 187             author_id = request['object_attributes']['author_id']  # map this to user name ...
 188             author = self.gitlab.getuser(author_id)
 189             if author:
 190                 author_name = author['username']
 191             else:
 192                 self.log.info("unexpected author_id {}".format(author_id))
 193                 author_name = author_id
 194
 195             target_project_id = request['object_attributes']['target_project_id']
 196             iid = request['object_attributes']['iid']
 197
 198             # If the MR is tagged 'never-close' ignore it
 199             if 'labels' in request:
 200                 for a_label in request['labels']:
 201                     if a_label['title'] == 'never-close':
 202                         self.log.info("Skipping never-close notice for {} MR".format(url))
 203                         return "OK"
 204
 205             msg_template = "Hi there ! {} has opened a new {}MR: \"{}\"\n{}/merge_requests/{}"
 206             msg = msg_template.format(author_name, wip, title, url, iid)
 207
 208             if 'OPEN_MRS' not in self.keys():
 209                 empty_dict = {}
 210                 self['OPEN_MRS'] = empty_dict
 211
 212             open_mrs = self['OPEN_MRS']
 213
 214             # jn:  iid references in this block used to be mr_id
 215             if (target_project_id, iid, rooms) not in open_mrs:
 216                 for a_room in rooms.split(','):
 217                     if self.config:
 218                         self.send(self.build_identifier(a_room + '@' + self.chatroom_host), msg)
 219
 220                 self.log.info("webhook: Saving ({}, {}, {})".format(target_project_id, iid, rooms))
 221                 open_mrs[(target_project_id, iid, rooms)] = True
 222                 self['OPEN_MRS'] = open_mrs
 223
 224         # TODO:  Add check if an MR has toggled the WIP indicator
 225         # (trigger on updates (what's that look like in request['object_attributes']['state'])
 226         # Then check in request['changes']['title']['previous'] starts with 'WIP:'
 227         # but not request['changes']['title']['current'], and vice versa
 228         # See https://gitlab.com/gitlab-org/gitlab-ce/issues/53529
 229
 230         return "OK"
 231
 232     def mr_status_msg(self, a_mr, author=None):
 233         """
 234         Create the merge request status message
 235         """
 236         self.log.info("mr_status_msg: a_mr: {}".format(a_mr))
 237
 238         # Only weed out MRs less than the soak time for the crontab output (where author==None)
 239         now = datetime.now(timezone.utc)
 240         creation_time = parser.parse(a_mr['created_at'], tzinfos=tzutc)
 241         if not author:
 242             self.log.info("times: {}, {}, {}".format(creation_time, self.soak_delta, now))
 243             if creation_time + self.soak_delta > now:
 244                 project_id = a_mr['target_project_id']
 245                 mr_id = a_mr['id']
 246                 soak_hours = self.config['CRONTAB_SOAK_HOURS']
 247                 info_template = "skipping: MR <{},{}> was opened less than {} hours ago"
 248                 info_msg = info_template.format(project_id, mr_id, soak_hours)
 249                 self.log.info(info_msg)
 250                 return None
 251
 252         str_open_since = deltastr(now - creation_time)
 253
 254         warning = ""
 255         if a_mr['work_in_progress']:
 256             warning = "still WIP"
 257         elif a_mr['merge_status'] != 'can_be_merged':
 258             warning = "there are merge conflicts"
 259
 260         if author:
 261             authored = (a_mr['author']['id'] == author)
 262         else:
 263             authored = False
 264
 265         msg = "{} (opened {})".format(a_mr['web_url'], str_open_since)
 266         upvotes = a_mr['upvotes']
 267         if upvotes >= 2:
 268             msg += ": Has 2+ upvotes and could be merged in now"
 269             if warning != "":
 270                 msg += " except {}.".format(warning)
 271             else:
 272                 msg += "."
 273
 274         elif upvotes == 1:
 275             award_emoji = self.gitlab.getawardemoji(a_mr['target_project_id'], a_mr['iid'])
 276             self.log.info("award_emoji: {}".format(award_emoji))
 277             already_approved = "someone"
 278             for an_emoji in award_emoji:
 279                 if an_emoji["name"] == "thumbsup":
 280                     already_approved = an_emoji["user"]["username"]
 281
 282             if authored:
 283                 msg += ": Your MR has been approved by " + already_approved + " and is waiting for another upvote"
 284             else:
 285                 msg += ": Approved by " + already_approved + " and waiting for another upvote"
 286             if warning != "":
 287                 msg += " but {}.".format(warning)
 288             else:
 289                 msg += "."
 290
 291         else:
 292             if authored:
 293                 msg += ": Your MR has no upvotes"
 294             else:
 295                 msg += ": No upvotes, please review"
 296             if warning != "":
 297                 msg += " but {}.".format(warning)
 298             else:
 299                 msg += "."
 300
 301         return {'creation_time': creation_time, 'msg': msg}
 302
 303     def crontab_hook(self, polled_time):
 304         """
 305         Send a scheduled message to the rooms margebot is watching
 306         about open MRs the room cares about.
 307         """
 308
 309         self.log.info("crontab_hook triggered at {}".format(polled_time))
 310
 311         reminder_msg = {}  # Map of reminder_msg['roomname@domain'] = [(created_at,msg)]
 312
 313         # initialize the reminders
 314         rooms = self.rooms()
 315         for a_room in rooms:
 316             self.log.info("poller: a_room.node: {}".format(a_room.node))
 317             reminder_msg[a_room.node] = []
 318
 319         still_open_mrs = {}
 320
 321         # Let's walk through the MRs we've seen already:
 322         open_mrs = self['OPEN_MRS']
 323
 324         for (project_id, mr_id, notify_rooms) in open_mrs:
 325
 326             # Lookup the MR from the project/id
 327             a_mr = self.gitlab.getmergerequest(project_id, mr_id)
 328             if not a_mr:
 329                 self.log.debug("Couldn't find project: {}, id: {}".format(project_id, mr_id))
 330                 continue
 331
 332             # If the MR is tagged 'never-close' ignore it
 333             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 334                 continue
 335
 336             self.log.info("a_mr: {} {} {} {}".format(project_id, mr_id, notify_rooms, a_mr['state']))
 337
 338             # If the MR is no longer open, skip to the next MR,
 339             # and don't include this MR in the next check
 340             if 'opened' not in a_mr['state']:
 341                 continue
 342             else:
 343                 still_open_mrs[(project_id, mr_id, notify_rooms)] = True
 344
 345             msg_dict = self.mr_status_msg(a_mr)
 346             if msg_dict is None:
 347                 continue
 348
 349             for a_room in notify_rooms.split(','):
 350                 if a_room in reminder_msg:
 351                     reminder_msg[a_room].append(msg_dict)
 352                 else:
 353                     self.log.error("{} not in reminder_msg (project_id={}, mr_id={})".format(a_room, project_id, mr_id))
 354
 355         self['OPEN_MRS'] = open_mrs
 356
 357         # Remind each of the rooms about open MRs
 358         for a_room, room_msg_list in reminder_msg.items():
 359             if room_msg_list != []:
 360                 if self.config:
 361                     to_room = a_room + '@' + self.config['CHATROOM_HOST']
 362                     msg = tenv().get_template('reviews.md').render(msg_list=room_msg_list)
 363                     self.send(self.build_identifier(to_room), msg)
 364
 365         self['OPEN_MRS'] = still_open_mrs
 366
 367     @botcmd(template="reviews")
 368     def reviews(self, msg, args):
 369         """
 370         Returns a list of MRs that are waiting for some luv.
 371         Also returns a list of MRs that have had enough luv but aren't merged in yet.
 372         """
 373         # Sending directly to Margbot:  sender in the form sender@....
 374         # Sending to a chatroom:  snder in the form room@rooms/sender
 375
 376         log_template = 'reviews: msg: {}, args: {}, \nmsg.frm: {}, \nmsg.msg.frm: {}, chatroom_host: {}'
 377         self.log.info(log_template.format(msg, args, msg.frm.__dict__, dir(msg.frm), self.config['CHATROOM_HOST']))
 378         self.log.info('reviews: bot mode: {}'.format(self._bot.mode))
 379
 380         if self._bot.mode == "xmpp":
 381             if msg.frm.domain == self.config['CHATROOM_HOST']:
 382                 sender = msg.frm.resource
 383             else:
 384                 sender = msg.frm.node
 385         else:
 386             sender = str(msg.frm).split('@')[0]
 387
 388         keys = self.keys()
 389         if 'OPEN_MRS' not in keys:
 390             self.log.error('OPEN_MRS not in {}'.format(keys))
 391             return "No MRs to review"
 392
 393         sender_gitlab_id = None
 394         sender_users = self.gitlab.getusers(search=(('username', sender)))
 395         if not sender_users:
 396             self.log.error('problem mapping {} to a gitlab user'.format(sender))
 397             sender_gitlab_id = None
 398         else:
 399             sender_gitlab_id = sender_users[0]['id']
 400
 401         # Walk through the MRs we've seen already:
 402         msg_list = []
 403         msg = ""
 404         still_open_mrs = {}
 405         open_mrs = self['OPEN_MRS']
 406         self.log.info('open_mrs: {}'.format(open_mrs))
 407         for (project, mr_id, notify_rooms) in open_mrs:
 408
 409             # Lookup the MR from the project/id
 410             a_mr = self.gitlab.getmergerequest(project, mr_id)
 411             if not a_mr:
 412                 self.log.debug("Couldn't find project: {}, id: {}".format(project, id))
 413                 continue
 414
 415             self.log.info('project: {}, id: {}, a_mr: {}'.format(project, id, a_mr))
 416
 417             # If the MR is tagged 'never-close' ignore it
 418             if 'labels' in a_mr and 'never-close' in a_mr['labels']:
 419                 continue
 420
 421             # If the MR is no longer open, skip to the next MR,
 422             # and don't include this MR in the next check
 423             if 'opened' not in a_mr['state']:
 424                 self.log.info('state not opened: {}'.format(a_mr['state']))
 425                 continue
 426             else:
 427                 still_open_mrs[(project, mr_id, notify_rooms)] = True
 428
 429             msg_dict = self.mr_status_msg(a_mr, author=sender_gitlab_id)
 430             if msg_dict is None:
 431                 continue
 432
 433             msg_list.append(msg_dict)
 434
 435         self['OPEN_MRS'] = still_open_mrs
 436
 437         return {'sender': sender, 'msg_list': msg_list}
 438
 439     @arg_botcmd('rooms', type=str, help="Comma-separated room list without @conference-room suffix")
 440     @arg_botcmd('repo', type=str, help="repo to start watching for MRs in NAMESPACE/PROJECT_NAME format")
 441     def watchrepo(self, msg, repo, rooms):
 442         """
 443         Add the margebot webhook to a repo, and prepopulate any open MRs in the repo with margebot
 444         """
 445         self.log.info("msg={}".format(msg))
 446         self.log.info("repo={}".format(repo))
 447         self.log.info("rooms={}".format(rooms))
 448
 449         # get the group/repo repo, error out if it doesn't exist
 450         project = self.gitlab.getproject(repo)
 451         if not project:
 452             msg = "Couldn't find repo {}".format(repo)
 453             self.log.info("watchrepo: {}".format(msg))
 454             return msg
 455
 456         self.log.info('project: {}'.format(project))
 457
 458         target_project_id = project['id']
 459
 460         # Check is the project already includes the margebot hook
 461         # If no hooks, will it return False or [] ?
 462         marge_hook = None
 463         hooks = self.gitlab.getprojecthooks(target_project_id)
 464         self.log.error("hooks = {}".format(hooks))
 465         if hooks is False:
 466             msg = "Couldn't find {} hooks".format(repo)
 467             self.log.error("watchrepo: {}".format(msg))
 468             return msg
 469         else:
 470             for a_hook in hooks:
 471                 self.log.info('a_hook: {}'.format(a_hook))
 472                 if a_hook['merge_requests_events'] and a_hook['url'].startswith(self.webhook_url):
 473                     marge_hook = a_hook
 474                     break
 475
 476         # If so replace it (or error out ?)
 477         url = "{}{}".format(self.webhook_url, rooms)  # webhooks_url will end in '/'
 478         if marge_hook:
 479
 480             old_rooms = marge_hook['url'].split(self.webhook_url, 1)[1]
 481             if old_rooms == rooms:
 482                 msg = "Already reporting {} MRs to the {} room(s)".format(repo, rooms)
 483                 self.log.info('watchrepo: {}'.format(msg))
 484                 return msg
 485             else:
 486                 hook_updated = self.gitlab.editprojecthook_extra(target_project_id, marge_hook['id'], url, merge_requests=True, extra_data={'enable_ssl_verification': True})
 487                 s_watch_msg = "Updating room list for {} MRs from {} to {}".format(repo, old_rooms, rooms)
 488                 s_action = "update"
 489         else:
 490             hook_updated = self.gitlab.addprojecthook_extra(target_project_id, url, merge_requests=True, extra_data={'enable_ssl_verification': True})
 491             s_watch_msg = "Now watching for new MRs in the {} repo to the {} room(s)".format(repo, rooms)
 492             s_action = "add"
 493
 494         if not hook_updated:
 495             msg = "Couldn't {} hook: {}".format(s_action, repo)
 496             self.log.error("watchrepo: {}".format(msg))
 497             return msg
 498
 499         open_mrs = self['OPEN_MRS']
 500         mr_count = 0
 501         # get the open MRs in the repo
 502
 503         # If updating the room list, walk through the existing MR list
 504         if s_action == "update":
 505             new_open_mrs = {}
 506             for (project_id, mr_id, old_rooms) in open_mrs:
 507                 # pragma pylint: disable=simplifiable-if-statement
 508                 if project_id == target_project_id:
 509                     new_open_mrs[(project_id, mr_id, rooms)] = True
 510                 else:
 511                     new_open_mrs[(project_id, mr_id, old_rooms)] = True
 512                 # pragma pylint: enable=simplifiable-if-statement
 513             open_mrs = new_open_mrs
 514
 515         # If adding a new repo, check for existing opened/reopened MRs in the repo.
 516         else:
 517             # For debugging the 'watchrepo didn't find my MR' issue.
 518             # mr_list = self.gitlab.getmergerequests(target_project_id, page=1, per_page=100)
 519             # self.log.error('juden: mr_list state: {}'.format(mr_list[0]['state']))
 520             for state in ['opened', 'reopened']:
 521                 page = 1
 522                 mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100, state=state)
 523                 while (mr_list is not False) and (mr_list != []):
 524                     for an_mr in mr_list:
 525                         mr_count += 1
 526                         self.log.info('watchrepo: an_mr WATS THE IID\n{}'.format(an_mr))
 527                         mr_id = an_mr['iid']
 528                         open_mrs[(target_project_id, mr_id, rooms)] = True
 529                     # Get the next page of MRs
 530                     page += 1
 531                     mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100)
 532
 533         self['OPEN_MRS'] = open_mrs
 534
 535         if mr_count == 0:
 536             mr_msg = "No open MRs were found in the repo."
 537         elif mr_count == 1:
 538             mr_msg = "1 open MR was found in the repo.  Run !reviews to see the updated MR list."
 539         else:
 540             mr_msg = "{} open MRs were found in the repo.  Run !reviews to see the updated MR list."
 541         return "{}\n{}".format(s_watch_msg, mr_msg)
 542
 543     # pragma pylint: disable=unused-argument
 544
 545     # Check Chucklebot for the chuckles
 546
 547     @re_botcmd(pattern=r".*", prefixed=True)
 548     def catchall(self, msg, args):
 549         """
 550         Don't have the bot complain about unknown commands if the first word in a msg is its name
 551         """
 552         return
 553
 554     # pragma pylint: enable=unused-argument