e81e00aca7bb3bb9b803b8ed65d91e40d0b52ac7
[margebot.git] / plugins / marge.py
1 """
2 Margebot: A Errbot Plugin for Gitlab MR reminders
3 """
4 import re
5 from datetime import datetime, timezone
6 from time import sleep
7 from dateutil import parser
8 from dateutil.tz import tzutc
9 from dateutil.relativedelta import relativedelta
10 from errbot import BotPlugin, botcmd, arg_botcmd, re_botcmd, webhook
11 from errbot.templating import tenv
12 from errcron.bot import CrontabMixin
13 import gitlab
14 import requests
15
16
17 def addprojecthook_extra(self, project_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
18 """
19 A copy parent addprojecthook with an extra_data field
20 """
21 data = {"id": project_id, "url": url}
22 if extra_data:
23 for ed_key, ed_value in extra_data.items():
24 data[ed_key] = ed_value
25 data['push_events'] = int(bool(push))
26 data['issues_events'] = int(bool(issues))
27 data['merge_requests_events'] = int(bool(merge_requests))
28 data['tag_push_events'] = int(bool(tag_push))
29 request = requests.post("{0}/{1}/hooks".format(self.projects_url, project_id),
30 headers=self.headers, data=data, verify=self.verify_ssl)
31 if request.status_code == 201:
32 return request.json()
33 return False
34
35 gitlab.Gitlab.addprojecthook_extra = addprojecthook_extra
36
37
38 def editprojecthook_extra(self, project_id, hook_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None):
39 """
40 A copy of the parent editprojecthook with an extra_data field
41 """
42 data = {"id": project_id, "hook_id": hook_id, "url": url}
43 if extra_data:
44 for ed_key, ed_value in extra_data.items():
45 data[ed_key] = ed_value
46 data['push_events'] = int(bool(push))
47 data['issues_events'] = int(bool(issues))
48 data['merge_requests_events'] = int(bool(merge_requests))
49 data['tag_push_events'] = int(bool(tag_push))
50 request = requests.put("{0}/{1}/hooks/{2}".format(self.projects_url, project_id, hook_id),
51 headers=self.headers, data=data, verify=self.verify_ssl)
52 return request.status_code == 200
53
54 gitlab.Gitlab.editprojecthook_extra = editprojecthook_extra
55
56
57 def deltastr(any_delta):
58 """
59 Output a datetime delta in the format "x days, y hours, z minutes ago"
60 """
61 l_delta = []
62 days = any_delta.days
63 hours = any_delta.seconds // 3600
64 mins = (any_delta.seconds // 60) % 60
65
66 for (key, val) in [("day", days), ("hour", hours), ("minute", mins)]:
67 if val == 1:
68 l_delta.append("1 " + key)
69 elif val > 1:
70 l_delta.append("{} {}s".format(val, key))
71
72 if l_delta == []:
73 retval = "now"
74 else:
75 retval = ", ".join(l_delta) + " ago"
76 return retval
77
78
79 class Marge(BotPlugin, CrontabMixin):
80 """
81 I remind you about merge requests
82
83 Use:
84 In gitlab:
85 Add a merge request webook of the form
86 'https://webookserver/margeboot/<rooms>'
87 to the projects you want tracked. <rooms> should be a
88 comma-separated list of short room names (anything before the '@')
89 that you want notified.
90 In errbot:
91 Add <roomname>@domain to the CHATROOM_PRESENCE list in config.py for
92 rooms margebot should join
93 """
94
95 CRONTAB = [
96 # Set in config now: '0 11,17 * * * .crontab_hook' # 7:00AM and 1:00PM EST warnings
97 ]
98
99 def __init__(self, *args, **kwargs):
100 self.git_host = None
101 self.chatroom_host = None
102 self.gitlab = None
103 self.soak_delta = None
104 self.webhook_url = None
105 super().__init__(*args, **kwargs)
106
107 def get_configuration_template(self):
108 """
109 GITLAB_HOST: Host name of your gitlab server
110 GITLAB_ADMIN_TOKEN: PAT from an admin's https://${GIT_HOST}/profile/personal_access_tokens.
111 CHATROOM_HOST: Chatroom host. Usually 'chatroom' + FQDN of Jabber server
112 CRONTAB: Schedule of automated merge request checks in '%M %H %d %m %w' format
113 VERIFY_SSL : True, False, or path to CA cert to verify cert
114 CRONTAB_SOAK_HOURS : Don't send out reminders about MRs opened less than this many hours
115 WEBHOOK_URL : URL to use for defining MR integration in gitlab
116 """
117 return {'GITLAB_HOST': 'gitlab.example.com',
118 'GITLAB_ADMIN_TOKEN': 'gitlab-admin-user-private-token',
119 'CHATROOM_HOST': 'conference.jabber.example.com',
120 'CRONTAB': '0 11,17 * * *',
121 'VERIFY_SSL': True,
122 'CRONTAB_SOAK_HOURS': 1,
123 'WEBHOOK_URL': 'https://webhooks.example.com:3142/margebot/'}
124
125 def check_configuration(self, configuration):
126 """
127 Check that the plugin has been configured properly
128 """
129 super().check_configuration(configuration)
130
131 def activate(self):
132 """
133 Initialization done when the plugin is activated
134 """
135 if not self.config:
136 self.log.info('Margebot is not configured. Forbid activation')
137 return
138 self.git_host = self.config['GITLAB_HOST']
139 self.chatroom_host = self.config['CHATROOM_HOST']
140 Marge.CRONTAB = ['{} .crontab_hook'.format(self.config['CRONTAB'])]
141 gitlab_auth_token = self.config['GITLAB_ADMIN_TOKEN']
142 verify_ssl = self.config['VERIFY_SSL']
143 self.gitlab = gitlab.Gitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl)
144 self.activate_crontab()
145
146 self.soak_delta = relativedelta(hours=self.config['CRONTAB_SOAK_HOURS'])
147 self.webhook_url = self.config['WEBHOOK_URL']
148 if self.webhook_url[-1] != '/':
149 self.webhook_url += '/'
150 super().activate()
151
152 def deactivate(self):
153 """
154 Anything that needs to be tore down when the plugin is deactivated goes here.
155 """
156 super().deactivate()
157
158 @webhook('/margebot/<rooms>/')
159 def gitlab_hook(self, request, rooms):
160 """
161 Webhook that listens on http://<server>:<port>/gitlab
162 """
163
164 self.log.info("webhook: request: {}, rooms: {}".format(request, rooms))
165 # self.log.info("state: {}".format(request['object_attributes']['state']))
166
167 # verify it's a merge request
168 if request['object_kind'] != 'merge_request':
169 self.log.error('unexpecting object_kind: {}'.format(request['object_kind']))
170 elif 'opened' in request['object_attributes']['state']:
171
172 if request['object_attributes']['work_in_progress']:
173 wip = "WIP "
174 else:
175 wip = ""
176 url = request['project']['homepage']
177 title = request['object_attributes']['title']
178
179 author_id = request['object_attributes']['author_id'] # map this to user name ...
180 author = self.gitlab.getuser(author_id)
181 if author:
182 author_name = author['username']
183 else:
184 self.log.info("unexpected author_id {}".format(author_id))
185 author_name = author_id
186
187 target_project_id = request['object_attributes']['target_project_id']
188 iid = request['object_attributes']['iid']
189
190 # If the MR is tagged 'never-close' ignore it
191 if 'labels' in request:
192 for a_label in request['labels']:
193 if a_label['title'] == 'never-close':
194 self.log.info("Skipping never-close notice for {} MR".format(url))
195 return "OK"
196
197 msg_template = "Hi there ! {} has opened a new {}MR: \"{}\"\n{}/merge_requests/{}"
198 msg = msg_template.format(author_name, wip, title, url, iid)
199
200 if 'OPEN_MRS' not in self.keys():
201 empty_dict = {}
202 self['OPEN_MRS'] = empty_dict
203
204 open_mrs = self['OPEN_MRS']
205
206 if (target_project_id, iid, rooms) not in open_mrs:
207 for a_room in rooms.split(','):
208 if self.config:
209 self.send(self.build_identifier(a_room + '@' + self.chatroom_host), msg)
210
211 self.log.info("webhook: Saving ({}, {}, {})".format(target_project_id, iid, rooms))
212 open_mrs[(target_project_id, iid, rooms)] = True
213 self['OPEN_MRS'] = open_mrs
214
215 # TODO: Add check if an MR has toggled the WIP indicator
216 # (trigger on updates (what's that look like in request['object_attributes']['state'])
217 # Then check in request['changes']['title']['previous'] starts with 'WIP:'
218 # but not request['changes']['title']['current'], and vice versa
219 # See https://gitlab.com/gitlab-org/gitlab-ce/issues/53529
220
221 return "OK"
222
223 def mr_status_msg(self, a_mr, author=None):
224 """
225 Create the merge request status message
226 """
227 self.log.info("mr_status_msg: a_mr: {}".format(a_mr))
228
229 # Only weed out MRs less than the soak time for the crontab output (where author==None)
230 now = datetime.now(timezone.utc)
231 creation_time = parser.parse(a_mr['created_at'], tzinfos=tzutc)
232 if not author:
233 self.log.info("times: {}, {}, {}".format(creation_time, self.soak_delta, now))
234 if creation_time + self.soak_delta > now:
235 project_id = a_mr['target_project_id']
236 mr_id = a_mr['id']
237 soak_hours = self.config['CRONTAB_SOAK_HOURS']
238 info_template = "skipping: MR <{},{}> was opened less than {} hours ago"
239 info_msg = info_template.format(project_id, mr_id, soak_hours)
240 self.log.info(info_msg)
241 return None
242
243 str_open_since = deltastr(now - creation_time)
244
245 warning = ""
246 if a_mr['work_in_progress']:
247 warning = "still WIP"
248 elif a_mr['merge_status'] != 'can_be_merged':
249 warning = "there are merge conflicts"
250
251 if author:
252 authored = (a_mr['author']['id'] == author)
253 else:
254 authored = False
255
256 # getapprovals is only available in GitLab 8.9 EE or greater
257 # (not the open source CE version)
258 # approvals = self.gitlab.getapprovals(a_mr['id'])
259 # approved = ""
260 # for approved in approvals['approved_by']:
261 # approved += "," + approvals['user']['name']
262
263 # See https://gitlab.com/gitlab-org/gitlab-ce/issues/35498
264 # awards = GET /projects/:id/merge_requests/:merge_request_iid/award_emoji
265 # for an award in awards:
266 # if name==??? and award_type==???:
267 # approved += "," + award["user"]["username"]
268
269 upvotes = a_mr['upvotes']
270 msg = "{} (opened {})".format(a_mr['web_url'], str_open_since)
271 if upvotes >= 2:
272 msg += ": Has 2+ upvotes and could be merged in now"
273 if warning != "":
274 msg += " except {}.".format(warning)
275 else:
276 msg += "."
277
278 elif upvotes == 1:
279 if authored:
280 msg += ": Your MR is waiting for another upvote"
281 else:
282 msg += ": Waiting for another upvote"
283 if warning != "":
284 msg += " but {}.".format(warning)
285 else:
286 msg += "."
287
288 else:
289 if authored:
290 msg += ": Your MR has no upvotes"
291 else:
292 msg += ": No upvotes, please review"
293 if warning != "":
294 msg += " but {}.".format(warning)
295 else:
296 msg += "."
297
298 return {'creation_time': creation_time, 'msg': msg}
299
300 def crontab_hook(self, polled_time):
301 """
302 Send a scheduled message to the rooms margebot is watching
303 about open MRs the room cares about.
304 """
305
306 self.log.info("crontab_hook triggered at {}".format(polled_time))
307
308 reminder_msg = {} # Map of reminder_msg['roomname@domain'] = [(created_at,msg)]
309
310 # initialize the reminders
311 rooms = self.rooms()
312 for a_room in rooms:
313 self.log.info("poller: a_room.node: {}".format(a_room.node))
314 reminder_msg[a_room.node] = []
315
316 still_open_mrs = {}
317
318 # Let's walk through the MRs we've seen already:
319 open_mrs = self['OPEN_MRS']
320
321 for (project_id, mr_id, notify_rooms) in open_mrs:
322
323 # Lookup the MR from the project/id
324 a_mr = self.gitlab.getmergerequest(project_id, mr_id)
325 if not a_mr:
326 self.log.debug("Couldn't find project: {}, id: {}".format(project_id, mr_id))
327 continue
328
329 # If the MR is tagged 'never-close' ignore it
330 if 'labels' in a_mr and 'never-close' in a_mr['labels']:
331 continue
332
333 self.log.info("a_mr: {} {} {} {}".format(project_id, mr_id, notify_rooms, a_mr['state']))
334
335 # If the MR is no longer open, skip to the next MR,
336 # and don't include this MR in the next check
337 if 'opened' not in a_mr['state']:
338 continue
339 else:
340 still_open_mrs[(project_id, mr_id, notify_rooms)] = True
341
342 msg_dict = self.mr_status_msg(a_mr)
343 if msg_dict is None:
344 continue
345
346 for a_room in notify_rooms.split(','):
347 if a_room in reminder_msg:
348 reminder_msg[a_room].append(msg_dict)
349 else:
350 self.log.error("{} not in reminder_msg (project_id={}, mr_id={})".format(a_room, project_id, mr_id))
351
352 self['OPEN_MRS'] = open_mrs
353
354 # Remind each of the rooms about open MRs
355 for a_room, room_msg_list in reminder_msg.items():
356 if room_msg_list != []:
357 if self.config:
358 to_room = a_room + '@' + self.config['CHATROOM_HOST']
359 msg = tenv().get_template('reviews.md').render(msg_list=room_msg_list)
360 self.send(self.build_identifier(to_room), msg)
361
362 self['OPEN_MRS'] = still_open_mrs
363
364 @botcmd(template="reviews")
365 def reviews(self, msg, args):
366 """
367 Returns a list of MRs that are waiting for some luv.
368 Also returns a list of MRs that have had enough luv but aren't merged in yet.
369 """
370 # Sending directly to Margbot: sender in the form sender@....
371 # Sending to a chatroom: snder in the form room@rooms/sender
372
373 log_template = 'reviews: msg: {}, args: {}, \nmsg.frm: {}, \nmsg.msg.frm: {}, chatroom_host: {}'
374 self.log.info(log_template.format(msg, args, msg.frm.__dict__, dir(msg.frm), self.config['CHATROOM_HOST']))
375 self.log.info('reviews: bot mode: {}'.format(self._bot.mode))
376
377 if self._bot.mode == "xmpp":
378 if msg.frm.domain == self.config['CHATROOM_HOST']:
379 sender = msg.frm.resource
380 else:
381 sender = msg.frm.node
382 else:
383 sender = str(msg.frm).split('@')[0]
384
385 keys = self.keys()
386 if 'OPEN_MRS' not in keys:
387 self.log.error('OPEN_MRS not in {}'.format(keys))
388 return "No MRs to review"
389
390 sender_gitlab_id = None
391 sender_users = self.gitlab.getusers(search=(('username', sender)))
392 if not sender_users:
393 self.log.error('problem mapping {} to a gitlab user'.format(sender))
394 sender_gitlab_id = None
395 else:
396 sender_gitlab_id = sender_users[0]['id']
397
398 # Walk through the MRs we've seen already:
399 msg_list = []
400 msg = ""
401 still_open_mrs = {}
402 open_mrs = self['OPEN_MRS']
403 self.log.info('open_mrs: {}'.format(open_mrs))
404 for (project, mr_id, notify_rooms) in open_mrs:
405
406 # Lookup the MR from the project/id
407 a_mr = self.gitlab.getmergerequest(project, mr_id)
408 if not a_mr:
409 self.log.debug("Couldn't find project: {}, id: {}".format(project, id))
410 continue
411
412 self.log.info('project: {}, id: {}, a_mr: {}'.format(project, id, a_mr))
413
414 # If the MR is tagged 'never-close' ignore it
415 if 'labels' in a_mr and 'never-close' in a_mr['labels']:
416 continue
417
418 # If the MR is no longer open, skip to the next MR,
419 # and don't include this MR in the next check
420 if 'opened' not in a_mr['state']:
421 self.log.info('state not opened: {}'.format(a_mr['state']))
422 continue
423 else:
424 still_open_mrs[(project, mr_id, notify_rooms)] = True
425
426 msg_dict = self.mr_status_msg(a_mr, author=sender_gitlab_id)
427 if msg_dict is None:
428 continue
429
430 msg_list.append(msg_dict)
431
432 self['OPEN_MRS'] = still_open_mrs
433
434 return {'sender': sender, 'msg_list': msg_list}
435
436 @arg_botcmd('rooms', type=str, help="Comma-separated room list without @conference-room suffix")
437 @arg_botcmd('repo', type=str, help="repo to start watching for MRs in NAMESPACE/PROJECT_NAME format")
438 def watchrepo(self, msg, repo, rooms):
439 """
440 Add the margebot webhook to a repo, and prepopulate any open MRs in the repo with margebot
441 """
442 self.log.info("msg={}".format(msg))
443 self.log.info("repo={}".format(repo))
444 self.log.info("rooms={}".format(rooms))
445
446 # get the group/repo repo, error out if it doesn't exist
447 project = self.gitlab.getproject(repo)
448 if not project:
449 msg = "Couldn't find repo {}".format(repo)
450 self.log.info("watchrepo: {}".format(msg))
451 return msg
452
453 self.log.info('project: {}'.format(project))
454
455 target_project_id = project['id']
456
457 # Check is the project already includes the margebot hook
458 # If no hooks, will it return False or [] ?
459 marge_hook = None
460 hooks = self.gitlab.getprojecthooks(target_project_id)
461 self.log.error("hooks = {}".format(hooks))
462 if hooks is False:
463 msg = "Couldn't find {} hooks".format(repo)
464 self.log.error("watchrepo: {}".format(msg))
465 return msg
466 else:
467 for a_hook in hooks:
468 self.log.info('a_hook: {}'.format(a_hook))
469 if a_hook['merge_requests_events'] and a_hook['url'].startswith(self.webhook_url):
470 marge_hook = a_hook
471 break
472
473 # If so replace it (or error out ?)
474 url = "{}{}".format(self.webhook_url, rooms) # webhooks_url will end in '/'
475 if marge_hook:
476
477 old_rooms = marge_hook['url'].split(self.webhook_url, 1)[1]
478 if old_rooms == rooms:
479 msg = "Already reporting {} MRs to the {} room(s)".format(repo, rooms)
480 self.log.info('watchrepo: {}'.format(msg))
481 return msg
482 else:
483 hook_updated = self.gitlab.editprojecthook_extra(target_project_id, marge_hook['id'], url, merge_requests=True, extra_data={'enable_ssl_verification': True})
484 s_watch_msg = "Updating room list for {} MRs from {} to {}".format(repo, old_rooms, rooms)
485 s_action = "update"
486 else:
487 hook_updated = self.gitlab.addprojecthook_extra(target_project_id, url, merge_requests=True, extra_data={'enable_ssl_verification': True})
488 s_watch_msg = "Now watching for new MRs in the {} repo to the {} room(s)".format(repo, rooms)
489 s_action = "add"
490
491 if not hook_updated:
492 msg = "Couldn't {} hook: {}".format(s_action, repo)
493 self.log.error("watchrepo: {}".format(msg))
494 return msg
495
496 open_mrs = self['OPEN_MRS']
497 mr_count = 0
498 # get the open MRs in the repo
499
500 # If updating the room list, walk through the existing MR list
501 if s_action == "update":
502 new_open_mrs = {}
503 for (project_id, mr_id, old_rooms) in open_mrs:
504 # pragma pylint: disable=simplifiable-if-statement
505 if project_id == target_project_id:
506 new_open_mrs[(project_id, mr_id, rooms)] = True
507 else:
508 new_open_mrs[(project_id, mr_id, old_rooms)] = True
509 # pragma pylint: enable=simplifiable-if-statement
510 open_mrs = new_open_mrs
511
512 # If adding a new repo, check for existing opened/reopened MRs in the repo.
513 else:
514 # For debugging the 'watchrepo didn't find my MR' issue.
515 # mr_list = self.gitlab.getmergerequests(target_project_id, page=1, per_page=100)
516 # self.log.error('juden: mr_list state: {}'.format(mr_list[0]['state']))
517 for state in ['opened', 'reopened']:
518 page = 1
519 mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100, state=state)
520 while (mr_list is not False) and (mr_list != []):
521 for an_mr in mr_list:
522 mr_count += 1
523 self.log.info('watchrepo: an_mr WATS THE IID\n{}'.format(an_mr))
524 mr_id = an_mr['iid']
525 open_mrs[(target_project_id, mr_id, rooms)] = True
526 # Get the next page of MRs
527 page += 1
528 mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100)
529
530 self['OPEN_MRS'] = open_mrs
531
532 if mr_count == 0:
533 mr_msg = "No open MRs were found in the repo."
534 elif mr_count == 1:
535 mr_msg = "1 open MR was found in the repo. Run !reviews to see the updated MR list."
536 else:
537 mr_msg = "{} open MRs were found in the repo. Run !reviews to see the updated MR list."
538 return "{}\n{}".format(s_watch_msg, mr_msg)
539
540 # pragma pylint: disable=unused-argument
541
542 @botcmd()
543 def xyzzy(self, msg, args):
544 """
545 Don't call this command...
546 """
547 yield "/me whispers \"All open MRs have been merged into master.\""
548 sleep(5)
549 yield "(just kidding)"
550
551 @re_botcmd(pattern=r"I blame marge(bot)?", prefixed=False, flags=re.IGNORECASE)
552 def dont_blame_margebot(self, msg, match):
553 """
554 margebot is innocent.
555 """
556 yield u"(\u300D\uFF9F\uFF9B\uFF9F)\uFF63NOOOooooo say it ain't so."
557
558 @re_botcmd(pattern=r"\u0028\u256F\u00B0\u25A1\u00B0\uFF09\u256F\uFE35\u0020\u253B(\u2501+)\u253B", prefixed=False)
559 def deflipped(self, msg, match):
560 """
561 Unflip a properly sized table
562 """
563 table_len = len(match.group(1))
564 deflip_table = u"\u252c" + (u"\u2500" * table_len) + u"\u252c \u30ce( \u309c-\u309c\u30ce)"
565 # yield u"\u252c\u2500\u2500\u252c \u30ce( \u309c-\u309c\u30ce)"
566 yield deflip_table
567
568 @re_botcmd(pattern=r"good bot", prefixed=False, flags=re.IGNORECASE)
569 def best_bot(self, msg, match):
570 """
571 margebot is the best.
572 """
573 yield "Best bot"
574
575 @re_botcmd(pattern=r"magfest", prefixed=False, flags=re.IGNORECASE)
576 def margefest(self, msg, args):
577 """
578 margefest4ever
579 """
580 return "More like MargeFest, amirite ?"
581
582 # ha the dev-infra room sez koji sooooooooooooooooooooooooooooooooooooooooooo much
583 # @re_botcmd(pattern=r"k+o+j+i+", prefixed=False, flags=re.IGNORECASE)
584 # def koji(self, msg, args):
585 # """
586 # More like daikaiju, amirite ?
587 # """
588 # return "More like kaiju, amirite ?"
589
590 @re_botcmd(pattern=r"peruvian chicken", prefixed=False, flags=re.IGNORECASE)
591 def booruvian(self, msg, args):
592 """
593 They put hard boiled eggs in their tamales too.
594 """
595 return "More like Booruvian chicken, amirite ?"
596
597 @re_botcmd(pattern=r"booruvian chicken", prefixed=False, flags=re.IGNORECASE)
598 def booihoohooruvian(self, msg, args):
599 """
600 That chicken I do not like.
601 """
602 return "More like Boohoohooruvian chicken, amirite ?"
603
604 @re_botcmd(pattern=r"margebot sucks", prefixed=False, flags=re.IGNORECASE)
605 def new_agenda_item(self, msg, args):
606 """
607 Bring it up with the committee
608 """
609 return "Bring it up with the Margebot steering committee."
610
611 @re_botcmd(pattern=r"jackie", prefixed=False, flags=re.IGNORECASE)
612 def jackie(self, msg, args):
613 """
614 Who dat ?
615 """
616 return "I don't know any Jackies. I'm calling security."
617
618 @re_botcmd(pattern=r".*", prefixed=True)
619 def catchall(self, msg, args):
620 """
621 Don't have the bot complain about unknown commands if the first word in a msg is its name
622 """
623 return
624
625 # pragma pylint: enable=unused-argument