From: JudeN Date: Fri, 26 Jan 2018 06:27:31 +0000 (-0500) Subject: Checkpoint commit for !watchrepo support - moar tests still needed X-Git-Url: https://pwan.org/git/?p=margebot.git;a=commitdiff_plain;h=59097b0207a31d66adbc0b3cb2275605f1b07922 Checkpoint commit for !watchrepo support - moar tests still needed --- diff --git a/.pylintrc b/.pylintrc index 0de8463..20e66a0 100644 --- a/.pylintrc +++ b/.pylintrc @@ -59,7 +59,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,line-too-long,no-name-in-module,input-error,missing-super-argument,no-self-use,too-many-locals,too-many-branches,import-error,locally-disabled,locally-enabled,too-many-ancestors,useless-super-delegation,logging-format-interpolation,too-many-statements +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,line-too-long,no-name-in-module,input-error,missing-super-argument,no-self-use,too-many-locals,too-many-branches,import-error,locally-disabled,locally-enabled,too-many-ancestors,useless-super-delegation,logging-format-interpolation,too-many-statements,too-many-arguments [REPORTS] diff --git a/plugins/marge.py b/plugins/marge.py index 7f66423..227e16a 100755 --- a/plugins/marge.py +++ b/plugins/marge.py @@ -6,10 +6,55 @@ from time import sleep from dateutil import parser from dateutil.tz import tzutc from dateutil.relativedelta import relativedelta -from errbot import BotPlugin, botcmd, re_botcmd, webhook +from errbot import BotPlugin, arg_botcmd, botcmd, re_botcmd, webhook from errbot.templating import tenv from errcron.bot import CrontabMixin import gitlab +import requests + + +class MargeGitlab(gitlab.Gitlab): + """ + Subclass gitlab.Gitlab so extra_data args can be added + to the addprojecthook() and editprojecthook() methods + """ + + def __init__(self, host, token="", oauth_token="", verify_ssl=True): + super().__init__(host, token, oauth_token, verify_ssl) + + def addprojecthook_extra(self, project_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None): + """ + A copy parent addprojecthook with an extra_data field + """ + data = {"id": project_id, "url": url} + if extra_data: + for ed_key, ed_value in extra_data.items(): + data[ed_key] = ed_value + data['push_events'] = int(bool(push)) + data['issues_events'] = int(bool(issues)) + data['merge_requests_events'] = int(bool(merge_requests)) + data['tag_push_events'] = int(bool(tag_push)) + request = requests.post("{0}/{1}/hooks".format(self.projects_url, project_id), + headers=self.headers, data=data, verify=self.verify_ssl) + if request.status_code == 201: + return request.json() + return False + + def editprojecthook_extra(self, project_id, hook_id, url, push=False, issues=False, merge_requests=False, tag_push=False, extra_data=None): + """ + A copy of the parent editprojecthook with an extra_data field + """ + data = {"id": project_id, "hook_id": hook_id, "url": url} + if extra_data: + for ed_key, ed_value in extra_data.items(): + data[ed_key] = ed_value + data['push_events'] = int(bool(push)) + data['issues_events'] = int(bool(issues)) + data['merge_requests_events'] = int(bool(merge_requests)) + data['tag_push_events'] = int(bool(tag_push)) + request = requests.put("{0}/{1}/hooks/{2}".format(self.projects_url, project_id, hook_id), + headers=self.headers, data=data, verify=self.verify_ssl) + return request.status_code == 200 def deltastr(any_delta): @@ -59,6 +104,7 @@ class Marge(BotPlugin, CrontabMixin): self.chatroom_host = None self.gitlab = None self.soak_delta = None + self.webhook_url = None super().__init__(*args, **kwargs) def get_configuration_template(self): @@ -69,13 +115,15 @@ class Marge(BotPlugin, CrontabMixin): CRONTAB: Schedule of automated merge request checks in '%M %H %d %m %w' format VERIFY_SSL : True, False, or path to CA cert to verify cert CRONTAB_SOAK_HOURS : Don't send out reminders about MRs opened less than this many hours + WEBHOOK_URL : URL to use for defining MR integration in gitlab """ return {'GITLAB_HOST': 'gitlab.example.com', 'GITLAB_ADMIN_TOKEN': 'gitlab-admin-user-private-token', 'CHATROOM_HOST': 'conference.jabber.example.com', 'CRONTAB': '0 11,17 * * *', 'VERIFY_SSL': True, - 'CRONTAB_SOAK_HOURS': 1} + 'CRONTAB_SOAK_HOURS': 1, + 'WEBHOOK_URL': 'https://webhooks.example.com:3142/margebot/'} def check_configuration(self, configuration): """ @@ -95,10 +143,14 @@ class Marge(BotPlugin, CrontabMixin): Marge.CRONTAB = ['{} .crontab_hook'.format(self.config['CRONTAB'])] gitlab_auth_token = self.config['GITLAB_ADMIN_TOKEN'] verify_ssl = self.config['VERIFY_SSL'] - self.gitlab = gitlab.Gitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl) + # self.gitlab = gitlab.Gitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl) + self.gitlab = MargeGitlab(self.git_host, gitlab_auth_token, verify_ssl=verify_ssl) self.activate_crontab() self.soak_delta = relativedelta(hours=self.config['CRONTAB_SOAK_HOURS']) + self.webhook_url = self.config['WEBHOOK_URL'] + if self.webhook_url[-1] != '/': + self.webhook_url += '/' super().activate() def deactivate(self): @@ -278,8 +330,6 @@ class Marge(BotPlugin, CrontabMixin): for a_room in notify_rooms.split(','): reminder_msg[a_room].append(msg_dict) - self['OPEN_MRS'] = open_mrs - # Remind each of the rooms about open MRs for a_room, room_msg_list in reminder_msg.items(): if room_msg_list != []: @@ -360,6 +410,112 @@ class Marge(BotPlugin, CrontabMixin): return {'sender': sender, 'msg_list': msg_list} + # ----------------------------------------------------------- + # webhook maintenance commands + + @arg_botcmd('rooms', type=str) + @arg_botcmd('repo', type=str) + def watchrepo(self, msg, repo, rooms): + """ + Add the margebot webhook to a repo, and prepopulate any open MRs in the repo with margebot + args: repo: gitlab repo name in the 'NAMESPACE/PROJECT_NAME' format + rooms: comma separates list of rooms to notify when the webhook triggers + """ + self.log.info("msg={}".format(msg)) + self.log.info("repo={}".format(repo)) + self.log.info("rooms={}".format(rooms)) + + # get the group/repo repo, error out if it doesn't exist + project = self.gitlab.getproject(repo) + if not project: + msg = "Couldn't find repo {}".format(repo) + self.log.info("watchrepo: {}".format(msg)) + return msg + + self.log.info('project: {}'.format(project)) + + target_project_id = project['id'] + + # Check is the project already includes the margebot hook + # If no hooks, will it return False or [] ? + marge_hook = None + hooks = self.gitlab.getprojecthooks(target_project_id) + if not hooks: + msg = "Couldn't find {} hooks".format(repo) + self.log.error("watchrepo: {}".format(msg)) + return msg + else: + for a_hook in hooks: + self.log.info('a_hook: {}'.format(a_hook)) + if a_hook['merge_requests_events'] and a_hook['url'].startswith(self.webhook_url): + marge_hook = a_hook + break + + # If so replace it (or error out ?) + url = "{}{}".format(self.webhook_url, rooms) # webhooks_url will end in '/' + if marge_hook: + + old_rooms = marge_hook['url'].split(self.webhook_url, 1)[1] + if old_rooms == rooms: + msg = "Already reporting {} MRs to the {} room(s)".format(repo, rooms) + self.log.info('watchrepo: {}'.format(msg)) + return msg + else: + hook_updated = self.gitlab.editprojecthook_extra(target_project_id, marge_hook['id'], url, merge_requests=True, extra_data={'enable_ssl_verification': False}) + s_watch_msg = "Updating room list for {} MRs from {} to {}".format(repo, old_rooms, rooms) + s_action = "update" + else: + hook_updated = self.gitlab.addprojecthook_extra(target_project_id, url, merge_requests=True, extra_data={'enable_ssl_verification': False}) + s_watch_msg = "Now watching for new MRs in the {} repo to the {} roomi(s)".format(repo, rooms) + s_action = "add" + + if not hook_updated: + msg = "Couldn't {} hook: {}".format(s_action, repo) + self.log.error("watchrepo: {}".format(msg)) + return msg + + open_mrs = self['OPEN_MRS'] + mr_count = 0 + # get the open MRs in the repo + + # If updating the room list, walk through the existing MR list + if s_action == "update": + new_open_mrs = {} + for (project_id, mr_id, old_rooms) in open_mrs: + # pragma pylint: disable=simplifiable-if-statement + if project_id == target_project_id: + new_open_mrs[(project_id, mr_id, rooms)] = True + else: + new_open_mrs[(project_id, mr_id, old_rooms)] = True + # pragma pylint: enable=simplifiable-if-statement + open_mrs = new_open_mrs + + # If adding a new repo, check for existing opened/reopened MRs in the repo. + else: + for state in ['opened', 'reopened']: + page = 1 + mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100, state=state) + while not mr_list and mr_list != []: + for an_mr in mr_list: + mr_count += 1 + self.log.info('watchrepo: an_mr WATS THE ID\n{}'.format(an_mr)) + mr_id = an_mr['id'] + open_mrs[(target_project_id, mr_id, rooms)] = True + # Get the next page of MRs + page += 1 + mr_list = self.gitlab.getmergerequests(target_project_id, page=page, per_page=100) + + self['OPEN_MRS'] = open_mrs + + if mr_count == 0: + mr_msg = "No open MRs were found in the repo." + elif mr_count == 1: + mr_msg = "1 open MR was found in the repo. Run !reviews to see the updated MR list." + else: + mr_msg = "{} open MRs were found in the repo. Run !reviews to see the updated MR list." + + return "{}\n{}".format(s_watch_msg, mr_msg) + # pragma pylint: disable=unused-argument @botcmd() def hello(self, msg, args): diff --git a/test-requirements.txt b/test-requirements.txt index 470f46e..c35295c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,4 @@ pytest -pytest-catchlog pytest-cov pytest-mock pytest-pep8 diff --git a/tests/test_marge.py b/tests/test_marge.py index 8aed0ba..8e9d450 100644 --- a/tests/test_marge.py +++ b/tests/test_marge.py @@ -13,11 +13,11 @@ from datetime import datetime import logging import json -import pytest import errbot from errbot.backends.test import testbot # pylint: disable=unused-import import gitlab from dateutil.relativedelta import relativedelta +import pytest from plugins.marge import deltastr @@ -33,7 +33,7 @@ class TestMarge(object): @pytest.fixture def margebot(self, testbot, monkeypatch, mocker): - testbot.push_message("!plugin config Marge {'CHATROOM_HOST': 'conference.test.com', 'GITLAB_HOST': 'gitlab.test.com', 'GITLAB_ADMIN_TOKEN': 'fake-token', 'CRONTAB': '0 * * * *', 'VERIFY_SSL': True, 'CRONTAB_SOAK_HOURS': 1}") + testbot.push_message("!plugin config Marge {'CHATROOM_HOST': 'conference.test.com', 'GITLAB_HOST': 'gitlab.test.com', 'GITLAB_ADMIN_TOKEN': 'fake-token', 'CRONTAB': '0 * * * *', 'VERIFY_SSL': True, 'CRONTAB_SOAK_HOURS': 1, 'WEBHOOK_URL': 'https://webhood.errbot.com:3142/margebot'}") testbot.pop_message() testbot.push_message("!plugin config Webserver {'HOST': '0.0.0.0', 'PORT':3141, 'SSL': {'certificate': '', 'enabled': False, 'host': '0.0.0.0', 'key': '', 'port': 3142}}") testbot.pop_message()