From b2bfdc454aaa861cbed618cafb0ed7a4a590712a Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 22 Sep 2023 15:46:19 -0400 Subject: [PATCH 01/12] test: remove jira mentions in tests These tests don't affect Jira anymore, so no need to talk about Jira. --- tests/test_pull_request_opened.py | 80 +++++++------------------------ 1 file changed, 17 insertions(+), 63 deletions(-) diff --git a/tests/test_pull_request_opened.py b/tests/test_pull_request_opened.py index 649b0101..c7920d65 100644 --- a/tests/test_pull_request_opened.py +++ b/tests/test_pull_request_opened.py @@ -34,7 +34,7 @@ def close_and_reopen_pr(pr): return pull_request_changed(prj) -def test_internal_pr_opened(fake_github, fake_jira): +def test_internal_pr_opened(fake_github): pr = fake_github.make_pull_request("openedx", user="nedbat") key, anything_happened = pull_request_changed(pr.as_json()) @@ -49,7 +49,7 @@ def test_internal_pr_opened(fake_github, fake_jira): assert anything_happened2 is False -def test_pr_in_private_repo_opened(fake_github, fake_jira): +def test_pr_in_private_repo_opened(fake_github): repo = fake_github.make_repo("edx", "some-private-repo", private=True) pr = repo.make_pull_request(user="some_contractor") key, anything_happened = pull_request_changed(pr.as_json()) @@ -62,7 +62,7 @@ def test_pr_in_private_repo_opened(fake_github, fake_jira): @pytest.mark.parametrize("user", ["tusbar", "feanil"]) -def test_pr_in_nocontrib_repo_opened(fake_github, fake_jira, user): +def test_pr_in_nocontrib_repo_opened(fake_github, user): repo = fake_github.make_repo("edx", "some-public-repo") pr = repo.make_pull_request(user=user) key, anything_happened = pull_request_changed(pr.as_json()) @@ -77,7 +77,7 @@ def test_pr_in_nocontrib_repo_opened(fake_github, fake_jira, user): assert pull_request_projects(pr.as_json()) == set() -def test_pr_opened_by_bot(fake_github, fake_jira): +def test_pr_opened_by_bot(fake_github): fake_github.make_user(login="some_bot", type="Bot") pr = fake_github.make_pull_request(user="some_bot") key, anything_happened = pull_request_changed(pr.as_json()) @@ -88,7 +88,7 @@ def test_pr_opened_by_bot(fake_github, fake_jira): assert pull_request_projects(pr.as_json()) == set() -def test_external_pr_opened_no_cla(fake_github, fake_jira): +def test_external_pr_opened_no_cla(fake_github): # No CLA, because this person is not in our database. fake_github.make_user(login="new_contributor", name="Newb Contributor") pr = fake_github.make_pull_request(owner="openedx", repo="edx-platform", user="new_contributor") @@ -98,7 +98,6 @@ def test_external_pr_opened_no_cla(fake_github, fake_jira): assert anything_happened is True assert issue_id is None - assert len(fake_jira.issues) == 0 # Check the GitHub comment that was created. pr_comments = pr.list_comments() @@ -126,10 +125,8 @@ def test_external_pr_opened_no_cla(fake_github, fake_jira): # re-opening it deleted it. assert len(pr.list_comments()) == 1 - assert len(fake_jira.issues) == 0 - -def test_external_pr_opened_with_cla(fake_github, fake_jira): +def test_external_pr_opened_with_cla(fake_github): pr = fake_github.make_pull_request(owner="openedx", repo="some-code", user="tusbar", number=11235) prj = pr.as_json() @@ -137,7 +134,6 @@ def test_external_pr_opened_with_cla(fake_github, fake_jira): assert anything_happened is True assert issue_id is None - assert len(fake_jira.issues) == 0 # Check the GitHub comment that was created. pr_comments = pr.list_comments() @@ -165,17 +161,15 @@ def test_external_pr_opened_with_cla(fake_github, fake_jira): # re-opening it deleted it. pr_comments = pr.list_comments() assert len(pr_comments) == 1 - assert len(fake_jira.issues) == 0 -def test_blended_pr_opened_with_cla(fake_github, fake_jira): +def test_blended_pr_opened_with_cla(fake_github): pr = fake_github.make_pull_request(owner="openedx", repo="some-code", user="tusbar", title="[BD-34] Something good") prj = pr.as_json() issue_id, anything_happened = pull_request_changed(prj) assert anything_happened is True assert issue_id is None - assert len(fake_jira.issues) == 0 # Check the GitHub comment that was created. pr_comments = pr.list_comments() @@ -197,7 +191,7 @@ def test_blended_pr_opened_with_cla(fake_github, fake_jira): assert pull_request_projects(pr.as_json()) == {settings.GITHUB_BLENDED_PROJECT} -def test_external_pr_rescanned(fake_github, fake_jira): +def test_external_pr_rescanned(fake_github): # Rescanning a pull request shouldn't do anything. # Make a pull request and process it. @@ -218,40 +212,8 @@ def test_external_pr_rescanned(fake_github, fake_jira): assert len(pr.list_comments()) == 1 -def test_title_change_changes_jira_project(fake_github, fake_jira): - """ - A blended developer opens a PR, but forgets to put "[BD]" in the title. - """ - # The developer makes a pull request, but forgets the right syntax in the title. - pr = fake_github.make_pull_request(user="tusbar", title="This is for BD-34") - - ospr_id, anything_happened = pull_request_changed(pr.as_json()) - - # An OSPR issue was not made. - assert ospr_id is None - assert anything_happened is True - - # Someone assigns an ad-hoc label to the PR. - pr.repo.add_label(name="pretty") - pr.labels.add("pretty") - - # The developer changes the title. - pr.title = "This is for [BD-34]." - issue_id, anything_happened = pull_request_changed(pr.as_json()) - - assert anything_happened is True - assert issue_id is None - - # The bot comment now mentions the new issue. - pr_comments = pr.list_comments() - assert len(pr_comments) == 1 - - # The pull request still has the ad-hoc label. - assert "pretty" in pr.labels - - @pytest.mark.parametrize("pr_type", ["normal", "blended", "nocla"]) -def test_draft_pr_opened(pr_type, fake_github, fake_jira, mocker): +def test_draft_pr_opened(pr_type, fake_github, mocker): # pylint: disable=too-many-statements # Set the GITHUB_STATUS_LABEL variable with a set() of labels that should map to jira issues. @@ -283,7 +245,6 @@ def test_draft_pr_opened(pr_type, fake_github, fake_jira, mocker): assert anything_happened is True assert issue_id is None - assert len(fake_jira.issues) == 0 # Check the GitHub comment that was created. pr_comments = pr.list_comments() @@ -313,8 +274,7 @@ def test_draft_pr_opened(pr_type, fake_github, fake_jira, mocker): pr.title = title2 issue_id2, _ = pull_request_changed(pr.as_json()) - assert issue_id2 == issue_id - assert len(fake_jira.issues) == 0 + assert issue_id2 is None pr_comments = pr.list_comments() assert len(pr_comments) == 1 @@ -326,8 +286,7 @@ def test_draft_pr_opened(pr_type, fake_github, fake_jira, mocker): pr.title = title1 issue_id3, _ = pull_request_changed(pr.as_json()) - assert issue_id3 == issue_id - assert len(fake_jira.issues) == 0 + assert issue_id3 is None pr_comments = pr.list_comments() assert len(pr_comments) == 1 @@ -336,14 +295,13 @@ def test_draft_pr_opened(pr_type, fake_github, fake_jira, mocker): assert 'click "Ready for Review"' in body -def test_handle_closed_pr(is_merged, fake_github, fake_jira): +def test_handle_closed_pr(is_merged, fake_github): pr = fake_github.make_pull_request(user="tusbar", number=11237, state="closed", merged=is_merged) prj = pr.as_json() issue_id1, anything_happened = pull_request_changed(prj) assert anything_happened is True assert issue_id1 is None - assert len(fake_jira.issues) == 0 # Check the GitHub comment that was created. pr_comments = pr.list_comments() @@ -364,38 +322,34 @@ def test_handle_closed_pr(is_merged, fake_github, fake_jira): assert pull_request_projects(pr.as_json()) == {settings.GITHUB_OSPR_PROJECT} # Rescan the pull request. - num_issues = len(fake_jira.issues) issue_id2, anything_happened2 = pull_request_changed(pr.as_json()) - assert issue_id2 == issue_id1 + assert issue_id2 is None assert anything_happened2 is False - # No Jira issue was created. - assert len(fake_jira.issues) == num_issues - # No new GitHub comment was created. assert len(pr.list_comments()) == 1 -def test_dont_add_internal_prs_to_project(fake_github, fake_jira): +def test_dont_add_internal_prs_to_project(fake_github): pr = fake_github.make_pull_request(owner="openedx", repo="credentials", user="nedbat") pull_request_changed(pr.as_json()) assert pull_request_projects(pr.as_json()) == set() -def test_add_external_prs_to_project(fake_github, fake_jira): +def test_add_external_prs_to_project(fake_github): pr = fake_github.make_pull_request(owner="openedx", repo="credentials", user="tusbar") pull_request_changed(pr.as_json()) assert pull_request_projects(pr.as_json()) == {settings.GITHUB_OSPR_PROJECT, ("openedx", 23)} -def test_dont_add_draft_prs_to_project(fake_github, fake_jira): +def test_dont_add_draft_prs_to_project(fake_github): pr = fake_github.make_pull_request(owner="openedx", repo="credentials", user="tusbar", draft=True) pull_request_changed(pr.as_json()) assert pull_request_projects(pr.as_json()) == {settings.GITHUB_OSPR_PROJECT} -def test_add_to_multiple_projects(fake_github, fake_jira): +def test_add_to_multiple_projects(fake_github): pr = fake_github.make_pull_request(owner="anotherorg", repo="multi-project", user="tusbar") pull_request_changed(pr.as_json()) assert pull_request_projects(pr.as_json()) == { From 1bcaba2cc6c50f975e2d218792c1f3571a22b8df Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 24 Sep 2023 07:01:37 -0400 Subject: [PATCH 02/12] refactor: remove more unneeded code --- openedx_webhooks/tasks/github_work.py | 15 --------------- tests/fake_jira.py | 19 ------------------- tests/test_fake_jira.py | 4 ---- 3 files changed, 38 deletions(-) delete mode 100644 openedx_webhooks/tasks/github_work.py diff --git a/openedx_webhooks/tasks/github_work.py b/openedx_webhooks/tasks/github_work.py deleted file mode 100644 index 82bdc269..00000000 --- a/openedx_webhooks/tasks/github_work.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Operations on GitHub data. -""" - -from typing import Any, Dict - -from openedx_webhooks.auth import get_github_session -from openedx_webhooks.utils import paginated_get - - -def get_repo_labels(repo: str) -> Dict[str, Dict[str, Any]]: - """Get a dict mapping label names to full label info.""" - url = f"/repos/{repo}/labels" - repo_labels = {lbl["name"]: lbl for lbl in paginated_get(url, session=get_github_session())} - return repo_labels diff --git a/tests/fake_jira.py b/tests/fake_jira.py index 4eaf68fa..bc0bb33a 100644 --- a/tests/fake_jira.py +++ b/tests/fake_jira.py @@ -2,7 +2,6 @@ import dataclasses import itertools -import re from dataclasses import dataclass, field from typing import Dict, List, Optional, Set @@ -215,21 +214,3 @@ def _post_issue_transitions(self, match, request, _context): assert issue is not None transition_id = request.json()["transition"]["id"] issue.status = self.TRANSITION_IDS[transition_id] - - @faker.route(r"/rest/api/2/search", "GET") - def _get_search(self, _match, request, _context): - """ - Implement the search endpoint. - """ - jql = request.qs["jql"][0] - # We only handle certain specific queries. - if bd_ids := re.findall(r'"Blended Project ID" ~ "(.*?)"', jql): - issues = [iss for iss in self.issues.values() if iss.blended_project_id in bd_ids] - else: - # We don't understand this query. - _context.status_code = 500 - return None - return { - "issues": [iss.as_json() for iss in issues], - "total": len(issues), - } diff --git a/tests/test_fake_jira.py b/tests/test_fake_jira.py index cc37e01c..bffa42f9 100644 --- a/tests/test_fake_jira.py +++ b/tests/test_fake_jira.py @@ -78,7 +78,3 @@ def test_no_such_put(self, fake_jira): def test_no_such_transitions(self, fake_jira): resp = requests.get("https://test.atlassian.net/rest/api/2/issue/XYZ-999/transitions") assert resp.status_code == 404 - - def test_baffling_search(self, fake_jira): - resp = requests.get("https://test.atlassian.net/rest/api/2/search?jql=xyzzy") - assert resp.status_code == 500 From ac764bfe5dabae6a4d1be5813f5019dd0e2fa5a7 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sun, 24 Sep 2023 07:02:05 -0400 Subject: [PATCH 03/12] style: get to clean mypy and ruff runs --- openedx_webhooks/__init__.py | 4 ++-- openedx_webhooks/github_views.py | 10 +++++----- openedx_webhooks/tasks/pr_tracking.py | 2 +- openedx_webhooks/utils.py | 6 ++++-- setup.cfg | 2 +- setup.py | 2 +- tests/fake_github.py | 2 +- tests/faker.py | 1 + tests/helpers.py | 2 +- tests/test_fake_github.py | 4 ++-- 10 files changed, 19 insertions(+), 16 deletions(-) diff --git a/openedx_webhooks/__init__.py b/openedx_webhooks/__init__.py index e1a0bbf6..03773849 100644 --- a/openedx_webhooks/__init__.py +++ b/openedx_webhooks/__init__.py @@ -37,7 +37,7 @@ def expand_config(name=None): def create_app(config=None): app = Flask(__name__) - app.wsgi_app = ProxyFix(app.wsgi_app) + app.wsgi_app = ProxyFix(app.wsgi_app) # type: ignore[method-assign] config = config or os.environ.get("OPENEDX_WEBHOOKS_CONFIG") or "default" # Instantiate the config object because we rely on the __init__ # function to translate config between heroku and what sqlalchemy wants @@ -70,7 +70,7 @@ def create_celery_app(app=None, config="worker"): app = app or create_app(config=config) celery.main = app.import_name celery.conf.update(app.config) - class ContextTask(celery.Task): + class ContextTask(celery.Task): # type: ignore[name-defined] abstract = True def __call__(self, *args, **kwargs): if "wsgi_environ" in kwargs: diff --git a/openedx_webhooks/github_views.py b/openedx_webhooks/github_views.py index d23a33a2..b24e1220 100644 --- a/openedx_webhooks/github_views.py +++ b/openedx_webhooks/github_views.py @@ -3,6 +3,7 @@ """ import logging +from typing import cast from flask import current_app as app from flask import Blueprint, jsonify, render_template, request @@ -37,7 +38,7 @@ def hook_receiver(): """ signature = request.headers.get("X-Hub-Signature") secret = app.config.get('GITHUB_WEBHOOKS_SECRET') - if not is_valid_payload(secret, signature, request.data): + if not is_valid_payload(secret, signature, request.data): # type: ignore[arg-type] msg = "Rejecting because signature doesn't match!" logging.info(msg) return msg, 403 @@ -157,7 +158,7 @@ def rescan(): :func:`~openedx_webhooks.tasks.github.pull_request_changed` must be idempotent. It could run many times over the same pull request. """ - repo = request.form.get("repo") + repo = cast(str, request.form.get("repo")) inline = bool(request.form.get("inline", False)) rescan_kwargs = dict( # pylint: disable=use-dict-literal @@ -174,7 +175,7 @@ def rescan(): org = repo[4:] return queue_task(rescan_organization_task, org, **rescan_kwargs) elif inline: - return jsonify(rescan_repository(repo, **rescan_kwargs)) + return jsonify(rescan_repository(repo, **rescan_kwargs)) # type: ignore[arg-type] else: return queue_task(rescan_repository_task, repo, **rescan_kwargs) @@ -204,8 +205,7 @@ def process_pr(): resp = jsonify({"error": "Pull request number required"}) resp.status_code = 400 return resp - num = int(num) - pr_resp = get_github_session().get("/repos/{repo}/pulls/{num}".format(repo=repo, num=num)) + pr_resp = get_github_session().get(f"/repos/{repo}/pulls/{num}") if not pr_resp.ok: resp = jsonify({"error": pr_resp.text}) resp.status_code = 400 diff --git a/openedx_webhooks/tasks/pr_tracking.py b/openedx_webhooks/tasks/pr_tracking.py index 5d0ea5fa..b6da7290 100644 --- a/openedx_webhooks/tasks/pr_tracking.py +++ b/openedx_webhooks/tasks/pr_tracking.py @@ -708,7 +708,7 @@ def edit_comment_on_pull_request(self, *, comment_body: str) -> None: resp = get_github_session().patch(url, json={"body": comment_body}) log_check_response(resp) - def delete_comment_on_pull_request(self, *, comment_id: int) -> None: + def delete_comment_on_pull_request(self, *, comment_id: str) -> None: url = f"/repos/{self.prid.full_name}/issues/comments/{comment_id}" logger.info(f"Deleting comment on PR {self.prid}") resp = get_github_session().delete(url) diff --git a/openedx_webhooks/utils.py b/openedx_webhooks/utils.py index 15caf850..ae5673cd 100644 --- a/openedx_webhooks/utils.py +++ b/openedx_webhooks/utils.py @@ -357,7 +357,8 @@ def github_pr_repo(issue): parent_ref = issue["fields"].get("parent") if not pr_repo and parent_ref: parent = get_jira_issue(parent_ref["key"]) - pr_repo = parent["fields"].get(custom_fields["Repo"]) + if parent is not None: + pr_repo = parent["fields"].get(custom_fields["Repo"]) return pr_repo @@ -367,7 +368,8 @@ def github_pr_num(issue): parent_ref = issue["fields"].get("parent") if not pr_num and parent_ref: parent = get_jira_issue(parent_ref["key"]) - pr_num = parent["fields"].get(custom_fields["PR Number"]) + if parent is not None: + pr_num = parent["fields"].get(custom_fields["PR Number"]) try: return int(pr_num) except Exception: # pylint: disable=broad-except diff --git a/setup.cfg b/setup.cfg index 2046f708..e810e5cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [mypy] ignore_missing_imports = True -#check_untyped_defs = True +check_untyped_defs = True [tool:pytest] markers = diff --git a/setup.py b/setup.py index 91dc6056..17e24a9e 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def is_requirement(line): def get_requirements(path): with open(path) as f: lines = f.readlines() - return [l.strip() for l in lines if is_requirement(l)] + return [line.strip() for line in lines if is_requirement(line)] version = '' diff --git a/tests/fake_github.py b/tests/fake_github.py index fb02d183..e3549050 100644 --- a/tests/fake_github.py +++ b/tests/fake_github.py @@ -130,7 +130,7 @@ def as_json(self, brief=False) -> Dict: "title": self.title, "user": self.user.as_json(), "body": self.body, - "labels": [self.repo.get_label(l).as_json() for l in sorted(self.labels)], + "labels": [self.repo.get_label(lbl).as_json() for lbl in sorted(self.labels)], "base": { "repo": self.repo.as_json(), "ref": self.ref, diff --git a/tests/faker.py b/tests/faker.py index 8ead8db1..04dbb94c 100644 --- a/tests/faker.py +++ b/tests/faker.py @@ -138,6 +138,7 @@ def reset_mock(self) -> None: """ Clear the `requests_made` history. """ + assert self.requests_mocker is not None self.requests_mocker.reset_mock() def assert_readonly(self) -> None: diff --git a/tests/helpers.py b/tests/helpers.py index f869e94c..ac608b63 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -33,7 +33,7 @@ def check_good_markdown(text: str) -> None: raise ValueError(f"Markdown has a link to a None url: {text!r}") -def check_issue_link_in_markdown(text: str, issue_id: str) -> None: +def check_issue_link_in_markdown(text: str, issue_id: str|None) -> None: """ Check that `text` has properly links to `issue_id`. diff --git a/tests/test_fake_github.py b/tests/test_fake_github.py index 155d9e04..cc40545e 100644 --- a/tests/test_fake_github.py +++ b/tests/test_fake_github.py @@ -155,7 +155,7 @@ def test_updating_labels_with_api(self, fake_github): assert resp.status_code == 200 prj = resp.json() assert prj["title"] == "Here is a pull request" - label_summary = [(l["name"], l["color"]) for l in prj["labels"]] + label_summary = [(lbl["name"], lbl["color"]) for lbl in prj["labels"]] assert label_summary == [ ("another label", "ededed"), ("bug", "d73a4a"), @@ -183,7 +183,7 @@ def test_updating_labels_elsewhere(self, fake_github): assert resp.status_code == 200 prj = resp.json() assert prj["title"] == "Here is a pull request" - label_summary = [(l["name"], l["color"]) for l in prj["labels"]] + label_summary = [(lbl["name"], lbl["color"]) for lbl in prj["labels"]] assert label_summary == [ ("another label", "ededed"), ("bug", "d73a4a"), From c577052c30a89480dec18f03c7be1485f50b6a76 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 13 Sep 2023 16:31:00 -0400 Subject: [PATCH 04/12] refactor: a settings object so future settings management will be easier --- openedx_webhooks/auth.py | 2 +- openedx_webhooks/bot_comments.py | 1 + openedx_webhooks/info.py | 2 +- openedx_webhooks/settings.py | 38 ++++++++++++++++----------- openedx_webhooks/tasks/pr_tracking.py | 4 +-- openedx_webhooks/ui.py | 2 +- openedx_webhooks/utils.py | 3 ++- tests/conftest.py | 9 ++++--- tests/test_pull_request_opened.py | 2 +- 9 files changed, 37 insertions(+), 26 deletions(-) diff --git a/openedx_webhooks/auth.py b/openedx_webhooks/auth.py index 32fb1615..155c8fed 100644 --- a/openedx_webhooks/auth.py +++ b/openedx_webhooks/auth.py @@ -5,7 +5,7 @@ import requests from urlobject import URLObject -from openedx_webhooks import settings +from openedx_webhooks.settings import settings class BaseUrlSession(requests.Session): diff --git a/openedx_webhooks/bot_comments.py b/openedx_webhooks/bot_comments.py index d601ed84..ddcbc64e 100644 --- a/openedx_webhooks/bot_comments.py +++ b/openedx_webhooks/bot_comments.py @@ -16,6 +16,7 @@ is_draft_pull_request, pull_request_has_cla, ) +from openedx_webhooks.settings import settings from openedx_webhooks.types import PrDict diff --git a/openedx_webhooks/info.py b/openedx_webhooks/info.py index b6178f97..f20d12aa 100644 --- a/openedx_webhooks/info.py +++ b/openedx_webhooks/info.py @@ -9,8 +9,8 @@ import yaml from glom import glom -from openedx_webhooks import settings from openedx_webhooks.auth import get_github_session +from openedx_webhooks.settings import settings from openedx_webhooks.types import GhProject, PrDict, PrCommentDict, PrId from openedx_webhooks.utils import ( memoize, diff --git a/openedx_webhooks/settings.py b/openedx_webhooks/settings.py index 58aeeae0..8155b7d9 100644 --- a/openedx_webhooks/settings.py +++ b/openedx_webhooks/settings.py @@ -1,21 +1,33 @@ """Settings for how the webhook should behave.""" import os -from typing import Optional +from types import SimpleNamespace +from typing import Mapping, Optional from openedx_webhooks.types import GhProject -# The Jira server to use. Missing or "" will become None, -# meaning don't use Jira at all. -JIRA_SERVER = os.environ.get("JIRA_SERVER", None) or None -JIRA_USER_EMAIL = os.environ.get("JIRA_USER_EMAIL", None) -JIRA_USER_TOKEN = os.environ.get("JIRA_USER_TOKEN", None) +settings = SimpleNamespace() -GITHUB_PERSONAL_TOKEN = os.environ.get("GITHUB_PERSONAL_TOKEN", None) +def settings_from_environment(env: Mapping) -> None: + # The Jira server to use. Missing or "" will become None, + # meaning don't use Jira at all. + settings.JIRA_SERVER = env.get("JIRA_SERVER", None) or None + settings.JIRA_USER_EMAIL = env.get("JIRA_USER_EMAIL", None) + settings.JIRA_USER_TOKEN = env.get("JIRA_USER_TOKEN", None) -def read_project_setting(setting_name: str) -> Optional[GhProject]: + settings.GITHUB_PERSONAL_TOKEN = env.get("GITHUB_PERSONAL_TOKEN", None) + + # The project all OSPRs should be added to. + # This should be in the form of org:num, like "openedx:19" + settings.GITHUB_OSPR_PROJECT = read_project_setting(env, "GITHUB_OSPR_PROJECT") + + # The project all Blended pull requests should be added to. + settings.GITHUB_BLENDED_PROJECT = read_project_setting(env, "GITHUB_BLENDED_PROJECT") + + +def read_project_setting(env: Mapping, setting_name: str) -> Optional[GhProject]: """Read a project spec from a setting. Project number NUM in org ORG is specified as ``ORG:NUM``. @@ -24,19 +36,15 @@ def read_project_setting(setting_name: str) -> Optional[GhProject]: ("ORG", NUM) if the setting is present. None if the setting is missing. """ - project = os.environ.get(setting_name, None) + project = env.get(setting_name, None) if project is not None: org, num = project.split(":") return (org, int(num)) return None -# The project all OSPRs should be added to. -# This should be in the form of org:num, like "openedx:19" -GITHUB_OSPR_PROJECT = read_project_setting("GITHUB_OSPR_PROJECT") - -# The project all Blended pull requests should be added to. -GITHUB_BLENDED_PROJECT = read_project_setting("GITHUB_BLENDED_PROJECT") +# Read our settings from the real environment. +settings_from_environment(os.environ) # Made-up values to use while testing. diff --git a/openedx_webhooks/tasks/pr_tracking.py b/openedx_webhooks/tasks/pr_tracking.py index b6da7290..ca8413eb 100644 --- a/openedx_webhooks/tasks/pr_tracking.py +++ b/openedx_webhooks/tasks/pr_tracking.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Set, Tuple, cast -from openedx_webhooks import settings +from openedx_webhooks.auth import get_github_session, get_jira_session from openedx_webhooks.bot_comments import ( BOT_COMMENT_INDICATORS, BOT_COMMENTS_FIRST, @@ -54,7 +54,7 @@ GITHUB_STATUS_LABELS, JIRA_CATEGORY_LABELS, ) -from openedx_webhooks.auth import get_github_session, get_jira_session +from openedx_webhooks.settings import settings from openedx_webhooks.tasks import logger from openedx_webhooks.tasks.jira_work import ( transition_jira_issue, diff --git a/openedx_webhooks/ui.py b/openedx_webhooks/ui.py index 8561237b..c4484b94 100644 --- a/openedx_webhooks/ui.py +++ b/openedx_webhooks/ui.py @@ -2,8 +2,8 @@ from flask import Blueprint, render_template -from openedx_webhooks import settings from openedx_webhooks.auth import get_github_session, get_jira_session +from openedx_webhooks.settings import settings from openedx_webhooks.utils import requires_auth ui = Blueprint('ui', __name__) diff --git a/openedx_webhooks/utils.py b/openedx_webhooks/utils.py index ae5673cd..8a4e5b8a 100644 --- a/openedx_webhooks/utils.py +++ b/openedx_webhooks/utils.py @@ -17,8 +17,9 @@ from flask import jsonify, request, Response, url_for from urlobject import URLObject -from openedx_webhooks import logger, settings +from openedx_webhooks import logger from openedx_webhooks.auth import get_github_session, get_jira_session +from openedx_webhooks.settings import settings from openedx_webhooks.types import JiraDict diff --git a/tests/conftest.py b/tests/conftest.py index 28d55765..b246020a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,9 +8,10 @@ import requests_mock import openedx_webhooks -import openedx_webhooks.utils import openedx_webhooks.info -from openedx_webhooks import settings +import openedx_webhooks.utils +import openedx_webhooks.settings +from openedx_webhooks.settings import settings from .fake_github import FakeGitHub from .fake_jira import FakeJira @@ -77,9 +78,9 @@ def pytest_addoption(parser): @pytest.fixture(autouse=True) def settings_for_tests(mocker): - for name, value in vars(settings.TestSettings).items(): + for name, value in vars(openedx_webhooks.settings.TestSettings).items(): if name.isupper(): - mocker.patch(f"openedx_webhooks.settings.{name}", value) + mocker.patch(f"openedx_webhooks.settings.settings.{name}", value) @pytest.fixture diff --git a/tests/test_pull_request_opened.py b/tests/test_pull_request_opened.py index c7920d65..d8789ffd 100644 --- a/tests/test_pull_request_opened.py +++ b/tests/test_pull_request_opened.py @@ -2,7 +2,6 @@ import pytest -from openedx_webhooks import settings from openedx_webhooks.bot_comments import ( BotComment, is_comment_kind, @@ -16,6 +15,7 @@ CLA_STATUS_PRIVATE, ) from openedx_webhooks.gh_projects import pull_request_projects +from openedx_webhooks.settings import settings from openedx_webhooks.tasks.github import pull_request_changed from .helpers import check_issue_link_in_markdown From b13a3223d9e59e7c77f4bd884c749f41706b266b Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 13 Sep 2023 17:30:54 -0400 Subject: [PATCH 05/12] refactor: test settings are now treated as environment variables --- openedx_webhooks/settings.py | 12 ++---------- tests/conftest.py | 10 ++++++---- tests/settings.py | 9 +++++++++ tests/test_auth.py | 9 +++++---- 4 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 tests/settings.py diff --git a/openedx_webhooks/settings.py b/openedx_webhooks/settings.py index 8155b7d9..48901c3c 100644 --- a/openedx_webhooks/settings.py +++ b/openedx_webhooks/settings.py @@ -7,10 +7,12 @@ from openedx_webhooks.types import GhProject +# The global settings object. settings = SimpleNamespace() def settings_from_environment(env: Mapping) -> None: + """Update `settings` from a dict of environment variables.""" # The Jira server to use. Missing or "" will become None, # meaning don't use Jira at all. settings.JIRA_SERVER = env.get("JIRA_SERVER", None) or None @@ -45,13 +47,3 @@ def read_project_setting(env: Mapping, setting_name: str) -> Optional[GhProject] # Read our settings from the real environment. settings_from_environment(os.environ) - - -# Made-up values to use while testing. -class TestSettings: - GITHUB_BLENDED_PROJECT = ("blendorg", 42) - GITHUB_OSPR_PROJECT = ("testorg", 17) - GITHUB_PERSONAL_TOKEN = "github_pat_FooBarBaz" - JIRA_SERVER = "https://test.atlassian.net" - JIRA_USER_EMAIL = "someone@megacorp.com" - JIRA_USER_TOKEN = "xyzzy-123-plugh" diff --git a/tests/conftest.py b/tests/conftest.py index b246020a..a5cb6eab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ import openedx_webhooks.settings from openedx_webhooks.settings import settings +from . import settings as test_settings from .fake_github import FakeGitHub from .fake_jira import FakeJira @@ -78,10 +79,11 @@ def pytest_addoption(parser): @pytest.fixture(autouse=True) def settings_for_tests(mocker): - for name, value in vars(openedx_webhooks.settings.TestSettings).items(): - if name.isupper(): - mocker.patch(f"openedx_webhooks.settings.settings.{name}", value) - + # This patch will restore the real settings when the fixture is done, even + # though the patch itself doesn't change any. + mocker.patch.dict(settings.__dict__, {}) + # Update the real settings from our test environment variables. + openedx_webhooks.settings.settings_from_environment(test_settings.__dict__) @pytest.fixture def fake_github(pytestconfig, mocker, requests_mocker, fake_repo_data): diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 00000000..ea914a61 --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,9 @@ +"""Made-up settings to use during tests.""" + +GITHUB_BLENDED_PROJECT = "blendorg:42" +GITHUB_OSPR_PROJECT = "testorg:17" +GITHUB_PERSONAL_TOKEN = "github_pat_FooBarBaz" + +JIRA_SERVER = "https://test.atlassian.net" +JIRA_USER_EMAIL = "someone@megacorp.com" +JIRA_USER_TOKEN = "xyzzy-123-plugh" diff --git a/tests/test_auth.py b/tests/test_auth.py index 8a64ed2f..24f92157 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,14 +1,15 @@ import base64 from openedx_webhooks.auth import get_github_session, get_jira_session -from openedx_webhooks.settings import TestSettings + +from . import settings as test_settings def test_get_github_session(fake_github): session = get_github_session() response = session.get("/user") headers = response.request.headers - assert headers["Authorization"] == f"token {TestSettings.GITHUB_PERSONAL_TOKEN}" + assert headers["Authorization"] == f"token {test_settings.GITHUB_PERSONAL_TOKEN}" assert response.url == "https://api.github.com/user" @@ -16,7 +17,7 @@ def test_get_jira_session(fake_jira): session = get_jira_session() response = session.get("/rest/api/2/field") headers = response.request.headers - user_token = f"{TestSettings.JIRA_USER_EMAIL}:{TestSettings.JIRA_USER_TOKEN}" + user_token = f"{test_settings.JIRA_USER_EMAIL}:{test_settings.JIRA_USER_TOKEN}" basic_auth = base64.b64encode(user_token.encode()).decode() assert headers["Authorization"] == f"Basic {basic_auth}" - assert response.url == f"{TestSettings.JIRA_SERVER}/rest/api/2/field" + assert response.url == f"{test_settings.JIRA_SERVER}/rest/api/2/field" From 724cfca258af2131111f8d5326284bb0cb23d440 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 22 Sep 2023 10:57:59 -0400 Subject: [PATCH 06/12] refactor: allow for multiple jira servers --- openedx_webhooks/auth.py | 13 ++++++++++--- openedx_webhooks/bot_comments.py | 1 - openedx_webhooks/info.py | 17 ++++++++++++++++- openedx_webhooks/settings.py | 6 ------ openedx_webhooks/types.py | 18 ++++++++++++++++++ tests/conftest.py | 4 ++-- .../openedx-webhooks-data/jira-info.yaml | 14 ++++++++++++++ tests/settings.py | 4 ---- tests/test_auth.py | 6 +++--- tests/test_info.py | 13 +++++++++++-- 10 files changed, 74 insertions(+), 22 deletions(-) create mode 100644 tests/repo_data/openedx/openedx-webhooks-data/jira-info.yaml diff --git a/openedx_webhooks/auth.py b/openedx_webhooks/auth.py index 155c8fed..da66a55d 100644 --- a/openedx_webhooks/auth.py +++ b/openedx_webhooks/auth.py @@ -26,12 +26,19 @@ def request(self, method, url, data=None, headers=None, **kwargs): ) -def get_jira_session(): +def get_jira_session(jira_nick): """ Get the Jira session to use, in an easily test-patchable way. + + `jira_nick` is a nickname for one of our configured Jira servers. """ - session = BaseUrlSession(base_url=settings.JIRA_SERVER) - session.auth = (settings.JIRA_USER_EMAIL, settings.JIRA_USER_TOKEN) + # Avoid a circular import. + from openedx_webhooks.info import get_jira_info + + jira_info = get_jira_info() + jira_server = jira_info[jira_nick.lower()] + session = BaseUrlSession(base_url=jira_server.server) + session.auth = (jira_server.email, jira_server.token) session.trust_env = False # prevent reading the local .netrc return session diff --git a/openedx_webhooks/bot_comments.py b/openedx_webhooks/bot_comments.py index ddcbc64e..d601ed84 100644 --- a/openedx_webhooks/bot_comments.py +++ b/openedx_webhooks/bot_comments.py @@ -16,7 +16,6 @@ is_draft_pull_request, pull_request_has_cla, ) -from openedx_webhooks.settings import settings from openedx_webhooks.types import PrDict diff --git a/openedx_webhooks/info.py b/openedx_webhooks/info.py index f20d12aa..daceadd1 100644 --- a/openedx_webhooks/info.py +++ b/openedx_webhooks/info.py @@ -11,7 +11,7 @@ from openedx_webhooks.auth import get_github_session from openedx_webhooks.settings import settings -from openedx_webhooks.types import GhProject, PrDict, PrCommentDict, PrId +from openedx_webhooks.types import GhProject, JiraServer, PrDict, PrCommentDict, PrId from openedx_webhooks.utils import ( memoize, memoize_timed, @@ -123,6 +123,21 @@ def get_orgs_file(): orgs[org_data["name"]] = org_data return orgs + +@memoize_timed(minutes=30) +def get_jira_info(): + jira_info = {} + for key, info in _read_yaml_data_file("jira-info.yaml").items(): + jira_info[key.lower()] = JiraServer(**info) + return jira_info + + +def get_jira_server_info(jira_nick: str) -> JiraServer: + jira_info = get_jira_info() + jira_server = jira_info[jira_nick.lower()] + return jira_server + + def is_internal_pull_request(pull_request: PrDict) -> bool: """ Is this pull request's author internal to the PR's GitHub org? diff --git a/openedx_webhooks/settings.py b/openedx_webhooks/settings.py index 48901c3c..26d8d872 100644 --- a/openedx_webhooks/settings.py +++ b/openedx_webhooks/settings.py @@ -13,12 +13,6 @@ def settings_from_environment(env: Mapping) -> None: """Update `settings` from a dict of environment variables.""" - # The Jira server to use. Missing or "" will become None, - # meaning don't use Jira at all. - settings.JIRA_SERVER = env.get("JIRA_SERVER", None) or None - settings.JIRA_USER_EMAIL = env.get("JIRA_USER_EMAIL", None) - settings.JIRA_USER_TOKEN = env.get("JIRA_USER_TOKEN", None) - settings.GITHUB_PERSONAL_TOKEN = env.get("GITHUB_PERSONAL_TOKEN", None) # The project all OSPRs should be added to. diff --git a/openedx_webhooks/types.py b/openedx_webhooks/types.py index 9779be8a..7697a3e9 100644 --- a/openedx_webhooks/types.py +++ b/openedx_webhooks/types.py @@ -35,3 +35,21 @@ def __str__(self): def org(self): org, _, _ = self.full_name.partition("/") return org + + +@dataclasses.dataclass(frozen=True) +class JiraServer: + """A Jira server and its credentials.""" + # The URL of the Jira server. + server: str + + # The user email and token to use for credentials. + email: str + token: str + + # A description of the server, suitable for the bot to comment, + # "I created an issue in {{ description }}." + description: str = "a Jira server" + + mapping: str | None = None + project: str | None = None diff --git a/tests/conftest.py b/tests/conftest.py index a5cb6eab..c5145ee2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -97,8 +97,8 @@ def fake_github(pytestconfig, mocker, requests_mocker, fake_repo_data): @pytest.fixture -def fake_jira(requests_mocker): - the_fake_jira = FakeJira(settings.JIRA_SERVER) +def fake_jira(requests_mocker, fake_repo_data): + the_fake_jira = FakeJira("https://test.atlassian.net") the_fake_jira.install_mocks(requests_mocker) return the_fake_jira diff --git a/tests/repo_data/openedx/openedx-webhooks-data/jira-info.yaml b/tests/repo_data/openedx/openedx-webhooks-data/jira-info.yaml new file mode 100644 index 00000000..04763772 --- /dev/null +++ b/tests/repo_data/openedx/openedx-webhooks-data/jira-info.yaml @@ -0,0 +1,14 @@ +# Jira connectivity information for the openedx-webhook + +Test1: + server: https://test.atlassian.net + email: jira-user@test1.com + token: asdasdasdasdasd + mapping: https://github.com/test1/.github/jira-mapping.yaml + description: the private Test1 Jira +AnotherTest: + server: https://anothertest.atlassian.net + email: jira-user@anothertest.com + token: xyzzyxyzzy + project: OPEN + description: the Another Test Jira diff --git a/tests/settings.py b/tests/settings.py index ea914a61..bf6ad529 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -3,7 +3,3 @@ GITHUB_BLENDED_PROJECT = "blendorg:42" GITHUB_OSPR_PROJECT = "testorg:17" GITHUB_PERSONAL_TOKEN = "github_pat_FooBarBaz" - -JIRA_SERVER = "https://test.atlassian.net" -JIRA_USER_EMAIL = "someone@megacorp.com" -JIRA_USER_TOKEN = "xyzzy-123-plugh" diff --git a/tests/test_auth.py b/tests/test_auth.py index 24f92157..2c4a8498 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -14,10 +14,10 @@ def test_get_github_session(fake_github): def test_get_jira_session(fake_jira): - session = get_jira_session() + session = get_jira_session("test1") response = session.get("/rest/api/2/field") headers = response.request.headers - user_token = f"{test_settings.JIRA_USER_EMAIL}:{test_settings.JIRA_USER_TOKEN}" + user_token = "jira-user@test1.com:asdasdasdasdasd" basic_auth = base64.b64encode(user_token.encode()).decode() assert headers["Authorization"] == f"Basic {basic_auth}" - assert response.url == f"{test_settings.JIRA_SERVER}/rest/api/2/field" + assert response.url == "https://test.atlassian.net/rest/api/2/field" diff --git a/tests/test_info.py b/tests/test_info.py index 1a062a22..f4042f8e 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -5,9 +5,11 @@ import pytest from openedx_webhooks.info import ( - get_people_file, - is_internal_pull_request, is_draft_pull_request, get_blended_project_id, + get_jira_info, + get_people_file, + is_draft_pull_request, + is_internal_pull_request, ) @@ -125,3 +127,10 @@ def test_check_people_missing_yaml_fields(): assert people[user].get('commiter') is None assert people[user].get('comments') is None assert people[user].get('before') is None + + +def test_jira_info(): + info = get_jira_info() + # These are specific items from our test jira-info.yaml file + assert info["test1"].server == "https://test.atlassian.net" + assert info["anothertest"].project == "OPEN" From a3e739a25069e4c42dba85eef05db76bed9be9f6 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 26 Sep 2023 15:36:14 -0400 Subject: [PATCH 07/12] refactor: remove old Jira behavior, change return values from work functions --- openedx_webhooks/info.py | 28 +--- openedx_webhooks/labels.py | 6 - openedx_webhooks/tasks/github.py | 42 +++--- openedx_webhooks/tasks/pr_tracking.py | 192 +++++++------------------- openedx_webhooks/types.py | 10 ++ tests/test_pull_request_closed.py | 21 ++- tests/test_pull_request_opened.py | 99 +++++-------- tests/test_rescan.py | 14 +- 8 files changed, 136 insertions(+), 276 deletions(-) diff --git a/openedx_webhooks/info.py b/openedx_webhooks/info.py index daceadd1..503962d8 100644 --- a/openedx_webhooks/info.py +++ b/openedx_webhooks/info.py @@ -4,13 +4,12 @@ import csv import logging import re -from typing import Dict, Iterable, Optional, Tuple, Union +from typing import Dict, Iterable, Optional import yaml from glom import glom from openedx_webhooks.auth import get_github_session -from openedx_webhooks.settings import settings from openedx_webhooks.types import GhProject, JiraServer, PrDict, PrCommentDict, PrId from openedx_webhooks.utils import ( memoize, @@ -267,31 +266,6 @@ def get_bot_comments(prid: PrId) -> Iterable[PrCommentDict]: yield comment -def get_jira_issue_key(pr: Union[PrId, PrDict]) -> Tuple[bool, Optional[str]]: - """ - Find mention of a Jira issue number in bot-authored comments. - - Returns: - on_our_jira (bool): is the Jira issue on the JIRA_SERVER? - issue_key (str): the id of the Jira issue. Can be None if no Jira issue - is on the pull request. - """ - if isinstance(pr, PrDict): - prid = PrId.from_pr_dict(pr) - else: - prid = pr - for comment in get_bot_comments(prid): - # search for the first occurrence of a JIRA ticket key in the comment body - match = re.search(r"(https://.*?)/browse/([A-Z]{2,}-\d+)\b", comment["body"]) - if match: - on_our_jira = (match[1] == settings.JIRA_SERVER) - jira_key = match[2] - return on_our_jira, jira_key - # If there is no jira id yet, return on_our_jira==True so that we will work - # on Jira to make new ids. - return True, None - - def get_catalog_info(repo_fullname: str) -> Dict: """Get the parsed catalog-info.yaml data from a repo, or {} if missing.""" yml = _read_github_file(repo_fullname, "catalog-info.yaml", not_there="{}") diff --git a/openedx_webhooks/labels.py b/openedx_webhooks/labels.py index 1fd72823..e5a53ac8 100644 --- a/openedx_webhooks/labels.py +++ b/openedx_webhooks/labels.py @@ -14,9 +14,3 @@ "blended", "open-source-contribution", } - -# These are categorization labels the bot assigns. - -JIRA_CATEGORY_LABELS = { - "blended", -} diff --git a/openedx_webhooks/tasks/github.py b/openedx_webhooks/tasks/github.py index cad0b49c..cf960ae5 100644 --- a/openedx_webhooks/tasks/github.py +++ b/openedx_webhooks/tasks/github.py @@ -4,7 +4,7 @@ import traceback -from typing import Dict, Optional, Tuple +from typing import Dict, Set from urlobject import URLObject @@ -16,9 +16,10 @@ current_support_state, desired_support_state, DryRunFixingActions, + FixResult, PrTrackingFixer, ) -from openedx_webhooks.types import PrDict +from openedx_webhooks.types import JiraId, PrDict from openedx_webhooks.utils import ( log_rate_limit, paginated_get, @@ -38,7 +39,7 @@ def pull_request_changed_task(_, pull_request): raise -def pull_request_changed(pr: PrDict, actions=None) -> Tuple[Optional[str], bool]: +def pull_request_changed(pr: PrDict, actions=None) -> FixResult: """ Process a pull request. @@ -52,10 +53,7 @@ def pull_request_changed(pr: PrDict, actions=None) -> Tuple[Optional[str], bool] As a result, it should not comment on the pull request without checking to see if it has *already* commented on the pull request. - Returns a 2-tuple. The first element in the tuple is the key of the JIRA - issue associated with the pull request, if any, as a string. The second - element in the tuple is a boolean indicating if this function did any - work, such as making a JIRA issue or commenting on the pull request. + Returns an object with details of the associated Jira issues. """ user = pr["user"]["login"] @@ -65,13 +63,10 @@ def pull_request_changed(pr: PrDict, actions=None) -> Tuple[Optional[str], bool] logger.info(f"Processing PR {repo}#{num} by @{user}...") desired = desired_support_state(pr) - if desired is not None: - current = current_support_state(pr) - fixer = PrTrackingFixer(pr, current, desired, actions=actions) - fixer.fix() - return fixer.result() - else: - return None, False + current = current_support_state(pr) + fixer = PrTrackingFixer(pr, current, desired, actions=actions) + fixer.fix() + return fixer.result() class PaginateCallback: @@ -137,7 +132,8 @@ def rescan_repository( state = "all" if allpr else "open" url = f"/repos/{repo}/pulls?state={state}" - changed: Dict[int, Optional[str]] = {} + changed: Dict[int, Set[JiraId]] = {} + errors: Dict[int, str] = {} dry_run_actions = {} # Pull requests before this will not be rescanned. Contractor messages @@ -167,12 +163,12 @@ def rescan_repository( resp.raise_for_status() pull_request = resp.json() - issue_key, anything_happened = pull_request_changed(pull_request, actions=actions) + result = pull_request_changed(pull_request, actions=actions) except Exception: # pylint: disable=broad-except - changed[pull_request["number"]] = traceback.format_exc() + errors[pull_request["number"]] = traceback.format_exc() else: - if anything_happened: - changed[pull_request["number"]] = issue_key + if result.changed_jira_issues: + changed[pull_request["number"]] = result.changed_jira_issues if dry_run: assert actions is not None dry_run_actions[pull_request["number"]] = actions.action_calls @@ -184,9 +180,11 @@ def rescan_repository( ), ) - info: Dict = {"repo": repo} - if changed: - info["changed"] = changed + info: Dict = { + "repo": repo, + "changed": changed, + "errors": errors, + } if dry_run_actions: info["dry_run_actions"] = dry_run_actions return info diff --git a/openedx_webhooks/tasks/pr_tracking.py b/openedx_webhooks/tasks/pr_tracking.py index ca8413eb..667343fe 100644 --- a/openedx_webhooks/tasks/pr_tracking.py +++ b/openedx_webhooks/tasks/pr_tracking.py @@ -8,7 +8,7 @@ import dataclasses import itertools from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Set, Tuple, cast +from typing import Dict, List, Optional, Set, Tuple, cast from openedx_webhooks.auth import get_github_session, get_jira_session from openedx_webhooks.bot_comments import ( @@ -39,7 +39,6 @@ from openedx_webhooks.info import ( get_blended_project_id, get_bot_comments, - get_jira_issue_key, get_people_file, is_bot_pull_request, is_draft_pull_request, @@ -52,7 +51,6 @@ from openedx_webhooks.labels import ( GITHUB_CATEGORY_LABELS, GITHUB_STATUS_LABELS, - JIRA_CATEGORY_LABELS, ) from openedx_webhooks.settings import settings from openedx_webhooks.tasks import logger @@ -60,10 +58,9 @@ transition_jira_issue, update_jira_issue, ) -from openedx_webhooks.types import GhProject, PrDict, PrId +from openedx_webhooks.types import GhProject, JiraId, PrDict, PrId from openedx_webhooks.utils import ( get_jira_custom_fields, - get_jira_issue, log_check_response, retry_get, sentry_extra_context, @@ -71,6 +68,30 @@ ) +@dataclass +class BotData: + """ + The data we store hidden on the first bot comment, to track our work. + """ + # Is this a draft pull request? + draft: bool = False + # The Jira issues associated with the pull request. + jira_issues: Set[JiraId] = field(default_factory=set) + + def asdict(self): + return { + "draft": self.draft, + "jira_issues": [ji.asdict() for ji in self.jira_issues], + } + + @classmethod + def fromdict(cls, d): + return cls( + draft=d.get("draft", False), + jira_issues=set(JiraId(**jd) for jd in d.get("jira_issues", [])), + ) + + @dataclass class PrCurrentInfo: """ @@ -85,24 +106,12 @@ class PrCurrentInfo: bot_survey_comment_id: Optional[str] = None # The last-seen state stored in the first bot comment. - last_seen_state: Dict = field(default_factory=dict) - # And aggregate of all data stored on all bot comments. - all_bot_state: Dict = field(default_factory=dict) - - # The Jira issue id mentioned on the PR if any. - jira_mentioned_id: Optional[str] = None - # Was the mentioned Jira issue on our Jira server? - on_our_jira: bool = False + bot_data: BotData = field(default_factory=BotData) # The actual Jira issue id. Can differ from jira_mentioned_id if the # issue was moved, or can be None if the issue has been deleted. jira_id: Optional[str] = None - jira_title: Optional[str] = None - jira_description: Optional[str] = None - jira_status: Optional[str] = None - jira_labels: Set[str] = field(default_factory=set) - # The actual set of labels on the pull request. github_labels: Set[str] = field(default_factory=set) @@ -112,10 +121,6 @@ class PrCurrentInfo: # The status of the cla check. cla_check: Optional[Dict[str, str]] = None - # Did the author make a change that could move us out of "Waiting on - # Author"? - author_acted: bool = False - @dataclass class PrDesiredInfo: @@ -139,9 +144,6 @@ class PrDesiredInfo: # The Jira status we want to set on an existing issue. Can be None if we # don't need to force a new status, but can leave the existing status. jira_status: Optional[str] = None - # If we're closing the pull request, we save away the previous Jira state - # in case we need to re-open it. - jira_previous_status: Optional[str] = None jira_labels: Set[str] = field(default_factory=set) @@ -156,6 +158,17 @@ class PrDesiredInfo: cla_check: Optional[Dict[str, str]] = None +@dataclass +class FixResult: + """ + Return value from PrTrackingFixer.result. + """ + # The Jira issues associated with the pull request. + jira_issues: Set[JiraId] = field(default_factory=set) + # The Jira issues that were created or changed. + changed_jira_issues: Set[JiraId] = field(default_factory=set) + + def current_support_state(pr: PrDict) -> PrCurrentInfo: """ Examine the world to determine what the current support state is. @@ -166,7 +179,7 @@ def current_support_state(pr: PrDict) -> PrCurrentInfo: full_bot_comments = list(get_bot_comments(prid)) if full_bot_comments: current.bot_comment0_text = cast(str, full_bot_comments[0]["body"]) - current.last_seen_state = extract_data_from_comment(current.bot_comment0_text) + current.bot_data = BotData.fromdict(extract_data_from_comment(current.bot_comment0_text)) for comment in full_bot_comments: body = comment["body"] for comment_id, snips in BOT_COMMENT_INDICATORS.items(): @@ -174,35 +187,15 @@ def current_support_state(pr: PrDict) -> PrCurrentInfo: current.bot_comments.add(comment_id) if comment_id == BotComment.SURVEY: current.bot_survey_comment_id = comment["id"] - current.all_bot_state.update(extract_data_from_comment(body)) - - on_our_jira, jira_id = get_jira_issue_key(prid) - current.jira_id = current.jira_mentioned_id = jira_id - current.on_our_jira = on_our_jira - if current.jira_id and current.on_our_jira: - issue = get_jira_issue(current.jira_id, missing_ok=True) - if issue is None: - # Issue has been deleted. Forget about it, and we'll make a new one. - current.jira_id = None - else: - current.jira_id = issue["key"] - current.jira_title = issue["fields"]["summary"] or "" - current.jira_description = issue["fields"]["description"] or "" - current.jira_status = issue["fields"]["status"]["name"] - current.jira_labels = set(issue["fields"]["labels"]) current.github_labels = set(lbl["name"] for lbl in pr["labels"]) current.github_projects = set(pull_request_projects(pr)) current.cla_check = cla_status_on_pr(pr) - if current.last_seen_state.get("draft", False) and not is_draft_pull_request(pr): - # It was a draft, but now isn't. The author acted. - current.author_acted = True - return current -def desired_support_state(pr: PrDict) -> Optional[PrDesiredInfo]: +def desired_support_state(pr: PrDict) -> PrDesiredInfo: """ Examine a pull request to decide what state we want the world to be in. """ @@ -287,7 +280,6 @@ def desired_support_state(pr: PrDict) -> Optional[PrDesiredInfo]: elif state == "merged": desired.jira_status = "Merged" elif state == "reopened": - desired.jira_status = "pre-close" # Not a real Jira status. desired.bot_comments_to_remove.add(BotComment.SURVEY) if state in ["closed", "merged"]: @@ -324,17 +316,19 @@ def __init__( self.prid = PrId.from_pr_dict(self.pr) self.actions = actions or FixingActions(self.prid) - self.last_seen_state = copy.deepcopy(current.last_seen_state) - self.happened = False + self.bot_data = copy.deepcopy(current.bot_data) + self.fix_result: FixResult = FixResult() - def result(self) -> Tuple[Optional[str], bool]: - return self.current.jira_id, self.happened + def result(self) -> FixResult: + return self.fix_result def fix(self) -> None: + """ + The main routine for making needed changes. + """ if self.desired.cla_check != self.current.cla_check: assert self.desired.cla_check is not None self.actions.set_cla_status(status=self.desired.cla_check) - self.happened = True if self.desired.is_ospr: self.fix_ospr() @@ -353,39 +347,13 @@ def fix_comments(self) -> None: self._add_bot_comments() def fix_ospr(self) -> None: - """ - The main routine for making needed changes. - """ self.actions.initial_state( current=json_safe_dict(self.current), desired=json_safe_dict(self.desired), ) # Draftiness - self.last_seen_state["draft"] = is_draft_pull_request(self.pr) - - if self.current.jira_id and self.current.on_our_jira: - # If the author acted, and we were waiting on the author, then we - # should set the status to this PR's usual initial status. - if self.current.author_acted and self.current.jira_status == "Waiting on Author": - self.desired.jira_status = self.desired.jira_initial_status - - # Check the state of the Jira issue. - if self.desired.jira_status is not None and self.desired.jira_status != self.current.jira_status: - if self.desired.jira_status == "pre-close": - self.desired.jira_status = self.current.all_bot_state.get( - "jira-pre-close", "Community Manager Review", - ) - elif self.desired.jira_status == "Rejected": - self.desired.jira_previous_status = self.current.jira_status - self.actions.transition_jira_issue( - jira_id=self.current.jira_id, jira_status=cast(str, self.desired.jira_status), - ) - self.current.jira_status = self.desired.jira_status - self.happened = True - - # Update the Jira issue information. - self._fix_jira_information() + self.bot_data.draft = is_draft_pull_request(self.pr) # Check the GitHub labels. self._fix_github_labels() @@ -398,15 +366,14 @@ def fix_ospr(self) -> None: self.actions.add_pull_request_to_project( pr_node_id=self.pr["node_id"], project=project ) - self.happened = True - def _make_jira_issue(self) -> None: + def _make_jira_issue(self) -> Dict: """ Make our desired Jira issue. """ assert self.desired.jira_project is not None user_name, institution = get_name_and_institution_for_pr(self.pr) - new_issue = self.actions.create_ospr_issue( + issue_data = self.actions.create_ospr_issue( pr_url=self.pr["html_url"], project=self.desired.jira_project, summary=self.desired.jira_title, @@ -415,53 +382,7 @@ def _make_jira_issue(self) -> None: user_name=user_name, institution=institution, ) - self.current.jira_id = new_issue["key"] - assert self.current.jira_id is not None - self.current.jira_status = new_issue["fields"]["status"]["name"] - self.current.jira_title = self.desired.jira_title - self.current.jira_description = self.desired.jira_description - self.current.jira_labels = self.desired.jira_labels - - if self.desired.jira_initial_status != self.current.jira_status: - assert self.desired.jira_initial_status is not None - self.actions.transition_jira_issue( - jira_id=self.current.jira_id, - jira_status=self.desired.jira_initial_status, - ) - self.current.jira_status = self.desired.jira_initial_status - - self.happened = True - - def _fix_jira_information(self) -> None: - """ - Update the information on the Jira issue. - """ - update_kwargs: Dict[str, Any] = {} - - if self.desired.jira_title != self.current.jira_title: - update_kwargs["summary"] = self.desired.jira_title - if self.desired.jira_description != self.current.jira_description: - update_kwargs["description"] = self.desired.jira_description - - desired_labels = set(self.desired.jira_labels) - ad_hoc_labels = self.current.jira_labels - JIRA_CATEGORY_LABELS - desired_labels.update(ad_hoc_labels) - if desired_labels != self.current.jira_labels: - update_kwargs["labels"] = list(self.desired.jira_labels) - - if update_kwargs: - assert self.current.jira_id is not None - try: - self.actions.update_jira_issue(jira_id=self.current.jira_id, **update_kwargs) - except: - logger.warning( - f"Couldn't update jira: {update_kwargs=}, {self.current.jira_description=}" - ) - raise - self.current.jira_title = self.desired.jira_title - self.current.jira_description = self.desired.jira_description - self.current.jira_labels = self.desired.jira_labels - self.happened = True + return issue_data def _fix_github_labels(self) -> None: """ @@ -469,8 +390,6 @@ def _fix_github_labels(self) -> None: Take care to preserve any label we've never heard of. """ desired_labels = set(self.desired.github_labels) - if self.current.jira_status: - desired_labels.add(self.current.jira_status.lower()) ad_hoc_labels = self.current.github_labels - GITHUB_CATEGORY_LABELS - GITHUB_STATUS_LABELS desired_labels.update(ad_hoc_labels) @@ -478,7 +397,6 @@ def _fix_github_labels(self) -> None: self.actions.update_labels_on_pull_request( labels=list(desired_labels), ) - self.happened = True def _fix_bot_comment(self) -> None: """ @@ -520,7 +438,7 @@ def _fix_bot_comment(self) -> None: needed_comments.remove(BotComment.END_OF_WIP) # BTW, we never have WELCOME_CLOSED in desired.bot_comments - comment_body += format_data_for_comment(self.last_seen_state) + comment_body += format_data_for_comment(self.bot_data.asdict()) if comment_body != self.current.bot_comment0_text: # If there are current-state comments, then we need to edit the @@ -529,7 +447,6 @@ def _fix_bot_comment(self) -> None: self.actions.edit_comment_on_pull_request(comment_body=comment_body) else: self.actions.add_comment_to_pull_request(comment_body=comment_body) - self.happened = True assert needed_comments == set(), f"Couldn't make first comments: {needed_comments}" @@ -545,17 +462,12 @@ def _add_bot_comments(self): if BotComment.SURVEY in needed_comments: body = github_end_survey_comment(self.pr) - if self.desired.jira_status == "Rejected": - # For close/re-open cycling, remember what Jira was before the close. - body += format_data_for_comment({"jira-pre-close": self.desired.jira_previous_status}) self.actions.add_comment_to_pull_request(comment_body=body) needed_comments.remove(BotComment.SURVEY) - self.happened = True if BotComment.SURVEY in self.desired.bot_comments_to_remove: if self.current.bot_survey_comment_id: self.actions.delete_comment_on_pull_request(comment_id=self.current.bot_survey_comment_id) - self.happened = True assert needed_comments == set(), f"Couldn't make comments: {needed_comments}" diff --git a/openedx_webhooks/types.py b/openedx_webhooks/types.py index 7697a3e9..e0fbe57d 100644 --- a/openedx_webhooks/types.py +++ b/openedx_webhooks/types.py @@ -53,3 +53,13 @@ class JiraServer: mapping: str | None = None project: str | None = None + + +@dataclasses.dataclass(frozen=True) +class JiraId: + """A JiraServer nickname and an issue key.""" + nick: str + key: str + + def asdict(self): + return dataclasses.asdict(self) diff --git a/tests/test_pull_request_closed.py b/tests/test_pull_request_closed.py index f6614f2f..d736f4de 100644 --- a/tests/test_pull_request_closed.py +++ b/tests/test_pull_request_closed.py @@ -34,20 +34,19 @@ def closed_pull_request(is_merged, fake_github, fake_jira): user="tusbar", title=random_text(), ) - issue_key, _ = pull_request_changed(pr.as_json()) - - assert issue_key is None + result = pull_request_changed(pr.as_json()) + assert not result.jira_issues pr.add_comment(user="nedbat", body="Please make some changes") pr.add_comment(user="tusbar", body="OK, I made the changes") pr.close(merge=is_merged) fake_jira.reset_mock() - return pr, issue_key + return pr def test_external_pr_closed(fake_jira, closed_pull_request): - pr, _ = closed_pull_request + pr = closed_pull_request pull_request_changed(pr.as_json()) pr_comments = pr.list_comments() @@ -58,24 +57,22 @@ def test_external_pr_closed(fake_jira, closed_pull_request): def test_external_pr_closed_but_issue_deleted(fake_jira, closed_pull_request): # A closing pull request, but its Jira issue has been deleted. - pr, old_issue_key = closed_pull_request - - issue_id, anything_happened = pull_request_changed(pr.as_json()) + pr = closed_pull_request - assert issue_id is None - assert anything_happened + result = pull_request_changed(pr.as_json()) + assert not result.jira_issues pr_comments = pr.list_comments() assert len(pr_comments) == 4 # 1 welcome, closed_pull_request makes two, 1 survey # We leave the old issue id in the comment. body = pr_comments[0].body - check_issue_link_in_markdown(body, old_issue_key) + check_issue_link_in_markdown(body, None) def test_external_pr_closed_but_issue_in_status(fake_jira, closed_pull_request): # The Jira issue associated with a closing pull request is already in the # status we want to move it to. - pr, _ = closed_pull_request + pr = closed_pull_request pull_request_changed(pr.as_json()) diff --git a/tests/test_pull_request_opened.py b/tests/test_pull_request_opened.py index d8789ffd..8a94498c 100644 --- a/tests/test_pull_request_opened.py +++ b/tests/test_pull_request_opened.py @@ -36,25 +36,22 @@ def close_and_reopen_pr(pr): def test_internal_pr_opened(fake_github): pr = fake_github.make_pull_request("openedx", user="nedbat") - key, anything_happened = pull_request_changed(pr.as_json()) + result = pull_request_changed(pr.as_json()) - assert key is None - assert anything_happened is True + assert not result.jira_issues assert len(pr.list_comments()) == 0 assert pr.status(CLA_CONTEXT) == CLA_STATUS_GOOD assert pull_request_projects(pr.as_json()) == set() - key, anything_happened2 = close_and_reopen_pr(pr) - assert key is None - assert anything_happened2 is False + result2 = close_and_reopen_pr(pr) + assert not result2.jira_issues def test_pr_in_private_repo_opened(fake_github): repo = fake_github.make_repo("edx", "some-private-repo", private=True) pr = repo.make_pull_request(user="some_contractor") - key, anything_happened = pull_request_changed(pr.as_json()) - assert key is None - assert anything_happened is True + result = pull_request_changed(pr.as_json()) + assert not result.jira_issues assert len(pr.list_comments()) == 0 # some_contractor has no cla, and even in a private repo we check. assert pr.status(CLA_CONTEXT) == CLA_STATUS_PRIVATE @@ -65,9 +62,8 @@ def test_pr_in_private_repo_opened(fake_github): def test_pr_in_nocontrib_repo_opened(fake_github, user): repo = fake_github.make_repo("edx", "some-public-repo") pr = repo.make_pull_request(user=user) - key, anything_happened = pull_request_changed(pr.as_json()) - assert key is None - assert anything_happened is True + result = pull_request_changed(pr.as_json()) + assert not result.jira_issues pr_comments = pr.list_comments() assert len(pr_comments) == 1 body = pr_comments[0].body @@ -80,9 +76,8 @@ def test_pr_in_nocontrib_repo_opened(fake_github, user): def test_pr_opened_by_bot(fake_github): fake_github.make_user(login="some_bot", type="Bot") pr = fake_github.make_pull_request(user="some_bot") - key, anything_happened = pull_request_changed(pr.as_json()) - assert key is None - assert anything_happened is True + result = pull_request_changed(pr.as_json()) + assert not result.jira_issues assert len(pr.list_comments()) == 0 assert pr.status(CLA_CONTEXT) == CLA_STATUS_BOT assert pull_request_projects(pr.as_json()) == set() @@ -94,16 +89,14 @@ def test_external_pr_opened_no_cla(fake_github): pr = fake_github.make_pull_request(owner="openedx", repo="edx-platform", user="new_contributor") prj = pr.as_json() - issue_id, anything_happened = pull_request_changed(prj) - - assert anything_happened is True - assert issue_id is None + result = pull_request_changed(prj) + assert not result.jira_issues # Check the GitHub comment that was created. pr_comments = pr.list_comments() assert len(pr_comments) == 1 body = pr_comments[0].body - check_issue_link_in_markdown(body, issue_id) + check_issue_link_in_markdown(body, None) assert "Thanks for the pull request, @new_contributor!" in body assert is_comment_kind(BotComment.NEED_CLA, body) assert is_comment_kind(BotComment.WELCOME, body) @@ -118,9 +111,8 @@ def test_external_pr_opened_no_cla(fake_github): assert pull_request_projects(pr.as_json()) == {settings.GITHUB_OSPR_PROJECT} # Test re-opening. - issue_id2, anything_happened2 = close_and_reopen_pr(pr) - assert issue_id2 == issue_id - assert anything_happened2 is True + result2 = close_and_reopen_pr(pr) + assert not result2.jira_issues # Now there is one comment: closing the PR added a survey comment, but # re-opening it deleted it. assert len(pr.list_comments()) == 1 @@ -130,16 +122,14 @@ def test_external_pr_opened_with_cla(fake_github): pr = fake_github.make_pull_request(owner="openedx", repo="some-code", user="tusbar", number=11235) prj = pr.as_json() - issue_id, anything_happened = pull_request_changed(prj) - - assert anything_happened is True - assert issue_id is None + result = pull_request_changed(prj) + assert not result.jira_issues # Check the GitHub comment that was created. pr_comments = pr.list_comments() assert len(pr_comments) == 1 body = pr_comments[0].body - check_issue_link_in_markdown(body, issue_id) + check_issue_link_in_markdown(body, None) assert "Thanks for the pull request, @tusbar!" in body assert is_comment_kind(BotComment.WELCOME, body) assert not is_comment_kind(BotComment.NEED_CLA, body) @@ -154,9 +144,8 @@ def test_external_pr_opened_with_cla(fake_github): assert pull_request_projects(pr.as_json()) == {settings.GITHUB_OSPR_PROJECT} # Test re-opening. - issue_id2, anything_happened2 = close_and_reopen_pr(pr) - assert issue_id2 == issue_id - assert anything_happened2 is True + result2 = close_and_reopen_pr(pr) + assert not result2.jira_issues # Now there is one comment: closing the PR added a survey comment, but # re-opening it deleted it. pr_comments = pr.list_comments() @@ -166,16 +155,14 @@ def test_external_pr_opened_with_cla(fake_github): def test_blended_pr_opened_with_cla(fake_github): pr = fake_github.make_pull_request(owner="openedx", repo="some-code", user="tusbar", title="[BD-34] Something good") prj = pr.as_json() - issue_id, anything_happened = pull_request_changed(prj) - - assert anything_happened is True - assert issue_id is None + result = pull_request_changed(prj) + assert not result.jira_issues # Check the GitHub comment that was created. pr_comments = pr.list_comments() assert len(pr_comments) == 1 body = pr_comments[0].body - check_issue_link_in_markdown(body, issue_id) + check_issue_link_in_markdown(body, None) assert "Thanks for the pull request, @tusbar!" in body has_project_link = "the [BD-34](https://thewiki/bd-34) project page" in body assert not has_project_link @@ -196,17 +183,15 @@ def test_external_pr_rescanned(fake_github): # Make a pull request and process it. pr = fake_github.make_pull_request(user="tusbar") - issue_id1, anything_happened1 = pull_request_changed(pr.as_json()) + result1 = pull_request_changed(pr.as_json()) - assert issue_id1 is None - assert anything_happened1 is True + assert not result1.jira_issues assert len(pr.list_comments()) == 1 # Rescan the pull request. - issue_id2, anything_happened2 = pull_request_changed(pr.as_json()) + result2 = pull_request_changed(pr.as_json()) - assert issue_id2 is None - assert anything_happened2 is False + assert not result2.jira_issues # No new GitHub comment was created. assert len(pr.list_comments()) == 1 @@ -241,10 +226,8 @@ def test_draft_pr_opened(pr_type, fake_github, mocker): pr = fake_github.make_pull_request(owner="openedx", repo="edx-platform", user="new_contributor", title=title1) prj = pr.as_json() - issue_id, anything_happened = pull_request_changed(prj) - - assert anything_happened is True - assert issue_id is None + result = pull_request_changed(prj) + assert not result.jira_issues # Check the GitHub comment that was created. pr_comments = pr.list_comments() @@ -272,9 +255,8 @@ def test_draft_pr_opened(pr_type, fake_github, mocker): # The author updates the PR, no longer draft. pr.title = title2 - issue_id2, _ = pull_request_changed(pr.as_json()) - - assert issue_id2 is None + result2 = pull_request_changed(pr.as_json()) + assert not result2.jira_issues pr_comments = pr.list_comments() assert len(pr_comments) == 1 @@ -284,9 +266,8 @@ def test_draft_pr_opened(pr_type, fake_github, mocker): # Oops, it goes back to draft! pr.title = title1 - issue_id3, _ = pull_request_changed(pr.as_json()) - - assert issue_id3 is None + result3 = pull_request_changed(pr.as_json()) + assert not result3.jira_issues pr_comments = pr.list_comments() assert len(pr_comments) == 1 @@ -298,16 +279,14 @@ def test_draft_pr_opened(pr_type, fake_github, mocker): def test_handle_closed_pr(is_merged, fake_github): pr = fake_github.make_pull_request(user="tusbar", number=11237, state="closed", merged=is_merged) prj = pr.as_json() - issue_id1, anything_happened = pull_request_changed(prj) - - assert anything_happened is True - assert issue_id1 is None + result = pull_request_changed(prj) + assert not result.jira_issues # Check the GitHub comment that was created. pr_comments = pr.list_comments() assert len(pr_comments) == 1 body = pr_comments[0].body - check_issue_link_in_markdown(body, issue_id1) + check_issue_link_in_markdown(body, None) if is_merged: assert "Although this pull request is already merged," in body else: @@ -322,10 +301,8 @@ def test_handle_closed_pr(is_merged, fake_github): assert pull_request_projects(pr.as_json()) == {settings.GITHUB_OSPR_PROJECT} # Rescan the pull request. - issue_id2, anything_happened2 = pull_request_changed(pr.as_json()) - - assert issue_id2 is None - assert anything_happened2 is False + result2 = pull_request_changed(pr.as_json()) + assert not result2.jira_issues # No new GitHub comment was created. assert len(pr.list_comments()) == 1 diff --git a/tests/test_rescan.py b/tests/test_rescan.py index 2d782096..9e5d49ee 100644 --- a/tests/test_rescan.py +++ b/tests/test_rescan.py @@ -63,12 +63,10 @@ def make_rescannable_repo(fake_github, org_name="an-org", repo_name="a-repo"): def test_rescan_repository(rescannable_repo, pull_request_changed_fn, allpr): ret = rescan_repository(rescannable_repo.full_name, allpr=allpr) changed = ret["changed"] - bad = False - for v in changed.values(): - if isinstance(v, str) and "Traceback" in v: # pragma: no cover - print(v) - bad = True - assert not bad + errors = ret["errors"] + for err in errors.values(): + print(err) + assert not errors # Look at the pull request numbers passed to pull_request_changed. Only the # external (even) numbers should be there. @@ -82,7 +80,7 @@ def test_rescan_repository(rescannable_repo, pull_request_changed_fn, allpr): # If we rescan again, nothing should happen. ret = rescan_repository(rescannable_repo.full_name, allpr=allpr) - assert "changed" not in ret + assert not ret["changed"] def test_rescan_repository_dry_run(rescannable_repo, fake_github, fake_jira, pull_request_changed_fn): @@ -187,7 +185,7 @@ def flaky_pull_request_changed(pr, actions): ret = rescan_repository(rescannable_repo.full_name, allpr=True) assert list(ret["changed"]) == [102, 106, 108, 110] - err = ret["changed"][108] + err = ret["errors"][108] assert err.startswith("Traceback (most recent call last):\n") assert " in flaky_pull_request_changed\n" in err assert "1/0 # BOOM" in err From 2912d85922aa63098c88244264b82fdea10c1738 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 26 Sep 2023 16:39:09 -0400 Subject: [PATCH 08/12] refactor: remove more unneeded jira support code --- openedx_webhooks/tasks/jira_work.py | 84 --------------------------- openedx_webhooks/tasks/pr_tracking.py | 4 -- openedx_webhooks/templates/main.html | 10 ---- openedx_webhooks/ui.py | 17 +----- openedx_webhooks/utils.py | 39 ------------- tests/fake_jira.py | 27 --------- tests/test_fake_jira.py | 4 -- 7 files changed, 2 insertions(+), 183 deletions(-) diff --git a/openedx_webhooks/tasks/jira_work.py b/openedx_webhooks/tasks/jira_work.py index e1d227ce..3d0fe84c 100644 --- a/openedx_webhooks/tasks/jira_work.py +++ b/openedx_webhooks/tasks/jira_work.py @@ -4,96 +4,12 @@ from typing import Any, Dict, List, Optional -import requests - from openedx_webhooks.auth import get_jira_session -from openedx_webhooks.tasks import logger from openedx_webhooks.utils import ( log_check_response, - sentry_extra_context, ) -def find_issues_for_pull_request(jira, pull_request_url): - """ - Find corresponding JIRA issues for a given GitHub pull request. - - Arguments: - jira (jira.JIRA): An authenticated JIRA API client session - pull_request_url (str) - - Returns: - jira.client.ResultList[jira.Issue] - """ - jql = 'project=OSPR AND cf[10904]="{}"'.format(pull_request_url) - return jira.search_issues(jql) - - -def transition_jira_issue(issue_key, status_name): - """ - Transition a Jira issue to a new status. - - Returns: - True if the issue was changed. - - """ - assert status_name is not None - transition_url = ( - "/rest/api/2/issue/{key}/transitions" - "?expand=transitions.fields".format(key=issue_key) - ) - transitions_resp = get_jira_session().get(transition_url) - log_check_response(transitions_resp, raise_for_status=False) - if transitions_resp.status_code == requests.codes.not_found: - # JIRA issue has been deleted - logger.info(f"Issue {issue_key} doesn't exist") - return False - transitions_resp.raise_for_status() - - transitions = transitions_resp.json()["transitions"] - sentry_extra_context({"transitions": transitions}) - - transition_id = None - for t in transitions: - if t["to"]["name"] == status_name: - transition_id = t["id"] - break - - if not transition_id: - # maybe the issue is *already* in the right status? - issue_url = "/rest/api/2/issue/{key}".format(key=issue_key) - issue_resp = get_jira_session().get(issue_url) - issue_resp.raise_for_status() - issue = issue_resp.json() - sentry_extra_context({"jira_issue": issue}) - current_status = issue["fields"]["status"]["name"] - if current_status == status_name: - logger.info(f"Issue {issue_key} is already in status {status_name}") - return False - - # nope, raise an error message - fail_msg = ( - "Issue {key} cannot be transitioned directly from status {curr_status} " - "to status {new_status}. Valid status transitions are: {valid}".format( - key=issue_key, - new_status=status_name, - curr_status=current_status, - valid=", ".join(t["to"]["name"] for t in transitions), - ) - ) - logger.error(fail_msg) - raise Exception(fail_msg) - - logger.info(f"Changing status on issue {issue_key} to {status_name}") - transition_resp = get_jira_session().post(transition_url, json={ - "transition": { - "id": transition_id, - } - }) - log_check_response(transition_resp) - return True - - def update_jira_issue( issue_key: str, summary: Optional[str]=None, diff --git a/openedx_webhooks/tasks/pr_tracking.py b/openedx_webhooks/tasks/pr_tracking.py index 667343fe..b765d755 100644 --- a/openedx_webhooks/tasks/pr_tracking.py +++ b/openedx_webhooks/tasks/pr_tracking.py @@ -55,7 +55,6 @@ from openedx_webhooks.settings import settings from openedx_webhooks.tasks import logger from openedx_webhooks.tasks.jira_work import ( - transition_jira_issue, update_jira_issue, ) from openedx_webhooks.types import GhProject, JiraId, PrDict, PrId @@ -594,9 +593,6 @@ def create_ospr_issue( new_issue["fields"]["status"] = {"name": "Needs Triage"} return new_issue - def transition_jira_issue(self, *, jira_id: str, jira_status: str) -> None: - transition_jira_issue(jira_id, jira_status) - def update_jira_issue(self, *, jira_id: str, **update_kwargs) -> None: update_jira_issue(jira_id, **update_kwargs) diff --git a/openedx_webhooks/templates/main.html b/openedx_webhooks/templates/main.html index 26669909..91574cb6 100644 --- a/openedx_webhooks/templates/main.html +++ b/openedx_webhooks/templates/main.html @@ -29,15 +29,5 @@

GitHub

  • Process Pull Request
  • Generate Error
  • -

    JIRA

    -
      -
    • - {% if jira_username %} - Authenticated as {{jira_username}}. - {% else %} - Not authenticated. - {% endif %} -
    • -
    diff --git a/openedx_webhooks/ui.py b/openedx_webhooks/ui.py index c4484b94..02d00b47 100644 --- a/openedx_webhooks/ui.py +++ b/openedx_webhooks/ui.py @@ -2,8 +2,7 @@ from flask import Blueprint, render_template -from openedx_webhooks.auth import get_github_session, get_jira_session -from openedx_webhooks.settings import settings +from openedx_webhooks.auth import get_github_session from openedx_webhooks.utils import requires_auth ui = Blueprint('ui', __name__) @@ -25,16 +24,4 @@ def index(): logger.error("Failed to process response: {}".format(gh_user_resp.text)) raise - jira_username = None - if settings.JIRA_SERVER: - jira_user_resp = get_jira_session().get("/rest/api/2/myself") - if jira_user_resp.ok: - try: - jira_username = jira_user_resp.json()["displayName"] - except Exception: - logger.error("Failed to process response: {}".format(jira_user_resp.text)) - raise - - return render_template("main.html", - github_username=github_username, jira_username=jira_username, - ) + return render_template("main.html", github_username=github_username) diff --git a/openedx_webhooks/utils.py b/openedx_webhooks/utils.py index 8a4e5b8a..eb9d2aee 100644 --- a/openedx_webhooks/utils.py +++ b/openedx_webhooks/utils.py @@ -350,42 +350,3 @@ def jira_get(*args, **kwargs): if resp.content: return resp return get_jira_session().get(*args, **kwargs) - - -def github_pr_repo(issue): - custom_fields = get_jira_custom_fields() - pr_repo = issue["fields"].get(custom_fields["Repo"]) - parent_ref = issue["fields"].get("parent") - if not pr_repo and parent_ref: - parent = get_jira_issue(parent_ref["key"]) - if parent is not None: - pr_repo = parent["fields"].get(custom_fields["Repo"]) - return pr_repo - - -def github_pr_num(issue): - custom_fields = get_jira_custom_fields() - pr_num = issue["fields"].get(custom_fields["PR Number"]) - parent_ref = issue["fields"].get("parent") - if not pr_num and parent_ref: - parent = get_jira_issue(parent_ref["key"]) - if parent is not None: - pr_num = parent["fields"].get(custom_fields["PR Number"]) - try: - return int(pr_num) - except Exception: # pylint: disable=broad-except - return None - - -def github_pr_url(issue): - """ - Return the pull request URL for the given JIRA issue, - or raise an exception if they can't be determined. - """ - pr_repo = github_pr_repo(issue) - pr_num = github_pr_num(issue) - if not pr_repo or not pr_num: - issue_key = issue["key"] - fail_msg = '{key} is missing "Repo" or "PR Number" fields'.format(key=issue_key) - raise Exception(fail_msg) - return "/repos/{repo}/pulls/{num}".format(repo=pr_repo, num=pr_num) diff --git a/tests/fake_jira.py b/tests/fake_jira.py index bc0bb33a..6c13618b 100644 --- a/tests/fake_jira.py +++ b/tests/fake_jira.py @@ -187,30 +187,3 @@ def _put_issue(self, match, request, context) -> None: context.status_code = 204 else: context.status_code = 404 - - @faker.route(r"/rest/api/2/issue/(?P\w+-\d+)/transitions") - def _get_issue_transitions(self, match, _request, context) -> Dict: - """Responds to the API endpoint for listing transitions between issue states.""" - if (issue := self.find_issue(match["key"])) is not None: - # The transitions don't include the transitions to the current state. - return { - "transitions": [ - {"id": id, "to": {"name": name}} - for name, id in self.TRANSITIONS.items() - if name != issue.status - ], - } - else: - # No such issue. - context.status_code = 404 - return {} - - @faker.route(r"/rest/api/2/issue/(?P\w+-\d+)/transitions", "POST") - def _post_issue_transitions(self, match, request, _context): - """ - Implement the POST to transition an issue to a new status. - """ - issue = self.find_issue(match["key"]) - assert issue is not None - transition_id = request.json()["transition"]["id"] - issue.status = self.TRANSITION_IDS[transition_id] diff --git a/tests/test_fake_jira.py b/tests/test_fake_jira.py index bffa42f9..eec14c0a 100644 --- a/tests/test_fake_jira.py +++ b/tests/test_fake_jira.py @@ -74,7 +74,3 @@ class TestBadRequests: def test_no_such_put(self, fake_jira): resp = requests.put("https://test.atlassian.net/rest/api/2/issue/XYZ-999") assert resp.status_code == 404 - - def test_no_such_transitions(self, fake_jira): - resp = requests.get("https://test.atlassian.net/rest/api/2/issue/XYZ-999/transitions") - assert resp.status_code == 404 From 2c19b494f223e7f4fb6395f4fb94590b194f10ee Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 27 Sep 2023 14:08:14 -0400 Subject: [PATCH 09/12] feat: adding a jira:xyz label creates a Jira issue --- .../20230927_140851_nedbat_label_for_jira.rst | 5 ++ openedx_webhooks/auth.py | 5 +- openedx_webhooks/bot_comments.py | 14 ++- openedx_webhooks/github_views.py | 1 + openedx_webhooks/tasks/jira_work.py | 5 +- openedx_webhooks/tasks/pr_tracking.py | 88 ++++++++++++------- .../templates/jira_issue_comment.md.j2 | 4 + openedx_webhooks/types.py | 2 +- openedx_webhooks/utils.py | 26 +++--- tests/conftest.py | 8 ++ tests/helpers.py | 16 ++-- tests/test_pull_request_opened.py | 66 ++++++++++++++ 12 files changed, 180 insertions(+), 60 deletions(-) create mode 100644 changelog.d/20230927_140851_nedbat_label_for_jira.rst create mode 100644 openedx_webhooks/templates/jira_issue_comment.md.j2 diff --git a/changelog.d/20230927_140851_nedbat_label_for_jira.rst b/changelog.d/20230927_140851_nedbat_label_for_jira.rst new file mode 100644 index 00000000..17a96d1d --- /dev/null +++ b/changelog.d/20230927_140851_nedbat_label_for_jira.rst @@ -0,0 +1,5 @@ +.. A new scriv changelog fragment. + +- Adding a label like ``jira:xyz`` to a pull request will look in a private + registry of Jira servers for one nicknamed ``xyz``, and then create a Jira + issue there to correspond to the pull request. diff --git a/openedx_webhooks/auth.py b/openedx_webhooks/auth.py index da66a55d..035a6b6a 100644 --- a/openedx_webhooks/auth.py +++ b/openedx_webhooks/auth.py @@ -33,10 +33,9 @@ def get_jira_session(jira_nick): `jira_nick` is a nickname for one of our configured Jira servers. """ # Avoid a circular import. - from openedx_webhooks.info import get_jira_info + from openedx_webhooks.info import get_jira_server_info - jira_info = get_jira_info() - jira_server = jira_info[jira_nick.lower()] + jira_server = get_jira_server_info(jira_nick) session = BaseUrlSession(base_url=jira_server.server) session.auth = (jira_server.email, jira_server.token) session.trust_env = False # prevent reading the local .netrc diff --git a/openedx_webhooks/bot_comments.py b/openedx_webhooks/bot_comments.py index d601ed84..3d43712e 100644 --- a/openedx_webhooks/bot_comments.py +++ b/openedx_webhooks/bot_comments.py @@ -13,10 +13,11 @@ from flask import render_template from openedx_webhooks.info import ( + get_jira_server_info, is_draft_pull_request, pull_request_has_cla, ) -from openedx_webhooks.types import PrDict +from openedx_webhooks.types import JiraId, PrDict class BotComment(Enum): @@ -152,6 +153,17 @@ def github_end_survey_comment(pull_request: PrDict) -> str: ) +def jira_issue_comment(pull_request: PrDict, jira_id: JiraId) -> str: # pylint: disable=unused-argument + """Render a comment about making a new Jira issue.""" + jira_server = get_jira_server_info(jira_id.nick) + return render_template( + "jira_issue_comment.md.j2", + server_url=jira_server.server, + server_description=jira_server.description, + key=jira_id.key, + ) + + def no_contributions_thanks(pull_request: PrDict) -> str: # pylint: disable=unused-argument """ Create a "no contributions" comment. diff --git a/openedx_webhooks/github_views.py b/openedx_webhooks/github_views.py index b24e1220..7776c344 100644 --- a/openedx_webhooks/github_views.py +++ b/openedx_webhooks/github_views.py @@ -91,6 +91,7 @@ def hook_receiver(): "converted_to_draft", "reopened", "enqueued", + "labeled", } def handle_pull_request_event(event): diff --git a/openedx_webhooks/tasks/jira_work.py b/openedx_webhooks/tasks/jira_work.py index 3d0fe84c..bdc1f32e 100644 --- a/openedx_webhooks/tasks/jira_work.py +++ b/openedx_webhooks/tasks/jira_work.py @@ -11,11 +11,12 @@ def update_jira_issue( + jira_nick: str, issue_key: str, summary: Optional[str]=None, description: Optional[str]=None, labels: Optional[List[str]]=None, - ): + ) -> None: """ Update some fields on a Jira issue. """ @@ -34,5 +35,5 @@ def update_jira_issue( # Contrary to the docs, if the bot is not an admin, the setting isn't ignored, # the request fails. url = f"/rest/api/2/issue/{issue_key}?notifyUsers={notify}" - resp = get_jira_session().put(url, json={"fields": fields}) + resp = get_jira_session(jira_nick).put(url, json={"fields": fields}) log_check_response(resp) diff --git a/openedx_webhooks/tasks/pr_tracking.py b/openedx_webhooks/tasks/pr_tracking.py index b765d755..9fc99bff 100644 --- a/openedx_webhooks/tasks/pr_tracking.py +++ b/openedx_webhooks/tasks/pr_tracking.py @@ -21,6 +21,7 @@ github_community_pr_comment, github_community_pr_comment_closed, github_end_survey_comment, + jira_issue_comment, no_contributions_thanks, ) from openedx_webhooks.cla_check import ( @@ -70,25 +71,19 @@ @dataclass class BotData: """ - The data we store hidden on the first bot comment, to track our work. + The data we store hidden in bot comments, to track our work. """ # Is this a draft pull request? draft: bool = False # The Jira issues associated with the pull request. jira_issues: Set[JiraId] = field(default_factory=set) - def asdict(self): - return { - "draft": self.draft, - "jira_issues": [ji.asdict() for ji in self.jira_issues], - } - - @classmethod - def fromdict(cls, d): - return cls( - draft=d.get("draft", False), - jira_issues=set(JiraId(**jd) for jd in d.get("jira_issues", [])), - ) + def update(self, data: dict) -> None: + """Add data from `data` to this BotData.""" + if "draft" in data: + self.draft = data["draft"] + if "jira_issues" in data: + self.jira_issues.update(JiraId(**jd) for jd in data["jira_issues"]) @dataclass @@ -137,6 +132,9 @@ class PrDesiredInfo: jira_title: Optional[str] = None jira_description: Optional[str] = None + # The Jira instances we want to have issues on. + jira_nicks: Set[str] = field(default_factory=set) + # The Jira status to start a new issue at. jira_initial_status: Optional[str] = None @@ -178,7 +176,7 @@ def current_support_state(pr: PrDict) -> PrCurrentInfo: full_bot_comments = list(get_bot_comments(prid)) if full_bot_comments: current.bot_comment0_text = cast(str, full_bot_comments[0]["body"]) - current.bot_data = BotData.fromdict(extract_data_from_comment(current.bot_comment0_text)) + current.bot_data.update(extract_data_from_comment(current.bot_comment0_text)) for comment in full_bot_comments: body = comment["body"] for comment_id, snips in BOT_COMMENT_INDICATORS.items(): @@ -186,6 +184,7 @@ def current_support_state(pr: PrDict) -> PrCurrentInfo: current.bot_comments.add(comment_id) if comment_id == BotComment.SURVEY: current.bot_survey_comment_id = comment["id"] + current.bot_data.update(extract_data_from_comment(body)) current.github_labels = set(lbl["name"] for lbl in pr["labels"]) current.github_projects = set(pull_request_projects(pr)) @@ -227,6 +226,10 @@ def desired_support_state(pr: PrDict) -> PrDesiredInfo: else: state = "closed" + # A label of jira:xyz means we want a Jira issue in the xyz Jira. + label_names = set(lbl["name"] for lbl in pr["labels"]) + desired.jira_nicks = {name.partition(":")[-1] for name in label_names if name.startswith("jira:")} + desired.jira_initial_status = "Needs Triage" desired.jira_title = pr["title"] desired.jira_description = pr["body"] or "" @@ -325,6 +328,13 @@ def fix(self) -> None: """ The main routine for making needed changes. """ + self.actions.initial_state( + current=json_safe_dict(self.current), + desired=json_safe_dict(self.desired), + ) + + self.fix_result.jira_issues = set(self.current.bot_data.jira_issues) + if self.desired.cla_check != self.current.cla_check: assert self.desired.cla_check is not None self.actions.set_cla_status(status=self.desired.cla_check) @@ -335,6 +345,12 @@ def fix(self) -> None: if self.desired.is_refused: self.fix_comments() + # Make needed Jira issues. + current_jira_nicks = {ji.nick for ji in self.current.bot_data.jira_issues} + for jira_nick in self.desired.jira_nicks: + if jira_nick not in current_jira_nicks: + self._make_jira_issue(jira_nick) + def fix_comments(self) -> None: fix_comment = True if self.pr["state"] == "closed" and self.current.bot_comments: @@ -346,11 +362,6 @@ def fix_comments(self) -> None: self._add_bot_comments() def fix_ospr(self) -> None: - self.actions.initial_state( - current=json_safe_dict(self.current), - desired=json_safe_dict(self.desired), - ) - # Draftiness self.bot_data.draft = is_draft_pull_request(self.pr) @@ -366,22 +377,32 @@ def fix_ospr(self) -> None: pr_node_id=self.pr["node_id"], project=project ) - def _make_jira_issue(self) -> Dict: + def _make_jira_issue(self, jira_nick) -> None: """ - Make our desired Jira issue. + Make a Jira issue in a particular Jira server. """ - assert self.desired.jira_project is not None user_name, institution = get_name_and_institution_for_pr(self.pr) - issue_data = self.actions.create_ospr_issue( + issue_data = self.actions.create_jira_issue( + jira_nick=jira_nick, pr_url=self.pr["html_url"], - project=self.desired.jira_project, + project="TODOXXX", # TODO: get the real project summary=self.desired.jira_title, description=self.desired.jira_description, labels=list(self.desired.jira_labels), user_name=user_name, institution=institution, ) - return issue_data + + jira_id = JiraId(jira_nick, issue_data["key"]) + self.current.bot_data.jira_issues.add(jira_id) + self.fix_result.jira_issues.add(jira_id) + self.fix_result.changed_jira_issues.add(jira_id) + + comment_body = jira_issue_comment(self.pr, jira_id) + comment_body += format_data_for_comment({ + "jira_issues": [jira_id.asdict()], + }) + self.actions.add_comment_to_pull_request(comment_body=comment_body) def _fix_github_labels(self) -> None: """ @@ -437,7 +458,9 @@ def _fix_bot_comment(self) -> None: needed_comments.remove(BotComment.END_OF_WIP) # BTW, we never have WELCOME_CLOSED in desired.bot_comments - comment_body += format_data_for_comment(self.bot_data.asdict()) + comment_body += format_data_for_comment({ + "draft": is_draft_pull_request(self.pr) + }) if comment_body != self.current.bot_comment0_text: # If there are current-state comments, then we need to edit the @@ -507,9 +530,9 @@ class DryRunFixingActions: def __init__(self): self.action_calls = [] - def create_ospr_issue(self, **kwargs): + def create_jira_issue(self, **kwargs): # This needs a special override because it has to return a Jira key. - self.action_calls.append(("create_ospr_issue", kwargs)) + self.action_calls.append(("create_jira_issue", kwargs)) return { "key": f"OSPR-{next(self.jira_ids)}", "fields": { @@ -543,8 +566,9 @@ def initial_state(self, *, current: Dict, desired: Dict) -> None: Does nothing when really fixing, but captures information for dry runs. """ - def create_ospr_issue( + def create_jira_issue( self, *, + jira_nick: str, pr_url: str, project: str, summary: Optional[str], @@ -554,12 +578,12 @@ def create_ospr_issue( institution: Optional[str], ) -> Dict: """ - Create a new OSPR or OSPR-like issue for a pull request. + Create a new Jira issue for a pull request. Returns the JSON describing the issue. """ - custom_fields = get_jira_custom_fields() + custom_fields = get_jira_custom_fields(jira_nick) new_issue = { "fields": { "project": { @@ -582,7 +606,7 @@ def create_ospr_issue( sentry_extra_context({"new_issue": new_issue}) logger.info(f"Creating new JIRA issue for PR {self.prid}...") - resp = get_jira_session().post("/rest/api/2/issue", json=new_issue) + resp = get_jira_session(jira_nick).post("/rest/api/2/issue", json=new_issue) log_check_response(resp) # Jira only sends the key. Put it into the JSON we started with, and diff --git a/openedx_webhooks/templates/jira_issue_comment.md.j2 b/openedx_webhooks/templates/jira_issue_comment.md.j2 new file mode 100644 index 00000000..af41eede --- /dev/null +++ b/openedx_webhooks/templates/jira_issue_comment.md.j2 @@ -0,0 +1,4 @@ +{% filter replace("\n", " ")|trim %} +{# The bot makes this comment when it creates a label-triggered Jira issue. #} +I've created issue [{{ key }}]({{ server_url }}/browse/{{ key }}) in {{ server_description }}. +{% endfilter %} diff --git a/openedx_webhooks/types.py b/openedx_webhooks/types.py index e0fbe57d..15d2d64d 100644 --- a/openedx_webhooks/types.py +++ b/openedx_webhooks/types.py @@ -20,7 +20,7 @@ @dataclasses.dataclass(frozen=True) class PrId: - """An id of a pull request, with three parts used by GitHub.""" + """An id of a pull request, with a repo full_name and an id.""" full_name: str number: int diff --git a/openedx_webhooks/utils.py b/openedx_webhooks/utils.py index eb9d2aee..c66b9ebd 100644 --- a/openedx_webhooks/utils.py +++ b/openedx_webhooks/utils.py @@ -19,7 +19,6 @@ from openedx_webhooks import logger from openedx_webhooks.auth import get_github_session, get_jira_session -from openedx_webhooks.settings import settings from openedx_webhooks.types import JiraDict @@ -306,21 +305,18 @@ def sentry_extra_context(data_dict): @memoize_timed(minutes=30) -def get_jira_custom_fields(): +def get_jira_custom_fields(jira_nick: str): """ Return a name-to-id mapping for the custom fields on JIRA. """ - if settings.JIRA_SERVER: - session = get_jira_session() - field_resp = session.get("/rest/api/2/field") - field_resp.raise_for_status() - fields = field_resp.json() - return {f["name"]: f["id"] for f in fields if f["custom"]} - else: - return {} + session = get_jira_session(jira_nick) + field_resp = session.get("/rest/api/2/field") + field_resp.raise_for_status() + fields = field_resp.json() + return {f["name"]: f["id"] for f in fields if f["custom"]} -def get_jira_issue(key: str, missing_ok: bool = False) -> Optional[JiraDict]: +def get_jira_issue(jira_nick: str, key: str, missing_ok: bool = False) -> Optional[JiraDict]: """ Get the dictionary for a Jira issue, from its key. @@ -333,20 +329,20 @@ def get_jira_issue(key: str, missing_ok: bool = False) -> Optional[JiraDict]: is missing. """ - resp = jira_get("/rest/api/2/issue/{key}".format(key=key)) + resp = jira_get(jira_nick, "/rest/api/2/issue/{key}".format(key=key)) if resp.status_code == 404 and missing_ok: return None log_check_response(resp) return resp.json() -def jira_get(*args, **kwargs): +def jira_get(jira_nick, *args, **kwargs): """ JIRA sometimes returns an empty response to a perfectly valid GET request, so this will retry it a few times if that happens. """ for _ in range(3): - resp = get_jira_session().get(*args, **kwargs) + resp = get_jira_session(jira_nick).get(*args, **kwargs) if resp.content: return resp - return get_jira_session().get(*args, **kwargs) + return get_jira_session(jira_nick).get(*args, **kwargs) diff --git a/tests/conftest.py b/tests/conftest.py index c5145ee2..42a4fa4b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -98,10 +98,18 @@ def fake_github(pytestconfig, mocker, requests_mocker, fake_repo_data): @pytest.fixture def fake_jira(requests_mocker, fake_repo_data): + """A FakeJira for the first server configured in our jira-info.yaml.""" the_fake_jira = FakeJira("https://test.atlassian.net") the_fake_jira.install_mocks(requests_mocker) return the_fake_jira +@pytest.fixture +def fake_jira2(requests_mocker, fake_repo_data): + """A FakeJira for the second server configured in our jira-info.yaml.""" + the_fake_jira = FakeJira("https://anothertest.atlassian.net") + the_fake_jira.install_mocks(requests_mocker) + return the_fake_jira + @pytest.fixture(autouse=True) def configure_flask_app(): diff --git a/tests/helpers.py b/tests/helpers.py index ac608b63..a153c526 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -3,6 +3,9 @@ import random import re +from openedx_webhooks.info import get_jira_server_info +from openedx_webhooks.types import JiraId + def check_good_markdown(text: str) -> None: """ @@ -33,21 +36,22 @@ def check_good_markdown(text: str) -> None: raise ValueError(f"Markdown has a link to a None url: {text!r}") -def check_issue_link_in_markdown(text: str, issue_id: str|None) -> None: +def check_issue_link_in_markdown(text: str, jira_id: JiraId|None) -> None: """ - Check that `text` has properly links to `issue_id`. + Check that `text` has properly formatted links to `jira_id`. Args: text: Markdown text. - issue_id: A JIRA issue id, which can be None. + jira_id: A JiraId (nick, key) pair, which can be None. Returns: Nothing. Will raise an exception with a failure message if something is wrong. """ - if issue_id is not None: - jira_link = "[{id}](https://test.atlassian.net/browse/{id})".format(id=issue_id) - assert jira_link in text, f"Markdown is missing a link to {issue_id}" + if jira_id is not None: + jira_server = get_jira_server_info(jira_id.nick) + jira_link = f"[{jira_id.key}]({jira_server.server}/browse/{jira_id.key})" + assert jira_link in text, f"Markdown is missing a link to {jira_id}" else: assert "/browse/" not in text, "Markdown links to JIRA when we have no issue id" diff --git a/tests/test_pull_request_opened.py b/tests/test_pull_request_opened.py index 8a94498c..43eaa6e7 100644 --- a/tests/test_pull_request_opened.py +++ b/tests/test_pull_request_opened.py @@ -332,3 +332,69 @@ def test_add_to_multiple_projects(fake_github): assert pull_request_projects(pr.as_json()) == { settings.GITHUB_OSPR_PROJECT, ("openedx", 23), ("anotherorg", 17), } + + +def test_jira_labelling(fake_github, fake_jira, fake_jira2): + # A PR with a "jira:" label makes a Jira issue. + pr = fake_github.make_pull_request("openedx", user="nedbat", title="Ned's PR") + pr.set_labels(["jira:test1"]) + assert len(pr.list_comments()) == 0 + + result = pull_request_changed(pr.as_json()) + assert len(result.jira_issues) == 1 + assert len(result.changed_jira_issues) == 1 + assert len(fake_jira.issues) == 1 + assert len(fake_jira2.issues) == 0 + pr_comments = pr.list_comments() + assert len(pr_comments) == 1 + body = pr_comments[-1].body + jira_id = result.changed_jira_issues.pop() + check_issue_link_in_markdown(body, jira_id) + assert "in the private Test1 Jira" in body + jira_issue = fake_jira.issues[jira_id.key] + assert jira_issue.summary == "Ned's PR" + + # Processing the pull request again won't make another issue. + result = pull_request_changed(pr.as_json()) + assert len(result.jira_issues) == 1 + assert len(result.changed_jira_issues) == 0 + assert len(fake_jira.issues) == 1 + assert len(fake_jira2.issues) == 0 + assert len(pr.list_comments()) == 1 + + +def test_jira_labelling_later(fake_github, fake_jira, fake_jira2): + # You can add the label later, and get a Jira issue. + # At first, no labels, so no Jira issues: + pr = fake_github.make_pull_request("openedx", user="nedbat", title="Yet another PR") + result = pull_request_changed(pr.as_json()) + assert len(result.jira_issues) == 0 + assert len(result.changed_jira_issues) == 0 + assert len(fake_jira.issues) == 0 + assert len(fake_jira2.issues) == 0 + + # Make a label, get an issue: + pr.set_labels(["jira:test1"]) + result = pull_request_changed(pr.as_json()) + assert len(result.jira_issues) == 1 + assert len(result.changed_jira_issues) == 1 + assert len(fake_jira.issues) == 1 + assert len(fake_jira2.issues) == 0 + pr_comments = pr.list_comments() + assert len(pr_comments) == 1 + + # You can add a second label for another Jira server. + pr.set_labels(["jira:AnotherTest"]) + result = pull_request_changed(pr.as_json()) + assert len(result.jira_issues) == 2 + assert len(result.changed_jira_issues) == 1 + assert len(fake_jira.issues) == 1 + assert len(fake_jira2.issues) == 1 + pr_comments = pr.list_comments() + assert len(pr_comments) == 2 + body = pr_comments[-1].body + jira_id = result.changed_jira_issues.pop() + check_issue_link_in_markdown(body, jira_id) + assert "in the Another Test Jira" in body + jira_issue = fake_jira2.issues[jira_id.key] + assert jira_issue.summary == "Yet another PR" From aeb10383f375f9fc05b2fd5c246b2d16a07f8259 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Wed, 27 Sep 2023 14:38:55 -0400 Subject: [PATCH 10/12] test: update the rescan tests Full disclosure: I haven't thought hard about what rescanning should do now, and haven't added the new label->Jira logic into the rescan tests. --- openedx_webhooks/tasks/github.py | 6 +++--- tests/test_rescan.py | 22 ++++++++-------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/openedx_webhooks/tasks/github.py b/openedx_webhooks/tasks/github.py index cf960ae5..621ff6b5 100644 --- a/openedx_webhooks/tasks/github.py +++ b/openedx_webhooks/tasks/github.py @@ -169,9 +169,9 @@ def rescan_repository( else: if result.changed_jira_issues: changed[pull_request["number"]] = result.changed_jira_issues - if dry_run: - assert actions is not None - dry_run_actions[pull_request["number"]] = actions.action_calls + if dry_run: + assert actions is not None + dry_run_actions[pull_request["number"]] = actions.action_calls if not dry_run: logger.info( diff --git a/tests/test_rescan.py b/tests/test_rescan.py index 9e5d49ee..88d3f4f8 100644 --- a/tests/test_rescan.py +++ b/tests/test_rescan.py @@ -73,10 +73,10 @@ def test_rescan_repository(rescannable_repo, pull_request_changed_fn, allpr): prnums = [c.args[0]["number"] for c in pull_request_changed_fn.call_args_list] if allpr: assert prnums == [102, 106, 108, 110] - assert changed == {102: None, 106: None, 108: None, 110: None} + assert changed == {} else: assert prnums == [102, 106] - assert set(changed.keys()) == {102, 106} + assert changed == {} # If we rescan again, nothing should happen. ret = rescan_repository(rescannable_repo.full_name, allpr=allpr) @@ -92,41 +92,36 @@ def test_rescan_repository_dry_run(rescannable_repo, fake_github, fake_jira, pul fake_jira.assert_readonly() # These are the OSPR tickets for the pull requests. - assert ret["changed"] == { - 102: None, - 106: None, - 108: None, - 110: None, - } + assert ret["changed"] == {} # Get the names of the actions. We won't worry about the details, those # are tested in the non-dry-run tests of rescanning pull requests. actions = {k: [name for name, kwargs in actions] for k, actions in ret["dry_run_actions"].items()} assert actions == { 102: [ - "set_cla_status", "initial_state", + "set_cla_status", "update_labels_on_pull_request", "add_comment_to_pull_request", "add_pull_request_to_project", ], 106: [ - "set_cla_status", "initial_state", + "set_cla_status", "update_labels_on_pull_request", "add_comment_to_pull_request", "add_pull_request_to_project", ], 108: [ - "set_cla_status", "initial_state", + "set_cla_status", "update_labels_on_pull_request", "add_comment_to_pull_request", "add_pull_request_to_project", ], 110: [ - "set_cla_status", "initial_state", + "set_cla_status", "update_labels_on_pull_request", "add_comment_to_pull_request", "add_pull_request_to_project", @@ -136,7 +131,6 @@ def test_rescan_repository_dry_run(rescannable_repo, fake_github, fake_jira, pul # The value returned should be json-encodable. json.dumps(ret) - @pytest.mark.parametrize("earliest, latest, nums", [ ("", "", [102, 106, 108, 110]), ("2019-06-01", "", [108, 110]), @@ -184,7 +178,7 @@ def flaky_pull_request_changed(pr, actions): mocker.patch("openedx_webhooks.tasks.github.pull_request_changed", flaky_pull_request_changed) ret = rescan_repository(rescannable_repo.full_name, allpr=True) - assert list(ret["changed"]) == [102, 106, 108, 110] + assert list(ret["changed"]) == [] err = ret["errors"][108] assert err.startswith("Traceback (most recent call last):\n") assert " in flaky_pull_request_changed\n" in err From 8ebae19afd9924211114e0222833dc43c4ff4a31 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 28 Sep 2023 13:03:29 -0400 Subject: [PATCH 11/12] refactor: go back to a simpler settings structure --- openedx_webhooks/auth.py | 2 +- openedx_webhooks/settings.py | 33 +++++++++------------------ openedx_webhooks/tasks/pr_tracking.py | 2 +- tests/conftest.py | 10 +++----- tests/settings.py | 6 +++-- tests/test_pull_request_opened.py | 2 +- 6 files changed, 21 insertions(+), 34 deletions(-) diff --git a/openedx_webhooks/auth.py b/openedx_webhooks/auth.py index 035a6b6a..d3cb707b 100644 --- a/openedx_webhooks/auth.py +++ b/openedx_webhooks/auth.py @@ -5,7 +5,7 @@ import requests from urlobject import URLObject -from openedx_webhooks.settings import settings +from openedx_webhooks import settings class BaseUrlSession(requests.Session): diff --git a/openedx_webhooks/settings.py b/openedx_webhooks/settings.py index 26d8d872..de184df9 100644 --- a/openedx_webhooks/settings.py +++ b/openedx_webhooks/settings.py @@ -1,29 +1,12 @@ """Settings for how the webhook should behave.""" import os -from types import SimpleNamespace -from typing import Mapping, Optional +from typing import Optional from openedx_webhooks.types import GhProject -# The global settings object. -settings = SimpleNamespace() - - -def settings_from_environment(env: Mapping) -> None: - """Update `settings` from a dict of environment variables.""" - settings.GITHUB_PERSONAL_TOKEN = env.get("GITHUB_PERSONAL_TOKEN", None) - - # The project all OSPRs should be added to. - # This should be in the form of org:num, like "openedx:19" - settings.GITHUB_OSPR_PROJECT = read_project_setting(env, "GITHUB_OSPR_PROJECT") - - # The project all Blended pull requests should be added to. - settings.GITHUB_BLENDED_PROJECT = read_project_setting(env, "GITHUB_BLENDED_PROJECT") - - -def read_project_setting(env: Mapping, setting_name: str) -> Optional[GhProject]: +def read_project_setting(setting_name: str) -> Optional[GhProject]: """Read a project spec from a setting. Project number NUM in org ORG is specified as ``ORG:NUM``. @@ -32,12 +15,18 @@ def read_project_setting(env: Mapping, setting_name: str) -> Optional[GhProject] ("ORG", NUM) if the setting is present. None if the setting is missing. """ - project = env.get(setting_name, None) + project = os.environ.get(setting_name, None) if project is not None: org, num = project.split(":") return (org, int(num)) return None -# Read our settings from the real environment. -settings_from_environment(os.environ) +GITHUB_PERSONAL_TOKEN = os.environ.get("GITHUB_PERSONAL_TOKEN", None) + +# The project all OSPRs should be added to. +# This should be in the form of org:num, like "openedx:19" +GITHUB_OSPR_PROJECT = read_project_setting("GITHUB_OSPR_PROJECT") + +# The project all Blended pull requests should be added to. +GITHUB_BLENDED_PROJECT = read_project_setting("GITHUB_BLENDED_PROJECT") diff --git a/openedx_webhooks/tasks/pr_tracking.py b/openedx_webhooks/tasks/pr_tracking.py index 9fc99bff..c3723bb1 100644 --- a/openedx_webhooks/tasks/pr_tracking.py +++ b/openedx_webhooks/tasks/pr_tracking.py @@ -53,7 +53,7 @@ GITHUB_CATEGORY_LABELS, GITHUB_STATUS_LABELS, ) -from openedx_webhooks.settings import settings +from openedx_webhooks import settings from openedx_webhooks.tasks import logger from openedx_webhooks.tasks.jira_work import ( update_jira_issue, diff --git a/tests/conftest.py b/tests/conftest.py index 42a4fa4b..9c47c3fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,8 +10,6 @@ import openedx_webhooks import openedx_webhooks.info import openedx_webhooks.utils -import openedx_webhooks.settings -from openedx_webhooks.settings import settings from . import settings as test_settings from .fake_github import FakeGitHub @@ -79,11 +77,9 @@ def pytest_addoption(parser): @pytest.fixture(autouse=True) def settings_for_tests(mocker): - # This patch will restore the real settings when the fixture is done, even - # though the patch itself doesn't change any. - mocker.patch.dict(settings.__dict__, {}) - # Update the real settings from our test environment variables. - openedx_webhooks.settings.settings_from_environment(test_settings.__dict__) + for name, value in vars(test_settings).items(): + if name.isupper(): + mocker.patch(f"openedx_webhooks.settings.{name}", value) @pytest.fixture def fake_github(pytestconfig, mocker, requests_mocker, fake_repo_data): diff --git a/tests/settings.py b/tests/settings.py index bf6ad529..88b69cde 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -1,5 +1,7 @@ """Made-up settings to use during tests.""" -GITHUB_BLENDED_PROJECT = "blendorg:42" -GITHUB_OSPR_PROJECT = "testorg:17" +# These should be in the in-memory form ready to patch into openedx_webhooks.settings + +GITHUB_BLENDED_PROJECT = ("blendorg", 42) +GITHUB_OSPR_PROJECT = ("testorg", 17) GITHUB_PERSONAL_TOKEN = "github_pat_FooBarBaz" diff --git a/tests/test_pull_request_opened.py b/tests/test_pull_request_opened.py index 43eaa6e7..cccd272a 100644 --- a/tests/test_pull_request_opened.py +++ b/tests/test_pull_request_opened.py @@ -14,8 +14,8 @@ CLA_STATUS_NO_CONTRIBUTIONS, CLA_STATUS_PRIVATE, ) +from openedx_webhooks import settings from openedx_webhooks.gh_projects import pull_request_projects -from openedx_webhooks.settings import settings from openedx_webhooks.tasks.github import pull_request_changed from .helpers import check_issue_link_in_markdown From b33718c87a6329108c25f7768fd2d231821f65b1 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 28 Sep 2023 13:20:23 -0400 Subject: [PATCH 12/12] refactor: we weren't using the jira initial status, remove it --- openedx_webhooks/tasks/pr_tracking.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/openedx_webhooks/tasks/pr_tracking.py b/openedx_webhooks/tasks/pr_tracking.py index c3723bb1..faf87e38 100644 --- a/openedx_webhooks/tasks/pr_tracking.py +++ b/openedx_webhooks/tasks/pr_tracking.py @@ -135,9 +135,6 @@ class PrDesiredInfo: # The Jira instances we want to have issues on. jira_nicks: Set[str] = field(default_factory=set) - # The Jira status to start a new issue at. - jira_initial_status: Optional[str] = None - # The Jira status we want to set on an existing issue. Can be None if we # don't need to force a new status, but can leave the existing status. jira_status: Optional[str] = None @@ -230,7 +227,6 @@ def desired_support_state(pr: PrDict) -> PrDesiredInfo: label_names = set(lbl["name"] for lbl in pr["labels"]) desired.jira_nicks = {name.partition(":")[-1] for name in label_names if name.startswith("jira:")} - desired.jira_initial_status = "Needs Triage" desired.jira_title = pr["title"] desired.jira_description = pr["body"] or "" @@ -270,12 +266,10 @@ def desired_support_state(pr: PrDict) -> PrDesiredInfo: if desired.is_ospr: # Some PR states mean we want to insist on a Jira status. if is_draft_pull_request(pr): - desired.jira_initial_status = "Waiting on Author" desired.bot_comments.add(BotComment.END_OF_WIP) if not has_signed_agreement: desired.bot_comments.add(BotComment.NEED_CLA) - desired.jira_initial_status = "Community Manager Review" if state == "closed": desired.jira_status = "Rejected"