From 3ac394e872d1112f0d0c65ef318ae42cbacc9821 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Fri, 24 Feb 2023 02:36:59 -0500 Subject: [PATCH] bugwarrior ini2toml: init Add a new subcommand to automatically convert bugwarriorrc to bugwarrior.toml. This commit adds the ini2toml package as a dependency and implements a custom profile to handle the idiosyncrasies of our bugwarriorrc format. In my opinion this makes the transition easy enough that we no longer need to document bugwarriorrc at all so I went ahead and dropped what little remained. The only "bug" I've noticed so far with ini2toml is that empty lines are not preserved. This is the standard behavior of ini2toml due to the `normalise_newlines` postprocessor, which removes all empty lines and adds one before each section. However, this seems to be due to the issue that upstream tomlkit adds lots of arbitrary empty lines (aka `Whitespace()` and leaving them all would be an even worse result. See https://github.com/sdispater/tomlkit/issues/48. This is about as far as I care to chase this bug down the rabbit hole at the moment. --- bugwarrior/command.py | 15 +- bugwarrior/config/__init__.py | 3 +- bugwarrior/config/ini2toml_plugin.py | 157 ++++++++++++++ bugwarrior/docs/common_configuration.rst | 21 +- bugwarrior/docs/configuration.rst | 8 - bugwarrior/docs/index.rst | 1 - setup.py | 3 + tests/config/example-bugwarrior.toml | 201 ++++++++++++++++++ .../config}/example-bugwarriorrc | 8 +- tests/config/test_load.py | 17 ++ tests/test_command.py | 21 +- tests/test_docs.py | 11 - 12 files changed, 436 insertions(+), 30 deletions(-) create mode 100644 bugwarrior/config/ini2toml_plugin.py delete mode 100644 bugwarrior/docs/configuration.rst create mode 100644 tests/config/example-bugwarrior.toml rename {bugwarrior/docs => tests/config}/example-bugwarriorrc (96%) diff --git a/bugwarrior/command.py b/bugwarrior/command.py index 7954cdc7..0723bdd9 100644 --- a/bugwarrior/command.py +++ b/bugwarrior/command.py @@ -7,8 +7,9 @@ import getpass import click +from ini2toml.api import Translator -from bugwarrior.config import get_keyring, load_config +from bugwarrior.config import get_keyring, get_config_path, load_config from bugwarrior.services import aggregate_issues, get_service from bugwarrior.db import ( get_defined_udas_as_strings, @@ -212,3 +213,15 @@ def uda(flavor): for uda in get_defined_udas_as_strings(conf, main_section): print(uda) print("# END Bugwarrior UDAs") + + +@cli.command() +@click.argument('rcfile', required=False, default=get_config_path(), + type=click.Path(exists=True)) +def ini2toml(rcfile): + """ Convert ini bugwarriorrc to toml and print result to stdout. """ + if os.path.splitext(rcfile)[-1] == '.toml': + raise SystemExit(f'{rcfile} is already toml!') + with open(rcfile, 'r') as f: + bugwarriorrc = f.read() + print(Translator().translate(bugwarriorrc, 'bugwarriorrc')) diff --git a/bugwarrior/config/__init__.py b/bugwarrior/config/__init__.py index a3a72730..cd980211 100644 --- a/bugwarrior/config/__init__.py +++ b/bugwarrior/config/__init__.py @@ -1,4 +1,4 @@ -from .load import BUGWARRIORRC, load_config +from .load import BUGWARRIORRC, get_config_path, load_config from .schema import (ConfigList, ExpandedPath, NoSchemeUrl, @@ -9,6 +9,7 @@ __all__ = [ # load 'BUGWARRIORRC', + 'get_config_path', 'load_config', # schema 'ConfigList', diff --git a/bugwarrior/config/ini2toml_plugin.py b/bugwarrior/config/ini2toml_plugin.py new file mode 100644 index 00000000..d14b7a9a --- /dev/null +++ b/bugwarrior/config/ini2toml_plugin.py @@ -0,0 +1,157 @@ +import logging +import re +import typing + +from ini2toml.types import IntermediateRepr, Translator +from pydantic import BaseModel + +from .schema import ConfigList +from ..services.activecollab2 import ActiveCollabProjects + +log = logging.getLogger(__name__) + +BOOLS = { + 'general': ['interactive', 'shorten', 'inline_links', 'annotation_links', + 'annotation_comments', 'annotation_newlines', + 'merge_annotations', 'merge_tags', 'replace_tags'], + 'bitbucket': ['filter_merge_requests', 'include_merge_requests', + 'project_owner_prefix'], + 'bts': ['udd', 'ignore_pending', 'udd_ignore_sponsor'], + 'bugzilla': ['ignore_cc', 'include_needinfos', 'force_rest', 'advanced'], + 'deck': ['import_labels_as_tags'], + 'gitbug': ['import_labels_as_tags'], + 'github': ['include_user_repos', 'import_labels_as_tags', + 'filter_pull_requests', 'exclude_pull_requests', + 'include_user_issues', 'involved_issues', 'project_owner_prefix' + ], + 'gitlab': ['filter_merge_requests', 'membership', 'owned', + 'import_labels_as_tags', 'include_merge_requests', + 'include_issues', 'include_todos', 'include_all_todos', + 'use_https', 'verify_ssl', 'project_owner_prefix'], + 'jira': ['import_labels_as_tags', 'import_sprints_as_tags', 'use_cookies', + 'verify_ssl'], + 'pagure': ['import_tags'], + 'phabricator': ['ignore_cc', 'ignore_author', 'ignore_owner', + 'ignore_reviewers', 'only_if_assigned'], + 'pivotaltracker': ['import_blockers', 'import_labels_as_tags', + 'only_if_author', 'only_if_assigned'], + 'redmine': ['verify_ssl'], + 'taiga': ['include_tasks'], + 'track': ['no_xmlrpc'], + 'trello': ['import_labels_as_tags'], + 'youtrack': ['anonymous', 'use_https', 'verify_ssl', 'incloud_instance', + 'import_tags'], +} + +INTEGERS = { + 'general': ['annotation_length', 'description_length'], + 'activecollab': ['user_id'], + 'activecollab2': ['user_id'], + 'gitbug': ['port'], + 'github': ['body_length'], + 'jira': ['body_length', 'version'], + 'pivotaltracker': ['user_id'], + 'redmine': ['issue_limit'], + 'youtrack': ['port', 'query_limit'], +} + +CONFIGLIST = { + 'general': ['targets', 'static_tags', 'static_fields'], + 'github': ['include_repos', 'exclude_repos', 'issue_urls'], + 'pivotaltracker': ['account_ids', 'exclude_projects', 'exclude_stories', 'exclude_tags'], + 'gitlab': ['include_repos', 'exclude_repos'], + 'bitbucket': ['include_repos', 'exclude_repos'], + 'phabricator': ['user_phids', 'project_phids'], + 'trello': ['include_boards', 'include_lists', 'exclude_lists'], + 'bts': ['packages', 'ignore_pkg', 'ignore_src'], + 'deck': ['include_board_ids', 'exclude_board_ids'], + 'bugzilla': ['open_statuses'], + 'pagure': ['include_repos', 'exclude_repos'], +} + + +def to_type(section: IntermediateRepr, key: str, converter: typing.Callable): + try: + val = section[key] + except KeyError: + pass + else: + section[key] = converter(val) + + +class BooleanModel(BaseModel): + """ + Use Pydantic to convert various strings to booleans. + + "True", "False", "yes", "no", etc. + Adapted from https://docs.pydantic.dev/usage/types/#booleans + """ + bool_value: bool + + +def to_bool(section: IntermediateRepr, key: str): + to_type(section, key, lambda val: BooleanModel(bool_value=val).bool_value) + + +def to_int(section: IntermediateRepr, key: str): + to_type(section, key, int) + + +def to_list(section: IntermediateRepr, key: str): + to_type(section, key, ConfigList.validate) + + +def process_values(doc: IntermediateRepr) -> IntermediateRepr: + for name, section in doc.items(): + if isinstance(name, str): + if name == 'general' or re.match(r'^flavor\.', name): + for k in INTEGERS['general']: + to_int(section, k) + for k in CONFIGLIST['general']: + to_list(section, k) + for k in BOOLS['general']: + to_bool(section, k) + for k in ['log.level', 'log.file']: + if k in section: + section.rename(k, k.replace('.', '_')) + elif name == 'hooks': + to_list(section, 'pre_import') + elif name == 'notifications': + to_bool(section, 'notifications') + to_bool(section, 'only_on_new_tasks') + else: # services + service = section['service'] + + # Validate and strip prefixes. + for key in section.keys(): + prefix = 'ado' if service == 'azuredevops' else service + if isinstance(key, str) and key != 'service': + newkey, subs = re.subn(f'^{prefix}\\.', '', key) + if subs != 1: + option = key.split('.').pop() + log.warning( + f"[{name}]\n{key} <-expected prefix " + f"'{prefix}': did you mean " + f"'{prefix}.{option}'?") + section.rename(key, newkey) + + to_bool(section, 'also_unassigned') + to_list(section, 'add_tags') + for k in INTEGERS.get(service, []): + to_int(section, k) + for k in CONFIGLIST.get(service, []): + to_list(section, k) + for k in BOOLS.get(service, []): + to_bool(section, k) + + if name == 'activecollab2' and 'projects' in section: + section['projects'] = ActiveCollabProjects.validate( + section['projects']) + + return doc + + +def activate(translator: Translator): + profile = translator["bugwarriorrc"] + profile.description = "Convert 'bugwarriorrc' files to 'bugwarrior.toml'" + profile.intermediate_processors.append(process_values) diff --git a/bugwarrior/docs/common_configuration.rst b/bugwarrior/docs/common_configuration.rst index f6accf0c..d1e0fe02 100644 --- a/bugwarrior/docs/common_configuration.rst +++ b/bugwarrior/docs/common_configuration.rst @@ -1,6 +1,24 @@ How to Configure ================ +Bugwarrior's configuration file is written in `toml `_. + +A basic configuration might look like this: + +.. code:: toml + + [general] + targets = my_github + + [my_github] + service = github + github.login = ralphbean + github.token = 123456 + github.username = ralphbean + +Main Section +------------ + Your :ref:`configuration file ` must include at least a ``[general]`` section including the following option: @@ -48,9 +66,6 @@ In addition to the ``[general]`` section, sections may be named ``bugwarrior pull``. This section will then be used rather than the ``[general]`` section. -A more-detailed example configuration can be found at -:ref:`example_configuration`. - .. _common_configuration_options: diff --git a/bugwarrior/docs/configuration.rst b/bugwarrior/docs/configuration.rst deleted file mode 100644 index 3a521f35..00000000 --- a/bugwarrior/docs/configuration.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _example_configuration: - -Example Configuration -====================== - -.. include:: example-bugwarriorrc - :code: ini - diff --git a/bugwarrior/docs/index.rst b/bugwarrior/docs/index.rst index 0ffb14c9..5e4e1c84 100644 --- a/bugwarrior/docs/index.rst +++ b/bugwarrior/docs/index.rst @@ -31,7 +31,6 @@ Contents using common_configuration services - configuration contributing faq diff --git a/setup.py b/setup.py index 332c670d..9aa9261b 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ install_requires=[ "click", "dogpile.cache>=0.5.3", + "ini2toml[full]", "jinja2>=2.7.2", "lockfile>=0.9.1", "pydantic[email]", @@ -103,5 +104,7 @@ azuredevops=bugwarrior.services.azuredevops:AzureDevopsService gitbug=bugwarrior.services.gitbug:GitBugService deck=bugwarrior.services.deck:NextcloudDeckService + [ini2toml.processing] + bugwarrior = bugwarrior.config.ini2toml_plugin:activate """, ) diff --git a/tests/config/example-bugwarrior.toml b/tests/config/example-bugwarrior.toml new file mode 100644 index 00000000..f237a132 --- /dev/null +++ b/tests/config/example-bugwarrior.toml @@ -0,0 +1,201 @@ +# Example bugwarriorrc +# General stuff. + +[general] +# Here you define a comma separated list of targets. Each of them must have a +# section below determining their properties, how to query them, etc. The name +# is just a symbol, and doesn't have any functional importance. +targets = [ + "activecollab", + "activecollab2", + "gitlab_config", + "jira_project", + "my_github", + "my_gmail", + "my_kanboard", + "my_phabricator", + "my_redmine", + "my_teamlab", + "moksha_trac", + "pivotaltracker", +] +# If unspecified, the default taskwarrior config will be used. +# taskrc = /path/to/.taskrc +# Setting this to true will shorten links with http://da.gd/ +shorten = false +# Setting this to True will include a link to the ticket in the description +inline_links = false +# Setting this to True will include a link to the ticket as an annotation +annotation_links = true +# Setting this to True will include issue comments and author name in task +# annotations +annotation_comments = true +# Setting this to False will strip newlines from comment annotations +annotation_newlines = false +# log_level specifies the verbosity. The default is DEBUG. +# log_level can be one of DEBUG, INFO, WARNING, ERROR, CRITICAL, DISABLED +log_level = "DEBUG" +# If log_file is specified, output will be redirected there. If it remains +# unspecified, output is sent to sys.stderr +log_file = "/var/log/bugwarrior.log" +# Configure the default description or annotation length. +# annotation_length = 45 +# Use hooks to run commands prior to importing from `bugwarrior pull`. +# `bugwarrior pull` will run the commands in the order that they are specified +# below. +# +# pre_import: The pre_import hook is invoked after all issues have been pulled +# from remote sources, but before they are synced to the TW db. If your +# pre_import script has a non-zero exit code, the `bugwarrior pull` command will +# exit early. + +[hooks] +pre_import = ["/home/someuser/backup.sh", "/home/someuser/sometask.sh"] +# This section is for configuring notifications when `bugwarrior pull` runs, +# and when issues are created, updated, or deleted by `bugwarrior pull`. +# Three backends are currently supported: +# +# - applescript macOS no external dependencies +# - gobject Linux python gobject must be installed +# +# [notifications] +# notifications = True +# backend = gobject +# only_on_new_tasks = True +# This is a github example. It says, "scrape every issue from every repository +# on http://github.com/ralphbean. It doesn't matter if ralphbean owns the issue +# or not." + +[my_github] +service = "github" +default_priority = "H" +add_tags = ["open_source"] +# This specifies that we should pull issues from repositories belonging +# to the 'ralphbean' github account. See the note below about +# 'github.username' and 'github.login'. They are different, and you need +# both. +username = "ralphbean" +# I want taskwarrior to include issues from all my repos, except these +# two because they're spammy or something. +exclude_repos = ["project_bar", "project_baz"] +# Working with a large number of projects, instead of excluding most of them I +# can also simply include just a limited set. +include_repos = ["project_foo", "project_foz"] +# Note that login and username can be different: I can login as me, but +# scrape issues from an organization's repos. +# +# - 'github.login' is the username you ask bugwarrior to +# login as. Set it to your account. +# - 'github.username' is the github entity you want to pull +# issues for. It could be you, or some other user entirely. +login = "ralphbean" +token = "123456" +# Here's an example of a trac target. + +[moksha_trac] +service = "trac" +base_uri = "fedorahosted.org/moksha" +username = "ralph" +password = "OMG_LULZ" +only_if_assigned = "ralph" +also_unassigned = true +default_priority = "H" +add_tags = ["work"] +# Example gitlab configuration containing individual priorities + +[gitlab_config] +service = "gitlab" +login = "ralphbean" +token = "OMG_LULZ" +host = "gitlab.com" +owned = true +default_issue_priority = "M" +default_todo_priority = "M" +default_mr_priority = "H" +# Here's an example of a jira project. The ``jira-python`` module is +# a bit particular, and jira deployments, like Bugzilla, tend to be +# reasonably customized. So YMMV. The ``base_uri`` must not have a +# have a trailing slash. In this case we fetch comments and +# cases from jira assigned to 'ralph' where the status is not closed or +# resolved. + +[jira_project] +service = "jira" +base_uri = "https://jira.example.org" +username = "ralph" +password = "OMG_LULZ" +query = "assignee = ralph and status != closed and status != resolved" +# Set this to your jira major version. We currently support only jira version +# 4 and 5(the default). You can find your particular version in the footer at +# the dashboard. +version = 5 +add_tags = ["enterprisey", "work"] +# This is a kanboard example. + +[my_kanboard] +service = "kanboard" +url = "https://kanboard.example.org" +username = "ralphbean" +# Your password or, even better, API token +password = "my_api_token" +# A custom query to search for open issues. By default, assigned and open +# tasks are queried. +query = "status:open assignee:me" +# Here's an example of a phabricator target + +[my_phabricator] +service = "phabricator" +# No need to specify credentials. They are gathered from ~/.arcrc +# Here's an example of a teamlab target. + +[my_teamlab] +service = "teamlab" +hostname = "teamlab.example.com" +login = "alice" +password = "secret" +# Here's an example of a redmine target. + +[my_redmine] +service = "redmine" +url = "http://redmine.example.org/" +key = "c0c4c014cafebabe" +add_tags = ["chiliproject"] + +[activecollab] +service = "activecollab" +url = "https://ac.example.org/api.php" +key = "your-api-key" +user_id = 15 +add_tags = ["php"] + +[activecollab2] +service = "activecollab2" +url = "http://ac.example.org/api.php" +key = "your-api-key" +user_id = 15 +projects = {1 = "first_project", 5 = "another_project"} + +[my_gmail] +service = "gmail" +query = "label:action OR label:readme" +login_name = "you@example.com" + +[pivotaltracker] +service = "pivotaltracker" +token = "your-api-key" +version = "v5" +user_id = 123456 +account_ids = ["first_account_id", "second_account_id"] +only_if_assigned = true +also_unassigned = false +only_if_author = false +import_labels_as_tags = true +label_template = "pivotal_{{label}}" +import_blockers = true +blocker_template = 'Description: {{description}} State: {{resolved}}\n' +annotation_template = "status: {{completed}} - MYDESC {{description}}" +exclude_projects = ["first_project_id", "second_project_id"] +exclude_stories = ["first_story_id", "second_story_id"] +exclude_tags = ["wont fix", "should fix"] +query = "mywork:1234 -has:label" + diff --git a/bugwarrior/docs/example-bugwarriorrc b/tests/config/example-bugwarriorrc similarity index 96% rename from bugwarrior/docs/example-bugwarriorrc rename to tests/config/example-bugwarriorrc index 4e61be89..ec6824f2 100644 --- a/bugwarrior/docs/example-bugwarriorrc +++ b/tests/config/example-bugwarriorrc @@ -11,7 +11,7 @@ targets = activecollab, activecollab2, gitlab_config, jira_project, my_github, m #taskrc = /path/to/.taskrc # Setting this to true will shorten links with http://da.gd/ -#shorten = False +shorten = False # Setting this to True will include a link to the ticket in the description inline_links = False @@ -202,9 +202,9 @@ pivotaltracker.only_if_author = False pivotaltracker.import_labels_as_tags = True pivotaltracker.label_template = pivotal_{{label}} pivotaltracker.import_blockers = True -pivotaltracker.blocker_template = "Description: {{description}} State: {{resolved}}\n" -pivotaltracker.annotation_template = "status: {{completed}} - MYDESC {{description}}" +pivotaltracker.blocker_template = Description: {{description}} State: {{resolved}}\n +pivotaltracker.annotation_template = status: {{completed}} - MYDESC {{description}} pivotaltracker.exclude_projects = first_project_id,second_project_id pivotaltracker.exclude_stories = first_story_id,second_story_id -pivotaltracker.exclude_tags = "wont fix", "should fix" +pivotaltracker.exclude_tags = wont fix, should fix pivotaltracker.query = mywork:1234 -has:label diff --git a/tests/config/test_load.py b/tests/config/test_load.py index 8a671f11..090db5a3 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -1,6 +1,7 @@ import configparser import itertools import os +import pathlib import textwrap from unittest import TestCase @@ -14,6 +15,22 @@ from ..base import ConfigTest +class ExampleTest(ConfigTest): + def setUp(self): + self.basedir = pathlib.Path(__file__).parent + super().setUp() + + def test_example_bugwarriorrc(self): + os.environ['BUGWARRIORRC'] = str( + self.basedir / 'example-bugwarriorrc') + load.load_config('general', False, False) + + def test_example_bugwarrior_toml(self): + os.environ['BUGWARRIORRC'] = str( + self.basedir / 'example-bugwarrior.toml') + load.load_config('general', False, False) + + class LoadTest(ConfigTest): def create(self, path): """ diff --git a/tests/test_command.py b/tests/test_command.py index 7da7b486..779f2906 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -1,6 +1,7 @@ import os import logging -from unittest import mock +import pathlib +from unittest import mock, TestCase from click.testing import CliRunner import tomli_w @@ -185,3 +186,21 @@ def test_legacy_cli(self): self.assertIn('Adding 1 tasks', logs) self.assertIn('Updating 0 tasks', logs) self.assertIn('Closing 0 tasks', logs) + + +class TestIni2Toml(TestCase): + def setUp(self): + super().setUp() + self.runner = CliRunner() + + def test_bugwarriorrc(self): + basedir = pathlib.Path(__file__).parent + result = self.runner.invoke( + command.cli, + args=('ini2toml', str(basedir / 'config/example-bugwarriorrc'))) + + self.assertEqual(result.exit_code, 0) + + self.maxDiff = None + with open(basedir / 'config/example-bugwarrior.toml', 'r') as f: + self.assertEqual(result.stdout, f.read()) diff --git a/tests/test_docs.py b/tests/test_docs.py index 9598a105..04cd2c19 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -9,10 +9,6 @@ import tempfile import unittest -from bugwarrior import config - -from .base import ConfigTest - DOCS_PATH = pathlib.Path(__file__).parent / '../bugwarrior/docs' try: @@ -82,10 +78,3 @@ def test_registered_services_are_documented(self): documented_services.add(re.sub(r'\.rst$', '', p)) self.assertEqual(registered_services, documented_services) - - -class ExampleBugwarriorrcTest(ConfigTest): - def test_example_bugwarriorrc(self): - os.environ['BUGWARRIORRC'] = os.path.join( - DOCS_PATH, 'example-bugwarriorrc') - config.load_config('general', False, False)