Test Markers as Test Fields

betelgeuse/__init__.py

@@ -423,7 +423,7 @@ def get_requirement_field_values(config, requirement):
 
 def update_testcase_fields(config, testcase):
     """Apply testcase fields default values and transformations."""
-    if testcase.docstring and not type(testcase.docstring) == str:
+    if testcase.docstring and not isinstance(testcase.docstring, str):
         testcase.docstring = testcase.docstring.decode('utf8')
 
     # Check if any field needs a default value
@@ -656,7 +656,7 @@ def test_case(
     testcases.append(properties)
 
     source_testcases = itertools.chain(*collector.collect_tests(
-        source_code_path, collect_ignore_path).values())
+        source_code_path, collect_ignore_path, config=config).values())
     for testcase in source_testcases:
         testcases.append(
             create_xml_testcase(config, testcase, automation_script_format))

betelgeuse/collector.py

@@ -6,6 +6,7 @@
 import os
 
 from betelgeuse.parser import parse_docstring
+from betelgeuse.parser import parse_markers
 from betelgeuse.source_generator import gen_source
 
 
@@ -21,7 +22,10 @@ def __init__(self, title, fields=None):
 class TestFunction(object):
     """Wrapper for ``ast.FunctionDef`` which parse docstring information."""
 
-    def __init__(self, function_def, parent_class=None, testmodule=None):
+    def __init__(
+            self, function_def, parent_class=None, testmodule=None,
+            config=None
+    ):
         """``ast.FunctionDef`` instance used to extract information."""
         #: The unparsed testcase docstring
         self.docstring = ast.get_docstring(function_def)
@@ -88,11 +92,20 @@ def __init__(self, function_def, parent_class=None, testmodule=None):
                 for decorator in self.parent_class_def.decorator_list
             ]
         self._parse_docstring()
+        self._parse_markers(config)
         self.junit_id = self._generate_junit_id()
 
         if 'id' not in self.fields:
             self.fields['id'] = self.junit_id
 
+    def _parse_markers(self, config=None):
+        """Parse module, class and function markers."""
+        markers = [self.module_def.marker_list,
+                   self.class_decorators,
+                   self.decorators]
+        if markers:
+            self.fields.update({'markers': parse_markers(markers, config)})
+
     def _parse_docstring(self):
         """Parse package, module, class and function docstrings."""
         if self.docstring is None:
@@ -137,29 +150,52 @@ def is_test_module(filename):
     return False
 
 
-def _get_tests(path):
+def _module_markers(module_def):
+    """Extract markers applied to testcases from the test module level.
+
+    The markers list would be collected from the pytestmark global variable.
+    """
+    markers = []
+    for node in module_def.body:
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                if isinstance(target, ast.Name) and target.id == 'pytestmark':
+                    if isinstance(node.value, ast.List):
+                        for item in node.value.elts:
+                            if isinstance(item, ast.Attribute):
+                                markers.append(item.attr)
+                    elif isinstance(node.value, ast.Attribute):
+                        markers.append(node.value.attr)
+    return markers or None
+
+
+def _get_tests(path, config=None):
     """Collect tests for the test module located at ``path``."""
     tests = []
     with open(path) as handler:
         root = ast.parse(handler.read())
         root.path = path  # TODO improve how to pass the path to TestFunction
+        # Updating test module with module level markers
+        root.__dict__['marker_list'] = _module_markers(root)
         for node in ast.iter_child_nodes(root):
             if isinstance(node, ast.ClassDef):
                 [
-                    tests.append(TestFunction(subnode, node, root))
+                    tests.append(TestFunction(subnode, node, root, config))
                     for subnode in ast.iter_child_nodes(node)
                     if isinstance(subnode, ast.FunctionDef) and
                     subnode.name.startswith('test_')
                 ]
             elif (isinstance(node, ast.FunctionDef) and
                     node.name.startswith('test_')):
-                tests.append(TestFunction(node, testmodule=root))
+                tests.append(TestFunction(
+                    node, testmodule=root, config=config))
     return tests
 
 
-def collect_tests(path, ignore_paths=None):
+def collect_tests(path, ignore_paths=None, config=None):
     """Walk ``path`` and collect test methods and functions found.
 
+    :param config: The config object of `config.BetelgeuseConfig`
     :param path: Either a file or directory path to look for test methods and
         functions.
     :return: A dict mapping a test module path and its test cases.
@@ -170,7 +206,7 @@ def collect_tests(path, ignore_paths=None):
     tests = collections.OrderedDict()
     if os.path.isfile(path) and path not in ignore_paths:
         if is_test_module(os.path.basename(path)):
-            tests[path] = _get_tests(path)
+            tests[path] = _get_tests(path, config)
             return tests
     for dirpath, _, filenames in os.walk(path):
         if dirpath in ignore_paths:
@@ -180,5 +216,5 @@ def collect_tests(path, ignore_paths=None):
             if path in ignore_paths:
                 continue
             if is_test_module(filename):
-                tests[path] = _get_tests(path)
+                tests[path] = _get_tests(path, config)
     return tests

betelgeuse/parser.py

@@ -1,4 +1,5 @@
 """Parsers for test docstrings."""
+import re
 from collections import namedtuple
 from io import StringIO
 from xml.dom import minidom
@@ -187,3 +188,44 @@ def parse_docstring(docstring=None):
             field_value = output
             fields_dict[field_name] = field_value
     return fields_dict
+
+
+def parse_markers(all_markers=None, config=None):
+    """Parse the markers from module, class and test level for a test.
+
+    This removes the mark prepended words and also pops out the
+    ignorable marker from the list received from the config object.
+
+    :returns string: Comma separated list of markers from all levels for a test
+    """
+    resolved_markers = []
+    ignore_list = getattr(config, 'MARKERS_IGNORE_LIST', None)
+
+    def _process_marker(_marker):
+
+        marker_name = re.findall(
+            r'(?:pytest\.mark\.)?([^(\s()]+)(?=\s*\(|\s*$)', _marker)
+        if marker_name:
+            marker_name = marker_name[0]
+
+        # ignoring the marker if in ignore list
+        if ignore_list and any(
+                re.fullmatch(
+                    ignore_word, marker_name
+                ) for ignore_word in ignore_list
+        ):
+            return
+
+        resolved_markers.append(marker_name)
+
+    for sec_marker in all_markers:
+        # If the marker is none
+        if not sec_marker:
+            continue
+        elif isinstance(sec_marker, list):
+            for marker in sec_marker:
+                _process_marker(marker)
+        else:
+            _process_marker(sec_marker)
+
+    return ', '.join(resolved_markers)

tests/data/test_sample.py

@@ -1,8 +1,11 @@
 # encoding=utf-8
 """Sample test module."""
 import unittest
+import pytest
 
 
+pytestmark = [pytest.mark.run_in_one_thread, pytest.mark.tier1]
+
 CONSTANT = 'contant-value'
 
 
@@ -72,3 +75,14 @@ def test_method(self):
 
     def test_without_docstring(self):  # noqa: D102
         pass
+
+
+@pytest.mark.on_prem_provisioning
+class TestclasswithMarkers:
+    """Class to verify tests markers are collected from class."""
+
+    @pytest.mark.skipif(2 == 3, reason='2 is not 3')
+    @pytest.mark.osp
+    def test_markers_sample(self):
+        """Test for markers at test level."""
+        assert True

tests/test_collector.py

@@ -11,7 +11,7 @@ def test_collect_tests(path):
     """Check if ``collect_tests`` 'tests/data'collect tests."""
     tests = collector.collect_tests(path)
     assert 'tests/data/test_sample.py' in tests
-    assert len(tests['tests/data/test_sample.py']) == 4
+    assert len(tests['tests/data/test_sample.py']) == 5
 
     # Check if we are not doing a specific python module collection
     if path.endswith('.py'):
@@ -30,7 +30,7 @@ def test_collect_ignore_path(ignore_path):
     tests = collector.collect_tests('tests/data', [ignore_path])
     assert 'tests/data/ignore_dir/test_ignore_dir.py' not in tests
     assert 'tests/data/test_sample.py' in tests
-    assert len(tests['tests/data/test_sample.py']) == 4
+    assert len(tests['tests/data/test_sample.py']) == 5
 
 
 @pytest.mark.parametrize('filename', ('test_module.py', 'module_test.py'))

tests/test_parser.py

@@ -1,6 +1,7 @@
 # coding=utf-8
 """Tests for :mod:`betelgeuse.parser`."""
 import pytest
+import mock
 
 from betelgeuse import parser
 
@@ -83,3 +84,29 @@ def test_parse_rst_special_characters():
         u'<p>String with special character like é</p>\n'
         u'</main>\n'
     )
+
+
+def test_parse_markers():
+    """
+    Test if the markers list is parsed.
+
+    List should be comma separated list of markers from all levels after
+    removing 'pytest.mark' text and ignore some markers.
+    """
+    _mod_markers = 'pytest.mark.destructive'
+    _class_markers = [
+        'pytest.mark.on_prem_provisioning',
+        "pytest.mark.usefixtures('cleandir')"
+    ]
+    _test_markers = [
+        "pytest.mark.parametrize('something', ['a', 'b'])",
+        'pytest.mark.skipif(not settings.robottelo.REPOS_HOSTING_URL)',
+        'pytest.mark.tier1'
+    ]
+    _all_markers = [_mod_markers, _class_markers, _test_markers]
+
+    expected = 'destructive, on_prem_provisioning, tier1'
+    config = mock.MagicMock()
+    config.MARKERS_IGNORE_LIST = [
+        'parametrize', 'skipif', 'usefixtures', 'skip_if_not_set']
+    assert parser.parse_markers(_all_markers, config=config) == expected

tests/test_source_generator.py

@@ -1,5 +1,6 @@
 """Tests for :mod:`betelgeuse.source_generator`."""
 from betelgeuse import collector
+import mock
 
 
 def test_source_generator():
@@ -37,3 +38,34 @@ def test_source_generator():
         '(lambda v: (v if v else None))'
         ')',
     ]
+
+
+def test_source_markers():
+    """Verifies if the test collection collects test markers."""
+    config = mock.Mock()
+    config.MARKERS_IGNORE_LIST = [
+        'parametrize', 'skipif', 'usefixtures', 'skip_if_not_set']
+    tests = collector.collect_tests('tests/data/test_sample.py', config=config)
+    marked_test = [
+        test for test in tests['tests/data/test_sample.py']
+        if test.name == 'test_markers_sample'
+    ].pop()
+    assert marked_test.fields['markers'] == ('run_in_one_thread, tier1, '
+                                             'on_prem_provisioning, osp')
+
+
+def test_source_singular_module_marker():
+    """Verifies the single module level marker is retrieved."""
+    mod_string = 'import pytest\n\npytestmark = pytest.mark.tier2' \
+                 '\n\ndef test_sing():\n\tpass'
+    with open('/tmp/test_singular.py', 'w') as tfile:
+        tfile.writelines(mod_string)
+
+    config = mock.Mock()
+    config.MARKERS_IGNORE_LIST = ['tier3']
+    tests = collector.collect_tests('/tmp/test_singular.py', config=config)
+    marked_test = [
+        test for test in tests['/tmp/test_singular.py']
+        if test.name == 'test_sing'
+    ].pop()
+    assert marked_test.fields['markers'] == 'tier2'