diff --git a/launchable/test_runners/dotnet.py b/launchable/test_runners/dotnet.py new file mode 100644 index 000000000..bac717eda --- /dev/null +++ b/launchable/test_runners/dotnet.py @@ -0,0 +1,76 @@ +import glob +import os +from typing import List + +import click + +from launchable.test_runners import launchable +from launchable.test_runners.nunit import nunit_parse_func +from launchable.testpath import TestPath + + +@launchable.subset +def subset(client): + """ + Alpha: Supports only Zero Input Subsetting + """ + if not client.is_get_tests_from_previous_sessions: + click.echo( + click.style( + "The dotnet profile only supports Zero Input Subsetting.\nMake sure to use `--get-tests-from-previous-sessions` opton", # noqa: E501 + fg="red"), + err=True) + + # ref: https://github.com/Microsoft/vstest-docs/blob/main/docs/filter.md + separator = "|" + prefix = "FullyQualifiedName=" + + if client.is_output_exclusion_rules: + separator = "&" + prefix = "FullyQualifiedName!=" + + def formatter(test_path: TestPath): + paths = [] + + for path in test_path: + t = path.get("type", "") + if t == 'Assembly': + continue + paths.append(path.get("name", "")) + + return prefix + ".".join(paths) + + def exclusion_output_handler(subset_tests: List[TestPath], rest_tests: List[TestPath]): + if client.rest: + with open(client.rest, "w+", encoding="utf-8") as fp: + fp.write(client.separator.join(formatter(t) for t in subset_tests)) + + click.echo(client.separator.join(formatter(t) for t in rest_tests)) + + client.separator = separator + client.formatter = formatter + client.exclusion_output_handler = exclusion_output_handler + client.run() + + +@click.argument('files', required=True, nargs=-1) +@launchable.record.tests +def record_tests(client, files): + """ + Alpha: Supports only NUnit report formats. + """ + for file in files: + match = False + for t in glob.iglob(file, recursive=True): + match = True + if os.path.isdir(t): + client.scan(t, "*.xml") + else: + client.report(t) + if not match: + click.echo("No matches found: {}".format(file), err=True) + + # Note: we support only Nunit test report format now. + # If we need to support another format e.g) JUnit, trc, then we'll add a option + client.parse_func = nunit_parse_func + client.run() diff --git a/launchable/test_runners/nunit.py b/launchable/test_runners/nunit.py index c3b7e9a4b..05cb3bcf4 100644 --- a/launchable/test_runners/nunit.py +++ b/launchable/test_runners/nunit.py @@ -42,6 +42,34 @@ def split_filepath(path: str) -> List[str]: ] +def nunit_parse_func(report: str): + events = [] + + # parse element into CaseEvent + def on_element(e: Element): + build_path(e) + if e.name == "test-case": + result = e.attrs.get('result') + status = CaseEvent.TEST_FAILED + if result == 'Passed': + status = CaseEvent.TEST_PASSED + elif result == 'Skipped': + status = CaseEvent.TEST_SKIPPED + + events.append(CaseEvent.create( + _replace_fixture_to_suite(e.tags['path']), # type: ignore + float(e.attrs['duration']), + status, + timestamp=str(e.tags['startTime']))) # timestamp is already iso-8601 formatted + + # the 'start-time' attribute is normally on but apparently not always, + # so we try to use the nearest ancestor as an approximate + SaxParser([TagMatcher.parse("*/@start-time={startTime}")], on_element).parse(report) + + # return the obtained events as a generator + return (x for x in events) + + @click.argument('report_xmls', type=click.Path(exists=True), required=True, nargs=-1) @launchable.subset def subset(client, report_xmls): @@ -82,34 +110,7 @@ def record_tests(client, report_xml): if not match: click.echo("No matches found: {}".format(root), err=True) - def parse_func(report: str): - events = [] - - # parse element into CaseEvent - def on_element(e: Element): - build_path(e) - if e.name == "test-case": - result = e.attrs.get('result') - status = CaseEvent.TEST_FAILED - if result == 'Passed': - status = CaseEvent.TEST_PASSED - elif result == 'Skipped': - status = CaseEvent.TEST_SKIPPED - - events.append(CaseEvent.create( - _replace_fixture_to_suite(e.tags['path']), # type: ignore - float(e.attrs['duration']), - status, - timestamp=str(e.tags['startTime']))) # timestamp is already iso-8601 formatted - - # the 'start-time' attribute is normally on but apparently not always, - # so we try to use the nearest ancestor as an approximate - SaxParser([TagMatcher.parse("*/@start-time={startTime}")], on_element).parse(report) - - # return the obtained events as a generator - return (x for x in events) - - client.parse_func = parse_func + client.parse_func = nunit_parse_func client.run() diff --git a/tests/data/dotnet/record_test_result.json b/tests/data/dotnet/record_test_result.json new file mode 100644 index 000000000..57315def5 --- /dev/null +++ b/tests/data/dotnet/record_test_result.json @@ -0,0 +1,115 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { + "type": "Assembly", + "name": "rocket-car-dotnet.dll" + }, + { + "type": "TestSuite", + "name": "rocket_car_dotnet" + }, + { + "type": "TestSuite", + "name": "ExampleTest" + }, + { + "type": "TestCase", + "name": "TestAdd" + } + ], + "duration": 0.006132, + "status": 1, + "stdout": "", + "stderr": "", + "created_at": "2023-05-15T 08:22:02Z", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Assembly", + "name": "rocket-car-dotnet.dll" + }, + { + "type": "TestSuite", + "name": "rocket_car_dotnet" + }, + { + "type": "TestSuite", + "name": "ExampleTest" + }, + { + "type": "TestCase", + "name": "TestDiv" + } + ], + "duration": 0.016795, + "status": 0, + "stdout": "", + "stderr": "", + "created_at": "2023-05-15T 08:22:02Z", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Assembly", + "name": "rocket-car-dotnet.dll" + }, + { + "type": "TestSuite", + "name": "rocket_car_dotnet" + }, + { + "type": "TestSuite", + "name": "ExampleTest" + }, + { + "type": "TestCase", + "name": "TestMul" + } + ], + "duration": 0.0003959, + "status": 0, + "stdout": "", + "stderr": "", + "created_at": "2023-05-15T 08:22:02Z", + "data": null + }, + { + "type": "case", + "testPath": [ + { + "type": "Assembly", + "name": "rocket-car-dotnet.dll" + }, + { + "type": "TestSuite", + "name": "rocket_car_dotnet" + }, + { + "type": "TestSuite", + "name": "ExampleTest" + }, + { + "type": "TestCase", + "name": "TestSub" + } + ], + "duration": 0.000308, + "status": 0, + "stdout": "", + "stderr": "", + "created_at": "2023-05-15T 08:22:02Z", + "data": null + } + ], + "testRunner": "dotnet", + "group": "", + "noBuild": false +} diff --git a/tests/data/dotnet/test-result.xml b/tests/data/dotnet/test-result.xml new file mode 100644 index 000000000..c6a04e2b8 --- /dev/null +++ b/tests/data/dotnet/test-result.xml @@ -0,0 +1,37 @@ + + + + + + + + Expected: 0.5d + But was: 0 + + at rocket_car_dotnet.ExampleTest.TestDiv() in /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/dotnet/ExampleTest.cs:line 35 + + + + + + Expected: 10 + But was: 25 + + at rocket_car_dotnet.ExampleTest.TestMul() in /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/dotnet/ExampleTest.cs:line 28 + + + + + + Expected: 2 + But was: 6 + + at rocket_car_dotnet.ExampleTest.TestSub() in /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/dotnet/ExampleTest.cs:line 21 + + + + + + + + diff --git a/tests/test_runners/test_dotnet.py b/tests/test_runners/test_dotnet.py new file mode 100644 index 000000000..1e003dc5c --- /dev/null +++ b/tests/test_runners/test_dotnet.py @@ -0,0 +1,105 @@ +import gzip +import json +import os +from pathlib import Path +from unittest import mock + +import responses # type: ignore + +from launchable.utils.http_client import get_base_url +from tests.cli_test_case import CliTestCase + + +class DotnetTest(CliTestCase): + test_files_dir = Path(__file__).parent.joinpath('../data/dotnet/').resolve() + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset(self): + mock_response = { + "testPaths": [ + [ + {"type": "Assembly", "name": "rocket-car-dotnet.dll"}, + {"type": "TestSuite", "name": "rocket_car_dotnet"}, + {"type": "TestSuite", "name": "ExampleTest"}, + {"type": "TestCase", "name": "TestSub"}, + ], + [ + {"type": "Assembly", "name": "rocket-car-dotnet.dll"}, + {"type": "TestSuite", "name": "rocket_car_dotnet"}, + {"type": "TestSuite", "name": "ExampleTest"}, + {"type": "TestCase", "name": "TestMul"}, + ], + ], + "rest": [ + [ + {"type": "Assembly", "name": "rocket-car-dotnet.dll"}, + {"type": "TestSuite", "name": "rocket_car_dotnet"}, + {"type": "TestSuite", "name": "ExampleTest"}, + {"type": "TestCase", "name": "TestAdd"}, + ], + [ + {"type": "Assembly", "name": "rocket-car-dotnet.dll"}, + {"type": "TestSuite", "name": "rocket_car_dotnet"}, + {"type": "TestSuite", "name": "ExampleTest"}, + {"type": "TestCase", "name": "TestDiv"}, + ], + ], + "testRunner": "dotnet", + "subsettingId": 123, + "summary": { + "subset": {"duration": 25, "candidates": 1, "rate": 25}, + "rest": {"duration": 78, "candidates": 3, "rate": 75} + }, + "isObservation": False, + } + + responses.replace(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), + self.organization, + self.workspace), + json=mock_response, + status=200) + + # dotnet profiles requires Zero Input Subsetting + result = self.cli('subset', '--target', '25%', '--session', self.session, 'dotnet') + self.assertEqual(result.exit_code, 1) + + result = self.cli( + 'subset', + '--target', + '25%', + '--session', + self.session, + '--get-tests-from-previous-sessions', + 'dotnet', + mix_stderr=False) + self.assertEqual(result.exit_code, 0) + + output = "FullyQualifiedName=rocket_car_dotnet.ExampleTest.TestSub|FullyQualifiedName=rocket_car_dotnet.ExampleTest.TestMul\n" # noqa: E501 + self.assertEqual(result.output, output) + + result = self.cli( + 'subset', + '--target', + '25%', + '--session', + self.session, + '--get-tests-from-previous-sessions', + '--output-exclusion-rules', + 'dotnet', + mix_stderr=False) + self.assertEqual(result.exit_code, 0) + output = "FullyQualifiedName!=rocket_car_dotnet.ExampleTest.TestAdd&FullyQualifiedName!=rocket_car_dotnet.ExampleTest.TestDiv\n" # noqa: E501 + self.assertEqual(result.output, output) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_tests(self): + result = self.cli('record', 'tests', '--session', self.session, + 'dotnet', str(self.test_files_dir) + "/test-result.xml") + self.assertEqual(result.exit_code, 0) + + payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) + expected = self.load_json_from_file(self.test_files_dir.joinpath("record_test_result.json")) + self.assert_json_orderless_equal(payload, expected)