Merge pull request #561 from launchableinc/dotnet-profile

Supports dotnet profile

launchable/test_runners/dotnet.py

@@ -0,0 +1,76 @@
+import glob
+import os
+from typing import List
+
+import click
+
+from launchable.test_runners import launchable
+from launchable.test_runners.nunit import nunit_parse_func
+from launchable.testpath import TestPath
+
+
+@launchable.subset
+def subset(client):
+    """
+    Alpha: Supports only Zero Input Subsetting
+    """
+    if not client.is_get_tests_from_previous_sessions:
+        click.echo(
+            click.style(
+                "The dotnet profile only supports Zero Input Subsetting.\nMake sure to use `--get-tests-from-previous-sessions` opton",  # noqa: E501
+                fg="red"),
+            err=True)
+
+    # ref: https://github.com/Microsoft/vstest-docs/blob/main/docs/filter.md
+    separator = "|"
+    prefix = "FullyQualifiedName="
+
+    if client.is_output_exclusion_rules:
+        separator = "&"
+        prefix = "FullyQualifiedName!="
+
+    def formatter(test_path: TestPath):
+        paths = []
+
+        for path in test_path:
+            t = path.get("type", "")
+            if t == 'Assembly':
+                continue
+            paths.append(path.get("name", ""))
+
+        return prefix + ".".join(paths)
+
+    def exclusion_output_handler(subset_tests: List[TestPath], rest_tests: List[TestPath]):
+        if client.rest:
+            with open(client.rest, "w+", encoding="utf-8") as fp:
+                fp.write(client.separator.join(formatter(t) for t in subset_tests))
+
+        click.echo(client.separator.join(formatter(t) for t in rest_tests))
+
+    client.separator = separator
+    client.formatter = formatter
+    client.exclusion_output_handler = exclusion_output_handler
+    client.run()
+
+
+@click.argument('files', required=True, nargs=-1)
+@launchable.record.tests
+def record_tests(client, files):
+    """
+    Alpha: Supports only NUnit report formats.
+    """
+    for file in files:
+        match = False
+        for t in glob.iglob(file, recursive=True):
+            match = True
+            if os.path.isdir(t):
+                client.scan(t, "*.xml")
+            else:
+                client.report(t)
+        if not match:
+            click.echo("No matches found: {}".format(file), err=True)
+
+    # Note: we support only Nunit test report format now.
+    # If we need to support another format e.g) JUnit, trc, then we'll add a option
+    client.parse_func = nunit_parse_func
+    client.run()

launchable/test_runners/nunit.py

@@ -42,6 +42,34 @@ def split_filepath(path: str) -> List[str]:
         ]
 
 
+def nunit_parse_func(report: str):
+    events = []
+
+    # parse <test-case> element into CaseEvent
+    def on_element(e: Element):
+        build_path(e)
+        if e.name == "test-case":
+            result = e.attrs.get('result')
+            status = CaseEvent.TEST_FAILED
+            if result == 'Passed':
+                status = CaseEvent.TEST_PASSED
+            elif result == 'Skipped':
+                status = CaseEvent.TEST_SKIPPED
+
+            events.append(CaseEvent.create(
+                _replace_fixture_to_suite(e.tags['path']),  # type: ignore
+                float(e.attrs['duration']),
+                status,
+                timestamp=str(e.tags['startTime'])))  # timestamp is already iso-8601 formatted
+
+    # the 'start-time' attribute is normally on <test-case> but apparently not always,
+    # so we try to use the nearest ancestor as an approximate
+    SaxParser([TagMatcher.parse("*/@start-time={startTime}")], on_element).parse(report)
+
+    # return the obtained events as a generator
+    return (x for x in events)
+
+
 @click.argument('report_xmls', type=click.Path(exists=True), required=True, nargs=-1)
 @launchable.subset
 def subset(client, report_xmls):
@@ -82,34 +110,7 @@ def record_tests(client, report_xml):
         if not match:
             click.echo("No matches found: {}".format(root), err=True)
 
-    def parse_func(report: str):
-        events = []
-
-        # parse <test-case> element into CaseEvent
-        def on_element(e: Element):
-            build_path(e)
-            if e.name == "test-case":
-                result = e.attrs.get('result')
-                status = CaseEvent.TEST_FAILED
-                if result == 'Passed':
-                    status = CaseEvent.TEST_PASSED
-                elif result == 'Skipped':
-                    status = CaseEvent.TEST_SKIPPED
-
-                events.append(CaseEvent.create(
-                    _replace_fixture_to_suite(e.tags['path']),  # type: ignore
-                    float(e.attrs['duration']),
-                    status,
-                    timestamp=str(e.tags['startTime'])))  # timestamp is already iso-8601 formatted
-
-        # the 'start-time' attribute is normally on <test-case> but apparently not always,
-        # so we try to use the nearest ancestor as an approximate
-        SaxParser([TagMatcher.parse("*/@start-time={startTime}")], on_element).parse(report)
-
-        # return the obtained events as a generator
-        return (x for x in events)
-
-    client.parse_func = parse_func
+    client.parse_func = nunit_parse_func
     client.run()
 
 

tests/data/dotnet/record_test_result.json

@@ -0,0 +1,115 @@
+{
+    "events": [
+        {
+            "type": "case",
+            "testPath": [
+                {
+                    "type": "Assembly",
+                    "name": "rocket-car-dotnet.dll"
+                },
+                {
+                    "type": "TestSuite",
+                    "name": "rocket_car_dotnet"
+                },
+                {
+                    "type": "TestSuite",
+                    "name": "ExampleTest"
+                },
+                {
+                    "type": "TestCase",
+                    "name": "TestAdd"
+                }
+            ],
+            "duration": 0.006132,
+            "status": 1,
+            "stdout": "",
+            "stderr": "",
+            "created_at": "2023-05-15T 08:22:02Z",
+            "data": null
+        },
+        {
+            "type": "case",
+            "testPath": [
+                {
+                    "type": "Assembly",
+                    "name": "rocket-car-dotnet.dll"
+                },
+                {
+                    "type": "TestSuite",
+                    "name": "rocket_car_dotnet"
+                },
+                {
+                    "type": "TestSuite",
+                    "name": "ExampleTest"
+                },
+                {
+                    "type": "TestCase",
+                    "name": "TestDiv"
+                }
+            ],
+            "duration": 0.016795,
+            "status": 0,
+            "stdout": "",
+            "stderr": "",
+            "created_at": "2023-05-15T 08:22:02Z",
+            "data": null
+        },
+        {
+            "type": "case",
+            "testPath": [
+                {
+                    "type": "Assembly",
+                    "name": "rocket-car-dotnet.dll"
+                },
+                {
+                    "type": "TestSuite",
+                    "name": "rocket_car_dotnet"
+                },
+                {
+                    "type": "TestSuite",
+                    "name": "ExampleTest"
+                },
+                {
+                    "type": "TestCase",
+                    "name": "TestMul"
+                }
+            ],
+            "duration": 0.0003959,
+            "status": 0,
+            "stdout": "",
+            "stderr": "",
+            "created_at": "2023-05-15T 08:22:02Z",
+            "data": null
+        },
+        {
+            "type": "case",
+            "testPath": [
+                {
+                    "type": "Assembly",
+                    "name": "rocket-car-dotnet.dll"
+                },
+                {
+                    "type": "TestSuite",
+                    "name": "rocket_car_dotnet"
+                },
+                {
+                    "type": "TestSuite",
+                    "name": "ExampleTest"
+                },
+                {
+                    "type": "TestCase",
+                    "name": "TestSub"
+                }
+            ],
+            "duration": 0.000308,
+            "status": 0,
+            "stdout": "",
+            "stderr": "",
+            "created_at": "2023-05-15T 08:22:02Z",
+            "data": null
+        }
+    ],
+    "testRunner": "dotnet",
+    "group": "",
+    "noBuild": false
+}

tests/data/dotnet/test-result.xml

@@ -0,0 +1,37 @@
+<test-run id="2" duration="0.023630900000000003" testcasecount="4" total="4" passed="1" failed="3" inconclusive="0" skipped="0" result="Failed" start-time="2023-05-15T 08:22:02Z" end-time="2023-05-15T 08:22:02Z">
+  <test-suite type="Assembly" name="rocket-car-dotnet.dll" fullname="/Users/yabuki-ryosuke/src/github.com/launchableinc/examples/dotnet/bin/Debug/net7.0/rocket-car-dotnet.dll" total="4" passed="1" failed="3" inconclusive="0" skipped="0" result="Failed" start-time="2023-05-15T 08:22:02Z" end-time="2023-05-15T 08:22:02Z" duration="0.0236309">
+    <test-suite type="TestSuite" name="rocket_car_dotnet" fullname="rocket_car_dotnet" total="4" passed="1" failed="3" inconclusive="0" skipped="0" result="Failed" start-time="2023-05-15T 08:22:02Z" end-time="2023-05-15T 08:22:02Z" duration="0.0236309">
+      <test-suite type="TestFixture" name="ExampleTest" fullname="rocket_car_dotnet.ExampleTest" total="4" passed="1" failed="3" inconclusive="0" skipped="0" result="Failed" start-time="2023-05-15T 08:22:02Z" end-time="2023-05-15T 08:22:02Z" duration="0.0236309">
+        <test-case name="TestAdd" fullname="rocket_car_dotnet.ExampleTest.TestAdd" methodname="TestAdd" classname="ExampleTest" result="Passed" start-time="2023-05-15T 08:22:02Z" end-time="2023-05-15T 08:22:02Z" duration="0.006132" asserts="0" />
+        <test-case name="TestDiv" fullname="rocket_car_dotnet.ExampleTest.TestDiv" methodname="TestDiv" classname="ExampleTest" result="Failed" start-time="2023-05-15T 08:22:02Z" end-time="2023-05-15T 08:22:02Z" duration="0.016795" asserts="0">
+          <failure>
+            <message>  Expected: 0.5d
+  But was:  0
+</message>
+            <stack-trace>   at rocket_car_dotnet.ExampleTest.TestDiv() in /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/dotnet/ExampleTest.cs:line 35
+</stack-trace>
+          </failure>
+        </test-case>
+        <test-case name="TestMul" fullname="rocket_car_dotnet.ExampleTest.TestMul" methodname="TestMul" classname="ExampleTest" result="Failed" start-time="2023-05-15T 08:22:02Z" end-time="2023-05-15T 08:22:02Z" duration="0.0003959" asserts="0">
+          <failure>
+            <message>  Expected: 10
+  But was:  25
+</message>
+            <stack-trace>   at rocket_car_dotnet.ExampleTest.TestMul() in /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/dotnet/ExampleTest.cs:line 28
+</stack-trace>
+          </failure>
+        </test-case>
+        <test-case name="TestSub" fullname="rocket_car_dotnet.ExampleTest.TestSub" methodname="TestSub" classname="ExampleTest" result="Failed" start-time="2023-05-15T 08:22:02Z" end-time="2023-05-15T 08:22:02Z" duration="0.000308" asserts="0">
+          <failure>
+            <message>  Expected: 2
+  But was:  6
+</message>
+            <stack-trace>   at rocket_car_dotnet.ExampleTest.TestSub() in /Users/yabuki-ryosuke/src/github.com/launchableinc/examples/dotnet/ExampleTest.cs:line 21
+</stack-trace>
+          </failure>
+        </test-case>
+      </test-suite>
+    </test-suite>
+    <errors />
+  </test-suite>
+</test-run>