From 80481640ea412f9eba3dcac3733f9aae76e5ee70 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 28 Jul 2023 15:01:42 -0700 Subject: [PATCH 1/3] Added nested type as a test case --- launchable/test_runners/nunit.py | 20 +++++++++++++++++++- tests/data/nunit/output-linux.xml | 11 +++++++---- tests/data/nunit/output-windows.xml | 3 +++ tests/data/nunit/record_test_result.json | 15 +++++++++++++++ tests/data/nunit/src/README.md | 3 +-- tests/data/nunit/src/Test.cs | 17 +++++++++++++++++ 6 files changed, 62 insertions(+), 7 deletions(-) diff --git a/launchable/test_runners/nunit.py b/launchable/test_runners/nunit.py index 05cb3bcf4..0d898c557 100644 --- a/launchable/test_runners/nunit.py +++ b/launchable/test_runners/nunit.py @@ -13,6 +13,24 @@ # common code between 'subset' & 'record tests' to build up test path from # nested s +""" + Nested class name handling in .NET + --------------------------------- + + Nested class 'Zot' in the following example gets the full name "Foo.Bar+Zot": + + namespace Foo { + class Bar { + class Zot { + }}} + + This is incontrast to how you refer to this class from the source code. For example, + "new Foo.Bar.Zot()" + + The subset command expects the list of tests to be passed to "nunit --testlist" option, + and this option expects test names to be in "Foo.Bar+Zot" format. + +""" def build_path(e: Element): pp = [] # type: TestPath @@ -32,7 +50,7 @@ def split_filepath(path: str) -> List[str]: else: return path.split('\\') - # "Assembly" type containts full path at a cutomer's environment + # "Assembly" type contains full path at a customer's environment # remove file path prefix in Assembly e.tags['path'] = [ {**path, 'name': split_filepath(path['name'])[-1]} diff --git a/tests/data/nunit/output-linux.xml b/tests/data/nunit/output-linux.xml index 6a9c45dbe..93ded10c7 100644 --- a/tests/data/nunit/output-linux.xml +++ b/tests/data/nunit/output-linux.xml @@ -1,8 +1,8 @@ - - - - + + + + @@ -19,6 +19,9 @@ + + + diff --git a/tests/data/nunit/output-windows.xml b/tests/data/nunit/output-windows.xml index 0e81cc7d0..2179cca43 100644 --- a/tests/data/nunit/output-windows.xml +++ b/tests/data/nunit/output-windows.xml @@ -19,6 +19,9 @@ + + + diff --git a/tests/data/nunit/record_test_result.json b/tests/data/nunit/record_test_result.json index 59f1f0bbe..987acb77f 100644 --- a/tests/data/nunit/record_test_result.json +++ b/tests/data/nunit/record_test_result.json @@ -46,6 +46,21 @@ "stderr": "", "data": null }, + { + "type": "case", + "testPath": [ + { "type": "Assembly", "name": "calc.dll" }, + { "type": "TestSuite", "name": "calc" }, + { "type": "TestSuite", "name": "Outer+Inner" }, + { "type": "TestCase", "name": "Test3" } + ], + "created_at": "2023-07-27T17:47:58.8159407Z", + "duration": 0.001215, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + }, { "type": "case", diff --git a/tests/data/nunit/src/README.md b/tests/data/nunit/src/README.md index c15baeecb..2558d3470 100644 --- a/tests/data/nunit/src/README.md +++ b/tests/data/nunit/src/README.md @@ -1,11 +1,10 @@ # Dev environment * [Install .NET SDK](https://dotnet.microsoft.com/download) on your platform of choice. - Specifically, .NET Core 3.1 * Install NUnit console runner from [the zip page](https://github.com/nunit/nunit-console/releases/tag/v3.12) or other formats of your choice * `dotnet build` will produce `bin/Debug/netcoreapp3.1/calc.dll` that contains all the code * `dotnet $NUNIT/bin/netcoreapp3.1/nunit3-console.dll` to run NUnit3 console runner. - More specifically, `dotnet $NUNIT/bin/netcoreapp3.1/nunit3-console.dll ./bin/Debug/netcoreapp3.1/calc.dll` will run tests + More specifically, `dotnet --roll-forward LatestMajor $NUNIT/bin/netcoreapp3.1/nunit3-console.dll ./bin/Debug/netcoreapp3.1/calc.dll` will run tests (with the most recent .NET SDK you have) and produce `TestResult.xml` that we capture as `../output.xml`. * `--explore=list.xml` to produec the test list. * Internal: See https://launchableinc.atlassian.net/l/c/RUoketE0 for more detailed notes. diff --git a/tests/data/nunit/src/Test.cs b/tests/data/nunit/src/Test.cs index 88c16cfc1..db966a27f 100644 --- a/tests/data/nunit/src/Test.cs +++ b/tests/data/nunit/src/Test.cs @@ -17,6 +17,23 @@ public void Test2() Assert.Fail(); } } + + public class Outer + { + public class Inner + { + [Test] + public void Test3() + { + // this is how you access the inner class, but ... + var x = new Outer.Inner(); + + // you get the different type name + Assert.That(typeof(Inner).Name, Is.EqualTo("Inner")); + Assert.That(typeof(Inner).FullName, Is.EqualTo("calc.Outer+Inner")); + } + } + } } namespace calc.sub From 95be615d49cf064fa7ca53f79eaa42f4b8d94ee8 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 28 Jul 2023 16:15:05 -0700 Subject: [PATCH 2/3] Working around a bug in NUnitXML.Logger Based on the investigation in https://launchableinc.atlassian.net/l/cp/0v12wUPy I believe NUnitXML.Logger mishandles nested types. So we need to work around that problem in ways that do not interfere with the handling of correct XML file. --- launchable/test_runners/nunit.py | 23 +++++++++++++++++- .../nunit-reporter-bug-with-nested-type.json | 24 +++++++++++++++++++ .../nunit-reporter-bug-with-nested-type.xml | 22 +++++++++++++++++ tests/test_runners/test_nunit.py | 14 +++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 tests/data/nunit/nunit-reporter-bug-with-nested-type.json create mode 100644 tests/data/nunit/nunit-reporter-bug-with-nested-type.xml diff --git a/launchable/test_runners/nunit.py b/launchable/test_runners/nunit.py index 0d898c557..3d303bff8 100644 --- a/launchable/test_runners/nunit.py +++ b/launchable/test_runners/nunit.py @@ -40,7 +40,28 @@ def build_path(e: Element): # s form a nested tree structure so capture those in path pp = pp + [{'type': e.attrs['type'], 'name': e.attrs['name']}] if e.name == "test-case": - pp = pp + [{'type': 'TestCase', 'name': e.attrs['name']}] + # work around a bug in NUnitXML.Logger. + # see nunit-reporter-bug-with-nested-type.xml test case + methodname = e.attrs['methodname'] + idx = methodname.rfind(".") + if idx >= 0: + # when things are going well, method name cannot contain '.' since it's not a valid character in a symbol. + # but when NUnitXML.Logger messes up, it ends up putting the class name and the method name, like + # 0: def split_filepath(path: str) -> List[str]: diff --git a/tests/data/nunit/nunit-reporter-bug-with-nested-type.json b/tests/data/nunit/nunit-reporter-bug-with-nested-type.json new file mode 100644 index 000000000..67d3340a7 --- /dev/null +++ b/tests/data/nunit/nunit-reporter-bug-with-nested-type.json @@ -0,0 +1,24 @@ +{ + "events": [ + { + "type": "case", + "testPath": [ + { "type": "Assembly", "name": "Launchable.NUnit.Test.dll" }, + { "type": "TestSuite", "name": "Launchable" }, + { "type": "TestSuite", "name": "NUnit" }, + { "type": "TestSuite", "name": "Test" }, + { "type": "TestSuite", "name": "Outer+Inner" }, + { "type": "TestCase", "name": "TheTest" } + ], + "created_at": "2023-07-28T 22:32:13Z", + "duration": 1.6e-05, + "status": 1, + "stdout": "", + "stderr": "", + "data": null + } + ], + "testRunner": "nunit", + "group": "", + "noBuild": false +} diff --git a/tests/data/nunit/nunit-reporter-bug-with-nested-type.xml b/tests/data/nunit/nunit-reporter-bug-with-nested-type.xml new file mode 100644 index 000000000..73842ff9f --- /dev/null +++ b/tests/data/nunit/nunit-reporter-bug-with-nested-type.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + diff --git a/tests/test_runners/test_nunit.py b/tests/test_runners/test_nunit.py index 0261478a1..56a1098aa 100644 --- a/tests/test_runners/test_nunit.py +++ b/tests/test_runners/test_nunit.py @@ -184,3 +184,17 @@ def test_record_test_on_windows(self): expected = self.load_json_from_file(self.test_files_dir.joinpath("record_test_result.json")) self.assert_json_orderless_equal(expected, payload) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_record_test_with_nunit_reporter_bug(self): + result = self.cli('record', 'tests', '--session', self.session, + 'nunit', str(self.test_files_dir) + "/nunit-reporter-bug-with-nested-type.xml") + self.assertEqual(result.exit_code, 0) + + payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) + + # turns out we collapse all TestFixtures to TestSuitest so the golden file has TestSuite=Outer+Inner, not TestFixture=Outer+Inner + expected = self.load_json_from_file(self.test_files_dir.joinpath("nunit-reporter-bug-with-nested-type.json")) + + self.assert_json_orderless_equal(expected, payload) From f99e6accdccdc972e3c75aac3bcb8cca75851070 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 28 Jul 2023 16:27:52 -0700 Subject: [PATCH 3/3] Making lint happy --- launchable/test_runners/nunit.py | 46 +++++++++++++++++--------------- tests/test_runners/test_nunit.py | 3 ++- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/launchable/test_runners/nunit.py b/launchable/test_runners/nunit.py index 3d303bff8..b0a706227 100644 --- a/launchable/test_runners/nunit.py +++ b/launchable/test_runners/nunit.py @@ -14,24 +14,25 @@ # nested s """ - Nested class name handling in .NET - --------------------------------- - - Nested class 'Zot' in the following example gets the full name "Foo.Bar+Zot": - - namespace Foo { - class Bar { - class Zot { - }}} - - This is incontrast to how you refer to this class from the source code. For example, - "new Foo.Bar.Zot()" - - The subset command expects the list of tests to be passed to "nunit --testlist" option, - and this option expects test names to be in "Foo.Bar+Zot" format. - +Nested class name handling in .NET +--------------------------------- + +Nested class 'Zot' in the following example gets the full name "Foo.Bar+Zot": + + namespace Foo { + class Bar { + class Zot { + }}} + +This is incontrast to how you refer to this class from the source code. For example, +"new Foo.Bar.Zot()" + +The subset command expects the list of tests to be passed to "nunit --testlist" option, +and this option expects test names to be in "Foo.Bar+Zot" format. + """ + def build_path(e: Element): pp = [] # type: TestPath if e.parent: @@ -47,13 +48,14 @@ def build_path(e: Element): if idx >= 0: # when things are going well, method name cannot contain '.' since it's not a valid character in a symbol. # but when NUnitXML.Logger messes up, it ends up putting the class name and the method name, like - # 0: def split_filepath(path: str) -> List[str]: diff --git a/tests/test_runners/test_nunit.py b/tests/test_runners/test_nunit.py index 56a1098aa..cfbe521ff 100644 --- a/tests/test_runners/test_nunit.py +++ b/tests/test_runners/test_nunit.py @@ -194,7 +194,8 @@ def test_record_test_with_nunit_reporter_bug(self): payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) - # turns out we collapse all TestFixtures to TestSuitest so the golden file has TestSuite=Outer+Inner, not TestFixture=Outer+Inner + # turns out we collapse all TestFixtures to TestSuitest so the golden file has TestSuite=Outer+Inner, + # not TestFixture=Outer+Inner expected = self.load_json_from_file(self.test_files_dir.joinpath("nunit-reporter-bug-with-nested-type.json")) self.assert_json_orderless_equal(expected, payload)