support pytest-subtests plugin

microsoft · Nov 21, 2024 · 3a18460 · 3a18460
1 parent 2fed954
commit 3a18460
Show file tree

Hide file tree

Showing 8 changed files with 141 additions and 7 deletions.
diff --git a/build/test-requirements.txt b/build/test-requirements.txt
@@ -36,3 +36,4 @@ pytest-json
 
 # for pytest-describe related tests
 pytest-describe
+pytest-subtests
diff --git a/python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py b/python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py
@@ -0,0 +1,8 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+def test_a(subtests):
+    with subtests.test(msg="test_a"):
+        assert 1 == 1
+    with subtests.test(msg="Second subtest"):
+        assert 2 == 1
diff --git a/python_files/tests/pytestadapter/expected_execution_test_output.py b/python_files/tests/pytestadapter/expected_execution_test_output.py
@@ -6,6 +6,7 @@
 TEST_ADD_FUNCTION = "unittest_folder/test_add.py::TestAddFunction::"
 SUCCESS = "success"
 FAILURE = "failure"
+SUBTEST_FAILURE = "subtest-failure"
 
 # This is the expected output for the unittest_folder execute tests
 # └── unittest_folder
@@ -734,3 +735,38 @@
         "subtest": None,
     },
 }
+
+# This is the expected output for the test_pytest_subtests_plugin.py file.
+# └── test_pytest_subtests_plugin.py
+#    └── test_a
+#       └── test_a [test_a]: subtest-success
+#       └── test_a [Second subtest]: subtest-failure
+test_pytest_subtests_plugin_path = TEST_DATA_PATH / "test_pytest_subtests_plugin.py"
+pytest_subtests_plugin_expected_execution_output = {
+    get_absolute_test_id(
+        "test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**",
+        test_pytest_subtests_plugin_path,
+    ): {
+        "test": get_absolute_test_id(
+            "test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**",
+            test_pytest_subtests_plugin_path,
+        ),
+        "outcome": "subtest-success",
+        "message": None,
+        "traceback": None,
+        "subtest": None,
+    },
+    get_absolute_test_id(
+        "test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**",
+        test_pytest_subtests_plugin_path,
+    ): {
+        "test": get_absolute_test_id(
+            "test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**",
+            test_pytest_subtests_plugin_path,
+        ),
+        "outcome": SUBTEST_FAILURE,
+        "message": "ERROR MESSAGE",
+        "traceback": None,
+        "subtest": None,
+    },
+}
diff --git a/python_files/tests/pytestadapter/test_execution.py b/python_files/tests/pytestadapter/test_execution.py
@@ -227,6 +227,34 @@ def test_pytest_execution(test_ids, expected_const):
     assert actual_result_dict == expected_const
 
 
+def test_pytest_subtest_plugin():
+    test_subtest_plugin_path = TEST_DATA_PATH / "test_pytest_subtests_plugin.py"
+
+    test_a_id = get_absolute_test_id(
+        "test_subtest_plugin_path::test_a",
+        test_subtest_plugin_path,
+    )
+    args = [test_a_id]
+    expected_const = expected_execution_test_output.pytest_subtests_plugin_expected_execution_output
+    actual = runner(args)
+    assert actual
+    actual_list: List[Dict[str, Dict[str, Any]]] = actual
+    assert len(actual_list) == len(expected_const)
+    actual_result_dict = {}
+    if actual_list is not None:
+        for actual_item in actual_list:
+            assert all(item in actual_item for item in ("status", "cwd", "result"))
+            assert actual_item.get("status") == "success"
+            assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
+            actual_result_dict.update(actual_item["result"])
+    for key in actual_result_dict:
+        if actual_result_dict[key]["outcome"] == "subtest-failure":
+            actual_result_dict[key]["message"] = "ERROR MESSAGE"
+        if actual_result_dict[key]["traceback"] is not None:
+            actual_result_dict[key]["traceback"] = "TRACEBACK"
+    assert actual_result_dict == expected_const
+
+
 def test_symlink_run():
     """Test to test pytest discovery with the command line arg --rootdir specified as a symlink path.
 

diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py
@@ -278,8 +278,19 @@ def pytest_report_teststatus(report, config):  # noqa: ARG001
             node_path = map_id_to_path[report.nodeid]
         except KeyError:
             node_path = cwd
-        # Calculate the absolute test id and use this as the ID moving forward.
+
         absolute_node_id = get_absolute_test_id(report.nodeid, node_path)
+        parent_test_name = report.nodeid.split("::")[-1]
+        if report.head_line and report.head_line != parent_test_name:
+            # add parent node to collected_tests_so_far to not double report
+            collected_tests_so_far.append(absolute_node_id)
+            # If the report has a head_line, then it is a pytest-subtest
+            # and we need to adjust the nodeid to reflect the subtest.
+            if report_value == "failure":
+                report_value = "subtest-failure"
+            elif report_value == "success":
+                report_value = "subtest-success"
+            absolute_node_id = absolute_node_id + "**{" + report.head_line + "}**"
         if absolute_node_id not in collected_tests_so_far:
             collected_tests_so_far.append(absolute_node_id)
             item_result = create_test_outcome(

diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts
@@ -242,7 +242,7 @@ export class PythonResultResolver implements ITestResultResolver {
                     }
                 } else if (testItem.outcome === 'subtest-failure') {
                     // split on [] or () based on how the subtest is setup.
-                    const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp);
+                    const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp, this.testProvider);
                     const parentTestItem = this.runIdToTestItem.get(parentTestCaseId);
                     const data = testItem;
                     // find the subtest's parent test item
@@ -280,7 +280,7 @@ export class PythonResultResolver implements ITestResultResolver {
                     }
                 } else if (testItem.outcome === 'subtest-success') {
                     // split on [] or () based on how the subtest is setup.
-                    const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp);
+                    const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp, this.testProvider);
                     const parentTestItem = this.runIdToTestItem.get(parentTestCaseId);
 
                     // find the subtest's parent test item

diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts
@@ -22,6 +22,7 @@ import {
 import { Deferred, createDeferred } from '../../../common/utils/async';
 import { createReaderPipe, generateRandomPipeName } from '../../../common/pipes/namedPipes';
 import { EXTENSION_ROOT_DIR } from '../../../constants';
+import { TestProvider } from '../../types';
 
 export function fixLogLines(content: string): string {
     const lines = content.split(/\r?\n/g);
@@ -464,10 +465,15 @@ export function createDiscoveryErrorPayload(
  * @param testName The full test name string.
  * @returns A tuple where the first item is the parent test name and the second item is the subtest section or `testName` if no subtest section exists.
  */
-export function splitTestNameWithRegex(testName: string): [string, string] {
+export function splitTestNameWithRegex(testName: string, testProvider: TestProvider): [string, string] {
     // If a match is found, return the parent test name and the subtest (whichever was captured between parenthesis or square brackets).
     // Otherwise, return the entire testName for the parent and entire testName for the subtest.
-    const regex = /^(.*?) ([\[(].*[\])])$/;
+    let regex: RegExp;
+    if (testProvider === 'pytest') {
+        regex = /^(.*?)\*\*{(.*?)}\*\*$/;
+    } else {
+        regex = /^(.*?) ([\[(].*[\])])$/;
+    }
     const match = testName.match(regex);
     if (match) {
         return [match[1].trim(), match[2] || match[3] || testName];

diff --git a/src/test/testing/testController/utils.unit.test.ts b/src/test/testing/testController/utils.unit.test.ts
@@ -106,7 +106,7 @@ ${data}${secondPayload}`;
         assert.deepStrictEqual(rpcContent.remainingRawData, '');
     });
 
-    suite('Test Controller Utils: Other', () => {
+    suite('Test Controller Utils: Unittest', () => {
         interface TestCase {
             name: string;
             input: string;
@@ -155,11 +155,55 @@ ${data}${secondPayload}`;
 
         testCases.forEach((testCase) => {
             test(`splitTestNameWithRegex: ${testCase.name}`, () => {
-                const splitResult = splitTestNameWithRegex(testCase.input);
+                const splitResult = splitTestNameWithRegex(testCase.input, 'unittest');
                 assert.deepStrictEqual(splitResult, [testCase.expectedParent, testCase.expectedSubtest]);
             });
         });
     });
+
+    suite('Test Controller Utils: Pytest', () => {
+        interface TestCase {
+            name: string;
+            input: string;
+            expectedParent: string;
+            expectedSubtest: string;
+        }
+
+        const testCases: Array<TestCase> = [
+            {
+                name: 'basic example',
+                input: 'test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**',
+                expectedParent: 'test_pytest_subtests_plugin.py::test_a',
+                expectedSubtest: 'test_a [Second subtest]',
+            },
+            {
+                name: 'duplicate name',
+                input: 'test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**',
+                expectedParent: 'test_pytest_subtests_plugin.py::test_a',
+                expectedSubtest: 'test_a [test_a]',
+            },
+            {
+                name: 'weird characters name',
+                input: 'test_pytest_subtests_plugin.py::L34Tc**{test_a111 [test_a]}**',
+                expectedParent: 'test_pytest_subtests_plugin.py::L34Tc',
+                expectedSubtest: 'test_a111 [test_a]',
+            },
+            {
+                name: 'name with stars',
+                input: 'test_pytest_subtests_plugin.py::L34Tc**{test_a111**** [test_a]}**',
+                expectedParent: 'test_pytest_subtests_plugin.py::L34Tc',
+                expectedSubtest: 'test_a111**** [test_a]',
+            },
+        ];
+
+        testCases.forEach((testCase) => {
+            test(`splitTestNameWithRegex: ${testCase.name}`, () => {
+                const splitResult = splitTestNameWithRegex(testCase.input, 'pytest');
+                assert.deepStrictEqual(splitResult, [testCase.expectedParent, testCase.expectedSubtest]);
+            });
+        });
+    });
+
     suite('Test Controller Utils: Args Mapping', () => {
         suite('addValueIfKeyNotExist', () => {
             test('should add key-value pair if key does not exist', () => {