Skip to content

Commit

Permalink
fix to support classes with parameterized tests inside (microsoft#23238)
Browse files Browse the repository at this point in the history 
fixes microsoft#22870
  • Loading branch information
@eleanorjboyd @anthonykim1
eleanorjboyd authored and anthonykim1 committed May 10, 2024
1 parent 50f0a12 commit 4d8b790
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 106 deletions.
15 changes: 8 additions & 7 deletions python_files/tests/pytestadapter/.data/parametrize_tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@
import pytest


# Testing pytest with parametrized tests. The first two pass, the third fails.
# The tests ids are parametrize_tests.py::test_adding[3+5-8] and so on.
@pytest.mark.parametrize( # test_marker--test_adding
"actual, expected", [("3+5", 8), ("2+4", 6), ("6+9", 16)]
)
def test_adding(actual, expected):
assert eval(actual) == expected
class TestClass:
# Testing pytest with parametrized tests. The first two pass, the third fails.
# The tests ids are parametrize_tests.py::test_adding[3+5-8] and so on.
@pytest.mark.parametrize( # test_marker--test_adding
"actual, expected", [("3+5", 8), ("2+4", 6), ("6+9", 16)]
)
def test_adding(self, actual, expected):
assert eval(actual) == expected


# Testing pytest with parametrized tests. All three pass.
Expand Down
139 changes: 83 additions & 56 deletions python_files/tests/pytestadapter/expected_discovery_test_output.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -511,10 +511,14 @@

# This is the expected output for the nested_folder tests.
# └── parametrize_tests.py
# └── test_adding
# └── [3+5-8]
# └── [2+4-6]
# └── [6+9-16]
# └── TestClass
# └── test_adding
# └── [3+5-8]
# └── [2+4-6]
# └── [6+9-16]
# └── test_string
# └── [hello]
# └── [complicated split [] ()]
parameterize_tests_path = TEST_DATA_PATH / "parametrize_tests.py"
parametrize_tests_expected_output = {
"name": ".data",
Expand All @@ -528,61 +532,69 @@
"id_": os.fspath(parameterize_tests_path),
"children": [
{
"name": "test_adding",
"name": "TestClass",
"path": os.fspath(parameterize_tests_path),
"type_": "function",
"id_": "parametrize_tests.py::test_adding",
"type_": "class",
"id_": "parametrize_tests.py::TestClass",
"children": [
{
"name": "[3+5-8]",
"path": os.fspath(parameterize_tests_path),
"lineno": find_test_line_number(
"test_adding[3+5-8]",
parameterize_tests_path,
),
"type_": "test",
"id_": get_absolute_test_id(
"parametrize_tests.py::test_adding[3+5-8]",
parameterize_tests_path,
),
"runID": get_absolute_test_id(
"parametrize_tests.py::test_adding[3+5-8]",
parameterize_tests_path,
),
},
{
"name": "[2+4-6]",
"name": "test_adding",
"path": os.fspath(parameterize_tests_path),
"lineno": find_test_line_number(
"test_adding[2+4-6]",
parameterize_tests_path,
),
"type_": "test",
"id_": get_absolute_test_id(
"parametrize_tests.py::test_adding[2+4-6]",
parameterize_tests_path,
),
"runID": get_absolute_test_id(
"parametrize_tests.py::test_adding[2+4-6]",
parameterize_tests_path,
),
},
{
"name": "[6+9-16]",
"path": os.fspath(parameterize_tests_path),
"lineno": find_test_line_number(
"test_adding[6+9-16]",
parameterize_tests_path,
),
"type_": "test",
"id_": get_absolute_test_id(
"parametrize_tests.py::test_adding[6+9-16]",
parameterize_tests_path,
),
"runID": get_absolute_test_id(
"parametrize_tests.py::test_adding[6+9-16]",
parameterize_tests_path,
),
"type_": "function",
"id_": "parametrize_tests.py::TestClass::test_adding",
"children": [
{
"name": "[3+5-8]",
"path": os.fspath(parameterize_tests_path),
"lineno": find_test_line_number(
"test_adding[3+5-8]",
parameterize_tests_path,
),
"type_": "test",
"id_": get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[3+5-8]",
parameterize_tests_path,
),
"runID": get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[3+5-8]",
parameterize_tests_path,
),
},
{
"name": "[2+4-6]",
"path": os.fspath(parameterize_tests_path),
"lineno": find_test_line_number(
"test_adding[2+4-6]",
parameterize_tests_path,
),
"type_": "test",
"id_": get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[2+4-6]",
parameterize_tests_path,
),
"runID": get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[2+4-6]",
parameterize_tests_path,
),
},
{
"name": "[6+9-16]",
"path": os.fspath(parameterize_tests_path),
"lineno": find_test_line_number(
"test_adding[6+9-16]",
parameterize_tests_path,
),
"type_": "test",
"id_": get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[6+9-16]",
parameterize_tests_path,
),
"runID": get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[6+9-16]",
parameterize_tests_path,
),
},
],
},
],
},
Expand Down Expand Up @@ -872,7 +884,15 @@
"id_": os.fspath(tests_path),
}
TEST_MULTI_CLASS_NEST_PATH = TEST_DATA_PATH / "test_multi_class_nest.py"

# This is the expected output for the nested_classes tests.
# └── test_multi_class_nest.py
# └── TestFirstClass
# └── TestSecondClass
# └── test_second
# └── test_first
# └── TestSecondClass2
# └── test_second2
# └── test_independent
nested_classes_expected_test_output = {
"name": ".data",
"path": TEST_DATA_PATH_STR,
Expand Down Expand Up @@ -984,6 +1004,13 @@
SYMLINK_FOLDER_PATH_TESTS_TEST_A = TEST_DATA_PATH / "symlink_folder" / "tests" / "test_a.py"
SYMLINK_FOLDER_PATH_TESTS_TEST_B = TEST_DATA_PATH / "symlink_folder" / "tests" / "test_b.py"

# This is the expected output for the symlink_folder tests.
# └── symlink_folder
# └── tests
# └── test_a.py
# └── test_a_function
# └── test_b.py
# └── test_b_function
symlink_expected_discovery_output = {
"name": "symlink_folder",
"path": str(SYMLINK_FOLDER_PATH),
Expand Down
26 changes: 18 additions & 8 deletions python_files/tests/pytestadapter/expected_execution_test_output.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,33 +370,40 @@
}
# This is the expected output for the nested_folder tests.
# └── parametrize_tests.py
# └── TestClass
# └── test_adding[3+5-8]: success
# └── test_adding[2+4-6]: success
# └── test_adding[6+9-16]: failure
parametrize_tests_path = TEST_DATA_PATH / "parametrize_tests.py"

parametrize_tests_expected_execution_output = {
get_absolute_test_id("parametrize_tests.py::test_adding[3+5-8]", parametrize_tests_path): {
get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[3+5-8]", parametrize_tests_path
): {
"test": get_absolute_test_id(
"parametrize_tests.py::test_adding[3+5-8]", parametrize_tests_path
"parametrize_tests.py::TestClass::test_adding[3+5-8]", parametrize_tests_path
),
"outcome": "success",
"message": None,
"traceback": None,
"subtest": None,
},
get_absolute_test_id("parametrize_tests.py::test_adding[2+4-6]", parametrize_tests_path): {
get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[2+4-6]", parametrize_tests_path
): {
"test": get_absolute_test_id(
"parametrize_tests.py::test_adding[2+4-6]", parametrize_tests_path
"parametrize_tests.py::TestClass::test_adding[2+4-6]", parametrize_tests_path
),
"outcome": "success",
"message": None,
"traceback": None,
"subtest": None,
},
get_absolute_test_id("parametrize_tests.py::test_adding[6+9-16]", parametrize_tests_path): {
get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[6+9-16]", parametrize_tests_path
): {
"test": get_absolute_test_id(
"parametrize_tests.py::test_adding[6+9-16]", parametrize_tests_path
"parametrize_tests.py::TestClass::test_adding[6+9-16]", parametrize_tests_path
),
"outcome": "failure",
"message": "ERROR MESSAGE",
Expand All @@ -407,11 +414,14 @@

# This is the expected output for the single parameterized tests.
# └── parametrize_tests.py
# └── TestClass
# └── test_adding[3+5-8]: success
single_parametrize_tests_expected_execution_output = {
get_absolute_test_id("parametrize_tests.py::test_adding[3+5-8]", parametrize_tests_path): {
get_absolute_test_id(
"parametrize_tests.py::TestClass::test_adding[3+5-8]", parametrize_tests_path
): {
"test": get_absolute_test_id(
"parametrize_tests.py::test_adding[3+5-8]", parametrize_tests_path
"parametrize_tests.py::TestClass::test_adding[3+5-8]", parametrize_tests_path
),
"outcome": "success",
"message": None,
Expand Down
10 changes: 5 additions & 5 deletions python_files/tests/pytestadapter/test_execution.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,20 +200,20 @@ def test_bad_id_error_execution():
expected_execution_test_output.dual_level_nested_folder_execution_expected_output,
),
(
["folder_a/folder_b/folder_a/test_nest.py::test_function"],
["folder_a/folder_b/folder_a/test_nest.py::test_function"], ##
expected_execution_test_output.double_nested_folder_expected_execution_output,
),
(
[
"parametrize_tests.py::test_adding[3+5-8]",
"parametrize_tests.py::test_adding[2+4-6]",
"parametrize_tests.py::test_adding[6+9-16]",
"parametrize_tests.py::TestClass::test_adding[3+5-8]", ##
"parametrize_tests.py::TestClass::test_adding[2+4-6]",
"parametrize_tests.py::TestClass::test_adding[6+9-16]",
],
expected_execution_test_output.parametrize_tests_expected_execution_output,
),
(
[
"parametrize_tests.py::test_adding[3+5-8]",
"parametrize_tests.py::TestClass::test_adding[3+5-8]",
],
expected_execution_test_output.single_parametrize_tests_expected_execution_output,
),
Expand Down
66 changes: 36 additions & 30 deletions python_files/vscode_pytest/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,37 @@ def build_test_tree(session: pytest.Session) -> TestNode:

for test_case in session.items:
test_node = create_test_node(test_case)
if hasattr(test_case, "callspec"): # This means it is a parameterized test.
function_name: str = ""
# parameterized test cases cut the repetitive part of the name off.
parent_part, parameterized_section = test_node["name"].split("[", 1)
test_node["name"] = "[" + parameterized_section
parent_path = os.fspath(get_node_path(test_case)) + "::" + parent_part
try:
function_name = test_case.originalname # type: ignore
function_test_node = function_nodes_dict[parent_path]
except AttributeError: # actual error has occurred
ERRORS.append(
f"unable to find original name for {test_case.name} with parameterization detected."
)
raise VSCodePytestError("Unable to find original name for parameterized test case")
except KeyError:
function_test_node: TestNode = create_parameterized_function_node(
function_name, get_node_path(test_case), test_case.nodeid
)
function_nodes_dict[parent_path] = function_test_node
function_test_node["children"].append(test_node)
# Check if the parent node of the function is file, if so create/add to this file node.
if isinstance(test_case.parent, pytest.File):
try:
parent_test_case = file_nodes_dict[test_case.parent]
except KeyError:
parent_test_case = create_file_node(test_case.parent)
file_nodes_dict[test_case.parent] = parent_test_case
if function_test_node not in parent_test_case["children"]:
parent_test_case["children"].append(function_test_node)
# If the parent is not a file, it is a class, add the function node as the test node to handle subsequent nesting.
test_node = function_test_node
if isinstance(test_case.parent, pytest.Class):
case_iter = test_case.parent
node_child_iter = test_node
Expand All @@ -423,7 +454,9 @@ def build_test_tree(session: pytest.Session) -> TestNode:
except KeyError:
test_class_node = create_class_node(case_iter)
class_nodes_dict[case_iter.nodeid] = test_class_node
test_class_node["children"].append(node_child_iter)
# Check if the class already has the child node. This will occur if the test is parameterized.
if node_child_iter not in test_class_node["children"]:
test_class_node["children"].append(node_child_iter)
# Iterate up.
node_child_iter = test_class_node
case_iter = case_iter.parent
Expand All @@ -442,35 +475,8 @@ def build_test_tree(session: pytest.Session) -> TestNode:
# Check if the class is already a child of the file node.
if test_class_node is not None and test_class_node not in test_file_node["children"]:
test_file_node["children"].append(test_class_node)
elif hasattr(test_case, "callspec"): # This means it is a parameterized test.
function_name: str = ""
# parameterized test cases cut the repetitive part of the name off.
parent_part, parameterized_section = test_node["name"].split("[", 1)
test_node["name"] = "[" + parameterized_section
parent_path = os.fspath(get_node_path(test_case)) + "::" + parent_part
try:
function_name = test_case.originalname # type: ignore
function_test_case = function_nodes_dict[parent_path]
except AttributeError: # actual error has occurred
ERRORS.append(
f"unable to find original name for {test_case.name} with parameterization detected."
)
raise VSCodePytestError("Unable to find original name for parameterized test case")
except KeyError:
function_test_case: TestNode = create_parameterized_function_node(
function_name, get_node_path(test_case), test_case.nodeid
)
function_nodes_dict[parent_path] = function_test_case
function_test_case["children"].append(test_node)
# Now, add the function node to file node.
try:
parent_test_case = file_nodes_dict[test_case.parent]
except KeyError:
parent_test_case = create_file_node(test_case.parent)
file_nodes_dict[test_case.parent] = parent_test_case
if function_test_case not in parent_test_case["children"]:
parent_test_case["children"].append(function_test_case)
else: # This includes test cases that are pytest functions or a doctests.
elif not hasattr(test_case, "callspec"):
# This includes test cases that are pytest functions or a doctests.
try:
parent_test_case = file_nodes_dict[test_case.parent]
except KeyError:
Expand Down

0 comments on commit 4d8b790

Please sign in to comment.