From ddbbc08e778b79a8a80ea37f1bbc1682e03eefe8 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Mon, 4 Nov 2024 15:15:46 +0100 Subject: [PATCH 1/3] refactor(ethereum_clis): move `TransitionTool.verify_fixture()` to `Statetest` & `Blocktest` --- src/ethereum_clis/blocktest.py | 39 ++ src/ethereum_clis/clis/geth.py | 346 +++++++++++------- src/ethereum_clis/ethereum_cli.py | 3 +- src/ethereum_clis/statetest.py | 39 ++ src/ethereum_clis/transition_tool.py | 22 +- src/ethereum_test_fixtures/__init__.py | 4 +- src/ethereum_test_fixtures/collector.py | 14 +- .../fixture_consumer.py | 58 +++ src/ethereum_test_fixtures/verify.py | 40 -- src/pytest_plugins/consume/direct/conftest.py | 40 +- .../consume/direct/test_via_direct.py | 30 +- src/pytest_plugins/filler/filler.py | 20 +- whitelist.txt | 4 + 13 files changed, 412 insertions(+), 247 deletions(-) create mode 100644 src/ethereum_clis/blocktest.py create mode 100644 src/ethereum_clis/statetest.py create mode 100644 src/ethereum_test_fixtures/fixture_consumer.py delete mode 100644 src/ethereum_test_fixtures/verify.py diff --git a/src/ethereum_clis/blocktest.py b/src/ethereum_clis/blocktest.py new file mode 100644 index 0000000000..dda681ba9d --- /dev/null +++ b/src/ethereum_clis/blocktest.py @@ -0,0 +1,39 @@ +""" +Abstract base class for `evm blocktest` subcommands. +""" + +from abc import abstractmethod +from pathlib import Path +from typing import Dict, List, Optional, Type + +from ethereum_test_exceptions import ExceptionMapper + +from .ethereum_cli import EthereumCLI + + +class Blocktest(EthereumCLI): + """ + Abstract base class for `evm blocktest` subcommands. + """ + + registered_tools: List[Type["Blocktest"]] = [] + default_tool: Optional[Type["Blocktest"]] = None + + blocktest_subcommand: Optional[str] = "blocktest" + + traces: List[List[List[Dict]]] | None = None + + @abstractmethod + def __init__( + self, + *, + exception_mapper: Optional[ExceptionMapper] = None, + binary: Optional[Path] = None, + trace: bool = False, + ): + """ + Abstract initialization method that all subclasses must implement. + """ + self.exception_mapper = exception_mapper + super().__init__(binary=binary) + self.trace = trace diff --git a/src/ethereum_clis/clis/geth.py b/src/ethereum_clis/clis/geth.py index 993690affa..37a728daf4 100644 --- a/src/ethereum_clis/clis/geth.py +++ b/src/ethereum_clis/clis/geth.py @@ -8,7 +8,7 @@ import textwrap from pathlib import Path from re import compile -from typing import Optional +from typing import List, Optional from ethereum_test_exceptions import ( EOFException, @@ -16,143 +16,13 @@ ExceptionMessage, TransactionException, ) -from ethereum_test_fixtures import BlockchainFixture, StateFixture +from ethereum_test_fixtures import BlockchainFixture, FixtureConsumer, FixtureFormat, StateFixture from ethereum_test_forks import Fork -from ..transition_tool import FixtureFormat, TransitionTool, dump_files_to_directory - - -class GethTransitionTool(TransitionTool): - """ - Go-ethereum `evm` Transition tool interface wrapper class. - """ - - default_binary = Path("evm") - detect_binary_pattern = compile(r"^evm(.exe)? version\b") - t8n_subcommand: Optional[str] = "t8n" - statetest_subcommand: Optional[str] = "statetest" - blocktest_subcommand: Optional[str] = "blocktest" - binary: Path - cached_version: Optional[str] = None - trace: bool - t8n_use_stream = True - - def __init__( - self, - *, - binary: Optional[Path] = None, - trace: bool = False, - ): - super().__init__(exception_mapper=GethExceptionMapper(), binary=binary, trace=trace) - args = [str(self.binary), str(self.t8n_subcommand), "--help"] - try: - result = subprocess.run(args, capture_output=True, text=True) - except subprocess.CalledProcessError as e: - raise Exception("evm process unexpectedly returned a non-zero status code: " f"{e}.") - except Exception as e: - raise Exception(f"Unexpected exception calling evm tool: {e}.") - self.help_string = result.stdout - - def is_fork_supported(self, fork: Fork) -> bool: - """ - Returns True if the fork is supported by the tool. - - If the fork is a transition fork, we want to check the fork it transitions to. - """ - return fork.transition_tool_name() in self.help_string - - def get_blocktest_help(self) -> str: - """ - Return the help string for the blocktest subcommand. - """ - args = [str(self.binary), "blocktest", "--help"] - try: - result = subprocess.run(args, capture_output=True, text=True) - except subprocess.CalledProcessError as e: - raise Exception("evm process unexpectedly returned a non-zero status code: " f"{e}.") - except Exception as e: - raise Exception(f"Unexpected exception calling evm tool: {e}.") - return result.stdout - - def is_verifiable( - self, - fixture_format: FixtureFormat, - ) -> bool: - """ - Returns whether the fixture format is verifiable by this Geth's evm tool. - """ - return fixture_format in {StateFixture, BlockchainFixture} - - def verify_fixture( - self, - fixture_format: FixtureFormat, - fixture_path: Path, - fixture_name: Optional[str] = None, - debug_output_path: Optional[Path] = None, - ): - """ - Executes `evm [state|block]test` to verify the fixture at `fixture_path`. - """ - command: list[str] = [str(self.binary)] - - if debug_output_path: - command += ["--debug", "--json", "--verbosity", "100"] - - if fixture_format == StateFixture: - assert self.statetest_subcommand, "statetest subcommand not set" - command.append(self.statetest_subcommand) - elif fixture_format == BlockchainFixture: - assert self.blocktest_subcommand, "blocktest subcommand not set" - command.append(self.blocktest_subcommand) - else: - raise Exception(f"Invalid test fixture format: {fixture_format}") - - if fixture_name and fixture_format == BlockchainFixture: - assert isinstance(fixture_name, str), "fixture_name must be a string" - command.append("--run") - command.append(fixture_name) - command.append(str(fixture_path)) - - result = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - if debug_output_path: - debug_fixture_path = debug_output_path / "fixtures.json" - # Use the local copy of the fixture in the debug directory - verify_fixtures_call = " ".join(command[:-1]) + f" {debug_fixture_path}" - verify_fixtures_script = textwrap.dedent( - f"""\ - #!/bin/bash - {verify_fixtures_call} - """ - ) - dump_files_to_directory( - str(debug_output_path), - { - "verify_fixtures_args.py": command, - "verify_fixtures_returncode.txt": result.returncode, - "verify_fixtures_stdout.txt": result.stdout.decode(), - "verify_fixtures_stderr.txt": result.stderr.decode(), - "verify_fixtures.sh+x": verify_fixtures_script, - }, - ) - shutil.copyfile(fixture_path, debug_fixture_path) - - if result.returncode != 0: - raise Exception( - f"EVM test failed.\n{' '.join(command)}\n\n Error:\n{result.stderr.decode()}" - ) - - if fixture_format == StateFixture: - result_json = json.loads(result.stdout.decode()) - if not isinstance(result_json, list): - raise Exception(f"Unexpected result from evm statetest: {result_json}") - else: - result_json = [] # there is no parseable format for blocktest output - return result_json +from ..blocktest import Blocktest +from ..ethereum_cli import EthereumCLI +from ..statetest import Statetest +from ..transition_tool import TransitionTool, dump_files_to_directory class GethExceptionMapper(ExceptionMapper): @@ -287,3 +157,207 @@ def _mapping_data(self): EOFException.INVALID_CODE_SECTION_INDEX, "err: invalid_code_section_index" ), ] + + +class GethFixtureConsumer(FixtureConsumer): + """ + A helper class + """ + + def __init__(self, binary: Path, trace: bool): + super().__init__(statetest_binary=binary, blocktest_binary=binary) + self.statetest = GethStatetest(binary=binary, trace=trace) + self.blocktest = GethBlocktest(binary=binary, trace=trace) + + def consume_fixture( + self, + fixture_format: FixtureFormat, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ): + """ + Executes the appropriate geth fixture consumer with the fixture at `fixture_path`. + """ + if fixture_format == StateFixture: + return self.statetest.consume(fixture_path, debug_output_path) + elif fixture_format == BlockchainFixture: + return self.blocktest.consume(fixture_path, fixture_name, debug_output_path) + else: + raise NotImplementedError( + f"Geth can not verify fixture format of type `{fixture_format}`" + ) + + +class GethEvm(EthereumCLI): + """ + go-ethereum `evm` base class. + """ + + default_binary = Path("evm") + detect_binary_pattern = compile(r"^evm(.exe)? version\b") + binary: Path + cached_version: Optional[str] = None + + def __init__(self, binary: Path, trace: bool = False, exception_mapper=GethExceptionMapper()): + self.binary = binary + self.trace = trace + self.exception_mapper = exception_mapper + + def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: + try: + return subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + except subprocess.CalledProcessError as e: + raise Exception(f"Command failed with non-zero status: {e}.") + except Exception as e: + raise Exception(f"Unexpected exception calling evm tool: {e}.") + + def _consume_debug_dump( + self, + command: List[str], + result: subprocess.CompletedProcess, + fixture_path: Path, + debug_output_path: Path, + ): + debug_fixture_path = debug_output_path / "fixtures.json" + consume_direct_call = " ".join(command[:-1]) + f" {debug_fixture_path}" + consume_direct_script = textwrap.dedent( + f"""\ + #!/bin/bash + {consume_direct_call} + """ + ) + dump_files_to_directory( + str(debug_output_path), + { + "consume_direct_args.py": command, + "consume_direct_returncode.txt": result.returncode, + "consume_direct_stdout.txt": result.stdout, + "consume_direct_stderr.txt": result.stderr, + "consume_direct.sh+x": consume_direct_script, + }, + ) + shutil.copyfile(fixture_path, debug_fixture_path) + + +class GethTransitionTool(GethEvm, TransitionTool): + """ + go-ethereum `evm` Transition tool interface wrapper class. + """ + + t8n_subcommand: Optional[str] = "t8n" + trace: bool + t8n_use_stream = True + + def __init__(self, *, binary: Path, trace: bool = False): + super().__init__(binary=binary, trace=trace) + help_command = [str(self.binary), str(self.t8n_subcommand), "--help"] + result = self._run_command(help_command) + self.help_string = result.stdout + + def is_fork_supported(self, fork: Fork) -> bool: + """ + Returns True if the fork is supported by the tool. + + If the fork is a transition fork, we want to check the fork it transitions to. + """ + return fork.transition_tool_name() in self.help_string + + +class GethBlocktest(GethEvm, Blocktest): + """ + Geth `evm blocktest` interface wrapper class. + """ + + blocktest_subcommand: str = "blocktest" + + def __init__(self, binary: Path, trace: bool): + super().__init__(binary=binary, trace=trace) + help_command = [str(self.binary), str(self.blocktest_subcommand), "--help"] + result = self._run_command(help_command) + self.help_string = result.stdout + + def help(self) -> str: + """ + Return the help string for the blocktest subcommand. + """ + return self.help_string + + def consume( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ): + """ + Run the blocktest subcommand to consume a blocktest fixture. + """ + command = [str(self.binary)] + if debug_output_path: + command += ["--debug", "--json", "--verbosity", "100"] + command += [self.blocktest_subcommand] + if fixture_name: + command += ["--run", fixture_name] + command.append(str(fixture_path)) + + result = self._run_command(command) + + if debug_output_path: + self._consume_debug_dump(command, result, fixture_path, debug_output_path) + + if result.returncode != 0: + raise Exception( + f"Unexpected exit code:\n{' '.join(command)}\n\n Error:\n{result.stderr}" + ) + + return [] # There is no parseable format for blocktest output + + +class GethStatetest(GethEvm, Statetest): + """ + Geth `evm statetest` interface wrapper class. + """ + + statetest_subcommand: str = "statetest" + + def __init__(self, binary: Path, trace: bool): + super().__init__(binary=binary, trace=trace) + help_command = [str(self.binary), str(self.statetest_subcommand), "--help"] + result = self._run_command(help_command) + self.help_string = result.stdout + + def help(self) -> str: + """ + Return the help string for the statetest subcommand. + """ + return self.help_string + + def consume( + self, + fixture_path: Path, + debug_output_path: Optional[Path] = None, + ): + """ + Run the statetest command to consume a state test fixture. + """ + command = [str(self.binary)] + if debug_output_path: + command += ["--debug", "--json", "--verbosity", "100"] + command += [self.statetest_subcommand, str(fixture_path)] + + result = self._run_command(command) + + if debug_output_path: + self._consume_debug_dump(command, result, fixture_path, debug_output_path) + + if result.returncode != 0: + raise Exception( + f"Unexpected exit code:\n{' '.join(command)}\n\n Error:\n{result.stderr}" + ) + + result_json = json.loads(result.stdout) + if not isinstance(result_json, list): + raise Exception(f"Unexpected result from evm statetest: {result_json}") + return result_json diff --git a/src/ethereum_clis/ethereum_cli.py b/src/ethereum_clis/ethereum_cli.py index 350761bd37..3ca2fe96ae 100644 --- a/src/ethereum_clis/ethereum_cli.py +++ b/src/ethereum_clis/ethereum_cli.py @@ -46,7 +46,7 @@ class EthereumCLI(ABC): cached_version: Optional[str] = None @abstractmethod - def __init__(self, *, binary: Optional[Path] = None, trace: bool = False): + def __init__(self, *, binary: Optional[Path] = None): """ Abstract initialization method that all subclasses must implement. """ @@ -61,7 +61,6 @@ def __init__(self, *, binary: Optional[Path] = None, trace: bool = False): if not binary: raise CLINotFoundInPath(binary=binary) self.binary = Path(binary) - self.trace = trace @classmethod def register_tool(cls, tool_subclass: Type[Any]): diff --git a/src/ethereum_clis/statetest.py b/src/ethereum_clis/statetest.py new file mode 100644 index 0000000000..13354085b5 --- /dev/null +++ b/src/ethereum_clis/statetest.py @@ -0,0 +1,39 @@ +""" +Abstract base class for `evm statetest` subcommands. +""" + +from abc import abstractmethod +from pathlib import Path +from typing import Dict, List, Optional, Type + +from ethereum_test_exceptions import ExceptionMapper + +from .ethereum_cli import EthereumCLI + + +class Statetest(EthereumCLI): + """ + Abstract base class for `evm statetest` subcommands. + """ + + registered_tools: List[Type["Statetest"]] = [] + default_tool: Optional[Type["Statetest"]] = None + + blocktest_subcommand: Optional[str] = "statetest" + + traces: List[List[List[Dict]]] | None = None + + @abstractmethod + def __init__( + self, + *, + exception_mapper: Optional[ExceptionMapper] = None, + binary: Optional[Path] = None, + trace: bool = False, + ): + """ + Abstract initialization method that all subclasses must implement. + """ + self.exception_mapper = exception_mapper + super().__init__(binary=binary) + self.trace = trace diff --git a/src/ethereum_clis/transition_tool.py b/src/ethereum_clis/transition_tool.py index 47c53e6409..b9c8d645e0 100644 --- a/src/ethereum_clis/transition_tool.py +++ b/src/ethereum_clis/transition_tool.py @@ -19,7 +19,6 @@ from requests_unixsocket import Session # type: ignore from ethereum_test_exceptions import ExceptionMapper -from ethereum_test_fixtures import FixtureFormat, FixtureVerifier from ethereum_test_forks import Fork from ethereum_test_types import Alloc, Environment, Transaction @@ -30,7 +29,7 @@ model_dump_config: Mapping = {"by_alias": True, "exclude_none": True} -class TransitionTool(EthereumCLI, FixtureVerifier): +class TransitionTool(EthereumCLI): """ Transition tool abstract base class which should be inherited by all transition tool implementations. @@ -42,11 +41,8 @@ class TransitionTool(EthereumCLI, FixtureVerifier): default_tool: Optional[Type["TransitionTool"]] = None t8n_subcommand: Optional[str] = None - statetest_subcommand: Optional[str] = None - blocktest_subcommand: Optional[str] = None cached_version: Optional[str] = None t8n_use_stream: bool = False - t8n_use_server: bool = False server_url: str process: Optional[subprocess.Popen] = None @@ -514,19 +510,3 @@ def evaluate( t8n_data=t8n_data, debug_output_path=debug_output_path, ) - - def verify_fixture( - self, - fixture_format: FixtureFormat, - fixture_path: Path, - fixture_name: Optional[str] = None, - debug_output_path: Optional[Path] = None, - ): - """ - Executes `evm [state|block]test` to verify the fixture at `fixture_path`. - - Currently only implemented by geth's evm. - """ - raise NotImplementedError( - "The `verify_fixture()` function is not supported by this tool. Use geth's evm tool." - ) diff --git a/src/ethereum_test_fixtures/__init__.py b/src/ethereum_test_fixtures/__init__.py index 455eac8a0b..527da017fa 100644 --- a/src/ethereum_test_fixtures/__init__.py +++ b/src/ethereum_test_fixtures/__init__.py @@ -10,8 +10,8 @@ from .blockchain import FixtureCommon as BlockchainFixtureCommon from .collector import FixtureCollector, TestInfo from .eof import Fixture as EOFFixture +from .fixture_consumer import FixtureConsumer from .state import Fixture as StateFixture -from .verify import FixtureVerifier FIXTURE_FORMATS: Dict[str, FixtureFormat] = { f.fixture_format_name: f # type: ignore @@ -31,7 +31,7 @@ "EOFFixture", "FixtureCollector", "FixtureFormat", - "FixtureVerifier", + "FixtureConsumer", "StateFixture", "TestInfo", ] diff --git a/src/ethereum_test_fixtures/collector.py b/src/ethereum_test_fixtures/collector.py index 24542223fb..a43433cc68 100644 --- a/src/ethereum_test_fixtures/collector.py +++ b/src/ethereum_test_fixtures/collector.py @@ -15,7 +15,7 @@ from .base import BaseFixture from .file import Fixtures -from .verify import FixtureVerifier +from .fixture_consumer import FixtureConsumer def strip_test_prefix(name: str) -> str: @@ -166,23 +166,23 @@ def dump_fixtures(self) -> None: raise TypeError("All fixtures in a single file must have the same format.") fixtures.collect_into_file(fixture_path) - def verify_fixture_files(self, evm_fixture_verification: FixtureVerifier) -> None: + def verify_fixture_files(self, evm_fixture_verification: FixtureConsumer) -> None: """ Runs `evm [state|block]test` on each fixture. """ for fixture_path, name_fixture_dict in self.all_fixtures.items(): for fixture_name, fixture in name_fixture_dict.items(): - if evm_fixture_verification.is_verifiable(fixture.__class__): + if evm_fixture_verification.is_consumable(fixture.__class__): info = self.json_path_to_test_item[fixture_path] - verify_fixtures_dump_dir = self._get_verify_fixtures_dump_dir(info) - evm_fixture_verification.verify_fixture( + consume_direct_dump_dir = self._get_consume_direct_dump_dir(info) + evm_fixture_verification.consume_fixture( fixture.__class__, fixture_path, fixture_name=None, - debug_output_path=verify_fixtures_dump_dir, + debug_output_path=consume_direct_dump_dir, ) - def _get_verify_fixtures_dump_dir( + def _get_consume_direct_dump_dir( self, info: TestInfo, ): diff --git a/src/ethereum_test_fixtures/fixture_consumer.py b/src/ethereum_test_fixtures/fixture_consumer.py new file mode 100644 index 0000000000..b93878217d --- /dev/null +++ b/src/ethereum_test_fixtures/fixture_consumer.py @@ -0,0 +1,58 @@ +""" +Ethereum test fixture consumer abstract class. +""" + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Optional + +from .base import FixtureFormat +from .blockchain import Fixture as BlockchainFixture +from .eof import Fixture as EOFFixture +from .state import Fixture as StateFixture + + +class FixtureConsumer(ABC): + """ + Abstract class for verifying Ethereum test fixtures. + """ + + def __init__( + self, + statetest_binary: Optional[Path] = None, + blocktest_binary: Optional[Path] = None, + eoftest_binary: Optional[Path] = None, + ): + self.statetest_binary = statetest_binary + self.blocktest_binary = blocktest_binary + self.eoftest_binary = eoftest_binary + + def is_consumable( + self, + fixture_format: FixtureFormat, + ) -> bool: + """ + Returns whether the fixture format is verifiable by this verifier. + """ + if fixture_format == StateFixture: + return self.statetest_binary is not None + elif fixture_format == BlockchainFixture: + return self.blocktest_binary is not None + elif fixture_format == EOFFixture: + return self.eoftest_binary is not None + return False + + @abstractmethod + def consume_fixture( + self, + fixture_format: FixtureFormat, + fixture_path: Path, + fixture_name: str | None = None, + debug_output_path: Path | None = None, + ): + """ + Test the client with the specified fixture using its direct consumer interface. + """ + raise NotImplementedError( + "The `consume_fixture()` function is not supported by this tool." + ) diff --git a/src/ethereum_test_fixtures/verify.py b/src/ethereum_test_fixtures/verify.py deleted file mode 100644 index 234834d9ba..0000000000 --- a/src/ethereum_test_fixtures/verify.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Ethereum test fixture verifyer abstract class. -""" - -from abc import ABC, abstractmethod -from pathlib import Path - -from .base import FixtureFormat - - -class FixtureVerifier(ABC): - """ - Abstract class for verifying Ethereum test fixtures. - """ - - def is_verifiable( - self, - fixture_format: FixtureFormat, - ) -> bool: - """ - Returns whether the fixture format is verifiable by this verifier. - """ - return False - - @abstractmethod - def verify_fixture( - self, - fixture_format: FixtureFormat, - fixture_path: Path, - fixture_name: str | None = None, - debug_output_path: Path | None = None, - ): - """ - Executes `evm [state|block]test` to verify the fixture at `fixture_path`. - - Currently only implemented by geth's evm. - """ - raise NotImplementedError( - "The `verify_fixture()` function is not supported by this tool. Use geth's evm tool." - ) diff --git a/src/pytest_plugins/consume/direct/conftest.py b/src/pytest_plugins/consume/direct/conftest.py index 894202082e..06052cf1d3 100644 --- a/src/pytest_plugins/consume/direct/conftest.py +++ b/src/pytest_plugins/consume/direct/conftest.py @@ -12,8 +12,9 @@ import pytest -from ethereum_clis import TransitionTool +from ethereum_clis.clis.geth import GethFixtureConsumer from ethereum_test_base_types import to_json +from ethereum_test_fixtures import FixtureConsumer from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream from ethereum_test_fixtures.file import Fixtures @@ -29,10 +30,7 @@ def pytest_addoption(parser): # noqa: D103 dest="evm_bin", type=Path, default=None, - help=( - "Path to an evm executable that provides `blocktest`. Default: First 'evm' entry in " - "PATH." - ), + help=("Path to a geth evm executable that provides `blocktest` or `statetest`."), ) consume_group.addoption( "--traces", @@ -53,47 +51,41 @@ def pytest_addoption(parser): # noqa: D103 def pytest_configure(config): # noqa: D103 - evm = TransitionTool.from_binary_path( - binary_path=config.getoption("evm_bin"), - # TODO: The verify_fixture() method doesn't currently use this option. - trace=config.getoption("evm_collect_traces"), + fixture_consumer = GethFixtureConsumer( + binary=config.getoption("evm_bin"), trace=config.getoption("evm_collect_traces") ) - try: - blocktest_help_string = evm.get_blocktest_help() - except NotImplementedError as e: - pytest.exit(str(e)) - config.evm = evm - config.evm_run_single_test = "--run" in blocktest_help_string + config.run_single_test = "--run" in fixture_consumer.blocktest.help() + config.fixture_consumer = fixture_consumer @pytest.fixture(autouse=True, scope="session") -def evm(request) -> Generator[TransitionTool, None, None]: +def fixture_consumer(request) -> Generator[FixtureConsumer, None, None]: """ - Returns the interface to the evm binary that will consume tests. + Returns the interface to the fixture verifier that will consume tests. """ - yield request.config.evm - request.config.evm.shutdown() + yield request.config.fixture_consumer + # request.config.fixture_consumer.shutdown() @pytest.fixture(scope="session") -def evm_run_single_test(request) -> bool: +def run_single_test(request) -> bool: """ Helper specifying whether to execute one test per fixture in each json file. """ - return request.config.evm_run_single_test + return request.config.run_single_test @pytest.fixture(scope="function") def test_dump_dir( - request, fixture_path: Path, fixture_name: str, evm_run_single_test: bool + request, fixture_path: Path, fixture_name: str, run_single_test: bool ) -> Optional[Path]: """ - The directory to write evm debug output to. + The directory to write debug output to. """ base_dump_dir = request.config.getoption("base_dump_dir") if not base_dump_dir: return None - if evm_run_single_test: + if run_single_test: if len(fixture_name) > 142: # ensure file name is not too long for eCryptFS fixture_name = fixture_name[:70] + "..." + fixture_name[-70:] diff --git a/src/pytest_plugins/consume/direct/test_via_direct.py b/src/pytest_plugins/consume/direct/test_via_direct.py index 98aec5ac5d..13a226846e 100644 --- a/src/pytest_plugins/consume/direct/test_via_direct.py +++ b/src/pytest_plugins/consume/direct/test_via_direct.py @@ -9,8 +9,7 @@ import pytest -from ethereum_clis import TransitionTool -from ethereum_test_fixtures import BlockchainFixture, StateFixture +from ethereum_test_fixtures import BlockchainFixture, FixtureConsumer, StateFixture from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream from ..decorator import fixture_format @@ -21,15 +20,15 @@ @fixture_format(BlockchainFixture) def test_blocktest( # noqa: D103 test_case: TestCaseIndexFile | TestCaseStream, - evm: TransitionTool, - evm_run_single_test: bool, + fixture_consumer: FixtureConsumer, + run_single_test: bool, fixture_path: Path, test_dump_dir: Optional[Path], ): fixture_name = None - if evm_run_single_test: + if run_single_test: fixture_name = re.escape(test_case.id) - evm.verify_fixture( + fixture_consumer.consume_fixture( test_case.format, fixture_path, fixture_name=fixture_name, @@ -39,8 +38,9 @@ def test_blocktest( # noqa: D103 @pytest.fixture(scope="function") def run_statetest( + run_single_test: bool, test_case: TestCaseIndexFile | TestCaseStream, - evm: TransitionTool, + fixture_consumer: FixtureConsumer, fixture_path: Path, test_dump_dir: Optional[Path], ): @@ -49,8 +49,10 @@ def run_statetest( """ # TODO: Check if all required results have been tested and delete test result data if so. # TODO: Can we group the tests appropriately so that this works more efficiently with xdist? + if run_single_test: + return if fixture_path not in statetest_results: - json_result = evm.verify_fixture( + json_result = fixture_consumer.consume_fixture( test_case.format, fixture_path, fixture_name=None, @@ -63,8 +65,20 @@ def run_statetest( @fixture_format(StateFixture) def test_statetest( # noqa: D103 test_case: TestCaseIndexFile | TestCaseStream, + fixture_consumer: FixtureConsumer, + run_single_test: bool, fixture_path: Path, + test_dump_dir: Optional[Path], ): + if run_single_test: + fixture_name = re.escape(test_case.id) + fixture_consumer.consume_fixture( + test_case.format, + fixture_path, + fixture_name=fixture_name, + debug_output_path=test_dump_dir, + ) + return test_result = [ test_result for test_result in statetest_results[fixture_path] diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py index d3033b65e0..9413b21830 100644 --- a/src/pytest_plugins/filler/filler.py +++ b/src/pytest_plugins/filler/filler.py @@ -21,8 +21,9 @@ from cli.gen_index import generate_fixtures_index from ethereum_clis import TransitionTool +from ethereum_clis.clis.geth import GethFixtureConsumer from ethereum_test_base_types import Alloc, ReferenceSpec -from ethereum_test_fixtures import BaseFixture, FixtureCollector, TestInfo +from ethereum_test_fixtures import BaseFixture, FixtureCollector, FixtureConsumer, TestInfo from ethereum_test_forks import Fork from ethereum_test_specs import SPEC_TYPES, BaseTest from ethereum_test_tools.utility.versioning import ( @@ -429,10 +430,11 @@ def do_fixture_verification( @pytest.fixture(autouse=True, scope="session") def evm_fixture_verification( + request: pytest.FixtureRequest, do_fixture_verification: bool, evm_bin: Path, verify_fixtures_bin: Path | None, -) -> Generator[TransitionTool | None, None, None]: +) -> Generator[FixtureConsumer | None, None, None]: """ Returns the configured evm binary for executing statetest and blocktest commands used to verify generated JSON fixtures. @@ -442,15 +444,19 @@ def evm_fixture_verification( return if not verify_fixtures_bin and evm_bin: verify_fixtures_bin = evm_bin - evm_fixture_verification = TransitionTool.from_binary_path(binary_path=verify_fixtures_bin) - if not evm_fixture_verification.blocktest_subcommand: + if not verify_fixtures_bin: + return + try: + evm_fixture_verification = GethFixtureConsumer( + binary=Path(verify_fixtures_bin), trace=request.config.getoption("evm_collect_traces") + ) + except Exception: pytest.exit( - "Only geth's evm tool is supported to verify fixtures: " + "Only geth's evm tool is currently supported to verify fixtures: " "Either remove --verify-fixtures or set --verify-fixtures-bin to a Geth evm binary.", returncode=pytest.ExitCode.USAGE_ERROR, ) yield evm_fixture_verification - evm_fixture_verification.shutdown() @pytest.fixture(scope="session") @@ -609,7 +615,7 @@ def generate_index(request) -> bool: # noqa: D103 def fixture_collector( request: pytest.FixtureRequest, do_fixture_verification: bool, - evm_fixture_verification: TransitionTool, + evm_fixture_verification: FixtureConsumer, filler_path: Path, base_dump_dir: Path | None, output_dir: Path, diff --git a/whitelist.txt b/whitelist.txt index 1e8486f13b..12ac8a17a5 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -512,6 +512,8 @@ makereport metafunc modifyitems monkeypatching +neth +nethtest nodeid noop oog @@ -558,6 +560,7 @@ tryfirst trylast usefixtures verifier +verifiers writelines xfail ZeroPaddedHexNumber @@ -723,6 +726,7 @@ callcode return delegatecall eofcreate +eoftest extcall extcalls extdelegatecall From 71c70032cf4d885c3361822935006c825152bf0a Mon Sep 17 00:00:00 2001 From: danceratopz Date: Mon, 4 Nov 2024 15:44:20 +0100 Subject: [PATCH 2/3] docs: update changelog --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6968108356..452b5f8612 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -56,6 +56,7 @@ Test fixtures for use by clients are available for each release on the [Github r - ✨ Fill test fixtures using EELS by default. EEST now uses the [`ethereum-specs-evm-resolver`](https://github.com/petertdavies/ethereum-spec-evm-resolver) with the EELS daemon ([#792](https://github.com/ethereum/execution-spec-tests/pull/792)). - 🔀 Move the `evm_transition_tool` package to `ethereum_clis` and derive the transition tool CL interfaces from a shared `EthereumCLI` class that can be reused for other sub-commands ([#894](https://github.com/ethereum/execution-spec-tests/pull/894)). +- 🔀 Refactor `ethereum_clis` and the `consume direct` interface to use `StateTest` and `Blocktest` classes called via the `FixtureConsumer` helper class ([#935](https://github.com/ethereum/execution-spec-tests/pull/935)). ### 📋 Misc From bf0f91b35083d1f5fa3576bcc06413260f06e231 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Mon, 4 Nov 2024 16:06:35 +0100 Subject: [PATCH 3/3] chore: clean-out currently unused code from `test_statetest` --- .../consume/direct/test_via_direct.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/pytest_plugins/consume/direct/test_via_direct.py b/src/pytest_plugins/consume/direct/test_via_direct.py index 13a226846e..62d67dbb64 100644 --- a/src/pytest_plugins/consume/direct/test_via_direct.py +++ b/src/pytest_plugins/consume/direct/test_via_direct.py @@ -38,7 +38,6 @@ def test_blocktest( # noqa: D103 @pytest.fixture(scope="function") def run_statetest( - run_single_test: bool, test_case: TestCaseIndexFile | TestCaseStream, fixture_consumer: FixtureConsumer, fixture_path: Path, @@ -49,8 +48,6 @@ def run_statetest( """ # TODO: Check if all required results have been tested and delete test result data if so. # TODO: Can we group the tests appropriately so that this works more efficiently with xdist? - if run_single_test: - return if fixture_path not in statetest_results: json_result = fixture_consumer.consume_fixture( test_case.format, @@ -65,20 +62,8 @@ def run_statetest( @fixture_format(StateFixture) def test_statetest( # noqa: D103 test_case: TestCaseIndexFile | TestCaseStream, - fixture_consumer: FixtureConsumer, - run_single_test: bool, fixture_path: Path, - test_dump_dir: Optional[Path], ): - if run_single_test: - fixture_name = re.escape(test_case.id) - fixture_consumer.consume_fixture( - test_case.format, - fixture_path, - fixture_name=fixture_name, - debug_output_path=test_dump_dir, - ) - return test_result = [ test_result for test_result in statetest_results[fixture_path]