From c50c5ad82797d39bd2c0685a886e15c22e18725e Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 19 Dec 2024 19:25:24 +0000 Subject: [PATCH 1/4] feat(clis): Add parameter `slow_request` to t8n evaluation --- src/ethereum_clis/transition_tool.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/ethereum_clis/transition_tool.py b/src/ethereum_clis/transition_tool.py index 1306e3e93e..2ec0b843e5 100644 --- a/src/ethereum_clis/transition_tool.py +++ b/src/ethereum_clis/transition_tool.py @@ -30,6 +30,9 @@ model_dump_config: Mapping = {"by_alias": True, "exclude_none": True} +NORMAL_SERVER_TIMEOUT = 20 +SLOW_REQUEST_TIMEOUT = 60 + class TransitionTool(EthereumCLI, FixtureVerifier): """ @@ -276,6 +279,7 @@ def _evaluate_filesystem( def _server_post( self, data: Dict[str, Any], + timeout: int, url_args: Dict[str, List[str] | str] = {}, retries: int = 5, ) -> Response: @@ -288,7 +292,7 @@ def _server_post( response = Session().post( f"{self.server_url}?{urlencode(url_args, doseq=True)}", json=data, - timeout=20, + timeout=timeout, ) break except ConnectionError as e: @@ -316,6 +320,7 @@ def _evaluate_server( *, t8n_data: TransitionToolData, debug_output_path: str = "", + timeout: int, ) -> TransitionToolOutput: """ Executes the transition tool sending inputs and outputs via a server. @@ -349,7 +354,9 @@ def _evaluate_server( }, ) - response = self._server_post(data=post_data, url_args=self._generate_post_args(t8n_data)) + response = self._server_post( + data=post_data, url_args=self._generate_post_args(t8n_data), timeout=timeout + ) output: TransitionToolOutput = TransitionToolOutput.model_validate(response.json()) if debug_output_path: @@ -496,6 +503,7 @@ def evaluate( eips: Optional[List[int]] = None, debug_output_path: str = "", state_test: bool = False, + slow_request: bool = False, ) -> TransitionToolOutput: """ Executes the relevant evaluate method as required by the `t8n` tool. @@ -524,7 +532,11 @@ def evaluate( if self.t8n_use_server: if not self.process: self.start_server() - return self._evaluate_server(t8n_data=t8n_data, debug_output_path=debug_output_path) + return self._evaluate_server( + t8n_data=t8n_data, + debug_output_path=debug_output_path, + timeout=SLOW_REQUEST_TIMEOUT if slow_request else NORMAL_SERVER_TIMEOUT, + ) if self.t8n_use_stream: return self._evaluate_stream(t8n_data=t8n_data, debug_output_path=debug_output_path) From c0e8acf4057bb2d73f149f55174ebd67f38fb76a Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 19 Dec 2024 19:26:06 +0000 Subject: [PATCH 2/4] feat(specs): Marked slow tests pass `slow_request` parameter --- src/ethereum_test_specs/blockchain.py | 19 +++++++++++++++---- src/ethereum_test_specs/helpers.py | 12 +++++++++++- src/ethereum_test_specs/state.py | 6 ++++-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/ethereum_test_specs/blockchain.py b/src/ethereum_test_specs/blockchain.py index 089bdee533..d785440e8f 100644 --- a/src/ethereum_test_specs/blockchain.py +++ b/src/ethereum_test_specs/blockchain.py @@ -45,7 +45,7 @@ from .base import BaseTest, verify_result from .debugging import print_traces -from .helpers import verify_transactions +from .helpers import is_slow_test, verify_transactions def environment_from_parent_header(parent: "FixtureHeader") -> "Environment": @@ -384,6 +384,7 @@ def generate_block_data( previous_env: Environment, previous_alloc: Alloc, eips: Optional[List[int]] = None, + slow: bool = False, ) -> Tuple[FixtureHeader, List[Transaction], List[Bytes] | None, Alloc, Environment]: """ Generate common block data for both make_fixture and make_hive_fixture. @@ -420,6 +421,7 @@ def generate_block_data( reward=fork.get_reward(env.number, env.timestamp), eips=eips, debug_output_path=self.get_next_transition_tool_output_path(), + slow_request=slow, ) try: @@ -528,6 +530,7 @@ def make_fixture( t8n: TransitionTool, fork: Fork, eips: Optional[List[int]] = None, + slow: bool = False, ) -> Fixture: """ Create a fixture from the blockchain test definition. @@ -552,6 +555,7 @@ def make_fixture( previous_env=env, previous_alloc=alloc, eips=eips, + slow=slow, ) fixture_block = FixtureBlockBase( header=header, @@ -610,6 +614,7 @@ def make_hive_fixture( t8n: TransitionTool, fork: Fork, eips: Optional[List[int]] = None, + slow: bool = False, ) -> EngineFixture: """ Create a hive fixture from the blocktest definition. @@ -623,7 +628,13 @@ def make_hive_fixture( for block in self.blocks: header, txs, requests, new_alloc, new_env = self.generate_block_data( - t8n=t8n, fork=fork, block=block, previous_env=env, previous_alloc=alloc, eips=eips + t8n=t8n, + fork=fork, + block=block, + previous_env=env, + previous_alloc=alloc, + eips=eips, + slow=slow, ) if block.rlp is None: fixture_payloads.append( @@ -701,9 +712,9 @@ def generate( """ t8n.reset_traces() if fixture_format == BlockchainEngineFixture: - return self.make_hive_fixture(t8n, fork, eips) + return self.make_hive_fixture(t8n, fork, eips, slow=is_slow_test(request)) elif fixture_format == BlockchainFixture: - return self.make_fixture(t8n, fork, eips) + return self.make_fixture(t8n, fork, eips, slow=is_slow_test(request)) raise Exception(f"Unknown fixture format: {fixture_format}") diff --git a/src/ethereum_test_specs/helpers.py b/src/ethereum_test_specs/helpers.py index 83195efcfa..5446d076ca 100644 --- a/src/ethereum_test_specs/helpers.py +++ b/src/ethereum_test_specs/helpers.py @@ -1,10 +1,11 @@ """ Helper functions """ - from dataclasses import dataclass from typing import Dict, List +import pytest + from ethereum_clis import Result from ethereum_test_exceptions import ExceptionBase, ExceptionMapper, UndefinedException from ethereum_test_types import Transaction @@ -150,3 +151,12 @@ def verify_transactions( verify_transaction_exception(exception_mapper=exception_mapper, info=info) return list(rejected_txs.keys()) + + +def is_slow_test(request: pytest.FixtureRequest) -> bool: + """ + Check if the test is slow + """ + if hasattr(request, "node"): + return request.node.get_closest_marker("slow") is not None + return False diff --git a/src/ethereum_test_specs/state.py b/src/ethereum_test_specs/state.py index 4595d2f6bb..029a2cbc6e 100644 --- a/src/ethereum_test_specs/state.py +++ b/src/ethereum_test_specs/state.py @@ -30,7 +30,7 @@ from .base import BaseTest from .blockchain import Block, BlockchainTest, Header from .debugging import print_traces -from .helpers import verify_transactions +from .helpers import is_slow_test, verify_transactions TARGET_BLOB_GAS_PER_BLOCK = 393216 @@ -124,6 +124,7 @@ def make_state_test_fixture( t8n: TransitionTool, fork: Fork, eips: Optional[List[int]] = None, + slow: bool = False, ) -> Fixture: """ Create a fixture from the state test definition. @@ -151,6 +152,7 @@ def make_state_test_fixture( eips=eips, debug_output_path=self.get_next_transition_tool_output_path(), state_test=True, + slow_request=slow, ) try: @@ -199,7 +201,7 @@ def generate( request=request, t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips ) elif fixture_format == StateFixture: - return self.make_state_test_fixture(t8n, fork, eips) + return self.make_state_test_fixture(t8n, fork, eips, slow=is_slow_test(request)) raise Exception(f"Unknown fixture format: {fixture_format}") From e6c58a0723d659c9289fda8c15741016548eaba0 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 19 Dec 2024 19:37:59 +0000 Subject: [PATCH 3/4] fix(clis): besu --- src/ethereum_clis/clis/besu.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ethereum_clis/clis/besu.py b/src/ethereum_clis/clis/besu.py index 840c8f8082..c3766d3b3f 100644 --- a/src/ethereum_clis/clis/besu.py +++ b/src/ethereum_clis/clis/besu.py @@ -109,6 +109,7 @@ def evaluate( eips: Optional[List[int]] = None, debug_output_path: str = "", state_test: bool = False, + slow_request: bool = False, ) -> TransitionToolOutput: """ Executes `evm t8n` with the specified arguments. From 049773f7b6ae501ad3bef631fd34babebd73b386 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 19 Dec 2024 19:45:54 +0000 Subject: [PATCH 4/4] docs: Changelog --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9663d36915..d6669d10aa 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -63,6 +63,7 @@ Test fixtures for use by clients are available for each release on the [Github r - ✨ Add the `eest make env` command that generates a default env file (`env.yaml`)([#996](https://github.com/ethereum/execution-spec-tests/pull/996)). - ✨ Generate Transaction Test type ([#933](https://github.com/ethereum/execution-spec-tests/pull/933)). - ✨ Add a default location for evm logs (`--evm-dump-dir`) when filling tests ([#999](https://github.com/ethereum/execution-spec-tests/pull/999)). +- ✨ Slow tests now have greater timeout when making a request to the T8N server ([#1037](https://github.com/ethereum/execution-spec-tests/pull/1037)). ### 🔧 EVM Tools