feat(fw): call evmone-eofparse on generated EOF fixtures in fill (#519

)

Co-authored-by: Dimitry Kh <[email protected]>
Co-authored-by: danceratopz <[email protected]>

docs/CHANGELOG.md

@@ -18,7 +18,8 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Libraries have been refactored to use `pydantic` for type checking in most test types ([#486](https://github.com/ethereum/execution-spec-tests/pull/486), [#501](https://github.com/ethereum/execution-spec-tests/pull/501), [#508](https://github.com/ethereum/execution-spec-tests/pull/508)).
 - ✨ Opcodes are now subscriptable and it's used to define the data portion of the opcode: `Op.PUSH1(1) == Op.PUSH1[1]  == b"\x60\x01"` ([#513](https://github.com/ethereum/execution-spec-tests/pull/513))
 - ✨ Added EOF fixture format ([#512](https://github.com/ethereum/execution-spec-tests/pull/512)).
-- ✨ Added `--traces` support when running with Hyperledger Besu ([#511](https://github.com/ethereum/execution-spec-tests/pull/511))
+- ✨ Verify filled EOF fixtures using `evmone-eofparse` during `fill` execution ([#519](https://github.com/ethereum/execution-spec-tests/pull/519)).
+- ✨ Added `--traces` support when running with Hyperledger Besu ([#511](https://github.com/ethereum/execution-spec-tests/pull/511)).
 
 ### 🔧 EVM Tools
 

setup.cfg

@@ -26,6 +26,7 @@ install_requires =
     ethereum@git+https://github.com/ethereum/execution-specs.git
     setuptools
     types-setuptools
+    bidict>=0.23,<1
     requests>=2.31.0,<3
     colorlog>=6.7.0,<7
     pytest>7.3.2,<8

src/ethereum_test_tools/exceptions/__init__.py

@@ -2,6 +2,7 @@
 Exceptions for invalid execution.
 """
 
+from .evmone_exceptions import EvmoneExceptionMapper
 from .exceptions import (
     BlockException,
     BlockExceptionInstanceOrList,
@@ -18,4 +19,5 @@
     "ExceptionInstanceOrList",
     "TransactionException",
     "TransactionExceptionInstanceOrList",
+    "EvmoneExceptionMapper",
 ]

src/ethereum_test_tools/exceptions/evmone_exceptions.py

@@ -0,0 +1,74 @@
+"""
+Evmone eof exceptions ENUM -> str mapper
+"""
+from dataclasses import dataclass
+
+from bidict import frozenbidict
+
+from .exceptions import EOFException
+
+
+@dataclass
+class ExceptionMessage:
+    """Defines a mapping between an exception and a message."""
+
+    exception: EOFException
+    message: str
+
+
+class EvmoneExceptionMapper:
+    """
+    Translate between EEST exceptions and error strings returned by evmone.
+    """
+
+    _mapping_data = (
+        # TODO EVMONE needs to differentiate when the section is missing in the header or body
+        ExceptionMessage(EOFException.MISSING_STOP_OPCODE, "err: no_terminating_instruction"),
+        ExceptionMessage(EOFException.MISSING_CODE_HEADER, "err: code_section_missing"),
+        ExceptionMessage(EOFException.MISSING_TYPE_HEADER, "err: type_section_missing"),
+        # TODO EVMONE these exceptions are too similar, this leeds to ambiguity
+        ExceptionMessage(EOFException.MISSING_TERMINATOR, "err: header_terminator_missing"),
+        ExceptionMessage(
+            EOFException.MISSING_HEADERS_TERMINATOR, "err: section_headers_not_terminated"
+        ),
+        ExceptionMessage(EOFException.INVALID_VERSION, "err: eof_version_unknown"),
+        ExceptionMessage(EOFException.INVALID_MAGIC, "err: invalid_prefix"),
+        ExceptionMessage(
+            EOFException.INVALID_FIRST_SECTION_TYPE, "err: invalid_first_section_type"
+        ),
+        ExceptionMessage(
+            EOFException.INVALID_SECTION_BODIES_SIZE, "err: invalid_section_bodies_size"
+        ),
+        ExceptionMessage(EOFException.INVALID_TYPE_SIZE, "err: invalid_type_section_size"),
+        ExceptionMessage(EOFException.INCOMPLETE_SECTION_SIZE, "err: incomplete_section_size"),
+        ExceptionMessage(EOFException.INCOMPLETE_SECTION_NUMBER, "err: incomplete_section_number"),
+        ExceptionMessage(EOFException.TOO_MANY_CODE_SECTIONS, "err: too_many_code_sections"),
+        ExceptionMessage(EOFException.ZERO_SECTION_SIZE, "err: zero_section_size"),
+    )
+
+    def __init__(self) -> None:
+        assert len(set(entry.exception for entry in self._mapping_data)) == len(
+            self._mapping_data
+        ), "Duplicate exception in _mapping_data"
+        assert len(set(entry.message for entry in self._mapping_data)) == len(
+            self._mapping_data
+        ), "Duplicate message in _mapping_data"
+        self.exception_to_message_map: frozenbidict = frozenbidict(
+            {entry.exception: entry.message for entry in self._mapping_data}
+        )
+
+    def exception_to_message(self, exception: EOFException) -> str:
+        """Takes an EOFException and returns a formatted string."""
+        message = self.exception_to_message_map.get(
+            exception,
+            f"No message defined for {exception}; please add it to {self.__class__.__name__}",
+        )
+        return message
+
+    def message_to_exception(self, exception_string: str) -> EOFException:
+        """Takes a string and tries to find matching exception"""
+        # TODO inform tester where to add the missing exception if get uses default
+        exception = self.exception_to_message_map.inverse.get(
+            exception_string, EOFException.UNDEFINED_EXCEPTION
+        )
+        return exception

src/ethereum_test_tools/exceptions/exceptions.py

@@ -193,6 +193,11 @@ class EOFException(ExceptionBase):
     Expect some exception, not yet known
     """
 
+    UNDEFINED_EXCEPTION = auto()
+    """
+    Indicates that exception string is not mapped to an exception enum
+    """
+
     UNKNOWN_VERSION = auto()
     """
     EOF container has an unknown version
@@ -277,6 +282,10 @@ class EOFException(ExceptionBase):
     """
     EOF container header has too many code sections
     """
+    MISSING_STOP_OPCODE = auto()
+    """
+    EOF container's code missing STOP bytecode at it's end
+    """
 
 
 """

src/ethereum_test_tools/spec/eof/eof_test.py

@@ -2,15 +2,126 @@
 Ethereum EOF test spec definition and filler.
 """
 
+import subprocess
+import warnings
+from pathlib import Path
+from shutil import which
+from subprocess import CompletedProcess
 from typing import Callable, ClassVar, Generator, List, Optional, Type
 
 from ethereum_test_forks import Fork
 from evm_transition_tool import FixtureFormats
 
 from ...common.base_types import Bytes
-from ...exceptions import EOFException
+from ...exceptions import EOFException, EvmoneExceptionMapper
 from ..base.base_test import BaseFixture, BaseTest
-from .types import Fixture
+from .types import Fixture, Result
+
+
+class EOFBaseException(Exception):
+    """
+    Base exception class for exceptions raised when verifying EOF code.
+    """
+
+    def __init__(self, message):
+        super().__init__(message)
+
+    @staticmethod
+    def format_code(code: Bytes, max_length=60) -> str:
+        """
+        Avoid printing long bytecode strings in the terminal upon test failure.
+        """
+        if len(code) > max_length:
+            half_length = max_length // 2 - 5  # Floor; adjust for ellipsis
+            return f"{code[:half_length].hex()}...{code[-half_length:].hex()}"
+        return code.hex()
+
+
+class UnexpectedEOFException(EOFBaseException):
+    """
+    Exception used when valid EOF code unexpectedly raises an exception in
+    eofparse.
+    """
+
+    def __init__(self, *, code: Bytes, got: str):
+        message = (
+            "Expected EOF code to be valid, but an exception occurred:\n"
+            f"   Code: {self.format_code(code)}\n"
+            "Expected: No Exception\n"
+            f"    Got: {got}"
+        )
+        super().__init__(message)
+
+
+class ExpectedEOFException(EOFBaseException):
+    """
+    Exception used when EOF code is expected to raise an exception, but
+    eofparse did not raise an exception.
+    """
+
+    def __init__(self, *, code: Bytes, expected: str):
+        message = (
+            "Expected EOF code to be invalid, but no exception was raised:\n"
+            f"    Code: {self.format_code(code)}\n"
+            f"Expected: {expected}\n"
+            "      Got: No Exception"
+        )
+        super().__init__(message)
+
+
+class EOFExceptionMismatch(EOFBaseException):
+    """
+    Exception used when the actual EOF exception differs from the expected one.
+    """
+
+    def __init__(self, code: Bytes, expected: str, got: str):
+        message = (
+            "EOF code raised a different exception than expected:\n"
+            f"    Code: {self.format_code(code)}\n"
+            f"Expected: {expected}\n"
+            f"     Got: {got}"
+        )
+        super().__init__(message)
+
+
+class EOFParse:
+    """evmone-eofparse binary."""
+
+    binary: Path
+
+    def __new__(cls):
+        """Make EOF binary a singleton."""
+        if not hasattr(cls, "instance"):
+            cls.instance = super(EOFParse, cls).__new__(cls)
+        return cls.instance
+
+    def __init__(
+        self,
+        binary: Optional[Path | str] = None,
+    ):
+        if binary is None:
+            which_path = which("evmone-eofparse")
+            if which_path is not None:
+                binary = Path(which_path)
+        if binary is None or not Path(binary).exists():
+            raise FileNotFoundError(
+                "`evmone-eofparse` binary executable not found/not executable."
+            )
+        self.binary = Path(binary)
+
+    def run(self, *args: str, input: str | None = None) -> CompletedProcess:
+        """Run evmone with the given arguments"""
+        result = subprocess.run(
+            [self.binary, *args],
+            capture_output=True,
+            text=True,
+            input=input,
+        )
+        if result.returncode not in [0, 1]:
+            raise Exception(
+                f"`{self.binary.name}` call failed with return code {result.returncode}."
+            )
+        return result
 
 
 class EOFTest(BaseTest):
@@ -35,7 +146,7 @@ def make_eof_test_fixture(
         """
         Generate the EOF test fixture.
         """
-        return Fixture(
+        fixture = Fixture(
             vectors={
                 "0": {
                     "code": self.data,
@@ -48,6 +159,51 @@ def make_eof_test_fixture(
                 }
             }
         )
+        try:
+            eof_parse = EOFParse()
+        except FileNotFoundError as e:
+            warnings.warn(f"{e} Skipping EOF fixture verification. Fixtures may be invalid!")
+            return fixture
+
+        for _, vector in fixture.vectors.items():
+            expected_result = vector.results.get(str(fork))
+            if expected_result is None:
+                raise Exception(f"EOF Fixture missing vector result for fork: {fork}")
+            result = eof_parse.run(input=str(vector.code))
+            self.verify_result(result, expected_result, vector.code)
+
+        return fixture
+
+    def verify_result(self, result: CompletedProcess, expected_result: Result, code: Bytes):
+        """
+        Checks that the reported exception string matches the expected error.
+        """
+        parser = EvmoneExceptionMapper()
+        actual_message = result.stdout.strip()
+        actual_exception = parser.message_to_exception(actual_message)
+
+        if expected_result.exception is None:
+            if "OK" in actual_message:
+                return
+            else:
+                raise UnexpectedEOFException(
+                    code=code, got=f"{actual_exception} ({actual_message})"
+                )
+
+        expected_exception = expected_result.exception
+        expected_message = parser.exception_to_message(expected_exception)
+
+        if "OK" in actual_message:
+            raise ExpectedEOFException(
+                code=code, expected=f"{expected_exception} ({expected_message})"
+            )
+
+        if expected_exception != actual_exception:
+            raise EOFExceptionMismatch(
+                code=code,
+                expected=f"{expected_exception} ({expected_message})",
+                got=f"{actual_exception} ({actual_message})",
+            )
 
     def generate(
         self,

src/pytest_plugins/test_filler/test_filler.py

@@ -427,7 +427,7 @@ def base_test_parametrizer_func(
         (see `pytest_parameter_name` in each implementation of BaseTest) in its function
         arguments.
 
-        When parametrizing, indirect must be used along with the fixture format as value.
+        When parametrize, indirect must be used along with the fixture format as value.
         """
         fixture_format = request.param
         assert isinstance(fixture_format, FixtureFormats)

tests/prague/eip3540_eof_v1/container.py

@@ -46,7 +46,7 @@
     Container(
         name="incomplete_magic",
         raw_bytes=bytes([0xEF]),
-        validity_error=EOFException.INCOMPLETE_MAGIC,
+        validity_error=EOFException.INVALID_MAGIC,
     ),
     Container(
         name="no_version",

tests/prague/eip3540_eof_v1/test_eof_example.py

@@ -6,7 +6,13 @@
 
 from ethereum_test_tools import EOFTestFiller
 from ethereum_test_tools import Opcodes as Op
-from ethereum_test_tools.eof.v1 import AutoSection, Container, Section
+from ethereum_test_tools.eof.v1 import (
+    AutoSection,
+    BytesConvertible,
+    Container,
+    EOFException,
+    Section,
+)
 from ethereum_test_tools.eof.v1.constants import NON_RETURNING_SECTION
 
 from .spec import EOF_FORK_NAME
@@ -125,3 +131,40 @@ def test_eof_example_custom_fields(eof_test: EOFTestFiller):
         data=eof_code,
         expect_exception=eof_code.validity_error,
     )
+
+
+@pytest.mark.parametrize(
+    "data_section_bytes",
+    ("0x01", "0xef"),
+)
+@pytest.mark.parametrize(
+    "code_section_code, exception",
+    [(Op.PUSH1(10) + Op.STOP, None), (Op.PUSH1(14), EOFException.MISSING_STOP_OPCODE)],
+)
+def test_eof_example_parameters(
+    eof_test: EOFTestFiller,
+    data_section_bytes: BytesConvertible,
+    code_section_code: BytesConvertible,
+    exception: EOFException,
+):
+    """
+    Example of python EOF classes
+    """
+    eof_code = Container(
+        name="parametrized_eof_example",
+        sections=[
+            Section.Code(
+                code=code_section_code,
+                code_inputs=0,
+                code_outputs=NON_RETURNING_SECTION,
+                max_stack_height=1,
+            ),
+            Section.Data(data_section_bytes),
+        ],
+        validity_error=exception,
+    )
+
+    eof_test(
+        data=eof_code,
+        expect_exception=eof_code.validity_error,
+    )