feat/pydantic: Use Pydantic library for types (#486)

* setup: Add pydantic, update mypy * refactor(fw): Use pydantic for base and common types Co-authored-by: danceratopz <[email protected]> * refactor(spec): Use pydantic types for test specs * refactor(fw): Fix all tests Co-authored-by: danceratopz <[email protected]> * refactor(fw): Cleanup exported types * refactor(evm_transition_tool): Return full dict on evaluate * fix(plugins): fix refactor issues * whilelist: Add pydantic * fix(tests): Pydantic required changes * changelog * fix(fw): Fixture block types with and without rlp * fix(fw): Transaction "to" field validation * feat(fw): add FixtureBlockBase json test * fix(fw): base fixture json generation * fix(fw): account for zero values in storage on empty account check * changelog * refactor(fw): ExceptionList as annotated type * fix(tests): exception lists * chore: fix minor typos in blobgasfee test docstrings * Apply suggestions from code review Co-authored-by: danceratopz <[email protected]> * fix(fw): Export Alloc Co-authored-by: danceratopz <[email protected]> * fix(fw): tox * fix(fw): add set/del item from Alloc * fix(tests): Use Alloc in blobgasfee tests Co-authored-by: danceratopz <[email protected]> * changelog * fix(fw): Alloc: don't pop on None assigment * fix(fw): Types: remove unused definitions --------- Co-authored-by: danceratopz <[email protected]>
ethereum · Apr 4, 2024 · 4a65686 · 4a65686
1 parent fc36d65
commit 4a65686
Show file tree

Hide file tree

Showing 47 changed files with 2,290 additions and 3,148 deletions.
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -14,6 +14,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 
 - 🐞 Fix incorrect `!=` operator for `FixedSizeBytes` ([#477](https://github.com/ethereum/execution-spec-tests/pull/477)).
 - ✨ Add Macro enum that represents byte sequence of Op instructions ([#457](https://github.com/ethereum/execution-spec-tests/pull/457))
+- ✨ Libraries have been refactored to use `pydantic` for type checking in most test types ([#486](https://github.com/ethereum/execution-spec-tests/pull/486)).
 
 ### 🔧 EVM Tools
 
@@ -22,6 +23,9 @@ Test fixtures for use by clients are available for each release on the [Github r
 - 🐞 Fix CI by using Golang 1.21 in Github Actions to build geth ([#484](https://github.com/ethereum/execution-spec-tests/pull/484)).
 - 💥 "Merge" has been renamed to "Paris" in the "network" field of the Blockchain tests, and in the "post" field of the State tests ([#480](https://github.com/ethereum/execution-spec-tests/pull/480)).
 - ✨ Port entry point scripts to use [click](https://click.palletsprojects.com) and add tests ([#483](https://github.com/ethereum/execution-spec-tests/pull/483)).
+- 💥 As part of the pydantic conversion, the fixtures have the following (possibly breaking) changes ([#486](https://github.com/ethereum/execution-spec-tests/pull/486)):
+  - State test field `transaction` now uses the proper zero-padded hex number format for fields `maxPriorityFeePerGas`, `maxFeePerGas`, and `maxFeePerBlobGas`
+  - Fixtures' hashes (in the `_info` field) are now calculated by removing the "_info" field entirely instead of it being set to an empty dict.
 
 ## 🔜 [v2.1.1](https://github.com/ethereum/execution-spec-tests/releases/tag/v2.1.1) - 2024-03-09
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -12,3 +12,4 @@ line-length = 99
 
 [tool.mypy]
 mypy_path = "$MYPY_CONFIG_FILE_DIR/stubs"
+plugins = ["pydantic.mypy"]
diff --git a/setup.cfg b/setup.cfg
@@ -34,6 +34,7 @@ install_requires =
     trie==2.1.1
     semver==3.0.1
     click>=8.0.0,<9
+    pydantic>=2.6.3
 
 [options.package_data]
 ethereum_test_tools =
@@ -61,7 +62,7 @@ test =
 
 lint =
     isort>=5.8,<6
-    mypy==0.982; implementation_name == "cpython"
+    mypy==0.991; implementation_name == "cpython"
     types-requests
     black==22.3.0; implementation_name == "cpython"
     flake8-spellcheck>=0.24,<0.25

diff --git a/src/ethereum_test_tools/__init__.py b/src/ethereum_test_tools/__init__.py
@@ -18,11 +18,10 @@
     AccessList,
     Account,
     Address,
-    Auto,
+    Alloc,
     EngineAPIError,
     Environment,
     Hash,
-    JSONEncoder,
     Removable,
     Storage,
     TestAddress,
@@ -39,9 +38,8 @@
     copy_opcode_cost,
     cost_memory_bytes,
     eip_2028_transaction_data_cost,
-    transaction_list_root,
 )
-from .exceptions import BlockException, ExceptionList, ExceptionType, TransactionException
+from .exceptions import BlockException, TransactionException
 from .reference_spec import ReferenceSpec, ReferenceSpecTypes
 from .spec import (
     SPEC_TYPES,
@@ -62,7 +60,7 @@
     "AccessList",
     "Account",
     "Address",
-    "Auto",
+    "Alloc",
     "BaseFixture",
     "BaseTest",
     "Block",
@@ -76,13 +74,10 @@
     "Conditional",
     "EngineAPIError",
     "Environment",
-    "ExceptionList",
-    "ExceptionType",
     "FixtureCollector",
     "Hash",
     "Header",
     "Initcode",
-    "JSONEncoder",
     "Opcode",
     "Macro",
     "OpcodeCallArg",
@@ -113,5 +108,4 @@
     "cost_memory_bytes",
     "eip_2028_transaction_data_cost",
     "eip_2028_transaction_data_cost",
-    "transaction_list_root",
 )
diff --git a/src/ethereum_test_tools/common/__init__.py b/src/ethereum_test_tools/common/__init__.py
@@ -1,6 +1,7 @@
 """
 Common definitions and types.
 """
+
 from .base_types import (
     Address,
     Bloom,
@@ -14,6 +15,7 @@
 from .constants import (
     AddrAA,
     AddrBB,
+    EmptyOmmersRoot,
     EmptyTrieRoot,
     EngineAPIError,
     TestAddress,
@@ -36,18 +38,11 @@
     AccessList,
     Account,
     Alloc,
-    Auto,
     Environment,
-    JSONEncoder,
     Removable,
     Storage,
     Transaction,
     Withdrawal,
-    alloc_to_accounts,
-    serialize_transactions,
-    str_or_none,
-    transaction_list_root,
-    withdrawals_root,
 )
 
 __all__ = (
@@ -57,16 +52,15 @@
     "AddrAA",
     "AddrBB",
     "Alloc",
-    "Auto",
     "Bloom",
     "Bytes",
     "EngineAPIError",
+    "EmptyOmmersRoot",
     "EmptyTrieRoot",
     "Environment",
     "Hash",
     "HeaderNonce",
     "HexNumber",
-    "JSONEncoder",
     "Number",
     "Removable",
     "Storage",
@@ -79,16 +73,11 @@
     "Withdrawal",
     "ZeroPaddedHexNumber",
     "add_kzg_version",
-    "alloc_to_accounts",
     "ceiling_division",
     "compute_create_address",
     "compute_create2_address",
     "copy_opcode_cost",
     "cost_memory_bytes",
     "eip_2028_transaction_data_cost",
-    "serialize_transactions",
-    "str_or_none",
     "to_json",
-    "transaction_list_root",
-    "withdrawals_root",
 )
diff --git a/src/ethereum_test_tools/common/base_types.py b/src/ethereum_test_tools/common/base_types.py
@@ -2,7 +2,14 @@
 Basic type primitives used to define other types.
 """
 
-from typing import ClassVar, SupportsBytes, Type, TypeVar
+from typing import Any, ClassVar, SupportsBytes, Type, TypeVar
+
+from pydantic import GetCoreSchemaHandler
+from pydantic_core.core_schema import (
+    PlainValidatorFunctionSchema,
+    no_info_plain_validator_function,
+    to_string_ser_schema,
+)
 
 from .conversions import (
     BytesConvertible,
@@ -12,12 +19,29 @@
     to_fixed_size_bytes,
     to_number,
 )
-from .json import JSONEncoder, SupportsJSON
 
 N = TypeVar("N", bound="Number")
 
 
-class Number(int, SupportsJSON):
+class ToStringSchema:
+    """
+    Type converter to add a simple pydantic schema that correctly parses and serializes the type.
+    """
+
+    @staticmethod
+    def __get_pydantic_core_schema__(
+        source_type: Any, handler: GetCoreSchemaHandler
+    ) -> PlainValidatorFunctionSchema:
+        """
+        Calls the class constructor without info and appends the serialization schema.
+        """
+        return no_info_plain_validator_function(
+            source_type,
+            serialization=to_string_ser_schema(),
+        )
+
+
+class Number(int, ToStringSchema):
     """
     Class that helps represent numbers in tests.
     """
@@ -34,12 +58,6 @@ def __str__(self) -> str:
         """
         return str(int(self))
 
-    def __json__(self, encoder: JSONEncoder) -> str:
-        """
-        Returns the JSON representation of the number.
-        """
-        return str(self)
-
     def hex(self) -> str:
         """
         Returns the hexadecimal representation of the number.
@@ -85,7 +103,10 @@ def hex(self) -> str:
         return "0x" + hex_str
 
 
-class Bytes(bytes, SupportsJSON):
+NumberBoundTypeVar = TypeVar("NumberBoundTypeVar", Number, HexNumber, ZeroPaddedHexNumber)
+
+
+class Bytes(bytes, ToStringSchema):
     """
     Class that helps represent bytes of variable length in tests.
     """
@@ -108,12 +129,6 @@ def __str__(self) -> str:
         """
         return self.hex()
 
-    def __json__(self, encoder: JSONEncoder) -> str:
-        """
-        Returns the JSON representation of the bytes.
-        """
-        return str(self)
-
     def hex(self, *args, **kwargs) -> str:
         """
         Returns the hexadecimal representation of the bytes.
@@ -130,6 +145,71 @@ def or_none(cls, input: "Bytes | BytesConvertible | None") -> "Bytes | None":
         return cls(input)
 
 
+S = TypeVar("S", bound="FixedSizeHexNumber")
+
+
+class FixedSizeHexNumber(int, ToStringSchema):
+    """
+    A base class that helps represent an integer as a fixed byte-length
+    hexadecimal number.
+
+    This class is used to dynamically generate subclasses of a specific byte
+    length.
+    """
+
+    byte_length: ClassVar[int]
+    max_value: ClassVar[int]
+
+    def __class_getitem__(cls, length: int) -> Type["FixedSizeHexNumber"]:
+        """
+        Creates a new FixedSizeHexNumber class with the given length.
+        """
+
+        class Sized(cls):  # type: ignore
+            byte_length = length
+            max_value = 2 ** (8 * length) - 1
+
+        return Sized
+
+    def __new__(cls, input: NumberConvertible | N):
+        """
+        Creates a new Number object.
+        """
+        i = to_number(input)
+        if i > cls.max_value:
+            raise ValueError(f"Value {i} is too large for {cls.byte_length} bytes")
+        if i < 0:
+            i += cls.max_value + 1
+            if i <= 0:
+                raise ValueError(f"Value {i} is too small for {cls.byte_length} bytes")
+        return super(FixedSizeHexNumber, cls).__new__(cls, i)
+
+    def __str__(self) -> str:
+        """
+        Returns the string representation of the number.
+        """
+        return self.hex()
+
+    def hex(self) -> str:
+        """
+        Returns the hexadecimal representation of the number.
+        """
+        if self == 0:
+            return "0x00"
+        hex_str = hex(self)[2:]
+        if len(hex_str) % 2 == 1:
+            return "0x0" + hex_str
+        return "0x" + hex_str
+
+
+class HashInt(FixedSizeHexNumber[32]):  # type: ignore
+    """
+    Class that helps represent hashes in tests.
+    """
+
+    pass
+
+
 T = TypeVar("T", bound="FixedSizeBytes")
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,3 +12,4 @@ line-length = 99

		[tool.mypy]
		mypy_path = "$MYPY_CONFIG_FILE_DIR/stubs"
		plugins = ["pydantic.mypy"]