Merge pull request #13873 from ethereum/foundry-prbmath-external-test

Foundry prbmath external test

.circleci/config.yml

@@ -691,7 +691,7 @@ defaults:
       name: t_native_test_ext_prb_math
       project: prb-math
       binary_type: native
-      image: cimg/node:18.16
+      image: cimg/rust:1.70
 
   - job_native_test_ext_elementfi: &job_native_test_ext_elementfi
       <<: *requires_b_ubu_static
@@ -1724,11 +1724,13 @@ workflows:
             - t_native_test_ext_yield_liquidator
             - t_native_test_ext_perpetual_pools
             - t_native_test_ext_uniswap
-            - t_native_test_ext_prb_math
             - t_native_test_ext_elementfi
             - t_native_test_ext_brink
             # NOTE: We are disabling gp2 tests due to constant failures.
             #- t_native_test_ext_gp2
+            # TODO: Dropping prb-math from the benchmarks since it is not implemented yet
+            # in the new Foundry external testing infrastructure.
+            # - t_native_test_ext_prb_math
             # NOTE: The external tests below were commented because they
             # depend on a specific version of hardhat which does not support shanghai EVM.
             #- t_native_test_ext_trident

scripts/externalTests/runners/base.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import os
+import subprocess
+from abc import ABCMeta
+from abc import abstractmethod
+from dataclasses import dataclass
+from dataclasses import field
+from pathlib import Path
+from shutil import rmtree
+from tempfile import mkdtemp
+from textwrap import dedent
+from typing import List
+from typing import Set
+
+from test_helpers import download_project
+from test_helpers import get_solc_short_version
+from test_helpers import parse_command_line
+from test_helpers import parse_custom_presets
+from test_helpers import parse_solc_version
+from test_helpers import replace_version_pragmas
+from test_helpers import settings_from_preset
+from test_helpers import SettingsPreset
+
+CURRENT_EVM_VERSION: str = "shanghai"
+
+@dataclass
+class TestConfig:
+    name: str
+    repo_url: str
+    ref_type: str
+    ref: str
+    compile_only_presets: List[SettingsPreset] = field(default_factory=list)
+    settings_presets: List[SettingsPreset] = field(default_factory=lambda: list(SettingsPreset))
+    evm_version: str = field(default=CURRENT_EVM_VERSION)
+
+    def selected_presets(self) -> Set[SettingsPreset]:
+        return set(self.compile_only_presets + self.settings_presets)
+
+
+class BaseRunner(metaclass=ABCMeta):
+    config: TestConfig
+    solc_binary_type: str
+    solc_binary_path: Path
+    presets: Set[SettingsPreset]
+
+    def __init__(self, argv, config: TestConfig):
+        args = parse_command_line(f"{config.name} external tests", argv)
+        self.config = config
+        self.solc_binary_type = args.solc_binary_type
+        self.solc_binary_path = args.solc_binary_path
+        self.presets = parse_custom_presets(args.selected_presets) if args.selected_presets else config.selected_presets()
+        self.env = os.environ.copy()
+        self.tmp_dir = mkdtemp(prefix=f"ext-test-{config.name}-")
+        self.test_dir = Path(self.tmp_dir) / "ext"
+
+    def setup_solc(self) -> str:
+        if self.solc_binary_type == "solcjs":
+            # TODO: add support to solc-js
+            raise NotImplementedError()
+        print("Setting up solc...")
+        solc_version_output = subprocess.check_output(
+            [self.solc_binary_path, "--version"],
+            shell=False,
+            encoding="utf-8"
+        ).split(":")[1]
+        return parse_solc_version(solc_version_output)
+
+    @staticmethod
+    def enter_test_dir(fn):
+        """Run a function inside the test directory"""
+
+        previous_dir = os.getcwd()
+        def f(self, *args, **kwargs):
+            try:
+                assert self.test_dir is not None
+                os.chdir(self.test_dir)
+                return fn(self, *args, **kwargs)
+            finally:
+                # Restore the previous directory after execute fn
+                os.chdir(previous_dir)
+        return f
+
+    def setup_environment(self):
+        """Configure the project build environment"""
+        print("Configuring Runner building environment...")
+        replace_version_pragmas(self.test_dir)
+
+    @enter_test_dir
+    def clean(self):
+        """Clean temporary directories"""
+        rmtree(self.tmp_dir)
+
+    @enter_test_dir
+    @abstractmethod
+    def configure(self):
+        raise NotImplementedError()
+
+    @enter_test_dir
+    @abstractmethod
+    def compile(self, preset: SettingsPreset):
+        raise NotImplementedError()
+
+    @enter_test_dir
+    @abstractmethod
+    def run_test(self):
+        raise NotImplementedError()
+
+def run_test(runner: BaseRunner):
+    print(f"Testing {runner.config.name}...\n===========================")
+    print(f"Selected settings presets: {' '.join(p.value for p in runner.presets)}")
+
+    # Configure solc compiler
+    solc_version = runner.setup_solc()
+    print(f"Using compiler version {solc_version}")
+
+    # Download project
+    download_project(runner.test_dir, runner.config.repo_url, runner.config.ref_type, runner.config.ref)
+
+    # Configure run environment
+    runner.setup_environment()
+
+    # Configure TestRunner instance
+    print(dedent(f"""\
+        Configuring runner's profiles with:
+        -------------------------------------
+        Binary type: {runner.solc_binary_type}
+        Compiler path: {runner.solc_binary_path}
+        -------------------------------------
+    """))
+    runner.configure()
+    for preset in runner.presets:
+        print("Running compile function...")
+        settings = settings_from_preset(preset, runner.config.evm_version)
+        print(dedent(f"""\
+            -------------------------------------
+            Settings preset: {preset.value}
+            Settings: {settings}
+            EVM version: {runner.config.evm_version}
+            Compiler version: {get_solc_short_version(solc_version)}
+            Compiler version (full): {solc_version}
+            -------------------------------------
+        """))
+        runner.compile(preset)
+        # TODO: COMPILE_ONLY should be a command-line option
+        if os.environ.get("COMPILE_ONLY") == "1" or preset in runner.config.compile_only_presets:
+            print("Skipping test function...")
+        else:
+            print("Running test function...")
+            runner.run_test()
+        # TODO: store_benchmark_report
+    runner.clean()
+    print("Done.")

scripts/externalTests/runners/foundry.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import os
+import re
+import subprocess
+from shutil import which
+from textwrap import dedent
+from typing import Optional
+
+from runners.base import BaseRunner
+from test_helpers import SettingsPreset
+from test_helpers import settings_from_preset
+
+def run_forge_command(command: str, env: Optional[dict] = None):
+    subprocess.run(
+        command.split(),
+        env=env if env is not None else os.environ.copy(),
+        check=True
+    )
+
+
+class FoundryRunner(BaseRunner):
+    """Configure and run Foundry-based projects"""
+
+    FOUNDRY_CONFIG_FILE = "foundry.toml"
+
+    def setup_environment(self):
+        super().setup_environment()
+        if which("forge") is None:
+            raise RuntimeError("Forge not found.")
+
+    @staticmethod
+    def profile_name(preset: SettingsPreset):
+        """Returns foundry profile name"""
+        # Replace - or + by underscore to avoid invalid toml syntax
+        return re.sub(r"(\-|\+)+", "_", preset.value)
+
+    @staticmethod
+    def profile_section(profile_fields: dict) -> str:
+        return dedent("""\
+            [profile.{name}]
+            gas_reports = ["*"]
+            auto_detect_solc = false
+            solc = "{solc}"
+            evm_version = "{evm_version}"
+            optimizer = {optimizer}
+            via_ir = {via_ir}
+
+            [profile.{name}.optimizer_details]
+            yul = {yul}
+        """).format(**profile_fields)
+
+    @BaseRunner.enter_test_dir
+    def configure(self):
+        """Configure forge tests profiles"""
+
+        profiles = []
+        for preset in self.presets:
+            settings = settings_from_preset(preset, self.config.evm_version)
+            profiles.append(self.profile_section({
+                "name": self.profile_name(preset),
+                "solc": self.solc_binary_path,
+                "evm_version": self.config.evm_version,
+                "optimizer": str(settings["optimizer"]["enabled"]).lower(),
+                "via_ir": str(settings["viaIR"]).lower(),
+                "yul": str(settings["optimizer"]["details"]["yul"]).lower(),
+            }))
+
+        with open(
+            file=self.test_dir / self.FOUNDRY_CONFIG_FILE,
+            mode="a",
+            encoding="utf-8",
+        ) as f:
+            for profile in profiles:
+                f.write(profile)
+
+        run_forge_command("forge install", self.env)
+
+    @BaseRunner.enter_test_dir
+    def compile(self, preset: SettingsPreset):
+        """Compile project"""
+
+        # Set the Foundry profile environment variable
+        self.env.update({"FOUNDRY_PROFILE": self.profile_name(preset)})
+        run_forge_command("forge build", self.env)
+
+    @BaseRunner.enter_test_dir
+    def run_test(self):
+        """Run project tests"""
+
+        run_forge_command("forge test --gas-report", self.env)