Merge branch 'main' into eof/eofcreate-memsize

hyperledger · Dec 6, 2024 · 7158914 · 7158914
2 parents b4fdfac + f8e93bf
commit 7158914
Show file tree

Hide file tree

Showing 61 changed files with 1,209 additions and 486 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,27 @@
 # Changelog
 
-## [Unreleased]
+## Unreleased
+
+### Breaking Changes
+
+### Upcoming Breaking Changes
+- Plugin API will be deprecating the BesuContext interface to be replaced with the ServiceManager interface.
+- `MetricSystem::createLabelledGauge` is deprecated and will be removed in a future release, replace it with `MetricSystem::createLabelledSuppliedGauge`
+- k8s (KUBERNETES) Nat method is now deprecated and will be removed in a future release
+- `--host-whitelist` has been deprecated in favor of `--host-allowlist` since 2020 and will be removed in a future release
+- Sunsetting features - for more context on the reasoning behind the deprecation of these features, including alternative options, read [this blog post](https://www.lfdecentralizedtrust.org/blog/sunsetting-tessera-and-simplifying-hyperledger-besu)
+  - Tessera privacy
+  - Smart-contract-based (onchain) permissioning
+  - Proof of Work consensus
+  - Fast Sync
+
+### Additions and Improvements
+- Proper support for `pending` block tag when calling `eth_estimateGas` and `eth_createAccessList` [#7951](https://github.com/hyperledger/besu/pull/7951)
+
+### Bug fixes
+-  Correct default parameters for frontier transactions in `eth_call` and `eth_estimateGas` [#7965](https://github.com/hyperledger/besu/pull/7965)
+
+## 24.12.0
 
 ### Breaking Changes
 - Removed Retesteth rpc service and commands [#7833](https://github.com/hyperledger/besu/pull/7783)
@@ -43,8 +64,10 @@
 - Add a method to check if a metric category is enabled to the plugin API [#7832](https://github.com/hyperledger/besu/pull/7832)
 - Add a new metric collector for counters which get their value from suppliers [#7894](https://github.com/hyperledger/besu/pull/7894)
 - Add account and state overrides to `eth_call` [#7801](https://github.com/hyperledger/besu/pull/7801) and `eth_estimateGas` [#7890](https://github.com/hyperledger/besu/pull/7890)
+- Add RPC WS options to specify password file for keystore and truststore [#7970](https://github.com/hyperledger/besu/pull/7970)
 - Prometheus Java Metrics library upgraded to version 1.3.3 [#7880](https://github.com/hyperledger/besu/pull/7880)
 - Add histogram to Prometheus metrics system [#7944](https://github.com/hyperledger/besu/pull/7944)
+- Improve newPayload and FCU logs [#7961](https://github.com/hyperledger/besu/pull/7961)
 
 ### Bug fixes
 - Fix registering new metric categories from plugins [#7825](https://github.com/hyperledger/besu/pull/7825)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -11,6 +11,9 @@ Having the following accounts is necessary for contributing code/issues to Besu.
 * To ask questions or chat with us, join our [Discord](https://discord.com/invite/hyperledger)
 * To edit pages in our wiki, you'll need a [Linux Foundation (LF) account].
 
+### Guidelines for Non-Code and other Trivial Contributions
+Please keep in mind that we do not accept non-code contributions like fixing comments, typos or some other trivial fixes. Although we appreciate the extra help, managing lots of these small contributions is unfeasible, and puts extra pressure in our continuous delivery systems (running all tests, etc). Feel free to open an issue pointing any of those errors, and we will batch them into a single change.
+
 ### Useful contributing links
 
 * [I just have a quick question](https://wiki.hyperledger.org/display/BESU/I+just+have+a+quick+question)

diff --git a/...sl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/...sl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java
@@ -396,9 +396,14 @@ TransactionSimulator provideTransactionSimulator(
         final Blockchain blockchain,
         final WorldStateArchive worldStateArchive,
         final ProtocolSchedule protocolSchedule,
+        final MiningConfiguration miningConfiguration,
         final ApiConfiguration apiConfiguration) {
       return new TransactionSimulator(
-          blockchain, worldStateArchive, protocolSchedule, apiConfiguration.getGasCap());
+          blockchain,
+          worldStateArchive,
+          protocolSchedule,
+          miningConfiguration,
+          apiConfiguration.getGasCap());
     }
 
     @Provides

diff --git a/...rc/test/java/org/hyperledger/besu/tests/acceptance/clique/CliqueMiningAcceptanceTest.java b/...rc/test/java/org/hyperledger/besu/tests/acceptance/clique/CliqueMiningAcceptanceTest.java
@@ -120,6 +120,11 @@ private void startClusterAndVerifyProducingBlocks(
       final BesuNode minerNode1, final BesuNode minerNode2, final BesuNode minerNode3) {
     cluster.start(minerNode1, minerNode2, minerNode3);
 
+    // verify nodes are fully connected otherwise blocks could not be propagated
+    minerNode1.verify(net.awaitPeerCount(2));
+    minerNode2.verify(net.awaitPeerCount(2));
+    minerNode3.verify(net.awaitPeerCount(2));
+
     // verify that we have started producing blocks
     waitForBlockHeight(minerNode1, 1);
     final var minerChainHead = minerNode1.execute(ethTransactions.block());

diff --git a/...est/java/org/hyperledger/besu/tests/acceptance/clique/CliqueProposeRpcAcceptanceTest.java b/...est/java/org/hyperledger/besu/tests/acceptance/clique/CliqueProposeRpcAcceptanceTest.java
@@ -33,6 +33,8 @@ public void shouldAddValidators() throws IOException {
     final BesuNode minerNode3 = besu.createCliqueNodeWithValidators("miner3", initialValidators);
     cluster.start(minerNode1, minerNode2, minerNode3);
 
+    waitForNodesConnectedAndInSync(minerNode1, minerNode2, minerNode3);
+
     cluster.verify(clique.validatorsEqual(minerNode1, minerNode2));
     minerNode1.execute(cliqueTransactions.createAddProposal(minerNode3));
     minerNode2.execute(cliqueTransactions.createAddProposal(minerNode3));
@@ -47,6 +49,8 @@ public void shouldRemoveValidators() throws IOException {
     final BesuNode minerNode3 = besu.createCliqueNodeWithValidators("miner3", initialValidators);
     cluster.start(minerNode1, minerNode2, minerNode3);
 
+    waitForNodesConnectedAndInSync(minerNode1, minerNode2, minerNode3);
+
     cluster.verify(clique.validatorsEqual(minerNode1, minerNode2, minerNode3));
     final Condition cliqueValidatorsChanged = clique.awaitSignerSetChange(minerNode1);
     minerNode1.execute(cliqueTransactions.createRemoveProposal(minerNode3));
@@ -63,6 +67,8 @@ public void shouldNotAddValidatorWhenInsufficientVotes() throws IOException {
     final BesuNode minerNode3 = besu.createCliqueNodeWithValidators("miner3", initialValidators);
     cluster.start(minerNode1, minerNode2, minerNode3);
 
+    waitForNodesConnectedAndInSync(minerNode1, minerNode2, minerNode3);
+
     cluster.verify(clique.validatorsEqual(minerNode1, minerNode2));
     minerNode1.execute(cliqueTransactions.createAddProposal(minerNode3));
     minerNode1.verify(blockchain.reachesHeight(minerNode1, 1));
@@ -76,6 +82,8 @@ public void shouldNotRemoveValidatorWhenInsufficientVotes() throws IOException {
     final BesuNode minerNode3 = besu.createCliqueNode("miner3");
     cluster.start(minerNode1, minerNode2, minerNode3);
 
+    waitForNodesConnectedAndInSync(minerNode1, minerNode2, minerNode3);
+
     cluster.verify(clique.validatorsEqual(minerNode1, minerNode2, minerNode3));
     minerNode1.execute(cliqueTransactions.createRemoveProposal(minerNode3));
     minerNode1.verify(blockchain.reachesHeight(minerNode1, 1));
@@ -90,6 +98,8 @@ public void shouldIncludeVoteInBlockHeader() throws IOException {
     final BesuNode minerNode3 = besu.createCliqueNodeWithValidators("miner3", initialValidators);
     cluster.start(minerNode1, minerNode2, minerNode3);
 
+    waitForNodesConnectedAndInSync(minerNode1, minerNode2, minerNode3);
+
     minerNode1.execute(cliqueTransactions.createAddProposal(minerNode3));
     minerNode1.verify(blockchain.reachesHeight(minerNode1, 1));
     minerNode1.verify(blockchain.beneficiaryEquals(minerNode3));
@@ -100,4 +110,18 @@ public void shouldIncludeVoteInBlockHeader() throws IOException {
     minerNode1.verify(blockchain.beneficiaryEquals(minerNode2));
     minerNode1.verify(clique.nonceVoteEquals(CLIQUE_NONCE_VOTE.DROP));
   }
+
+  private void waitForNodesConnectedAndInSync(
+      final BesuNode minerNode1, final BesuNode minerNode2, final BesuNode minerNode3) {
+    // verify nodes are fully connected otherwise blocks could not be propagated
+    minerNode1.verify(net.awaitPeerCount(2));
+    minerNode2.verify(net.awaitPeerCount(2));
+    minerNode3.verify(net.awaitPeerCount(2));
+
+    // verify that the miner started producing blocks and all other nodes are syncing from it
+    waitForBlockHeight(minerNode1, 1);
+    final var minerChainHead = minerNode1.execute(ethTransactions.block());
+    minerNode2.verify(blockchain.minimumHeight(minerChainHead.getNumber().longValue()));
+    minerNode3.verify(blockchain.minimumHeight(minerChainHead.getNumber().longValue()));
+  }
 }
diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/RpcWebsocketOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/RpcWebsocketOptions.java
@@ -37,6 +37,35 @@
 
 /** This class represents the WebSocket options for the RPC. */
 public class RpcWebsocketOptions {
+
+  static class KeystorePasswordOptions {
+    @CommandLine.Option(
+        names = {"--rpc-ws-ssl-keystore-password"},
+        paramLabel = "<PASSWORD>",
+        description = "Password for the WebSocket RPC keystore file")
+    private String rpcWsKeyStorePassword;
+
+    @CommandLine.Option(
+        names = {"--rpc-ws-ssl-keystore-password-file"},
+        paramLabel = "<FILE>",
+        description = "File containing the password for WebSocket keystore.")
+    private String rpcWsKeystorePasswordFile;
+  }
+
+  static class TruststorePasswordOptions {
+    @CommandLine.Option(
+        names = {"--rpc-ws-ssl-truststore-password"},
+        paramLabel = "<PASSWORD>",
+        description = "Password for the WebSocket RPC truststore file")
+    private String rpcWsTrustStorePassword;
+
+    @CommandLine.Option(
+        names = {"--rpc-ws-ssl-truststore-password-file"},
+        paramLabel = "<FILE>",
+        description = "File containing the password for WebSocket truststore.")
+    private String rpcWsTruststorePasswordFile;
+  }
+
   @CommandLine.Option(
       names = {"--rpc-ws-authentication-jwt-algorithm"},
       description =
@@ -131,12 +160,6 @@ public class RpcWebsocketOptions {
       description = "Path to the keystore file for the WebSocket RPC service")
   private String rpcWsKeyStoreFile = null;
 
-  @CommandLine.Option(
-      names = {"--rpc-ws-ssl-keystore-password"},
-      paramLabel = "<PASSWORD>",
-      description = "Password for the WebSocket RPC keystore file")
-  private String rpcWsKeyStorePassword = null;
-
   @CommandLine.Option(
       names = {"--rpc-ws-ssl-key-file"},
       paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP,
@@ -167,12 +190,6 @@ public class RpcWebsocketOptions {
       description = "Path to the truststore file for the WebSocket RPC service")
   private String rpcWsTrustStoreFile = null;
 
-  @CommandLine.Option(
-      names = {"--rpc-ws-ssl-truststore-password"},
-      paramLabel = "<PASSWORD>",
-      description = "Password for the WebSocket RPC truststore file")
-  private String rpcWsTrustStorePassword = null;
-
   @CommandLine.Option(
       names = {"--rpc-ws-ssl-trustcert-file"},
       paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP,
@@ -185,6 +202,12 @@ public class RpcWebsocketOptions {
       description = "Type of the truststore (JKS, PKCS12, PEM)")
   private String rpcWsTrustStoreType = null;
 
+  @CommandLine.ArgGroup(exclusive = true, multiplicity = "1")
+  private KeystorePasswordOptions keystorePasswordOptions;
+
+  @CommandLine.ArgGroup(exclusive = true, multiplicity = "1")
+  private TruststorePasswordOptions truststorePasswordOptions;
+
   /** Default Constructor. */
   public RpcWebsocketOptions() {}
 
@@ -292,7 +315,7 @@ private void checkOptionDependencies(final Logger logger, final CommandLine comm
             commandLine,
             "--rpc-ws-ssl-keystore-file",
             rpcWsKeyStoreFile == null,
-            List.of("--rpc-ws-ssl-keystore-password"));
+            List.of("--rpc-ws-ssl-keystore-password", "--rpc-ws-ssl-keystore-password-file"));
       }
     }
 
@@ -302,7 +325,7 @@ private void checkOptionDependencies(final Logger logger, final CommandLine comm
           commandLine,
           "--rpc-ws-ssl-truststore-file",
           rpcWsTrustStoreFile == null,
-          List.of("--rpc-ws-ssl-truststore-password"));
+          List.of("--rpc-ws-ssl-truststore-password", "--rpc-ws-ssl-truststore-password-file"));
     }
 
     if (isRpcWsAuthenticationEnabled) {
@@ -343,16 +366,27 @@ public WebSocketConfiguration webSocketConfiguration(
     webSocketConfiguration.setTimeoutSec(wsTimoutSec);
     webSocketConfiguration.setSslEnabled(isRpcWsSslEnabled);
     webSocketConfiguration.setKeyStorePath(rpcWsKeyStoreFile);
-    webSocketConfiguration.setKeyStorePassword(rpcWsKeyStorePassword);
     webSocketConfiguration.setKeyStoreType(rpcWsKeyStoreType);
     webSocketConfiguration.setClientAuthEnabled(isRpcWsClientAuthEnabled);
     webSocketConfiguration.setTrustStorePath(rpcWsTrustStoreFile);
-    webSocketConfiguration.setTrustStorePassword(rpcWsTrustStorePassword);
     webSocketConfiguration.setTrustStoreType(rpcWsTrustStoreType);
     webSocketConfiguration.setKeyPath(rpcWsKeyFile);
     webSocketConfiguration.setCertPath(rpcWsCertFile);
     webSocketConfiguration.setTrustCertPath(rpcWsTrustCertFile);
 
+    if (keystorePasswordOptions != null) {
+      webSocketConfiguration.setKeyStorePassword(keystorePasswordOptions.rpcWsKeyStorePassword);
+      webSocketConfiguration.setKeyStorePasswordFile(
+          keystorePasswordOptions.rpcWsKeystorePasswordFile);
+    }
+
+    if (truststorePasswordOptions != null) {
+      webSocketConfiguration.setTrustStorePassword(
+          truststorePasswordOptions.rpcWsTrustStorePassword);
+      webSocketConfiguration.setTrustStorePasswordFile(
+          truststorePasswordOptions.rpcWsTruststorePasswordFile);
+    }
+
     return webSocketConfiguration;
   }
 

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
@@ -626,7 +626,11 @@ public BesuController build() {
 
     transactionSimulator =
         new TransactionSimulator(
-            blockchain, worldStateArchive, protocolSchedule, apiConfiguration.getGasCap());
+            blockchain,
+            worldStateArchive,
+            protocolSchedule,
+            miningConfiguration,
+            apiConfiguration.getGasCap());
 
     final var consensusContext =
         createConsensusContext(blockchain, worldStateArchive, protocolSchedule);

diff --git a/besu/src/main/java/org/hyperledger/besu/services/TransactionSimulationServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/TransactionSimulationServiceImpl.java
@@ -18,7 +18,6 @@
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.datatypes.Transaction;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
-import org.hyperledger.besu.ethereum.mainnet.ImmutableTransactionValidationParams;
 import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams;
 import org.hyperledger.besu.ethereum.mainnet.ValidationResult;
 import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult;
@@ -35,11 +34,6 @@
 /** TransactionSimulationServiceImpl */
 @Unstable
 public class TransactionSimulationServiceImpl implements TransactionSimulationService {
-  private static final TransactionValidationParams SIMULATOR_ALLOWING_EXCEEDING_BALANCE =
-      ImmutableTransactionValidationParams.builder()
-          .from(TransactionValidationParams.transactionSimulator())
-          .isAllowExceedingBalance(true)
-          .build();
   private Blockchain blockchain;
   private TransactionSimulator transactionSimulator;
 
@@ -57,46 +51,50 @@ public void init(final Blockchain blockchain, final TransactionSimulator transac
     this.transactionSimulator = transactionSimulator;
   }
 
-  @Override
-  public Optional<TransactionSimulationResult> simulate(
-      final Transaction transaction,
-      final Hash blockHash,
-      final OperationTracer operationTracer,
-      final boolean isAllowExceedingBalance) {
-    return simulate(
-        transaction, Optional.empty(), blockHash, operationTracer, isAllowExceedingBalance);
-  }
-
   @Override
   public Optional<TransactionSimulationResult> simulate(
       final Transaction transaction,
       final Optional<AccountOverrideMap> maybeAccountOverrides,
-      final Hash blockHash,
+      final Optional<Hash> maybeBlockHash,
       final OperationTracer operationTracer,
       final boolean isAllowExceedingBalance) {
 
     final CallParameter callParameter = CallParameter.fromTransaction(transaction);
 
-    final var maybeBlockHeader =
-        blockchain.getBlockHeader(blockHash).or(() -> blockchain.getBlockHeaderSafe(blockHash));
+    if (maybeBlockHash.isPresent()) {
+      final Hash blockHash = maybeBlockHash.get();
+
+      final var maybeBlockHeader =
+          blockchain.getBlockHeader(blockHash).or(() -> blockchain.getBlockHeaderSafe(blockHash));
+
+      if (maybeBlockHeader.isEmpty()) {
+        return Optional.of(
+            new TransactionSimulationResult(
+                transaction,
+                TransactionProcessingResult.invalid(
+                    ValidationResult.invalid(TransactionInvalidReason.BLOCK_NOT_FOUND))));
+      }
 
-    if (maybeBlockHeader.isEmpty()) {
-      return Optional.of(
-          new TransactionSimulationResult(
-              transaction,
-              TransactionProcessingResult.invalid(
-                  ValidationResult.invalid(TransactionInvalidReason.BLOCK_NOT_FOUND))));
+      return transactionSimulator
+          .process(
+              callParameter,
+              isAllowExceedingBalance
+                  ? TransactionValidationParams.transactionSimulatorAllowExceedingBalance()
+                  : TransactionValidationParams.transactionSimulator(),
+              operationTracer,
+              maybeBlockHeader.get())
+          .map(res -> new TransactionSimulationResult(transaction, res.result()));
     }
 
     return transactionSimulator
-        .process(
+        .processOnPending(
             callParameter,
             maybeAccountOverrides,
             isAllowExceedingBalance
-                ? SIMULATOR_ALLOWING_EXCEEDING_BALANCE
+                ? TransactionValidationParams.transactionSimulatorAllowExceedingBalance()
                 : TransactionValidationParams.transactionSimulator(),
             operationTracer,
-            maybeBlockHeader.get())
+            transactionSimulator.simulatePendingBlockHeader())
         .map(res -> new TransactionSimulationResult(transaction, res.result()));
   }
 }
diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml
@@ -123,10 +123,12 @@ rpc-ws-authentication-jwt-public-key-file="none"
 rpc-ws-ssl-enabled=false
 rpc-ws-ssl-keystore-file="none.pfx"
 rpc-ws-ssl-keystore-password="none.passwd"
+rpc-ws-ssl-keystore-password-file="none.txt"
 rpc-ws-ssl-keystore-type="none"
 rpc-ws-ssl-client-auth-enabled=false
 rpc-ws-ssl-truststore-file="none.pfx"
 rpc-ws-ssl-truststore-password="none.passwd"
+rpc-ws-ssl-truststore-password-file="none.txt"
 rpc-ws-ssl-truststore-type="none"
 rpc-ws-ssl-key-file="none.pfx"
 rpc-ws-ssl-cert-file="none.pfx"

diff --git a/...est/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java b/...est/java/org/hyperledger/besu/consensus/clique/blockcreation/CliqueMinerExecutorTest.java
@@ -154,7 +154,7 @@ public void extraDataCreatedOnEpochBlocksContainsValidators() {
   }
 
   @Test
-  public void extraDataForNonEpochBlocksDoesNotContainValidaors() {
+  public void extraDataForNonEpochBlocksDoesNotContainValidators() {
     final Bytes vanityData = generateRandomVanityData();
 
     final MiningConfiguration miningConfiguration = createMiningConfiguration(vanityData);