From 0f241206d00c340c7891a3fe82df0b8bf382474a Mon Sep 17 00:00:00 2001 From: garyschulte Date: Wed, 10 Jan 2024 15:52:31 -0800 Subject: [PATCH 01/56] Bump to nex release snapshot 24.1.1 (#6383) * release next snapshot 24.1.1 Signed-off-by: garyschulte * move recent changelog items to 24.1.1-SNAPSHOT Signed-off-by: garyschulte --------- Signed-off-by: garyschulte --- CHANGELOG.md | 46 ++++++++++++++++++++++++++++++++++++++++------ gradle.properties | 2 +- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f51e8b4c2e..524184c1f05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,26 +1,59 @@ # Changelog -## 24.1.0-SNAPSHOT +## 24.1.1-SNAPSHOT ### Breaking Changes - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) +### Deprecations + +### Additions and Improvements +- Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) +- Disable transaction handling when the node is not in sync, to avoid unnecessary transaction validation work [#6302](https://github.com/hyperledger/besu/pull/6302) + +### Bug fixes +- INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) +- Fix Besu Docker images with `openjdk-latest` tags since 23.10.3 using UID 1001 instead of 1000 for the `besu` user [#6360](https://github.com/hyperledger/besu/pull/6360) + +### Download Links + + +## 24.1.0 + +### Breaking Changes + ### Deprecations - Forest pruning (`pruning-enabled` options) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230) ### Additions and Improvements - Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212) -- New `Sequenced` transaction pool. The pool is an evolution of the `legacy` pool and is likely to be more suitable to enterprise or permissioned chains than the `layered` transaction pool. Select to use this pool with `--tx-pool=sequenced`. Supports the same options as the `legacy` pool [#6211](https://github.com/hyperledger/besu/issues/6211) +- New `Sequenced` transaction pool. The pool is an evolution of the `legacy` pool and is likely to be more suitable to enterprise or permissioned chains than the `layered` transaction pool. Select to use this pool with `--tx-pool=sequenced`. Supports the same options as the `legacy` pool [#6274](https://github.com/hyperledger/besu/issues/6274) - Set Ethereum Classic mainnet activation block for Spiral network upgrade [#6267](https://github.com/hyperledger/besu/pull/6267) - Add custom genesis file name to config overview if specified [#6297](https://github.com/hyperledger/besu/pull/6297) - Update Gradle plugins and replace unmaintained License Gradle Plugin with the actively maintained Gradle License Report [#6275](https://github.com/hyperledger/besu/pull/6275) -- Disable transaction handling when the node is not in sync, to avoid unnecessary transaction validation work [#6302](https://github.com/hyperledger/besu/pull/6302) -- Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) ### Bug fixes -- INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) -- Fix Besu Docker images with `openjdk-latest` tags since 23.10.3 using UID 1001 instead of 1000 for the `besu` user [#6360](https://github.com/hyperledger/besu/pull/6360) +- Hotfix for selfdestruct preimages on bonsai [#6359]((https://github.com/hyperledger/besu/pull/6359) +- Fix trielog shipping issue during self destruct [#6340]((https://github.com/hyperledger/besu/pull/6340) +- mitigation for trielog failure [#6315]((https://github.com/hyperledger/besu/pull/6315) + +### Download Links +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.0/besu-24.1.0.zip / sha256 TBA +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.0/besu-24.1.0.tar.gz / sha256 TBA + + +## 23.10.3-hotfix +This is a hotfix for a selfdestruct defect that occurred on mainnet at block [18947893](https://etherscan.io/block/18947893) + +### Bug fixes +- Hotfix for selfdestruct preimages on bonsai [#6359]((https://github.com/hyperledger/besu/pull/6359) +- mitigation for trielog failure [#6315]((https://github.com/hyperledger/besu/pull/6315) + +### Download Links +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3-hotfix/besu-23.10.3-hotfix.zip / sha256 1c37762909858a40eca749fb85b77fb4d1e918f247aff56d518144828bd85378 +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3-hotfix/besu-23.10.3-hotfix.tar.gz / sha256 8e38e9fd0c16e049aa324effc96f9ec31dc06e82ea4995e9dd75d571394667af + ## 23.10.3 @@ -68,6 +101,7 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3/besu-23.10.3 - Force tx replacement price bump to zero when zero base fee market is configured or `--min-gas-price` is set to 0. This allows for easier tx replacement in networks where there is not gas price. [#6079](https://github.com/hyperledger/besu/pull/6079) - Introduce the possibility to limit the time spent selecting pending transactions during block creation, using the new experimental option `Xblock-txs-selection-max-time` on PoS and PoW networks (by default set to 5000ms) or `Xpoa-block-txs-selection-max-time` on PoA networks (by default 75% of the min block time) [#6044](https://github.com/hyperledger/besu/pull/6044) - Remove LowestInvalidNonceCache from `legacy` transaction pool to make it more private networks friendly [#6148](https://github.com/hyperledger/besu/pull/6148) +- Optimization: Delete leftPad when capturing the stack before and after a frame execution [#6102](https://github.com/hyperledger/besu/pull/6102) ### Bug fixes - Upgrade netty to address CVE-2023-44487, CVE-2023-34462 [#6100](https://github.com/hyperledger/besu/pull/6100) diff --git a/gradle.properties b/gradle.properties index 206d0b7c702..75ee1d04ee8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=24.1.0-SNAPSHOT +version=24.1.1-SNAPSHOT org.gradle.welcome=never # Set exports/opens flags required by Google Java Format and ErrorProne plugins. (JEP-396) From b6b2b79c536c7d4c520bc908b542ba80c820635f Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Wed, 10 Jan 2024 18:21:10 -0700 Subject: [PATCH 02/56] Correct Tangerine Whistle definition in Fluent EVM APIs. (#6382) The fluent API incorrectly added the code size limit in Tangerine Whistle instead of first adding it in Spurious Dragon. Signed-off-by: Danno Ferrin --- CHANGELOG.md | 1 + .../main/java/org/hyperledger/besu/evm/fluent/EVMExecutor.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 524184c1f05..9ce64c213aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) - Fix Besu Docker images with `openjdk-latest` tags since 23.10.3 using UID 1001 instead of 1000 for the `besu` user [#6360](https://github.com/hyperledger/besu/pull/6360) +- Fluent EVM API definition for Tangerine Whistle had incorrect code size validation configured [#6382](https://github.com/hyperledger/besu/pull/6382) ### Download Links diff --git a/evm/src/main/java/org/hyperledger/besu/evm/fluent/EVMExecutor.java b/evm/src/main/java/org/hyperledger/besu/evm/fluent/EVMExecutor.java index 9946b6f57a8..0a43a6d359a 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/fluent/EVMExecutor.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/fluent/EVMExecutor.java @@ -221,7 +221,7 @@ public static EVMExecutor tangerineWhistle(final EvmConfiguration evmConfigurati final EVMExecutor executor = new EVMExecutor(MainnetEVMs.tangerineWhistle(evmConfiguration)); executor.precompileContractRegistry = MainnetPrecompiledContracts.frontier(executor.evm.getGasCalculator()); - executor.contractValidationRules = List.of(MaxCodeSizeRule.of(0x6000)); + executor.contractValidationRules = List.of(); executor.initialNonce = 0; return executor; } From f21b705b3f16237c5eedee35e0587818a99474ca Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Thu, 11 Jan 2024 14:15:45 +1100 Subject: [PATCH 03/56] [MINOR] Fix pki tests condition check on mac (#6387) Signed-off-by: Gabriel-Trintinalia --- .../HardwareKeyStoreFileWrapperTest.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pki/src/test/java/org/hyperledger/besu/pki/keystore/HardwareKeyStoreFileWrapperTest.java b/pki/src/test/java/org/hyperledger/besu/pki/keystore/HardwareKeyStoreFileWrapperTest.java index 0dde591cf50..31eb03126c5 100644 --- a/pki/src/test/java/org/hyperledger/besu/pki/keystore/HardwareKeyStoreFileWrapperTest.java +++ b/pki/src/test/java/org/hyperledger/besu/pki/keystore/HardwareKeyStoreFileWrapperTest.java @@ -22,11 +22,13 @@ import java.nio.file.Path; import java.security.Provider; import java.security.Security; -import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.stream.Stream; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.OS; @@ -37,13 +39,19 @@ public class HardwareKeyStoreFileWrapperTest extends BaseKeyStoreFileWrapperTest private static final String crl = "/keystore/partner1client1/crl.pem"; private static final String configName = "NSScrypto-partner1client1"; private static final String validKeystorePassword = "test123"; + private static KeyStoreWrapperTestParameter keyStoreWrapperTestParameter; - public static Collection data() { - return Arrays.asList( + @BeforeAll + public static void setup() { + keyStoreWrapperTestParameter = new KeyStoreWrapperTestParameter( "HardwareKeyStoreWrapper[PKCS11 keystore/truststore]", true, - CryptoTestUtil.isNSSLibInstalled() ? getHardwareKeyStoreWrapper(configName) : null)); + CryptoTestUtil.isNSSLibInstalled() ? getHardwareKeyStoreWrapper(configName) : null); + } + + public static Collection data() { + return List.of(keyStoreWrapperTestParameter); } private static KeyStoreWrapper getHardwareKeyStoreWrapper(final String cfgName) { @@ -58,11 +66,10 @@ private static KeyStoreWrapper getHardwareKeyStoreWrapper(final String cfgName) .map(provider -> new HardwareKeyStoreWrapper(validKeystorePassword, provider, crlPath)) .orElseGet(() -> new HardwareKeyStoreWrapper(validKeystorePassword, path, crlPath)); } catch (final Exception e) { - if (OS.MAC.isCurrentOs()) { - // nss3 is difficult to setup on mac correctly, don't let it break unit tests for dev - // machines. - System.out.println("Failed to initialize hardware keystore " + e.getLocalizedMessage()); - } + // nss3 is difficult to setup on mac, don't let it break unit tests for dev machines. + Assumptions.assumeFalse( + OS.MAC.isCurrentOs(), + "Failed to initialize hardware keystore: " + e.getLocalizedMessage()); // Not a mac, probably a production build. Full failure. throw new PkiException("Failed to initialize hardware keystore", e); } From f721c1c2c26bf695edbbba6ba612c5fb312aa57c Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 11 Jan 2024 10:45:41 +0100 Subject: [PATCH 04/56] Upgrade dependencies (#6377) * Bump com.github.oshi:oshi-core to 6.4.10 Signed-off-by: Fabio Di Fabio * Bump com.github.tomakehurst to org.wiremock 3.3.1 Signed-off-by: Fabio Di Fabio * Bump com.google.auto.service:auto-service to 1.1.1 Signed-off-by: Fabio Di Fabio * Bump com.google.dagger group to 2.50 Signed-off-by: Fabio Di Fabio * Bump com.graphql-java:graphql-java to 21.3 Signed-off-by: Fabio Di Fabio * Bump com.splunk.logging:splunk-library-javalogging to 1.11.8 Signed-off-by: Fabio Di Fabio * Bump com.squareup.okhttp3:okhttp to 4.12.0 Signed-off-by: Fabio Di Fabio * Bump commons-io:commons-io to 2.15.1 Signed-off-by: Fabio Di Fabio * Bump dnsjava:dnsjava to 3.5.3 Signed-off-by: Fabio Di Fabio * Bump info.picocli group to 4.7.5 Signed-off-by: Fabio Di Fabio * Bump io.grpc group to 1.60.1 Signed-off-by: Fabio Di Fabio * Bump io.kubernetes:client-java to 18.0.1 Signed-off-by: Fabio Di Fabio * Bump io.netty group to 4.1.104.Final Signed-off-by: Fabio Di Fabio * Bump net.java.dev.jna:jna to 5.14.0 Signed-off-by: Fabio Di Fabio * Bump org.apache.commons:commons-compress to 1.25.0 Signed-off-by: Fabio Di Fabio * Bump org.apache.commons:commons-lang3 to 3.14.0 Signed-off-by: Fabio Di Fabio * Bump org.apache.commons:commons-text to 1.11.0 Signed-off-by: Fabio Di Fabio * Bump org.apache.logging.log4j group to 2.22.1 Signed-off-by: Fabio Di Fabio * Redorder io.tmio group Signed-off-by: Fabio Di Fabio * Bump org.assertj:assertj-core to 3.25.1 Signed-off-by: Fabio Di Fabio * Bump org.bouncycastle group to 1.77 Signed-off-by: Fabio Di Fabio * Bump org.fusesource.jansi:jansi to 2.4.1 Signed-off-by: Fabio Di Fabio * Bump org.immutables group 2.10.0 Signed-off-by: Fabio Di Fabio * Bump org.java-websocket:Java-WebSocket to 1.5.5 Signed-off-by: Fabio Di Fabio * Bump org.jetbrains.kotlin:kotlin-stdlib to 1.9.22 Signed-off-by: Fabio Di Fabio * Bump org.junit.jupiter group to 5.10.1 Signed-off-by: Fabio Di Fabio * Bump org.jupnp group to 2.7.1 Signed-off-by: Fabio Di Fabio * Bump org.rocksdb:rocksdbjni to 8.9.1 Signed-off-by: Fabio Di Fabio * Bump org.slf4j group to 2.0.10 Signed-off-by: Fabio Di Fabio * Bump org.springframework.security:spring-security-crypto to 6.2.1 Signed-off-by: Fabio Di Fabio * Bump org.testcontainers:testcontainers to 1.19.3 Signed-off-by: Fabio Di Fabio * Bump org.web3j group to 4.10.3 Signed-off-by: Fabio Di Fabio * Bump org.xerial.snappy:snappy-java to 1.1.10.5 Signed-off-by: Fabio Di Fabio * Regenerate gradle verification metadata Signed-off-by: Fabio Di Fabio * Update commons-codec:commons-codec to 1.16.0 Signed-off-by: Fabio Di Fabio * Update org.junit.vintage:junit-vintage-engine to 5.10.1 Signed-off-by: Fabio Di Fabio * Update CHANGELOG Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + acceptance-tests/dsl/build.gradle | 2 +- acceptance-tests/tests/build.gradle | 2 +- gradle/allowed-licenses.json | 4 + gradle/license-normalizer-bundle.json | 2 +- gradle/verification-metadata.xml | 2892 +++++++++---------------- gradle/versions.gradle | 109 +- 7 files changed, 1090 insertions(+), 1922 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ce64c213aa..912122e786b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Additions and Improvements - Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) - Disable transaction handling when the node is not in sync, to avoid unnecessary transaction validation work [#6302](https://github.com/hyperledger/besu/pull/6302) +- Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/acceptance-tests/dsl/build.gradle b/acceptance-tests/dsl/build.gradle index 96a6a1c1bc0..6c9090fe9b9 100644 --- a/acceptance-tests/dsl/build.gradle +++ b/acceptance-tests/dsl/build.gradle @@ -26,7 +26,6 @@ dependencies { implementation project(':testutil') implementation project(':util') - implementation 'com.github.tomakehurst:wiremock-jre8' implementation 'com.google.guava:guava' implementation 'com.google.dagger:dagger' annotationProcessor 'com.google.dagger:dagger-compiler' @@ -45,6 +44,7 @@ dependencies { implementation 'org.web3j:abi' implementation 'org.web3j:besu' implementation 'org.web3j:crypto' + implementation 'org.wiremock:wiremock' implementation 'org.testcontainers:testcontainers' implementation 'org.junit.jupiter:junit-jupiter' diff --git a/acceptance-tests/tests/build.gradle b/acceptance-tests/tests/build.gradle index 464805779c8..b20b96b2735 100644 --- a/acceptance-tests/tests/build.gradle +++ b/acceptance-tests/tests/build.gradle @@ -55,7 +55,6 @@ dependencies { testImplementation project(':testutil') testImplementation project(':util') - testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone' testImplementation 'commons-io:commons-io' testImplementation 'io.grpc:grpc-all' testImplementation 'io.grpc:grpc-core' @@ -84,6 +83,7 @@ dependencies { testImplementation 'org.web3j:abi' testImplementation 'org.web3j:besu' testImplementation 'org.web3j:core' + testImplementation 'org.wiremock:wiremock' testRuntimeOnly 'org.junit.vintage:junit-vintage-engine' } diff --git a/gradle/allowed-licenses.json b/gradle/allowed-licenses.json index a5d4048b224..25270190806 100644 --- a/gradle/allowed-licenses.json +++ b/gradle/allowed-licenses.json @@ -60,6 +60,10 @@ "moduleLicense": "Eclipse Public License - v 1.0", "moduleVersion": "4.13.2", "moduleName": "junit:junit" + }, + { + "moduleName": "org.jetbrains.kotlin:kotlin-stdlib-common", + "moduleVersion": "1.9.22" } ] } \ No newline at end of file diff --git a/gradle/license-normalizer-bundle.json b/gradle/license-normalizer-bundle.json index 341b7ab5f62..8c0af450fd3 100644 --- a/gradle/license-normalizer-bundle.json +++ b/gradle/license-normalizer-bundle.json @@ -50,9 +50,9 @@ { "bundleName" : "Apache-2.0", "licenseNamePattern" : ".*Apache License,?( Version)? 2.*" }, { "bundleName" : "Apache-2.0", "licenseUrlPattern" : ".*(www\\.)?opensource\\.org/licenses/Apache-2\\.0.*" }, { "bundleName" : "Apache-2.0", "licenseUrlPattern" : ".*www\\.apache\\.org/licenses/LICENSE-2\\.0.*" }, - { "bundleName" : "LGPL-2.1-only", "licenseUrlPattern" : ".*www\\.gnu\\.org/licenses/old-licenses/lgpl-2\\.1\\.html" }, { "bundleName" : "Apache-2.0", "licenseFileContentPattern" : ".*Apache License,?( Version)? 2.*" }, { "bundleName" : "Apache-1.1", "licenseFileContentPattern" : ".*Apache Software License, Version 1\\.1.*" }, + { "bundleName" : "LGPL-2.1-only", "licenseUrlPattern" : ".*www\\.gnu\\.org/licenses/old-licenses/lgpl-2\\.1\\.html" }, { "bundleName" : "CC0-1.0", "licenseNamePattern" : "CC0(( |-)1(\\.0)?)?" }, { "bundleName" : "CC0-1.0", "licenseUrlPattern" : ".*(www\\.)?creativecommons\\.org/publicdomain/zero/1\\.0/" }, { "bundleName" : "CDDL-1.0", "licenseFileContentPattern" : ".*CDDL.*1\\.0" }, diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 8265a2810c4..d0cfe4bd752 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -78,9 +78,12 @@ - - - + + + + + + @@ -133,9 +136,9 @@ - - - + + + @@ -153,11 +156,6 @@ - - - - - @@ -183,11 +181,6 @@ - - - - - @@ -208,11 +201,6 @@ - - - - - @@ -229,11 +217,6 @@ - - - - - @@ -250,6 +233,14 @@ + + + + + + + + @@ -290,6 +281,14 @@ + + + + + + + + @@ -327,14 +326,6 @@ - - - - - - - - @@ -343,12 +334,15 @@ - - - + + + - - + + + + + @@ -356,9 +350,9 @@ - - - + + + @@ -372,12 +366,26 @@ - - - + + + + + + + + + + + + + + + + + - - + + @@ -385,9 +393,9 @@ - - - + + + @@ -414,33 +422,33 @@ - - - + + + - - + + - - - + + + - - - + + + - - + + - - - + + + - - + + @@ -523,11 +531,6 @@ - - - - - @@ -584,27 +587,17 @@ - - - - - - - - - - - + + + - - - - + + - - - + + + @@ -615,35 +608,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -689,14 +653,6 @@ - - - - - - - - @@ -705,15 +661,10 @@ - - - - - - - - + + + @@ -736,6 +687,14 @@ + + + + + + + + @@ -806,54 +765,36 @@ - - - - - - - - - - - - - - - - + + + - - + + - - - + + + - - - - - - - + + - - - + + + - - + + - - - + + + - - + + @@ -864,11 +805,6 @@ - - - - - @@ -909,11 +845,6 @@ - - - - - @@ -922,11 +853,6 @@ - - - - - @@ -947,11 +873,6 @@ - - - - - @@ -960,11 +881,6 @@ - - - - - @@ -1072,9 +988,6 @@ - - - @@ -1281,31 +1194,23 @@ - - - - - - - - - - - + + + - - + + - - - + + + - - + + - - + + @@ -1316,12 +1221,15 @@ - - - + + + - - + + + + + @@ -1329,49 +1237,18 @@ - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - + + + + + @@ -1446,17 +1323,6 @@ - - - - - - - - - - - @@ -1465,6 +1331,17 @@ + + + + + + + + + + + @@ -1473,19 +1350,6 @@ - - - - - - - - - - - - - @@ -1494,25 +1358,19 @@ - - - - - - - - + + + - - - - - + + + + @@ -1523,6 +1381,17 @@ + + + + + + + + + + + @@ -1539,12 +1408,12 @@ - - - + + + - - + + @@ -1556,6 +1425,9 @@ + + + @@ -1584,15 +1456,10 @@ - - - - - - - - + + + @@ -1610,35 +1477,36 @@ - - - - - - - - + + + - - - - - - - - + + + - - - + + + + + + + + + + + + + + @@ -1649,41 +1517,36 @@ - - - - - - + + + - - - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -1694,155 +1557,140 @@ - - - + + + - - + + - - - - - - + + + - - - - + + - - - + + + - - + + - - - + + + - - + + - - - - - - + + + - - - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - - - - + + + - - - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -1853,61 +1701,46 @@ - - - - - - + + + - - - - + + - - - - - - + + + - - - - + + - - - + + + - - - + + + - - + + - - - - - - + + + - - - - + + - - - + + + @@ -1920,12 +1753,12 @@ - - - + + + - - + + @@ -1933,12 +1766,12 @@ - - - + + + - - + + @@ -1949,12 +1782,12 @@ - - - + + + - - + + @@ -1965,20 +1798,20 @@ - - - + + + - - + + - - - + + + - - + + @@ -1990,13 +1823,18 @@ - - - + + + + + + + + @@ -2005,49 +1843,44 @@ - - - - - - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -2058,28 +1891,28 @@ - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -2087,12 +1920,12 @@ - - - + + + - - + + @@ -2104,13 +1937,18 @@ - - - + + + + + + + + @@ -2119,17 +1957,12 @@ - - - - - - - - + + + - - + + @@ -2137,9 +1970,9 @@ - - - + + + @@ -2147,22 +1980,12 @@ - - - - - - - - - - - - - + + + - - + + @@ -2170,12 +1993,12 @@ - - - + + + - - + + @@ -2186,23 +2009,23 @@ - - - + + + - - + + - - - + + + - - + + - - + + @@ -2226,12 +2049,12 @@ - - - + + + - - + + @@ -2239,95 +2062,83 @@ - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - - + + - - - + + + - - + + - - + + - - + + - - - + + + - - - - - - - - - - - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -2365,14 +2176,6 @@ - - - - - - - - @@ -2428,14 +2231,6 @@ - - - - - - - - @@ -2458,14 +2253,6 @@ - - - - - - - - @@ -2477,14 +2264,6 @@ - - - - - - - - @@ -2507,14 +2286,6 @@ - - - - - - - - @@ -2548,14 +2319,6 @@ - - - - - - - - @@ -2567,14 +2330,6 @@ - - - - - - - - @@ -2586,14 +2341,6 @@ - - - - - - - - @@ -2618,14 +2365,6 @@ - - - - - - - - @@ -2634,14 +2373,6 @@ - - - - - - - - @@ -2750,11 +2481,6 @@ - - - - - @@ -2763,11 +2489,6 @@ - - - - - @@ -2784,11 +2505,6 @@ - - - - - @@ -2805,11 +2521,6 @@ - - - - - @@ -2836,9 +2547,7 @@ - - - + @@ -3003,16 +2712,6 @@ - - - - - - - - - - @@ -3029,11 +2728,6 @@ - - - - - @@ -3042,11 +2736,6 @@ - - - - - @@ -3057,11 +2746,6 @@ - - - - - @@ -3080,11 +2764,6 @@ - - - - - @@ -3119,11 +2798,6 @@ - - - - - @@ -3140,11 +2814,6 @@ - - - - - @@ -3153,11 +2822,6 @@ - - - - - @@ -3171,19 +2835,14 @@ - - - + + + - - - - - - - - + + + @@ -3210,14 +2869,6 @@ - - - - - - - - @@ -3268,12 +2919,12 @@ - - - + + + - - + + @@ -3289,9 +2940,9 @@ - - - + + + @@ -3304,38 +2955,33 @@ - - - - - - - - + + + - - - + + + - - + + - - - + + + - - + + - - - + + + @@ -3343,11 +2989,6 @@ - - - - - @@ -3364,36 +3005,20 @@ - - - - - - - - - - - - - - - - - - - + + + - - + + - - - + + + - - + + @@ -3446,11 +3071,6 @@ - - - - - @@ -3459,16 +3079,6 @@ - - - - - - - - - - @@ -3479,11 +3089,6 @@ - - - - - @@ -3532,14 +3137,6 @@ - - - - - - - - @@ -3549,6 +3146,9 @@ + + + @@ -3570,6 +3170,9 @@ + + + @@ -3600,11 +3203,6 @@ - - - - - @@ -3615,11 +3213,6 @@ - - - - - @@ -3660,38 +3253,17 @@ - - - + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - + + + @@ -3730,9 +3302,9 @@ - - - + + + @@ -3751,38 +3323,38 @@ - - - + + + - - + + - - - + + + - - - + + + - - + + - - - + + + - - + + - - - + + + @@ -3793,112 +3365,51 @@ - - - + + + - - - + + + - - - + + + - - - - + + - - - - - - - - - - - + + + - - - + + + - - - - + + - - - + + + - - + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -3957,22 +3468,12 @@ - - - - - - - - + + + - - - - - - - + + @@ -3996,15 +3497,10 @@ - - - - - - - - + + + @@ -4025,25 +3521,20 @@ - - - - - - - - + + + - - - + + + - - + + @@ -4097,9 +3588,9 @@ - - - + + + @@ -4120,55 +3611,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -4198,9 +3640,9 @@ - - - + + + @@ -4259,205 +3701,186 @@ - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - - - - + + + - - - + + + - - + + - - - + + + - - - - - - + + + - - - + + + - - - - + + - - - + + + - - + + - - - + + + - - + + - - - - - - + + + - - - + + + - - + + - - - + + + - - - - - - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - - - - + + + - - - + + + - - + + - - - + + + - - - - + + - - + + + + @@ -5195,15 +4618,10 @@ - - - - - - - - + + + @@ -5237,14 +4655,6 @@ - - - - - - - - @@ -5266,11 +4676,6 @@ - - - - - @@ -5323,9 +4728,6 @@ - - - @@ -5334,9 +4736,6 @@ - - - @@ -5345,9 +4744,6 @@ - - - @@ -5356,9 +4752,6 @@ - - - @@ -5367,9 +4760,6 @@ - - - @@ -5378,54 +4768,36 @@ - - - - - - - - + + + - - - - - - - - + + + - - - - - - - - - - - + + + - - - + + + @@ -5473,17 +4845,12 @@ - - - + + + - - - - - - - + + @@ -5539,14 +4906,6 @@ - - - - - - - - @@ -5555,12 +4914,15 @@ - - - + + + + + + - - + + @@ -5581,14 +4943,6 @@ - - - - - - - - @@ -5597,6 +4951,16 @@ + + + + + + + + + + @@ -5605,36 +4969,28 @@ - - - - - - - - - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -5650,36 +5006,28 @@ - - - - - - - - - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -5714,14 +5062,6 @@ - - - - - - - - @@ -5851,6 +5191,9 @@ + + + @@ -5858,18 +5201,10 @@ - - - - - - - - - - - + + + @@ -5885,18 +5220,10 @@ - - - - - - - - - - - + + + @@ -5912,18 +5239,10 @@ - - - - - - - - - - - + + + @@ -5931,15 +5250,15 @@ - - - + + + - - + + - - + + @@ -5950,23 +5269,12 @@ - - - - - - - - - - - - - - + + + - - + + @@ -5974,12 +5282,15 @@ - - - + + + + + + - - + + @@ -5990,25 +5301,19 @@ - - - - - - - - - - - - - - + + + + + + + + @@ -6022,34 +5327,21 @@ - - - + + + - - - - - - - - - - - - - - - + + + + + - - - @@ -6067,47 +5359,27 @@ - - - - - - - - - - - - - + + + - - - - - - - - + + + - - - - - @@ -6126,11 +5398,6 @@ - - - - - @@ -6139,11 +5406,6 @@ - - - - - @@ -6181,11 +5443,6 @@ - - - - - @@ -6202,11 +5459,6 @@ - - - - - @@ -6246,6 +5498,17 @@ + + + + + + + + + + + @@ -6280,12 +5543,12 @@ - - - + + + - - + + @@ -6296,6 +5559,11 @@ + + + + + @@ -6365,17 +5633,12 @@ - - - - - - + + + - - - - + + @@ -6394,35 +5657,25 @@ - - - - - - + + + - - - - + + - - - + + + - - - + + + - - - - - - - + + @@ -6435,14 +5688,9 @@ - - - - - - - - + + + @@ -6476,11 +5724,6 @@ - - - - - @@ -6496,99 +5739,28 @@ - - - + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - - + + + @@ -6599,6 +5771,17 @@ + + + + + + + + + + + @@ -6607,30 +5790,15 @@ - - - - - - - - - - - + + + - - - - - - - + + - - - - + + @@ -6641,6 +5809,17 @@ + + + + + + + + + + + @@ -6657,17 +5836,15 @@ - - - + + + - - + + - - - - + + @@ -6678,30 +5855,15 @@ - - - - - - - - - - - + + + - - - - + + - - - - - - - + + @@ -6709,6 +5871,17 @@ + + + + + + + + + + + @@ -6717,12 +5890,15 @@ - - - + + + - - + + + + + @@ -6741,12 +5917,15 @@ - - - + + + + + + - - + + @@ -6757,14 +5936,6 @@ - - - - - - - - @@ -6786,11 +5957,6 @@ - - - - - @@ -6799,46 +5965,52 @@ - - - + + + + + + + + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - - + + + - - + + @@ -6860,14 +6032,6 @@ - - - - - - - - diff --git a/gradle/versions.gradle b/gradle/versions.gradle index f239370ab7a..45626801fc0 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -30,14 +30,11 @@ dependencyManagement { dependency 'com.github.ben-manes.caffeine:caffeine:3.1.8' - dependencySet(group: 'com.github.tomakehurst', version: '2.35.0') { - entry'wiremock-jre8-standalone' - entry'wiremock-jre8' - } + dependency 'com.github.oshi:oshi-core:6.4.10' - dependency 'com.google.auto.service:auto-service:1.0.1' + dependency 'com.google.auto.service:auto-service:1.1.1' - dependencySet(group: 'com.google.dagger', version: '2.45') { + dependencySet(group: 'com.google.dagger', version: '2.50') { entry'dagger-compiler' entry'dagger' } @@ -51,37 +48,37 @@ dependencyManagement { dependency 'com.google.guava:guava:31.1-jre' - dependency 'com.graphql-java:graphql-java:20.1' + dependency 'com.graphql-java:graphql-java:21.3' - dependency 'com.splunk.logging:splunk-library-javalogging:1.11.5' + dependency 'com.splunk.logging:splunk-library-javalogging:1.11.8' - dependency 'com.squareup.okhttp3:okhttp:4.10.0' + dependency 'com.squareup.okhttp3:okhttp:4.12.0' - dependency 'commons-codec:commons-codec:1.15' + dependency 'commons-codec:commons-codec:1.16.0' - dependency 'commons-io:commons-io:2.11.0' + dependency 'commons-io:commons-io:2.15.1' - dependency 'dnsjava:dnsjava:3.5.2' + dependency 'dnsjava:dnsjava:3.5.3' - dependencySet(group: 'info.picocli', version: '4.7.1') { + dependencySet(group: 'info.picocli', version: '4.7.5') { entry 'picocli' entry 'picocli-codegen' } - dependencySet(group: 'io.grpc', version: '1.59.0') { + dependencySet(group: 'io.grpc', version: '1.60.1') { entry 'grpc-all' entry 'grpc-core' entry 'grpc-netty' entry 'grpc-stub' } - dependency 'io.kubernetes:client-java:18.0.0' + dependency 'io.kubernetes:client-java:18.0.1' - dependency 'io.netty:netty-all:4.1.100.Final' + dependency 'io.netty:netty-all:4.1.104.Final' dependency 'io.netty:netty-tcnative-boringssl-static:2.0.62.Final' - dependency group: 'io.netty', name: 'netty-transport-native-epoll', version:'4.1.100.Final', classifier: 'linux-x86_64' - dependency group: 'io.netty', name: 'netty-transport-native-kqueue', version:'4.1.100.Final', classifier: 'osx-x86_64' - dependency 'io.netty:netty-transport-native-unix-common:4.1.100.Final' + dependency group: 'io.netty', name: 'netty-transport-native-epoll', version:'4.1.104.Final', classifier: 'linux-x86_64' + dependency group: 'io.netty', name: 'netty-transport-native-kqueue', version:'4.1.104.Final', classifier: 'osx-x86_64' + dependency 'io.netty:netty-transport-native-unix-common:4.1.104.Final' dependency 'io.opentelemetry:opentelemetry-api:1.24.0' dependency 'io.opentelemetry:opentelemetry-exporter-otlp:1.24.0' @@ -108,6 +105,20 @@ dependencyManagement { dependency 'io.reactivex.rxjava2:rxjava:2.2.21' + dependencySet(group: 'io.tmio', version: '2.4.2') { + entry 'tuweni-bytes' + entry 'tuweni-config' + entry 'tuweni-concurrent' + entry 'tuweni-crypto' + entry 'tuweni-devp2p' + entry 'tuweni-dns-discovery' + entry 'tuweni-io' + entry 'tuweni-net' + entry 'tuweni-rlp' + entry 'tuweni-toml' + entry 'tuweni-units' + } + dependencySet(group: 'io.vertx', version: '4.3.5') { entry 'vertx-auth-jwt' entry 'vertx-codegen' @@ -121,43 +132,29 @@ dependencyManagement { dependency 'junit:junit:4.13.2' - dependency 'net.java.dev.jna:jna:5.13.0' + dependency 'net.java.dev.jna:jna:5.14.0' - dependency 'org.apache.commons:commons-compress:1.23.0' - dependency 'org.apache.commons:commons-lang3:3.12.0' - dependency 'org.apache.commons:commons-text:1.10.0' + dependency 'org.apache.commons:commons-compress:1.25.0' + dependency 'org.apache.commons:commons-lang3:3.14.0' + dependency 'org.apache.commons:commons-text:1.11.0' - dependencySet(group: 'org.apache.logging.log4j', version: '2.20.0') { + dependencySet(group: 'org.apache.logging.log4j', version: '2.22.1') { entry 'log4j-api' entry 'log4j-core' entry 'log4j-jul' entry 'log4j-slf4j2-impl' } - dependencySet(group: 'io.tmio', version: '2.4.2') { - entry 'tuweni-bytes' - entry 'tuweni-config' - entry 'tuweni-concurrent' - entry 'tuweni-crypto' - entry 'tuweni-devp2p' - entry 'tuweni-dns-discovery' - entry 'tuweni-io' - entry 'tuweni-net' - entry 'tuweni-rlp' - entry 'tuweni-toml' - entry 'tuweni-units' - } - - dependency 'org.assertj:assertj-core:3.24.2' + dependency 'org.assertj:assertj-core:3.25.1' dependency 'org.awaitility:awaitility:4.2.0' - dependencySet(group: 'org.bouncycastle', version: '1.76') { + dependencySet(group: 'org.bouncycastle', version: '1.77') { entry'bcpkix-jdk18on' entry'bcprov-jdk18on' } - dependency 'org.fusesource.jansi:jansi:2.4.0' + dependency 'org.fusesource.jansi:jansi:2.4.1' dependency 'org.openjdk.jol:jol-core:0.17' dependency 'tech.pegasys:jc-kzg-4844:0.8.0' @@ -170,16 +167,16 @@ dependencyManagement { entry 'blake2bf' } - dependencySet(group: 'org.immutables', version: '2.9.3') { + dependencySet(group: 'org.immutables', version: '2.10.0') { entry 'value-annotations' entry 'value' } - dependency 'org.java-websocket:Java-WebSocket:1.5.3' + dependency 'org.java-websocket:Java-WebSocket:1.5.5' - dependency 'org.jetbrains.kotlin:kotlin-stdlib:1.8.10' + dependency 'org.jetbrains.kotlin:kotlin-stdlib:1.9.22' - dependencySet(group: 'org.junit.jupiter', version: '5.8.2') { + dependencySet(group: 'org.junit.jupiter', version: '5.10.1') { entry 'junit-jupiter' entry 'junit-jupiter-api' entry 'junit-jupiter-engine' @@ -188,9 +185,9 @@ dependencyManagement { dependency 'org.junit.platform:junit-platform-runner:1.9.2' - dependency 'org.junit.vintage:junit-vintage-engine:5.9.2' + dependency 'org.junit.vintage:junit-vintage-engine:5.10.1' - dependencySet(group: 'org.jupnp', version:'2.7.0') { + dependencySet(group: 'org.jupnp', version:'2.7.1') { entry 'org.jupnp.support' entry 'org.jupnp' } @@ -207,31 +204,33 @@ dependencyManagement { dependency 'org.owasp.encoder:encoder:1.2.3' - dependency 'org.rocksdb:rocksdbjni:8.3.2' + dependency 'org.rocksdb:rocksdbjni:8.9.1' - dependencySet(group: 'org.slf4j', version:'2.0.7') { + dependencySet(group: 'org.slf4j', version:'2.0.10') { entry 'slf4j-api' entry 'slf4j-nop' } - dependency 'org.springframework.security:spring-security-crypto:6.0.2' + dependency 'org.springframework.security:spring-security-crypto:6.2.1' - dependency 'org.testcontainers:testcontainers:1.17.6' + dependency 'org.testcontainers:testcontainers:1.19.3' dependency 'org.web3j:quorum:4.9.5' - dependencySet(group: 'org.web3j', version: '4.9.7') { + dependencySet(group: 'org.web3j', version: '4.10.3') { entry 'abi' entry 'besu' entry 'core' entry 'crypto' } - dependency 'org.xerial.snappy:snappy-java:1.1.9.1' + dependencySet(group: 'org.wiremock', version: '3.3.1') { + entry 'wiremock' + } + + dependency 'org.xerial.snappy:snappy-java:1.1.10.5' dependency 'org.yaml:snakeyaml:2.0' dependency 'tech.pegasys.discovery:discovery:22.2.0' - - dependency 'com.github.oshi:oshi-core:6.4.1' } } From 945a44c598a83003c0bc36d5ccce5c48cf5e3425 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Thu, 11 Jan 2024 09:01:01 -0800 Subject: [PATCH 05/56] add a fallback for docker detection on Mac (#6356) Signed-off-by: garyschulte --- .../java/org/hyperledger/besu/nat/docker/DockerDetector.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nat/src/main/java/org/hyperledger/besu/nat/docker/DockerDetector.java b/nat/src/main/java/org/hyperledger/besu/nat/docker/DockerDetector.java index 5504def9c8d..e7f1cf2328b 100644 --- a/nat/src/main/java/org/hyperledger/besu/nat/docker/DockerDetector.java +++ b/nat/src/main/java/org/hyperledger/besu/nat/docker/DockerDetector.java @@ -33,6 +33,8 @@ public Optional detect() { return stream .filter(line -> line.contains("/docker")) .findFirst() + // fallback to looking for /.dockerenv in case we are running on Docker for Mac + .or(() -> Optional.ofNullable(Files.exists(Paths.get("/.dockerenv")) ? "docker" : null)) .map(__ -> NatMethod.DOCKER); } catch (IOException e) { return Optional.empty(); From c66054633d1a8cc38bbcb22e6f3e31e61464874a Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 12 Jan 2024 01:53:09 +0100 Subject: [PATCH 06/56] Fix test flackyness of acceptanceTestsPermissioning (#6384) Signed-off-by: Fabio Di Fabio Signed-off-by: Sally MacFarlane Co-authored-by: Sally MacFarlane --- .circleci/config.yml | 6 +-- .../dsl/condition/eth/EthConditions.java | 4 ++ .../condition/eth/SyncingStatusCondition.java | 40 +++++++++++++++++++ .../acceptance/dsl/node/cluster/Cluster.java | 7 +--- .../eth/EthSyncingTransaction.java | 40 +++++++++++++++++++ .../dsl/transaction/eth/EthTransactions.java | 4 ++ ...rtContractPermissioningAcceptanceTest.java | 5 +++ ...ContractPermissioningV2AcceptanceTest.java | 5 +++ .../simple_permissioning_genesis.json | 2 +- .../simple_permissioning_v2_genesis.json | 2 +- 10 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/SyncingStatusCondition.java create mode 100644 acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthSyncingTransaction.java diff --git a/.circleci/config.yml b/.circleci/config.yml index bcd81e45cbe..2f008df2578 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -269,7 +269,7 @@ jobs: - capture_test_logs acceptanceTestsPermissioning: - executor: besu_executor_xl + executor: besu_executor_med steps: - prepare - attach_workspace: @@ -278,7 +278,7 @@ jobs: name: AcceptanceTests (Non-Mainnet) no_output_timeout: 20m command: | - ./gradlew --no-daemon acceptanceTestPermissioning + ./gradlew --no-daemon --max-workers=1 acceptanceTestPermissioning - capture_test_results - capture_test_logs @@ -428,12 +428,12 @@ workflows: - acceptanceTestsPermissioning: requires: - assemble - - acceptanceTestsCliqueBft - buildDocker: requires: - assemble - buildArm64Docker: requires: + - buildDocker - assemble - publish: filters: diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/EthConditions.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/EthConditions.java index c4ee1347278..d3a86bafa86 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/EthConditions.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/EthConditions.java @@ -77,6 +77,10 @@ public Condition miningStatus(final boolean isMining) { return new MiningStatusCondition(transactions.mining(), isMining); } + public Condition syncingStatus(final boolean isSyncing) { + return new SyncingStatusCondition(transactions.syncing(), isSyncing); + } + public Condition expectNewPendingTransactions( final BigInteger filterId, final List transactionHashes) { return new NewPendingTransactionFilterChangesCondition( diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/SyncingStatusCondition.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/SyncingStatusCondition.java new file mode 100644 index 00000000000..5e01a0d4a90 --- /dev/null +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/SyncingStatusCondition.java @@ -0,0 +1,40 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.tests.acceptance.dsl.condition.eth; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.tests.acceptance.dsl.WaitUtils; +import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition; +import org.hyperledger.besu.tests.acceptance.dsl.node.Node; +import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthSyncingTransaction; + +public class SyncingStatusCondition implements Condition { + + private final EthSyncingTransaction transaction; + private final boolean syncingMiningStatus; + + public SyncingStatusCondition( + final EthSyncingTransaction transaction, final boolean syncingStatus) { + this.transaction = transaction; + this.syncingMiningStatus = syncingStatus; + } + + @Override + public void verify(final Node node) { + WaitUtils.waitFor( + 10, () -> assertThat(node.execute(transaction)).isEqualTo(syncingMiningStatus)); + } +} diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java index 16a02874262..b454894e066 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java @@ -86,11 +86,8 @@ public void start(final List nodes) { final Optional bootnode = selectAndStartBootnode(nodes); nodes.parallelStream() - .filter( - node -> { - LOG.info("starting non-bootnode {}", node.getName()); - return bootnode.map(boot -> boot != node).orElse(true); - }) + .filter(node -> bootnode.map(boot -> boot != node).orElse(true)) + .peek(node -> LOG.info("starting non-bootnode {}", node.getName())) .forEach(this::startNode); if (clusterConfiguration.isAwaitPeerDiscovery()) { diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthSyncingTransaction.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthSyncingTransaction.java new file mode 100644 index 00000000000..21b628abef2 --- /dev/null +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthSyncingTransaction.java @@ -0,0 +1,40 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.tests.acceptance.dsl.transaction.eth; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.tests.acceptance.dsl.transaction.NodeRequests; +import org.hyperledger.besu.tests.acceptance.dsl.transaction.Transaction; + +import java.io.IOException; + +import org.web3j.protocol.core.methods.response.EthSyncing; + +public class EthSyncingTransaction implements Transaction { + + EthSyncingTransaction() {} + + @Override + public Boolean execute(final NodeRequests node) { + try { + EthSyncing response = node.eth().ethSyncing().send(); + assertThat(response).isNotNull(); + return response.isSyncing(); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthTransactions.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthTransactions.java index 84882b822d0..b8b72052d0b 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthTransactions.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthTransactions.java @@ -73,6 +73,10 @@ public EthMiningTransaction mining() { return new EthMiningTransaction(); } + public EthSyncingTransaction syncing() { + return new EthSyncingTransaction(); + } + public EthNewPendingTransactionFilterTransaction newPendingTransactionsFilter() { return new EthNewPendingTransactionFilterTransaction(); } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningAcceptanceTest.java index c5fbb693043..e9442bc0077 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningAcceptanceTest.java @@ -49,6 +49,11 @@ public void setUp() { permissionedNode.verify(admin.addPeer(bootnode)); permissionedNode.verify(admin.addPeer(allowedNode)); + + allowedNode.verify(eth.syncingStatus(false)); + bootnode.verify(eth.syncingStatus(false)); + permissionedNode.verify(eth.syncingStatus(false)); + forbiddenNode.verify(eth.syncingStatus(false)); } @Test diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java index 72aab9cb7e4..b04bad5644d 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java @@ -46,6 +46,11 @@ public void setUp() { permissionedNode.execute(allowNode(permissionedNode)); permissionedNode.verify(connectionIsAllowed(permissionedNode)); + + allowedNode.verify(eth.syncingStatus(false)); + bootnode.verify(eth.syncingStatus(false)); + permissionedNode.verify(eth.syncingStatus(false)); + forbiddenNode.verify(eth.syncingStatus(false)); } @Test diff --git a/acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_genesis.json b/acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_genesis.json index 04ee1a4882e..5e8bc92a46a 100644 --- a/acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_genesis.json +++ b/acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_genesis.json @@ -4,7 +4,7 @@ "londonBlock": 0, "zeroBaseFee": true, "ethash": { - "fixeddifficulty": 100 + "fixeddifficulty": 500 } }, "nonce": "0x42", diff --git a/acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_v2_genesis.json b/acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_v2_genesis.json index b7ec620f310..aab9a35cdd3 100644 --- a/acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_v2_genesis.json +++ b/acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_v2_genesis.json @@ -4,7 +4,7 @@ "londonBlock": 0, "zeroBaseFee": true, "ethash": { - "fixeddifficulty": 100 + "fixeddifficulty": 500 } }, "nonce": "0x42", From d918baa4ae93be24a7a4988226d23f07cd190599 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 12 Jan 2024 02:27:59 +0100 Subject: [PATCH 07/56] Upgrade `com.fasterxml.jackson` dependencies (#6378) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + gradle/verification-metadata.xml | 166 ++++++------------ gradle/versions.gradle | 2 +- .../besu/testutil/JsonTestParameters.java | 9 +- 4 files changed, 60 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 912122e786b..64aeb93ab02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) - Disable transaction handling when the node is not in sync, to avoid unnecessary transaction validation work [#6302](https://github.com/hyperledger/besu/pull/6302) - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) +- Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index d0cfe4bd752..30d55eb6840 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -131,14 +131,9 @@ - - - - - - - - + + + @@ -181,6 +176,11 @@ + + + + + @@ -225,20 +225,12 @@ - - - + + + - - - - - - - - - - + + @@ -273,20 +265,12 @@ - - - + + + - - - - - - - - - - + + @@ -318,84 +302,55 @@ - - - + + + - - + + - - - + + + - - - - - - - + + - - - - - - - - - - + + - - - + + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - - - - - - - - - - - - - - - - - + + + @@ -1392,14 +1347,6 @@ - - - - - - - - @@ -5301,11 +5248,6 @@ - - - - - @@ -5351,14 +5293,6 @@ - - - - - - - - diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 45626801fc0..b497109d782 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -22,7 +22,7 @@ dependencyManagement { entry 'antlr4-runtime' } - dependencySet(group:'com.fasterxml.jackson.core', version:'2.14.2') { + dependencySet(group:'com.fasterxml.jackson.core', version:'2.16.1') { entry 'jackson-databind' entry 'jackson-datatype' entry 'jackson-datatype-jdk8' diff --git a/testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java b/testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java index f4fc7c49a36..69b21b37c8b 100644 --- a/testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java +++ b/testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java @@ -37,6 +37,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonFactoryBuilder; +import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; @@ -121,7 +123,12 @@ public interface Generator { } private static final ObjectMapper objectMapper = - new ObjectMapper().registerModule(new Jdk8Module()); + new ObjectMapper( + new JsonFactoryBuilder() + .streamReadConstraints( + StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build()) + .build()) + .registerModule(new Jdk8Module()); // The type to which the json file is directly mapped private final Class jsonFileMappedType; From f146e7808317fded0b363b23379f60c51fe1e8c7 Mon Sep 17 00:00:00 2001 From: daniellehrner Date: Fri, 12 Jan 2024 03:02:33 +0100 Subject: [PATCH 08/56] Use mining beneficiary from protocol spec in TraceServiceImpl (#6390) * use mining beneficiary from protocol spec Signed-off-by: Daniel Lehrner --------- Signed-off-by: Daniel Lehrner Co-authored-by: Sally MacFarlane --- CHANGELOG.md | 1 + .../java/org/hyperledger/besu/services/TraceServiceImpl.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64aeb93ab02..f420def0cb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) - Fix Besu Docker images with `openjdk-latest` tags since 23.10.3 using UID 1001 instead of 1000 for the `besu` user [#6360](https://github.com/hyperledger/besu/pull/6360) - Fluent EVM API definition for Tangerine Whistle had incorrect code size validation configured [#6382](https://github.com/hyperledger/besu/pull/6382) +- Correct mining beneficiary for Clique networks in TraceServiceImpl [#6390](https://github.com/hyperledger/besu/pull/6390) ### Download Links diff --git a/besu/src/main/java/org/hyperledger/besu/services/TraceServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/TraceServiceImpl.java index 3ab2961734f..43af4e3866e 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/TraceServiceImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/TraceServiceImpl.java @@ -216,7 +216,7 @@ private List trace( worldUpdater, header, transaction, - header.getCoinbase(), + protocolSpec.getMiningBeneficiaryCalculator().calculateBeneficiary(header), tracer, new CachingBlockHashLookup(header, blockchain), false, From 33df79dd4ae5a25713659ad7f61cdda33bb64380 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 12 Jan 2024 04:50:15 +0100 Subject: [PATCH 09/56] Make transaction pool configurable in acceptance tests (#6380) * Make transaction pool configurable in acceptance tests (#33) Signed-off-by: Fabio Di Fabio * Make BesuNode::getMiningParameters public Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio Co-authored-by: Sally MacFarlane --- .../besu/tests/acceptance/dsl/node/BesuNode.java | 15 ++++++++++++++- .../dsl/node/ProcessBesuNodeRunner.java | 14 +++++++++++--- .../acceptance/dsl/node/ThreadBesuNodeRunner.java | 1 + .../node/configuration/BesuNodeConfiguration.java | 8 ++++++++ .../BesuNodeConfigurationBuilder.java | 11 ++++++++++- .../dsl/node/configuration/BesuNodeFactory.java | 1 + .../tests/acceptance/dsl/privacy/PrivacyNode.java | 1 + 7 files changed, 46 insertions(+), 5 deletions(-) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java index c8613e9a4ba..ba00ac8f22f 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Util; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; @@ -100,6 +101,7 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable private final String name; private MiningParameters miningParameters; + private TransactionPoolConfiguration txPoolConfiguration; private final List runCommand; private PrivacyParameters privacyParameters = PrivacyParameters.DEFAULT; private final JsonRpcConfiguration jsonRpcConfiguration; @@ -135,6 +137,7 @@ public BesuNode( final String name, final Optional dataPath, final MiningParameters miningParameters, + final TransactionPoolConfiguration txPoolConfiguration, final JsonRpcConfiguration jsonRpcConfiguration, final Optional engineRpcConfiguration, final WebSocketConfiguration webSocketConfiguration, @@ -184,6 +187,7 @@ public BesuNode( () -> this.keyPair = KeyPairUtil.loadKeyPair(homeDirectory)); this.name = name; this.miningParameters = miningParameters; + this.txPoolConfiguration = txPoolConfiguration; this.jsonRpcConfiguration = jsonRpcConfiguration; this.engineRpcConfiguration = engineRpcConfiguration; this.webSocketConfiguration = webSocketConfiguration; @@ -661,7 +665,7 @@ public void setBootnodes(final List bootnodes) { this.bootnodes.addAll(bootnodes); } - MiningParameters getMiningParameters() { + public MiningParameters getMiningParameters() { return miningParameters; } @@ -669,6 +673,15 @@ public void setMiningParameters(final MiningParameters miningParameters) { this.miningParameters = miningParameters; } + public TransactionPoolConfiguration getTransactionPoolConfiguration() { + return txPoolConfiguration; + } + + public void setTransactionPoolConfiguration( + final TransactionPoolConfiguration txPoolConfiguration) { + this.txPoolConfiguration = txPoolConfiguration; + } + public PrivacyParameters getPrivacyParameters() { return privacyParameters; } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java index f01c58549bc..538cb89e126 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java @@ -17,8 +17,10 @@ import static com.google.common.base.Preconditions.checkState; import static java.nio.charset.StandardCharsets.UTF_8; +import org.hyperledger.besu.cli.options.TransactionPoolOptions; import org.hyperledger.besu.cli.options.unstable.NetworkingOptions; import org.hyperledger.besu.ethereum.api.jsonrpc.ipc.JsonRpcIpcConfiguration; +import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; @@ -98,6 +100,15 @@ public void startNode(final BesuNode node) { params.add("--p2p-port"); params.add(node.getP2pPort()); + params.addAll( + TransactionPoolOptions.fromConfig( + ImmutableTransactionPoolConfiguration.builder() + .from(node.getTransactionPoolConfiguration()) + .strictTransactionReplayProtectionEnabled( + node.isStrictTxReplayProtectionEnabled()) + .build()) + .getCLIOptions()); + if (node.getMiningParameters().isMiningEnabled()) { params.add("--miner-enabled"); params.add("--miner-coinbase"); @@ -391,9 +402,6 @@ public void startNode(final BesuNode node) { params.add("--auto-log-bloom-caching-enabled"); params.add("false"); - params.add("--strict-tx-replay-protection-enabled"); - params.add(Boolean.toString(node.isStrictTxReplayProtectionEnabled())); - final String level = System.getProperty("root.log.level"); if (level != null) { params.add("--logging=" + level); diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java index 501674ea383..429c91d2b99 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java @@ -182,6 +182,7 @@ public void startNode(final BesuNode node) { final TransactionPoolConfiguration txPoolConfig = ImmutableTransactionPoolConfiguration.builder() + .from(node.getTransactionPoolConfiguration()) .strictTransactionReplayProtectionEnabled(node.isStrictTxReplayProtectionEnabled()) .build(); diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java index 3a61749881a..0c0d66be6a8 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; @@ -39,6 +40,7 @@ public class BesuNodeConfiguration { private final String name; private final Optional dataPath; private final MiningParameters miningParameters; + private final TransactionPoolConfiguration transactionPoolConfiguration; private final JsonRpcConfiguration jsonRpcConfiguration; private final Optional engineRpcConfiguration; private final WebSocketConfiguration webSocketConfiguration; @@ -74,6 +76,7 @@ public class BesuNodeConfiguration { final String name, final Optional dataPath, final MiningParameters miningParameters, + final TransactionPoolConfiguration transactionPoolConfiguration, final JsonRpcConfiguration jsonRpcConfiguration, final Optional engineRpcConfiguration, final WebSocketConfiguration webSocketConfiguration, @@ -106,6 +109,7 @@ public class BesuNodeConfiguration { final Map environment) { this.name = name; this.miningParameters = miningParameters; + this.transactionPoolConfiguration = transactionPoolConfiguration; this.jsonRpcConfiguration = jsonRpcConfiguration; this.engineRpcConfiguration = engineRpcConfiguration; this.webSocketConfiguration = webSocketConfiguration; @@ -147,6 +151,10 @@ public MiningParameters getMiningParameters() { return miningParameters; } + public TransactionPoolConfiguration getTransactionPoolConfiguration() { + return transactionPoolConfiguration; + } + public JsonRpcConfiguration getJsonRpcConfiguration() { return jsonRpcConfiguration; } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java index 2a7f16d24fd..c20e7ec6d11 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java @@ -35,6 +35,7 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; @@ -63,7 +64,8 @@ public class BesuNodeConfigurationBuilder { .mutableInitValues( MutableInitValues.builder().coinbase(AddressHelpers.ofValue(1)).build()) .build(); - + private TransactionPoolConfiguration transactionPoolConfiguration = + TransactionPoolConfiguration.DEFAULT; private JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); private JsonRpcConfiguration engineRpcConfiguration = JsonRpcConfiguration.createEngineDefault(); private WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); @@ -128,6 +130,12 @@ public BesuNodeConfigurationBuilder miningConfiguration(final MiningParameters m return this; } + public BesuNodeConfigurationBuilder transactionPoolConfiguration( + final TransactionPoolConfiguration transactionPoolConfiguration) { + this.transactionPoolConfiguration = transactionPoolConfiguration; + return this; + } + public BesuNodeConfigurationBuilder jsonRpcConfiguration( final JsonRpcConfiguration jsonRpcConfiguration) { this.jsonRpcConfiguration = jsonRpcConfiguration; @@ -503,6 +511,7 @@ public BesuNodeConfiguration build() { name, dataPath, miningParameters, + transactionPoolConfiguration, jsonRpcConfiguration, Optional.of(engineRpcConfiguration), webSocketConfiguration, diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java index 4b24071ab33..ed26587812c 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java @@ -64,6 +64,7 @@ public BesuNode create(final BesuNodeConfiguration config) throws IOException { config.getName(), config.getDataPath(), config.getMiningParameters(), + config.getTransactionPoolConfiguration(), config.getJsonRpcConfiguration(), config.getEngineRpcConfiguration(), config.getWebSocketConfiguration(), diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java index 3d397e24e27..4ec4396d6b5 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java @@ -100,6 +100,7 @@ public PrivacyNode( besuConfig.getName(), besuConfig.getDataPath(), besuConfig.getMiningParameters(), + besuConfig.getTransactionPoolConfiguration(), besuConfig.getJsonRpcConfiguration(), besuConfig.getEngineRpcConfiguration(), besuConfig.getWebSocketConfiguration(), From 540f5a4c768a4c810e2e16a4480dc9fa9d319935 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 12 Jan 2024 10:21:49 +0100 Subject: [PATCH 10/56] Introduce TransactionEvaluationContext to pass data between selectors and plugin (#6381) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../txselection/BlockTransactionSelector.java | 91 +++++++++++-------- .../TransactionEvaluationContext.java | 66 ++++++++++++++ .../AbstractTransactionSelector.java | 10 +- .../AllAcceptingTransactionSelector.java | 9 +- .../BlobPriceTransactionSelector.java | 11 ++- .../BlockSizeTransactionSelector.java | 13 +-- ...nPriorityFeePerGasTransactionSelector.java | 11 ++- .../selectors/PriceTransactionSelector.java | 29 +++--- .../ProcessingResultTransactionSelector.java | 10 +- .../AbstractBlockTransactionSelectorTest.java | 39 +++++--- ...orityFeePerGasTransactionSelectorTest.java | 33 ++++--- plugin-api/build.gradle | 2 +- .../PluginTransactionSelector.java | 17 ++-- .../TransactionEvaluationContext.java | 57 ++++++++++++ 15 files changed, 277 insertions(+), 122 deletions(-) create mode 100644 ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/TransactionEvaluationContext.java create mode 100644 plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/TransactionEvaluationContext.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f420def0cb8..abcc2ab3c85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Additions and Improvements - Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) - Disable transaction handling when the node is not in sync, to avoid unnecessary transaction validation work [#6302](https://github.com/hyperledger/besu/pull/6302) +- Introduce TransactionEvaluationContext to pass data between transaction selectors and plugin, during block creation [#6381](https://github.com/hyperledger/besu/pull/6381) - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java index 4a0707e76f7..944cf426d73 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java @@ -228,26 +228,42 @@ private TransactionSelectionResult evaluateTransaction( final PendingTransaction pendingTransaction) { checkCancellation(); - final Stopwatch evaluationTimer = Stopwatch.createStarted(); + final TransactionEvaluationContext evaluationContext = + createTransactionEvaluationContext(pendingTransaction); - TransactionSelectionResult selectionResult = evaluatePreProcessing(pendingTransaction); + TransactionSelectionResult selectionResult = evaluatePreProcessing(evaluationContext); if (!selectionResult.selected()) { - return handleTransactionNotSelected(pendingTransaction, selectionResult, evaluationTimer); + return handleTransactionNotSelected(evaluationContext, selectionResult); } final WorldUpdater txWorldStateUpdater = blockWorldStateUpdater.updater(); final TransactionProcessingResult processingResult = processTransaction(pendingTransaction, txWorldStateUpdater); - var postProcessingSelectionResult = - evaluatePostProcessing(pendingTransaction, processingResult); + var postProcessingSelectionResult = evaluatePostProcessing(evaluationContext, processingResult); if (postProcessingSelectionResult.selected()) { - return handleTransactionSelected( - pendingTransaction, processingResult, txWorldStateUpdater, evaluationTimer); + return handleTransactionSelected(evaluationContext, processingResult, txWorldStateUpdater); } return handleTransactionNotSelected( - pendingTransaction, postProcessingSelectionResult, txWorldStateUpdater, evaluationTimer); + evaluationContext, postProcessingSelectionResult, txWorldStateUpdater); + } + + private TransactionEvaluationContext createTransactionEvaluationContext( + final PendingTransaction pendingTransaction) { + final Wei transactionGasPriceInBlock = + blockSelectionContext + .feeMarket() + .getTransactionPriceCalculator() + .price( + pendingTransaction.getTransaction(), + blockSelectionContext.processableBlockHeader().getBaseFee()); + + return new TransactionEvaluationContext( + pendingTransaction, + Stopwatch.createStarted(), + transactionGasPriceInBlock, + blockSelectionContext.miningParameters().getMinTransactionGasPrice()); } /** @@ -256,21 +272,20 @@ private TransactionSelectionResult evaluateTransaction( * it then processes it through external selectors. If the transaction is selected by all * selectors, it returns SELECTED. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection session data. * @return The result of the transaction selection process. */ private TransactionSelectionResult evaluatePreProcessing( - final PendingTransaction pendingTransaction) { + final TransactionEvaluationContext evaluationContext) { for (var selector : transactionSelectors) { TransactionSelectionResult result = - selector.evaluateTransactionPreProcessing( - pendingTransaction, transactionSelectionResults); + selector.evaluateTransactionPreProcessing(evaluationContext, transactionSelectionResults); if (!result.equals(SELECTED)) { return result; } } - return pluginTransactionSelector.evaluateTransactionPreProcessing(pendingTransaction); + return pluginTransactionSelector.evaluateTransactionPreProcessing(evaluationContext); } /** @@ -279,24 +294,24 @@ private TransactionSelectionResult evaluatePreProcessing( * whether the transaction should be included in a block. If the transaction is selected by all * selectors, it returns SELECTED. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection session data. * @param processingResult The result of the transaction processing. * @return The result of the transaction selection process. */ private TransactionSelectionResult evaluatePostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionProcessingResult processingResult) { for (var selector : transactionSelectors) { TransactionSelectionResult result = selector.evaluateTransactionPostProcessing( - pendingTransaction, transactionSelectionResults, processingResult); + evaluationContext, transactionSelectionResults, processingResult); if (!result.equals(SELECTED)) { return result; } } return pluginTransactionSelector.evaluateTransactionPostProcessing( - pendingTransaction, processingResult); + evaluationContext, processingResult); } /** @@ -328,18 +343,16 @@ private TransactionProcessingResult processTransaction( * receipt, updating the TransactionSelectionResults with the selected transaction, and notifying * the external transaction selector. * - * @param pendingTransaction The pending transaction. + * @param evaluationContext The current selection session data. * @param processingResult The result of the transaction processing. * @param txWorldStateUpdater The world state updater. - * @param evaluationTimer tracks the evaluation elapsed time * @return The result of the transaction selection process. */ private TransactionSelectionResult handleTransactionSelected( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionProcessingResult processingResult, - final WorldUpdater txWorldStateUpdater, - final Stopwatch evaluationTimer) { - final Transaction transaction = pendingTransaction.getTransaction(); + final WorldUpdater txWorldStateUpdater) { + final Transaction transaction = evaluationContext.getTransaction(); final long gasUsedByTransaction = transaction.getGasLimit() - processingResult.getGasRemaining(); @@ -363,13 +376,14 @@ private TransactionSelectionResult handleTransactionSelected( transaction.getType(), processingResult, worldState, cumulativeGasUsed); transactionSelectionResults.updateSelected( - pendingTransaction.getTransaction(), receipt, gasUsedByTransaction, blobGasUsed); + transaction, receipt, gasUsedByTransaction, blobGasUsed); } } if (tooLate) { // even if this tx passed all the checks, it is too late to include it in this block, // so we need to treat it as not selected + final var evaluationTimer = evaluationContext.getEvaluationTimer(); // check if this tx took too much to evaluate, and in case remove it from the pool final TransactionSelectionResult timeoutSelectionResult; @@ -395,15 +409,15 @@ private TransactionSelectionResult handleTransactionSelected( // do not rely on the presence of this result, since by the time it is added, the code // reading it could have been already executed by another thread return handleTransactionNotSelected( - pendingTransaction, timeoutSelectionResult, txWorldStateUpdater, evaluationTimer); + evaluationContext, timeoutSelectionResult, txWorldStateUpdater); } - pluginTransactionSelector.onTransactionSelected(pendingTransaction, processingResult); + pluginTransactionSelector.onTransactionSelected(evaluationContext, processingResult); blockWorldStateUpdater = worldState.updater(); LOG.atTrace() .setMessage("Selected {} for block creation, evaluated in {}") .addArgument(transaction::toTraceLog) - .addArgument(evaluationTimer) + .addArgument(evaluationContext.getPendingTransaction()) .log(); return SELECTED; } @@ -413,36 +427,35 @@ private TransactionSelectionResult handleTransactionSelected( * TransactionSelectionResults with the unselected transaction, and notifies the external * transaction selector. * - * @param pendingTransaction The unselected pending transaction. + * @param evaluationContext The current selection session data. * @param selectionResult The result of the transaction selection process. - * @param evaluationTimer tracks the evaluation elapsed time * @return The result of the transaction selection process. */ private TransactionSelectionResult handleTransactionNotSelected( - final PendingTransaction pendingTransaction, - final TransactionSelectionResult selectionResult, - final Stopwatch evaluationTimer) { + final TransactionEvaluationContext evaluationContext, + final TransactionSelectionResult selectionResult) { + + final var pendingTransaction = evaluationContext.getPendingTransaction(); transactionSelectionResults.updateNotSelected( - pendingTransaction.getTransaction(), selectionResult); - pluginTransactionSelector.onTransactionNotSelected(pendingTransaction, selectionResult); + evaluationContext.getTransaction(), selectionResult); + pluginTransactionSelector.onTransactionNotSelected(evaluationContext, selectionResult); LOG.atTrace() .setMessage("Not selected {} for block creation with result {}, evaluated in {}") .addArgument(pendingTransaction::toTraceLog) .addArgument(selectionResult) - .addArgument(evaluationTimer) + .addArgument(evaluationContext.getEvaluationTimer()) .log(); return selectionResult; } private TransactionSelectionResult handleTransactionNotSelected( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResult selectionResult, - final WorldUpdater txWorldStateUpdater, - final Stopwatch evaluationTimer) { + final WorldUpdater txWorldStateUpdater) { txWorldStateUpdater.revert(); - return handleTransactionNotSelected(pendingTransaction, selectionResult, evaluationTimer); + return handleTransactionNotSelected(evaluationContext, selectionResult); } private void checkCancellation() { diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/TransactionEvaluationContext.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/TransactionEvaluationContext.java new file mode 100644 index 00000000000..20609ebdc84 --- /dev/null +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/TransactionEvaluationContext.java @@ -0,0 +1,66 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.blockcreation.txselection; + +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; + +import com.google.common.base.Stopwatch; + +public class TransactionEvaluationContext + implements org.hyperledger.besu.plugin.services.txselection.TransactionEvaluationContext< + PendingTransaction> { + private final org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction + pendingTransaction; + private final Stopwatch evaluationTimer; + private final Wei transactionGasPrice; + private final Wei minGasPrice; + + public TransactionEvaluationContext( + final PendingTransaction pendingTransaction, + final Stopwatch evaluationTimer, + final Wei transactionGasPrice, + final Wei minGasPrice) { + this.pendingTransaction = pendingTransaction; + this.evaluationTimer = evaluationTimer; + this.transactionGasPrice = transactionGasPrice; + this.minGasPrice = minGasPrice; + } + + public Transaction getTransaction() { + return pendingTransaction.getTransaction(); + } + + @Override + public PendingTransaction getPendingTransaction() { + return pendingTransaction; + } + + @Override + public Stopwatch getEvaluationTimer() { + return evaluationTimer; + } + + @Override + public Wei getTransactionGasPrice() { + return transactionGasPrice; + } + + @Override + public Wei getMinGasPrice() { + return minGasPrice; + } +} diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AbstractTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AbstractTransactionSelector.java index 205e803e820..0e7f610a1f3 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AbstractTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AbstractTransactionSelector.java @@ -15,8 +15,8 @@ package org.hyperledger.besu.ethereum.blockcreation.txselection.selectors; import org.hyperledger.besu.ethereum.blockcreation.txselection.BlockSelectionContext; +import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionEvaluationContext; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.plugin.data.TransactionSelectionResult; @@ -34,25 +34,25 @@ public AbstractTransactionSelector(final BlockSelectionContext context) { /** * Evaluates a transaction in the context of other transactions in the same block. * - * @param pendingTransaction The transaction to be evaluated within a block. + * @param evaluationContext The current selection session data. * @param blockTransactionResults The results of other transaction evaluations in the same block. * @return The result of the transaction evaluation */ public abstract TransactionSelectionResult evaluateTransactionPreProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults blockTransactionResults); /** * Evaluates a transaction considering other transactions in the same block and a transaction * processing result. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection session data. * @param blockTransactionResults The results of other transaction evaluations in the same block. * @param processingResult The result of transaction processing. * @return The result of the transaction evaluation */ public abstract TransactionSelectionResult evaluateTransactionPostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults blockTransactionResults, final TransactionProcessingResult processingResult); } diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AllAcceptingTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AllAcceptingTransactionSelector.java index 9032ddca575..a2b8d57ff29 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AllAcceptingTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AllAcceptingTransactionSelector.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.plugin.data.TransactionProcessingResult; import org.hyperledger.besu.plugin.data.TransactionSelectionResult; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; +import org.hyperledger.besu.plugin.services.txselection.TransactionEvaluationContext; /** A TransactionSelector that unconditionally selects all transactions. */ public class AllAcceptingTransactionSelector implements PluginTransactionSelector { @@ -29,25 +30,25 @@ private AllAcceptingTransactionSelector() {} /** * Always selects the transaction in the pre-processing stage. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection context. * @return Always SELECTED. */ @Override public TransactionSelectionResult evaluateTransactionPreProcessing( - final PendingTransaction pendingTransaction) { + final TransactionEvaluationContext evaluationContext) { return TransactionSelectionResult.SELECTED; } /** * Always selects the transaction in the post-processing stage. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection context. * @param processingResult The result of the transaction processing. * @return Always SELECTED. */ @Override public TransactionSelectionResult evaluateTransactionPostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionProcessingResult processingResult) { return TransactionSelectionResult.SELECTED; } diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/BlobPriceTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/BlobPriceTransactionSelector.java index 56a17c28434..96135bea214 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/BlobPriceTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/BlobPriceTransactionSelector.java @@ -15,9 +15,9 @@ package org.hyperledger.besu.ethereum.blockcreation.txselection.selectors; import org.hyperledger.besu.ethereum.blockcreation.txselection.BlockSelectionContext; +import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionEvaluationContext; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; import org.hyperledger.besu.ethereum.core.Transaction; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.plugin.data.TransactionSelectionResult; @@ -39,14 +39,15 @@ public BlobPriceTransactionSelector(final BlockSelectionContext context) { /** * Evaluates a transaction considering its blob price. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection session data. * @param ignored The results of other transaction evaluations in the same block. * @return The result of the transaction selection. */ @Override public TransactionSelectionResult evaluateTransactionPreProcessing( - final PendingTransaction pendingTransaction, final TransactionSelectionResults ignored) { - if (transactionBlobPriceBelowMin(pendingTransaction.getTransaction())) { + final TransactionEvaluationContext evaluationContext, + final TransactionSelectionResults ignored) { + if (transactionBlobPriceBelowMin(evaluationContext.getTransaction())) { return TransactionSelectionResult.BLOB_PRICE_BELOW_CURRENT_MIN; } return TransactionSelectionResult.SELECTED; @@ -54,7 +55,7 @@ public TransactionSelectionResult evaluateTransactionPreProcessing( @Override public TransactionSelectionResult evaluateTransactionPostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults blockTransactionResults, final TransactionProcessingResult processingResult) { // All necessary checks were done in the pre-processing method, so nothing to do here. diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/BlockSizeTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/BlockSizeTransactionSelector.java index 96f357546f3..168107f5950 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/BlockSizeTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/BlockSizeTransactionSelector.java @@ -15,9 +15,9 @@ package org.hyperledger.besu.ethereum.blockcreation.txselection.selectors; import org.hyperledger.besu.ethereum.blockcreation.txselection.BlockSelectionContext; +import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionEvaluationContext; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; import org.hyperledger.besu.ethereum.core.Transaction; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.plugin.data.TransactionSelectionResult; @@ -40,20 +40,21 @@ public BlockSizeTransactionSelector(final BlockSelectionContext context) { * Evaluates a transaction considering other transactions in the same block. If the transaction is * too large for the block returns a selection result based on block occupancy. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection session data. * @param transactionSelectionResults The results of other transaction evaluations in the same * block. * @return The result of the transaction selection. */ @Override public TransactionSelectionResult evaluateTransactionPreProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults transactionSelectionResults) { + if (transactionTooLargeForBlock( - pendingTransaction.getTransaction(), transactionSelectionResults)) { + evaluationContext.getTransaction(), transactionSelectionResults)) { LOG.atTrace() .setMessage("Transaction {} too large to select for block creation") - .addArgument(pendingTransaction::toTraceLog) + .addArgument(evaluationContext.getPendingTransaction()::toTraceLog) .log(); if (blockOccupancyAboveThreshold(transactionSelectionResults)) { LOG.trace("Block occupancy above threshold, completing operation"); @@ -70,7 +71,7 @@ public TransactionSelectionResult evaluateTransactionPreProcessing( @Override public TransactionSelectionResult evaluateTransactionPostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults blockTransactionResults, final TransactionProcessingResult processingResult) { // All necessary checks were done in the pre-processing method, so nothing to do here. diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/MinPriorityFeePerGasTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/MinPriorityFeePerGasTransactionSelector.java index 91085f4f020..1a46242ffe2 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/MinPriorityFeePerGasTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/MinPriorityFeePerGasTransactionSelector.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.blockcreation.txselection.BlockSelectionContext; +import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionEvaluationContext; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; @@ -36,7 +37,7 @@ public MinPriorityFeePerGasTransactionSelector(final BlockSelectionContext conte /** * Evaluates a transaction before processing. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection session data. * @param transactionSelectionResults The results of other transaction evaluations in the same * block. * @return TransactionSelectionResult. If the priority fee is below the minimum, it returns an @@ -44,9 +45,9 @@ public MinPriorityFeePerGasTransactionSelector(final BlockSelectionContext conte */ @Override public TransactionSelectionResult evaluateTransactionPreProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults transactionSelectionResults) { - if (isPriorityFeePriceBelowMinimum(pendingTransaction)) { + if (isPriorityFeePriceBelowMinimum(evaluationContext.getPendingTransaction())) { return TransactionSelectionResult.PRIORITY_FEE_PER_GAS_BELOW_CURRENT_MIN; } return TransactionSelectionResult.SELECTED; @@ -74,13 +75,13 @@ private boolean isPriorityFeePriceBelowMinimum(final PendingTransaction pendingT /** * No evaluation is performed post-processing. * - * @param pendingTransaction The processed transaction. + * @param evaluationContext The current selection session data. * @param processingResult The result of the transaction processing. * @return Always returns SELECTED. */ @Override public TransactionSelectionResult evaluateTransactionPostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults blockTransactionResults, final TransactionProcessingResult processingResult) { return TransactionSelectionResult.SELECTED; diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/PriceTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/PriceTransactionSelector.java index defd75cb77c..d677ea9358f 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/PriceTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/PriceTransactionSelector.java @@ -14,10 +14,9 @@ */ package org.hyperledger.besu.ethereum.blockcreation.txselection.selectors; -import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.blockcreation.txselection.BlockSelectionContext; +import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionEvaluationContext; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; -import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.plugin.data.TransactionSelectionResult; @@ -41,14 +40,15 @@ public PriceTransactionSelector(final BlockSelectionContext context) { * Evaluates a transaction considering its price. If the transaction's current price is below the * minimum, it returns a selection result indicating the reason. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection session data. * @param ignored The results of other transaction evaluations in the same block. * @return The result of the transaction selection. */ @Override public TransactionSelectionResult evaluateTransactionPreProcessing( - final PendingTransaction pendingTransaction, final TransactionSelectionResults ignored) { - if (transactionCurrentPriceBelowMin(pendingTransaction)) { + final TransactionEvaluationContext evaluationContext, + final TransactionSelectionResults ignored) { + if (transactionCurrentPriceBelowMin(evaluationContext)) { return TransactionSelectionResult.CURRENT_TX_PRICE_BELOW_MIN; } return TransactionSelectionResult.SELECTED; @@ -56,7 +56,7 @@ public TransactionSelectionResult evaluateTransactionPreProcessing( @Override public TransactionSelectionResult evaluateTransactionPostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults blockTransactionResults, final TransactionProcessingResult processingResult) { // All necessary checks were done in the pre-processing method, so nothing to do here. @@ -66,30 +66,25 @@ public TransactionSelectionResult evaluateTransactionPostProcessing( /** * Checks if the transaction's current price is below the minimum. * - * @param pendingTransaction The transaction to be checked. + * @param evaluationContext The current selection session data. * @return True if the transaction's current price is below the minimum, false otherwise. */ - private boolean transactionCurrentPriceBelowMin(final PendingTransaction pendingTransaction) { - final Transaction transaction = pendingTransaction.getTransaction(); + private boolean transactionCurrentPriceBelowMin( + final TransactionEvaluationContext evaluationContext) { + final PendingTransaction pendingTransaction = evaluationContext.getPendingTransaction(); // Priority txs are exempt from this check if (!pendingTransaction.hasPriority()) { - // since the minGasPrice can change at runtime, we need to recheck it everytime - final Wei transactionGasPriceInBlock = - context - .feeMarket() - .getTransactionPriceCalculator() - .price(transaction, context.processableBlockHeader().getBaseFee()); if (context .miningParameters() .getMinTransactionGasPrice() - .compareTo(transactionGasPriceInBlock) + .compareTo(evaluationContext.getTransactionGasPrice()) > 0) { LOG.atTrace() .setMessage( "Current gas price of {} is {} and lower than the configured minimum {}, skipping") .addArgument(pendingTransaction::toTraceLog) - .addArgument(transactionGasPriceInBlock::toHumanReadableString) + .addArgument(evaluationContext.getTransactionGasPrice()::toHumanReadableString) .addArgument( context.miningParameters().getMinTransactionGasPrice()::toHumanReadableString) .log(); diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/ProcessingResultTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/ProcessingResultTransactionSelector.java index 8a2778eda03..bb89b965a39 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/ProcessingResultTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/ProcessingResultTransactionSelector.java @@ -15,9 +15,9 @@ package org.hyperledger.besu.ethereum.blockcreation.txselection.selectors; import org.hyperledger.besu.ethereum.blockcreation.txselection.BlockSelectionContext; +import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionEvaluationContext; import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionSelectionResults; import org.hyperledger.besu.ethereum.core.Transaction; -import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.ethereum.mainnet.ValidationResult; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; @@ -41,7 +41,7 @@ public ProcessingResultTransactionSelector(final BlockSelectionContext context) @Override public TransactionSelectionResult evaluateTransactionPreProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults blockTransactionResults) { // All checks depend on processingResult and will be done in the post-processing method, so // nothing to do here. @@ -53,20 +53,20 @@ public TransactionSelectionResult evaluateTransactionPreProcessing( * result. If the processing result is invalid, it determines the selection result for the invalid * result. * - * @param pendingTransaction The transaction to be evaluated. + * @param evaluationContext The current selection session data. * @param blockTransactionResults The results of other transaction evaluations in the same block. * @param processingResult The processing result of the transaction. * @return The result of the transaction selection. */ @Override public TransactionSelectionResult evaluateTransactionPostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResults blockTransactionResults, final TransactionProcessingResult processingResult) { if (processingResult.isInvalid()) { return transactionSelectionResultForInvalidResult( - pendingTransaction.getTransaction(), processingResult.getValidationResult()); + evaluationContext.getTransaction(), processingResult.getValidationResult()); } return TransactionSelectionResult.SELECTED; } diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java index 4e7b3eae5eb..0b0b2cd7831 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java @@ -83,6 +83,7 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; +import org.hyperledger.besu.plugin.services.txselection.TransactionEvaluationContext; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.util.number.Percentage; @@ -581,17 +582,25 @@ public void transactionSelectionPluginShouldWork_PreProcessing() { new PluginTransactionSelector() { @Override public TransactionSelectionResult evaluateTransactionPreProcessing( - final PendingTransaction pendingTransaction) { - if (pendingTransaction.getTransaction().equals(notSelectedTransient)) + final TransactionEvaluationContext + evaluationContext) { + if (evaluationContext + .getPendingTransaction() + .getTransaction() + .equals(notSelectedTransient)) return PluginTransactionSelectionResult.GENERIC_PLUGIN_INVALID_TRANSIENT; - if (pendingTransaction.getTransaction().equals(notSelectedInvalid)) + if (evaluationContext + .getPendingTransaction() + .getTransaction() + .equals(notSelectedInvalid)) return PluginTransactionSelectionResult.GENERIC_PLUGIN_INVALID; return SELECTED; } @Override public TransactionSelectionResult evaluateTransactionPostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext + evaluationContext, final org.hyperledger.besu.plugin.data.TransactionProcessingResult processingResult) { return SELECTED; @@ -645,13 +654,15 @@ public void transactionSelectionPluginShouldWork_PostProcessing() { new PluginTransactionSelector() { @Override public TransactionSelectionResult evaluateTransactionPreProcessing( - final PendingTransaction pendingTransaction) { + final TransactionEvaluationContext + evaluationContext) { return SELECTED; } @Override public TransactionSelectionResult evaluateTransactionPostProcessing( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext + evaluationContext, final org.hyperledger.besu.plugin.data.TransactionProcessingResult processingResult) { // the transaction with max gas +1 should fail @@ -711,13 +722,14 @@ public void transactionSelectionPluginShouldBeNotifiedWhenTransactionSelectionCo selector.buildTransactionListForBlock(); - ArgumentCaptor argumentCaptor = - ArgumentCaptor.forClass(PendingTransaction.class); + @SuppressWarnings("unchecked") + ArgumentCaptor> argumentCaptor = + ArgumentCaptor.forClass(TransactionEvaluationContext.class); // selected transaction must be notified to the selector verify(transactionSelector) .onTransactionSelected(argumentCaptor.capture(), any(TransactionProcessingResult.class)); - PendingTransaction selected = argumentCaptor.getValue(); + PendingTransaction selected = argumentCaptor.getValue().getPendingTransaction(); assertThat(selected.getTransaction()).isEqualTo(transaction); // unselected transaction must be notified to the selector with correct reason @@ -725,7 +737,7 @@ public void transactionSelectionPluginShouldBeNotifiedWhenTransactionSelectionCo .onTransactionNotSelected( argumentCaptor.capture(), eq(TransactionSelectionResult.invalid(invalidReason.toString()))); - PendingTransaction rejectedTransaction = argumentCaptor.getValue(); + PendingTransaction rejectedTransaction = argumentCaptor.getValue().getPendingTransaction(); assertThat(rejectedTransaction.getTransaction()).isEqualTo(invalidTransaction); } @@ -931,9 +943,10 @@ private void internalBlockSelectionTimeoutSimulation( final BiFunction> tooLate = (p, t) -> invocation -> { - final org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction ptx = - invocation.getArgument(0); - if (ptx.getTransaction().equals(p)) { + final org.hyperledger.besu.ethereum.blockcreation.txselection + .TransactionEvaluationContext + ctx = invocation.getArgument(0); + if (ctx.getTransaction().equals(p)) { Thread.sleep(t); } else { Thread.sleep(fastProcessingTxTime); diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/MinPriorityFeePerGasTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/MinPriorityFeePerGasTransactionSelectorTest.java index a86dc73ee46..94d3abb105c 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/MinPriorityFeePerGasTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/MinPriorityFeePerGasTransactionSelectorTest.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.blockcreation.txselection.BlockSelectionContext; +import org.hyperledger.besu.ethereum.blockcreation.txselection.TransactionEvaluationContext; import org.hyperledger.besu.ethereum.blockcreation.txselection.selectors.AbstractTransactionSelector; import org.hyperledger.besu.ethereum.blockcreation.txselection.selectors.MinPriorityFeePerGasTransactionSelector; import org.hyperledger.besu.ethereum.core.MiningParameters; @@ -29,6 +30,7 @@ import org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction; import org.hyperledger.besu.plugin.data.TransactionSelectionResult; +import com.google.common.base.Stopwatch; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -56,44 +58,47 @@ public void initialize() { @Test public void shouldNotSelectWhen_PriorityFeePerGas_IsLessThan_MinPriorityFeePerGas() { - var transaction = mockTransactionWithPriorityFee(minPriorityFeeParameter - 1); + var transaction = mockTransactionEvaluationContext(minPriorityFeeParameter - 1); assertSelectionResult( transaction, TransactionSelectionResult.PRIORITY_FEE_PER_GAS_BELOW_CURRENT_MIN); } @Test public void shouldSelectWhen_PriorityFeePerGas_IsEqual_MinPriorityFeePerGas() { - var transaction = mockTransactionWithPriorityFee(minPriorityFeeParameter); + var transaction = mockTransactionEvaluationContext(minPriorityFeeParameter); assertSelectionResult(transaction, TransactionSelectionResult.SELECTED); } @Test public void shouldSelectWhen_PriorityFeePerGas_IsGreaterThan_MinPriorityFeePerGas() { - var transaction = mockTransactionWithPriorityFee(minPriorityFeeParameter + 1); + var transaction = mockTransactionEvaluationContext(minPriorityFeeParameter + 1); assertSelectionResult(transaction, TransactionSelectionResult.SELECTED); } @Test public void shouldSelectWhenPrioritySender() { - var prioritySenderTransaction = mockTransactionWithPriorityFee(minPriorityFeeParameter - 1); + final var evaluationContext = mockTransactionEvaluationContext(minPriorityFeeParameter - 1); assertSelectionResult( - prioritySenderTransaction, - TransactionSelectionResult.PRIORITY_FEE_PER_GAS_BELOW_CURRENT_MIN); - when(prioritySenderTransaction.hasPriority()).thenReturn(true); - assertSelectionResult(prioritySenderTransaction, TransactionSelectionResult.SELECTED); + evaluationContext, TransactionSelectionResult.PRIORITY_FEE_PER_GAS_BELOW_CURRENT_MIN); + when(evaluationContext.getPendingTransaction().hasPriority()).thenReturn(true); + assertSelectionResult(evaluationContext, TransactionSelectionResult.SELECTED); } private void assertSelectionResult( - final PendingTransaction transaction, final TransactionSelectionResult expectedResult) { - var actualResult = transactionSelector.evaluateTransactionPreProcessing(transaction, null); + final TransactionEvaluationContext evaluationContext, + final TransactionSelectionResult expectedResult) { + var actualResult = + transactionSelector.evaluateTransactionPreProcessing(evaluationContext, null); assertThat(actualResult).isEqualTo(expectedResult); } - private PendingTransaction mockTransactionWithPriorityFee(final int priorityFeePerGas) { - PendingTransaction mockTransaction = mock(PendingTransaction.class); + private TransactionEvaluationContext mockTransactionEvaluationContext( + final int priorityFeePerGas) { + PendingTransaction pendingTransaction = mock(PendingTransaction.class); Transaction transaction = mock(Transaction.class); - when(mockTransaction.getTransaction()).thenReturn(transaction); + when(pendingTransaction.getTransaction()).thenReturn(transaction); when(transaction.getEffectivePriorityFeePerGas(any())).thenReturn(Wei.of(priorityFeePerGas)); - return mockTransaction; + return new TransactionEvaluationContext( + pendingTransaction, Stopwatch.createStarted(), Wei.ONE, Wei.ONE); } } diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index d271094604d..1dcfd00f7ee 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'N583pqJipDs4kJkgL0cPq9PBsYdsLzvUlu2I8Kk+w7g=' + knownHash = 'IGq+V3KaStHCRFkeK3KwPxJYKO4RX9YM1O4JYITk8S8=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/PluginTransactionSelector.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/PluginTransactionSelector.java index 716a05e7d8b..3d152c370ca 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/PluginTransactionSelector.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/PluginTransactionSelector.java @@ -39,39 +39,40 @@ default BlockAwareOperationTracer getOperationTracer() { * Method called to decide whether a transaction is added to a block. The result can also indicate * that no further transactions can be added to the block. * - * @param pendingTransaction candidate transaction + * @param evaluationContext The current selection context * @return TransactionSelectionResult that indicates whether to include the transaction */ TransactionSelectionResult evaluateTransactionPreProcessing( - PendingTransaction pendingTransaction); + TransactionEvaluationContext evaluationContext); /** * Method called to decide whether a processed transaction is added to a block. The result can * also indicate that no further transactions can be added to the block. * - * @param pendingTransaction candidate transaction + * @param evaluationContext The current selection context * @param processingResult the transaction processing result * @return TransactionSelectionResult that indicates whether to include the transaction */ TransactionSelectionResult evaluateTransactionPostProcessing( - PendingTransaction pendingTransaction, TransactionProcessingResult processingResult); + TransactionEvaluationContext evaluationContext, + TransactionProcessingResult processingResult); /** * Method called when a transaction is selected to be added to a block. * - * @param pendingTransaction The transaction that has been selected. + * @param evaluationContext The current selection context * @param processingResult The result of processing the selected transaction. */ default void onTransactionSelected( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionProcessingResult processingResult) {} /** * Method called when a transaction is not selected to be added to a block. * - * @param pendingTransaction The transaction that has not been selected. + * @param evaluationContext The current selection context * @param transactionSelectionResult The transaction selection result */ default void onTransactionNotSelected( - final PendingTransaction pendingTransaction, + final TransactionEvaluationContext evaluationContext, final TransactionSelectionResult transactionSelectionResult) {} } diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/TransactionEvaluationContext.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/TransactionEvaluationContext.java new file mode 100644 index 00000000000..83aa7b91e7c --- /dev/null +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txselection/TransactionEvaluationContext.java @@ -0,0 +1,57 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.plugin.services.txselection; + +import org.hyperledger.besu.datatypes.PendingTransaction; +import org.hyperledger.besu.datatypes.Wei; + +import com.google.common.base.Stopwatch; + +/** + * This interface defines the context for evaluating a transaction. It provides methods to get the + * pending transaction, the evaluation timer, and the transaction gas price. + * + * @param the type of the pending transaction + */ +public interface TransactionEvaluationContext { + + /** + * Gets the pending transaction. + * + * @return the pending transaction + */ + PT getPendingTransaction(); + + /** + * Gets the stopwatch used for timing the evaluation. + * + * @return the evaluation timer + */ + Stopwatch getEvaluationTimer(); + + /** + * Gets the gas price of the transaction. + * + * @return the transaction gas price + */ + Wei getTransactionGasPrice(); + + /** + * Gets the min gas price for block inclusion + * + * @return the min gas price + */ + Wei getMinGasPrice(); +} From 013d1c955365ca5aa826e0ccd1d01e83d59f509c Mon Sep 17 00:00:00 2001 From: Thabokani <149070269+Thabokani@users.noreply.github.com> Date: Sat, 13 Jan 2024 00:58:54 +0800 Subject: [PATCH 11/56] Fix typo in SUPPORT.md (#6395) Signed-off-by: Thabokani <149070269+Thabokani@users.noreply.github.com> Co-authored-by: Fabio Di Fabio --- SUPPORT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SUPPORT.md b/SUPPORT.md index 16a9bccd6ae..a9eb54acb86 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -7,7 +7,7 @@ Welcome to the Besu repository! The following links are a set of guidelines for Having Github, Discord, and Linux Foundation accounts is necessary for obtaining support for Besu through the community channels, wiki and issue management. * If you want to raise an issue, you can do so [on the github issue tab](https://github.com/hyperledger/besu/issues). * Hyperledger Discord requires a [Discord account]. -* The Hyperlegder wiki also requires a [Linux Foundation (LF) account] in order to edit pages. +* The Hyperledger wiki also requires a [Linux Foundation (LF) account] in order to edit pages. ### Useful support links From b65c633860c8f7f0fa2dc34c0bae98e5ef00f1d1 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Sat, 13 Jan 2024 03:47:30 +1000 Subject: [PATCH 12/56] reduce machine size (#6392) Signed-off-by: Sally MacFarlane Co-authored-by: Fabio Di Fabio --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2f008df2578..6d08e515a18 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -233,7 +233,7 @@ jobs: - capture_test_logs acceptanceTestsCliqueBft: - executor: besu_executor_xl + executor: besu_executor_med steps: - prepare - attach_workspace: @@ -242,7 +242,7 @@ jobs: name: AcceptanceTests (Non-Mainnet) no_output_timeout: 20m command: | - ./gradlew --no-daemon acceptanceTestCliqueBft + ./gradlew --no-daemon --max-workers=1 acceptanceTestCliqueBft - capture_test_results - capture_test_logs From 9fbe4fbbb2e82671ddc8b7fe127b6d8706b2fc7c Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Sun, 14 Jan 2024 22:50:45 +0100 Subject: [PATCH 13/56] Upgrade Guava dependency (#6396) * Bump com.google.guava:guava to 33.0.0 Signed-off-by: Fabio Di Fabio * Update pending tx estimated memory size after Guava update Signed-off-by: Fabio Di Fabio * Rebuilt Gradle validation metadata Signed-off-by: Fabio Di Fabio * Update CHANGELOG Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + ...ingTransactionEstimatedMemorySizeTest.java | 2 +- gradle/verification-metadata.xml | 96 +++++++++---------- gradle/versions.gradle | 2 +- 4 files changed, 47 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abcc2ab3c85..bb8d3cb9bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Introduce TransactionEvaluationContext to pass data between transaction selectors and plugin, during block creation [#6381](https://github.com/hyperledger/besu/pull/6381) - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) +- Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java index c9c81543a63..20b94f5bf8c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java @@ -58,7 +58,7 @@ public class PendingTransactionEstimatedMemorySizeTest extends BaseTransactionPo private static final Set> SHARED_CLASSES = Set.of(SignatureAlgorithm.class, TransactionType.class); private static final Set COMMON_CONSTANT_FIELD_PATHS = - Set.of(".value.ctor", ".hashNoSignature"); + Set.of(".value.ctor", ".hashNoSignature", ".signature.encoded.delegate"); private static final Set EIP1559_EIP4844_CONSTANT_FIELD_PATHS = Sets.union(COMMON_CONSTANT_FIELD_PATHS, Set.of(".gasPrice")); private static final Set FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS = diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 30d55eb6840..58a562fff88 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -760,14 +760,6 @@ - - - - - - - - @@ -777,9 +769,6 @@ - - - @@ -792,6 +781,14 @@ + + + + + + + + @@ -808,11 +805,6 @@ - - - - - @@ -828,6 +820,11 @@ + + + + + @@ -920,12 +917,12 @@ - - - + + + - - + + @@ -944,16 +941,22 @@ + + + + + + + + + + + - - - - - @@ -964,6 +967,11 @@ + + + + + @@ -998,14 +1006,6 @@ - - - - - - - - @@ -3492,22 +3492,6 @@ - - - - - - - - - - - - - - - - @@ -3523,8 +3507,16 @@ - - + + + + + + + + + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index b497109d782..c27af91f09b 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -46,7 +46,7 @@ dependencyManagement { entry 'error_prone_test_helpers' } - dependency 'com.google.guava:guava:31.1-jre' + dependency 'com.google.guava:guava:33.0.0-jre' dependency 'com.graphql-java:graphql-java:21.3' From 39d7042ca3f41091a7d24d12bfca6c85fe5b4dd8 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Mon, 15 Jan 2024 10:09:37 +1000 Subject: [PATCH 14/56] log a different warning if pruning is enabled with BONSAI (#6401) Signed-off-by: Sally MacFarlane --- CHANGELOG.md | 2 +- .../org/hyperledger/besu/cli/BesuCommand.java | 12 ++++++++++-- .../org/hyperledger/besu/cli/BesuCommandTest.java | 15 +++++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb8d3cb9bff..13f26a25781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ ### Breaking Changes ### Deprecations -- Forest pruning (`pruning-enabled` options) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230) +- Forest pruning (`pruning-enabled` option) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230) ### Additions and Improvements - Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 9842d52f875..d2fb5243313 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -148,6 +148,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract; import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract; @@ -2063,8 +2064,15 @@ && isOptionSet(commandLine, "--sync-min-peers")) { } if (isPruningEnabled()) { - logger.warn( - "Forest pruning is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format."); + if (dataStorageOptions + .toDomainObject() + .getDataStorageFormat() + .equals(DataStorageFormat.BONSAI)) { + logger.warn("Forest pruning is ignored with Bonsai data storage format."); + } else { + logger.warn( + "Forest pruning is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format."); + } } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index ba0e5b2754a..dc56c34845d 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3840,8 +3840,8 @@ public void pruningParametersAreCaptured() throws Exception { } @Test - public void pruningLogsDeprecationWarning() { - parseCommand("--pruning-enabled"); + public void pruningLogsDeprecationWarningWithForest() { + parseCommand("--pruning-enabled", "--data-storage-format=FOREST"); verify(mockControllerBuilder).isPruningEnabled(true); @@ -3854,6 +3854,17 @@ public void pruningLogsDeprecationWarning() { + " To save disk space consider switching to Bonsai data storage format.")); } + @Test + public void pruningLogsIgnoredWarningWithBonsai() { + parseCommand("--pruning-enabled", "--data-storage-format=BONSAI"); + + verify(mockControllerBuilder).isPruningEnabled(true); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + verify(mockLogger).warn(contains("Forest pruning is ignored with Bonsai data storage format.")); + } + @Test public void devModeOptionMustBeUsed() throws Exception { parseCommand("--network", "dev"); From 724e3d040e74204c9ace7844a0651cbdebd671f9 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 16 Jan 2024 00:27:57 +0100 Subject: [PATCH 15/56] Upgrade Mockito (#6397) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../ethereum/api/handlers/HandlerFactory.java | 4 +- .../internal/methods/DebugTraceBlock.java | 23 ++-- .../JsonRpcHttpServiceHostAllowlistTest.java | 64 +++++---- .../jsonrpc/JsonRpcHttpServiceLoginTest.java | 62 ++++----- .../JsonRpcHttpServiceRpcApisTest.java | 129 +++++++++--------- .../api/jsonrpc/JsonRpcHttpServiceTest.java | 86 ++++++------ .../jsonrpc/JsonRpcHttpServiceTestBase.java | 88 +++++++----- .../JsonRpcHttpServiceTlsClientAuthTest.java | 64 +++++---- ...RpcHttpServiceTlsMisconfigurationTest.java | 64 +++++---- .../jsonrpc/JsonRpcHttpServiceTlsTest.java | 64 +++++---- .../DebugStandardTraceBlockToFileTest.java | 28 ++-- .../internal/methods/DebugTraceBlockTest.java | 45 +++--- .../backwardsync/BackwardSyncAlgSpec.java | 4 +- .../rlpx/connections/netty/DeFramerTest.java | 13 +- .../evm/operations/ChainIdOperationTest.java | 3 +- gradle/verification-metadata.xml | 46 +++---- gradle/versions.gradle | 2 +- 18 files changed, 396 insertions(+), 394 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13f26a25781..d520eba5425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) - Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) +- Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java index 2d3639d52f6..03465f70f21 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import io.opentelemetry.api.trace.Tracer; @@ -35,7 +36,8 @@ public static Handler timeout( assert methods != null && globalOptions != null; return TimeoutHandler.handler( Optional.of(globalOptions), - methods.keySet().stream().collect(Collectors.toMap(String::new, ignored -> globalOptions))); + methods.keySet().stream() + .collect(Collectors.toMap(Function.identity(), ignored -> globalOptions))); } public static Handler authentication( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java index 270f176c7a8..6dab0436a84 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java @@ -46,15 +46,15 @@ public class DebugTraceBlock implements JsonRpcMethod { private static final Logger LOG = LoggerFactory.getLogger(DebugTraceBlock.class); private final Supplier blockTracerSupplier; private final BlockHeaderFunctions blockHeaderFunctions; - private final BlockchainQueries blockchain; + private final BlockchainQueries blockchainQueries; public DebugTraceBlock( final Supplier blockTracerSupplier, final BlockHeaderFunctions blockHeaderFunctions, - final BlockchainQueries blockchain) { + final BlockchainQueries blockchainQueries) { this.blockTracerSupplier = blockTracerSupplier; this.blockHeaderFunctions = blockHeaderFunctions; - this.blockchain = blockchain; + this.blockchainQueries = blockchainQueries; } @Override @@ -79,18 +79,17 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { .map(TransactionTraceParams::traceOptions) .orElse(TraceOptions.DEFAULT); - if (this.blockchain.blockByHash(block.getHeader().getParentHash()).isPresent()) { + if (this.blockchainQueries.blockByHash(block.getHeader().getParentHash()).isPresent()) { final Collection results = Tracer.processTracing( - blockchain, + blockchainQueries, Optional.of(block.getHeader()), - mutableWorldState -> { - return blockTracerSupplier - .get() - .trace(mutableWorldState, block, new DebugOperationTracer(traceOptions)) - .map(BlockTrace::getTransactionTraces) - .map(DebugTraceTransactionResult::of); - }) + mutableWorldState -> + blockTracerSupplier + .get() + .trace(mutableWorldState, block, new DebugOperationTracer(traceOptions)) + .map(BlockTrace::getTransactionTraces) + .map(DebugTraceTransactionResult::of)) .orElse(null); return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), results); } else { diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java index da7319282fb..f433bcdc977 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java @@ -17,7 +17,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -98,38 +97,37 @@ public void initServerAndClient() throws Exception { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); service = createJsonRpcHttpService(); service.start().join(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java index 7b9eac24788..aa1b582174f 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java @@ -19,7 +19,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.util.Lists.list; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -129,37 +128,36 @@ public static void initServerAndClient() throws Exception { new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - genesisConfigOptions, - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig(genesisConfigOptions), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.empty(), - Optional.empty(), - JSON_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + genesisConfigOptions, + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig(genesisConfigOptions), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.empty(), + Optional.empty(), + JSON_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); service = createJsonRpcHttpService(); jwtAuth = service.authenticationService.get().getJwtAuthProvider(); service.start().join(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java index 6870c4fed03..8323f61b2c5 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java @@ -17,7 +17,6 @@ import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.config.StubGenesisConfigOptions; @@ -201,37 +200,36 @@ private JsonRpcHttpService createJsonRpcHttpServiceWithRpcApis(final JsonRpcConf supportedCapabilities.add(EthProtocol.ETH63); final Map rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - NETWORK_ID, - new StubGenesisConfigOptions(), - mock(P2PNetwork.class), - blockchainQueries, - mock(Synchronizer.class), - ProtocolScheduleFixture.MAINNET, - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - config.getRpcApis(), - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + NETWORK_ID, + new StubGenesisConfigOptions(), + mock(P2PNetwork.class), + blockchainQueries, + mock(Synchronizer.class), + ProtocolScheduleFixture.MAINNET, + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + config.getRpcApis(), + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); final JsonRpcHttpService jsonRpcHttpService = new JsonRpcHttpService( vertx, @@ -302,8 +300,7 @@ private JsonRpcHttpService createJsonRpcHttpService( final WebSocketConfiguration webSocketConfiguration, final P2PNetwork p2pNetwork, final MetricsConfiguration metricsConfiguration, - final NatService natService) - throws Exception { + final NatService natService) { final Set supportedCapabilities = new HashSet<>(); supportedCapabilities.add(EthProtocol.ETH62); supportedCapabilities.add(EthProtocol.ETH63); @@ -311,37 +308,36 @@ private JsonRpcHttpService createJsonRpcHttpService( webSocketConfiguration.setPort(0); final Map rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - NETWORK_ID, - new StubGenesisConfigOptions(), - p2pNetwork, - blockchainQueries, - mock(Synchronizer.class), - ProtocolScheduleFixture.MAINNET, - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - jsonRpcConfiguration.getRpcApis(), - mock(PrivacyParameters.class), - jsonRpcConfiguration, - webSocketConfiguration, - metricsConfiguration, - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + NETWORK_ID, + new StubGenesisConfigOptions(), + p2pNetwork, + blockchainQueries, + mock(Synchronizer.class), + ProtocolScheduleFixture.MAINNET, + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + jsonRpcConfiguration.getRpcApis(), + mock(PrivacyParameters.class), + jsonRpcConfiguration, + webSocketConfiguration, + metricsConfiguration, + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); final JsonRpcHttpService jsonRpcHttpService = new JsonRpcHttpService( vertx, @@ -425,8 +421,7 @@ public RequestBody createNetServicesRequestBody() { "{\"jsonrpc\":\"2.0\",\"id\":" + Json.encode(id) + ",\"method\":\"net_services\"}", JSON); } - public JsonRpcHttpService getJsonRpcHttpService(final boolean[] enabledNetServices) - throws Exception { + public JsonRpcHttpService getJsonRpcHttpService(final boolean[] enabledNetServices) { JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java index f9c9b14ccd0..901f20ff754 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java @@ -17,10 +17,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Address; @@ -1389,65 +1386,68 @@ public void disabledMethod() throws Exception { + "\"}", JSON); - when(rpcMethods.get(any(String.class))).thenReturn(null); - when(rpcMethods.containsKey(any(String.class))).thenReturn(false); + try (var unused = disableRpcMethod(methodName)) { - try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { - assertThat(resp.code()).isEqualTo(200); - final JsonObject json = new JsonObject(resp.body().string()); - final RpcErrorType expectedError = RpcErrorType.METHOD_NOT_ENABLED; - testHelper.assertValidJsonRpcError( - json, id, expectedError.getCode(), expectedError.getMessage()); + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonObject json = new JsonObject(resp.body().string()); + final RpcErrorType expectedError = RpcErrorType.METHOD_NOT_ENABLED; + testHelper.assertValidJsonRpcError( + json, id, expectedError.getCode(), expectedError.getMessage()); + } } - - verify(rpcMethods).containsKey(methodName); - verify(rpcMethods).get(methodName); - - reset(rpcMethods); } @Test public void exceptionallyHandleJsonSingleRequest() throws Exception { + final String methodName = "foo"; final JsonRpcMethod jsonRpcMethod = mock(JsonRpcMethod.class); - when(jsonRpcMethod.getName()).thenReturn("foo"); + when(jsonRpcMethod.getName()).thenReturn(methodName); when(jsonRpcMethod.response(any())).thenThrow(new RuntimeException("test exception")); - doReturn(jsonRpcMethod).when(rpcMethods).get("foo"); + try (var unused = addRpcMethod(methodName, jsonRpcMethod)) { - final RequestBody body = - RequestBody.create("{\"jsonrpc\":\"2.0\",\"id\":\"666\",\"method\":\"foo\"}", JSON); + final RequestBody body = + RequestBody.create( + "{\"jsonrpc\":\"2.0\",\"id\":\"666\",\"method\":\"" + methodName + "\"}", JSON); - try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { - assertThat(resp.code()).isEqualTo(200); - final JsonObject json = new JsonObject(resp.body().string()); - final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; - testHelper.assertValidJsonRpcError( - json, "666", expectedError.getCode(), expectedError.getMessage()); + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonObject json = new JsonObject(resp.body().string()); + final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; + testHelper.assertValidJsonRpcError( + json, "666", expectedError.getCode(), expectedError.getMessage()); + } } } @Test public void exceptionallyHandleJsonBatchRequest() throws Exception { + final String methodName = "foo"; final JsonRpcMethod jsonRpcMethod = mock(JsonRpcMethod.class); - when(jsonRpcMethod.getName()).thenReturn("foo"); + when(jsonRpcMethod.getName()).thenReturn(methodName); when(jsonRpcMethod.response(any())).thenThrow(new RuntimeException("test exception")); - doReturn(jsonRpcMethod).when(rpcMethods).get("foo"); - final RequestBody body = - RequestBody.create( - "[{\"jsonrpc\":\"2.0\",\"id\":\"000\",\"method\":\"web3_clientVersion\"}," - + "{\"jsonrpc\":\"2.0\",\"id\":\"111\",\"method\":\"foo\"}," - + "{\"jsonrpc\":\"2.0\",\"id\":\"222\",\"method\":\"net_version\"}]", - JSON); - - try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { - assertThat(resp.code()).isEqualTo(200); - final JsonArray array = new JsonArray(resp.body().string()); - testHelper.assertValidJsonRpcResult(array.getJsonObject(0), "000"); - final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; - testHelper.assertValidJsonRpcError( - array.getJsonObject(1), "111", expectedError.getCode(), expectedError.getMessage()); - testHelper.assertValidJsonRpcResult(array.getJsonObject(2), "222"); + try (var unused = addRpcMethod(methodName, jsonRpcMethod)) { + + final RequestBody body = + RequestBody.create( + "[{\"jsonrpc\":\"2.0\",\"id\":\"000\",\"method\":\"web3_clientVersion\"}," + + "{\"jsonrpc\":\"2.0\",\"id\":\"111\",\"method\":\"" + + methodName + + "\"}," + + "{\"jsonrpc\":\"2.0\",\"id\":\"222\",\"method\":\"net_version\"}]", + JSON); + + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonArray array = new JsonArray(resp.body().string()); + testHelper.assertValidJsonRpcResult(array.getJsonObject(0), "000"); + final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; + testHelper.assertValidJsonRpcError( + array.getJsonObject(1), "111", expectedError.getCode(), expectedError.getMessage()); + testHelper.assertValidJsonRpcResult(array.getJsonObject(2), "222"); + } } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java index 580af7b7467..257dbf16679 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.ethereum.api.jsonrpc; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -72,8 +71,9 @@ public class JsonRpcHttpServiceTestBase { protected final JsonRpcTestHelper testHelper = new JsonRpcTestHelper(); private static final Vertx vertx = Vertx.vertx(); - protected static Map rpcMethods; + private static Map disabledRpcMethods; + private static Set addedRpcMethods; protected static JsonRpcHttpService service; protected static OkHttpClient client; protected static String baseUrl; @@ -106,39 +106,41 @@ public static void initServerAndClient() throws Exception { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID), - EvmConfiguration.DEFAULT), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - JSON_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - ethPeersMock, - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID), + EvmConfiguration.DEFAULT), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + JSON_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + ethPeersMock, + vertx, + mock(ApiConfiguration.class), + Optional.empty()); + disabledRpcMethods = new HashMap<>(); + addedRpcMethods = new HashSet<>(); + service = createJsonRpcHttpService(createLimitedJsonRpcConfig()); service.start().join(); @@ -189,6 +191,22 @@ protected Request buildGetRequest(final String path) { return new Request.Builder().get().url(baseUrl + path).build(); } + protected AutoCloseable disableRpcMethod(final String methodName) { + disabledRpcMethods.put(methodName, rpcMethods.remove(methodName)); + return () -> resetRpcMethods(); + } + + protected AutoCloseable addRpcMethod(final String methodName, final JsonRpcMethod method) { + rpcMethods.put(methodName, method); + addedRpcMethods.add(methodName); + return () -> resetRpcMethods(); + } + + protected void resetRpcMethods() { + disabledRpcMethods.forEach(rpcMethods::put); + addedRpcMethods.forEach(rpcMethods::remove); + } + /** Tears down the HTTP server. */ @AfterAll public static void shutdownServer() { diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java index 3a9a5577a1d..70cc3d58f84 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java @@ -21,7 +21,6 @@ import static org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration.Builder.aTlsClientAuthConfiguration; import static org.hyperledger.besu.ethereum.api.tls.TlsConfiguration.Builder.aTlsConfiguration; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -112,38 +111,37 @@ public void initServer() throws Exception { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - Collections.emptyMap(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + Collections.emptyMap(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); System.setProperty("javax.net.ssl.trustStore", CLIENT_AS_CA_CERT.getKeyStoreFile().toString()); System.setProperty( diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java index 695dec93269..ee069f03232 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java @@ -20,7 +20,6 @@ import static org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration.Builder.aTlsClientAuthConfiguration; import static org.hyperledger.besu.ethereum.api.tls.TlsConfiguration.Builder.aTlsConfiguration; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -100,38 +99,37 @@ public void beforeEach() { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - Collections.emptyMap(), - tempDir.getRoot(), - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + Collections.emptyMap(), + tempDir.getRoot(), + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); } @AfterEach diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java index 587d142b7f8..295c7b91bd7 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java @@ -20,7 +20,6 @@ import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; import static org.hyperledger.besu.ethereum.api.tls.TlsConfiguration.Builder.aTlsConfiguration; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -101,38 +100,37 @@ public void initServer() throws Exception { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - Collections.emptyMap(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + Collections.emptyMap(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); service = createJsonRpcHttpService(createJsonRpcConfig()); service.start().join(); baseUrl = service.url(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java index 42ec4c697f5..2cf7dcbfc27 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java @@ -18,11 +18,12 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; @@ -30,28 +31,23 @@ import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.nio.file.Path; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.function.Function; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.mockito.Answers; public class DebugStandardTraceBlockToFileTest { // this tempDir is deliberately static @TempDir private static Path folder; - private final WorldStateArchive archive = - mock(WorldStateArchive.class, Answers.RETURNS_DEEP_STUBS); private final Blockchain blockchain = mock(Blockchain.class); - private final BlockchainQueries blockchainQueries = - spy(new BlockchainQueries(blockchain, archive)); + private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); private final TransactionTracer transactionTracer = mock(TransactionTracer.class); private final DebugStandardTraceBlockToFile debugStandardTraceBlockToFile = new DebugStandardTraceBlockToFile(() -> transactionTracer, blockchainQueries, folder); @@ -76,20 +72,26 @@ public void shouldTraceTheTransactionUsingTheTransactionTracer() { new JsonRpcRequestContext( new JsonRpcRequest("2.0", "debug_standardTraceBlockToFile", params)); - final List paths = new ArrayList<>(); - paths.add("path-1"); - - when(blockchainQueries.getBlockchain()).thenReturn(blockchain); + final List paths = List.of("path-1"); when(blockchain.getBlockByHash(block.getHash())).thenReturn(Optional.of(block)); when(blockchain.getBlockHeader(genesis.getHash())).thenReturn(Optional.of(genesis.getHeader())); + when(blockchainQueries.getBlockchain()).thenReturn(blockchain); + + when(blockchainQueries.getAndMapWorldState(any(), any())) + .thenAnswer( + invocationOnMock -> { + Function> mapper = + invocationOnMock.getArgument(1); + return mapper.apply(mock(Tracer.TraceableState.class)); + }); when(transactionTracer.traceTransactionToFile( any(MutableWorldState.class), eq(block.getHash()), any(), any())) .thenReturn(paths); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugStandardTraceBlockToFile.response(request); - final List result = (ArrayList) response.getResult(); + final List result = (List) response.getResult(); assertThat(result.size()).isEqualTo(1); } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java index 60841c24ef4..5da556182bc 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java @@ -18,9 +18,8 @@ import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Wei; @@ -35,32 +34,25 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; import org.hyperledger.besu.ethereum.api.query.BlockWithMetadata; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.Collection; import java.util.Collections; import java.util.Optional; import java.util.OptionalLong; +import java.util.function.Function; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; -import org.mockito.Answers; -import org.mockito.Mockito; public class DebugTraceBlockTest { private final BlockTracer blockTracer = mock(BlockTracer.class); - private final WorldStateArchive archive = - mock(WorldStateArchive.class, Answers.RETURNS_DEEP_STUBS); - private final Blockchain blockchain = mock(Blockchain.class); - private final BlockchainQueries blockchainQueries = - spy(new BlockchainQueries(blockchain, archive)); + private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); private final DebugTraceBlock debugTraceBlock = new DebugTraceBlock(() -> blockTracer, new MainnetBlockHeaderFunctions(), blockchainQueries); @@ -127,22 +119,25 @@ public void shouldReturnCorrectResponse() { when(transaction2Trace.getResult()).thenReturn(transaction2Result); when(transaction1Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); when(transaction2Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); - when(blockTracer.trace(any(Tracer.TraceableState.class), Mockito.eq(block), any())) + when(blockTracer.trace(any(Tracer.TraceableState.class), eq(block), any())) .thenReturn(Optional.of(blockTrace)); - when(blockchain.getBlockHeader(parentBlock.getHash())) - .thenReturn(Optional.of(parentBlock.getHeader())); - doAnswer( - invocation -> - Optional.of( - new BlockWithMetadata<>( - parentBlock.getHeader(), - Collections.emptyList(), - Collections.emptyList(), - parentBlock.getHeader().getDifficulty(), - parentBlock.calculateSize()))) - .when(blockchainQueries) - .blockByHash(parentBlock.getHash()); + when(blockchainQueries.blockByHash(parentBlock.getHash())) + .thenReturn( + Optional.of( + new BlockWithMetadata<>( + parentBlock.getHeader(), + Collections.emptyList(), + Collections.emptyList(), + parentBlock.getHeader().getDifficulty(), + parentBlock.calculateSize()))); + when(blockchainQueries.getAndMapWorldState(eq(parentBlock.getHash()), any())) + .thenAnswer( + invocationOnMock -> { + Function> mapper = + invocationOnMock.getArgument(1); + return mapper.apply(mock(Tracer.TraceableState.class)); + }); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugTraceBlock.response(request); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java index 8a1a0a226e8..83355ecc291 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java @@ -165,7 +165,7 @@ public void shouldAwokeWhenTTDReachedAndReady() throws Exception { ttdCaptor.getValue().onTTDReached(true); - voidCompletableFuture.get(100, TimeUnit.MILLISECONDS); + voidCompletableFuture.get(200, TimeUnit.MILLISECONDS); assertThat(voidCompletableFuture).isCompleted(); verify(context.getSyncState()).unsubscribeTTDReached(88L); @@ -192,7 +192,7 @@ public void shouldAwokeWhenConditionReachedAndReady() throws Exception { completionCaptor.getValue().onInitialSyncCompleted(); - voidCompletableFuture.get(100, TimeUnit.MILLISECONDS); + voidCompletableFuture.get(200, TimeUnit.MILLISECONDS); assertThat(voidCompletableFuture).isCompleted(); verify(context.getSyncState()).unsubscribeTTDReached(88L); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java index dfafb002f49..f5030fd9ad6 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java @@ -65,6 +65,7 @@ import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelId; import io.netty.channel.ChannelPipeline; @@ -196,7 +197,7 @@ public void decode_handlesHello() throws ExecutionException, InterruptedExceptio assertThat(out).isEmpty(); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); // Next message should be pushed out final PingMessage nextMessage = PingMessage.get(); @@ -204,7 +205,7 @@ public void decode_handlesHello() throws ExecutionException, InterruptedExceptio when(framer.deframe(eq(nextData))) .thenReturn(new RawMessage(nextMessage.getCode(), nextMessage.getData())) .thenReturn(null); - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); deFramer.decode(ctx, nextData, out); assertThat(out.size()).isEqualTo(1); } @@ -246,7 +247,7 @@ public void decode_handlesHelloFromPeerWithAdvertisedPortOf0() assertThat(peerConnection.getPeer().getEnodeURL()).isEqualTo(expectedEnode); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); // Next message should be pushed out final PingMessage nextMessage = PingMessage.get(); @@ -254,7 +255,7 @@ public void decode_handlesHelloFromPeerWithAdvertisedPortOf0() when(framer.deframe(eq(nextData))) .thenReturn(new RawMessage(nextMessage.getCode(), nextMessage.getData())) .thenReturn(null); - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); deFramer.decode(ctx, nextData, out); assertThat(out.size()).isEqualTo(1); } @@ -292,7 +293,7 @@ public void decode_handlesUnexpectedPeerId() { assertThat(out).isEmpty(); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); } @Test @@ -321,7 +322,7 @@ public void decode_handlesNoSharedCaps() { assertThat(out).isEmpty(); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); } @Test diff --git a/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java b/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java index 7202be03ce3..e8d2c3d02db 100644 --- a/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java +++ b/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java @@ -27,7 +27,6 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -51,7 +50,7 @@ static Iterable params() { void shouldReturnChainId(final String chainIdString, final int expectedGas) { Bytes32 chainId = Bytes32.fromHexString(chainIdString); ChainIdOperation operation = new ChainIdOperation(new ConstantinopleGasCalculator(), chainId); - final ArgumentCaptor arg = ArgumentCaptor.forClass(UInt256.class); + final ArgumentCaptor arg = ArgumentCaptor.forClass(Bytes.class); when(messageFrame.getRemainingGas()).thenReturn(100L); operation.execute(messageFrame, null); Mockito.verify(messageFrame).getRemainingGas(); diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 58a562fff88..906d4a2ee27 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2858,12 +2858,12 @@ - - - + + + - - + + @@ -2874,17 +2874,17 @@ - - - + + + - - + + - - - + + + @@ -5316,20 +5316,20 @@ - - - + + + - - + + - - - + + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index c27af91f09b..747a386cb38 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -192,7 +192,7 @@ dependencyManagement { entry 'org.jupnp' } - dependencySet(group: 'org.mockito', version:'4.11.0') { + dependencySet(group: 'org.mockito', version:'5.8.0') { entry 'mockito-core' entry 'mockito-junit-jupiter' } From 2c1d3d28410d6b0f3f55b67e0f2ad8cb7e111d5b Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Tue, 16 Jan 2024 10:53:41 +1000 Subject: [PATCH 16/56] cliqueBft AT task flakiness - extend no_output_timeout (#6406) * revert machine change and extend timeout instead Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d08e515a18..065a0280d04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,7 +240,7 @@ jobs: at: ~/project - run: name: AcceptanceTests (Non-Mainnet) - no_output_timeout: 20m + no_output_timeout: 30m command: | ./gradlew --no-daemon --max-workers=1 acceptanceTestCliqueBft - capture_test_results From c316a6dfb5ea354bad50194840362c6622910b0e Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Tue, 16 Jan 2024 13:53:55 +1100 Subject: [PATCH 17/56] Remove deprecated `--privacy-onchain-groups-enabled` option (#6411) Signed-off-by: Gabriel-Trintinalia --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 21 +------- .../util/ConfigOptionSearchAndRunHandler.java | 11 ---- .../hyperledger/besu/cli/BesuCommandTest.java | 51 ------------------- .../src/test/resources/everything_config.toml | 1 - 5 files changed, 3 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d520eba5425..504d120a9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Breaking Changes - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) +- The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) ### Deprecations diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index d2fb5243313..c51d03d4305 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -22,7 +22,6 @@ import static org.hyperledger.besu.cli.DefaultCommandValues.getDefaultBesuDataPath; import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPRECATION_WARNING_MSG; import static org.hyperledger.besu.cli.util.CommandLineUtils.isOptionSet; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; import static org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration.DEFAULT_GRAPHQL_HTTP_PORT; @@ -953,13 +952,6 @@ static class PrivacyOptionGroup { names = {"--privacy-flexible-groups-enabled"}, description = "Enable flexible privacy groups (default: ${DEFAULT-VALUE})") private final Boolean isFlexiblePrivacyGroupsEnabled = false; - - @Option( - hidden = true, - names = {"--privacy-onchain-groups-enabled"}, - description = - "!!DEPRECATED!! Use `--privacy-flexible-groups-enabled` instead. Enable flexible (onchain) privacy groups (default: ${DEFAULT-VALUE})") - private final Boolean isOnchainPrivacyGroupsEnabled = false; } // Metrics Option Group @@ -1716,8 +1708,7 @@ private void validatePluginOptions() { } if (unstablePrivacyPluginOptions.isPrivacyPluginEnabled() - && (privacyOptionGroup.isFlexiblePrivacyGroupsEnabled - || privacyOptionGroup.isOnchainPrivacyGroupsEnabled)) { + && privacyOptionGroup.isFlexiblePrivacyGroupsEnabled) { throw new ParameterException( commandLine, "Privacy Plugin can not be used with flexible privacy groups"); } @@ -2056,13 +2047,6 @@ && isOptionSet(commandLine, "--sync-min-peers")) { "--security-module=" + DEFAULT_SECURITY_MODULE); } - if (Boolean.TRUE.equals(privacyOptionGroup.isOnchainPrivacyGroupsEnabled)) { - logger.warn( - DEPRECATION_WARNING_MSG, - "--privacy-onchain-groups-enabled", - "--privacy-flexible-groups-enabled"); - } - if (isPruningEnabled()) { if (dataStorageOptions .toDomainObject() @@ -2750,8 +2734,7 @@ private PrivacyParameters privacyParameters() { privacyParametersBuilder.setMultiTenancyEnabled( privacyOptionGroup.isPrivacyMultiTenancyEnabled); privacyParametersBuilder.setFlexiblePrivacyGroupsEnabled( - privacyOptionGroup.isFlexiblePrivacyGroupsEnabled - || privacyOptionGroup.isOnchainPrivacyGroupsEnabled); + privacyOptionGroup.isFlexiblePrivacyGroupsEnabled); privacyParametersBuilder.setPrivacyPluginEnabled( unstablePrivacyPluginOptions.isPrivacyPluginEnabled()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java index 17cf2e649c9..5866d43cde6 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java @@ -55,7 +55,6 @@ public ConfigOptionSearchAndRunHandler( public List handle(final ParseResult parseResult) throws ParameterException { final CommandLine commandLine = parseResult.commandSpec().commandLine(); final Optional configFile = findConfigFile(parseResult, commandLine); - validatePrivacyOptions(parseResult, commandLine); commandLine.setDefaultValueProvider(createDefaultValueProvider(commandLine, configFile)); commandLine.setExecutionStrategy(resultHandler); commandLine.setParameterExceptionHandler(parameterExceptionHandler); @@ -64,16 +63,6 @@ public List handle(final ParseResult parseResult) throws ParameterExcept return new ArrayList<>(); } - private void validatePrivacyOptions( - final ParseResult parseResult, final CommandLine commandLine) { - if (parseResult.hasMatchedOption("--privacy-onchain-groups-enabled") - && parseResult.hasMatchedOption("--privacy-flexible-groups-enabled")) { - throw new ParameterException( - commandLine, - "The `--privacy-onchain-groups-enabled` option is deprecated and you should only use `--privacy-flexible-groups-enabled`"); - } - } - private Optional findConfigFile( final ParseResult parseResult, final CommandLine commandLine) { if (parseResult.hasMatchedOption("--config-file") diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index dc56c34845d..26296556302 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -29,7 +29,6 @@ import static org.hyperledger.besu.cli.config.NetworkName.MORDOR; import static org.hyperledger.besu.cli.config.NetworkName.SEPOLIA; import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPRECATION_WARNING_MSG; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ENGINE; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; @@ -1977,16 +1976,6 @@ public void ethStatsContactOptionCannotBeUsedWithoutEthStatsServerProvided() { "The `--ethstats-contact` requires ethstats server URL to be provided. Either remove --ethstats-contact or provide a URL (via --ethstats=nodename:secret@host:port)"); } - @Test - public void privacyOnchainGroupsEnabledCannotBeUsedWithPrivacyFlexibleGroupsEnabled() { - parseCommand("--privacy-onchain-groups-enabled", "--privacy-flexible-groups-enabled"); - Mockito.verifyNoInteractions(mockRunnerBuilder); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "The `--privacy-onchain-groups-enabled` option is deprecated and you should only use `--privacy-flexible-groups-enabled`"); - } - @Test public void parsesValidBonsaiTrieLimitBackLayersOption() { parseCommand("--data-storage-format", "BONSAI", "--bonsai-historical-block-limit", "11"); @@ -4203,46 +4192,6 @@ public void flexiblePrivacyGroupEnabledFlagDefaultValueIsFalse() { assertThat(privacyParameters.isFlexiblePrivacyGroupsEnabled()).isEqualTo(false); } - @Test - public void onchainPrivacyGroupEnabledFlagValueIsSet() { - parseCommand( - "--privacy-enabled", - "--privacy-public-key-file", - ENCLAVE_PUBLIC_KEY_PATH, - "--privacy-onchain-groups-enabled", - "--min-gas-price", - "0"); - - final ArgumentCaptor privacyParametersArgumentCaptor = - ArgumentCaptor.forClass(PrivacyParameters.class); - - verify(mockControllerBuilder).privacyParameters(privacyParametersArgumentCaptor.capture()); - verify(mockControllerBuilder).build(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - - final PrivacyParameters privacyParameters = privacyParametersArgumentCaptor.getValue(); - assertThat(privacyParameters.isFlexiblePrivacyGroupsEnabled()).isEqualTo(true); - } - - @Test - public void onchainPrivacyGroupEnabledOptionIsDeprecated() { - parseCommand( - "--privacy-enabled", - "--privacy-public-key-file", - ENCLAVE_PUBLIC_KEY_PATH, - "--privacy-onchain-groups-enabled", - "--min-gas-price", - "0"); - - verify(mockLogger) - .warn( - DEPRECATION_WARNING_MSG, - "--privacy-onchain-groups-enabled", - "--privacy-flexible-groups-enabled"); - } - @Test public void flexiblePrivacyGroupEnabledFlagValueIsSet() { parseCommand( diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index e516060da86..c653f3f60fd 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -169,7 +169,6 @@ privacy-enabled=false privacy-multi-tenancy-enabled=true privacy-marker-transaction-signing-key-file="./signerKey" privacy-enable-database-migration=false -privacy-onchain-groups-enabled=false privacy-flexible-groups-enabled=false # Transaction Pool From 25f8e57a36b673a44953f63cc2e0f400e08be444 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 17 Jan 2024 10:06:06 +0100 Subject: [PATCH 18/56] Upgrade tech.pegasys.discovery:discovery (#6414) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../internal/ENRResponsePacketDataTest.java | 48 +++++++---- gradle/allowed-licenses.json | 5 ++ gradle/verification-metadata.xml | 86 ++++++++++--------- gradle/versions.gradle | 2 +- 5 files changed, 87 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 504d120a9fe..e09db4de540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) - Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) - Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) +- Upgrade `tech.pegasys.discovery:discovery` [#6414](https://github.com/hyperledger/besu/pull/6414) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java index 72bb407d162..7ba47bd97cc 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java @@ -20,6 +20,8 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.crypto.SECP256K1; import org.apache.tuweni.units.bigints.UInt64; import org.ethereum.beacon.discovery.schema.EnrField; import org.ethereum.beacon.discovery.schema.IdentitySchema; @@ -34,8 +36,10 @@ public void serializeDeserialize() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -48,7 +52,8 @@ public void serializeDeserialize() { new EnrField(EnrField.TCP, 8080), new EnrField(EnrField.TCP_V6, 8080), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); assertThat(nodeRecord.getNodeId()).isEqualTo(nodeId); @@ -72,8 +77,10 @@ public void readFrom() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -82,7 +89,8 @@ public void readFrom() { new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); assertThat(nodeRecord.getNodeId()).isEqualTo(nodeId); @@ -109,8 +117,10 @@ public void writeTo() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -119,7 +129,8 @@ public void writeTo() { new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); assertThat(nodeRecord.getNodeId()).isEqualTo(nodeId); @@ -144,8 +155,10 @@ public void readFrom_withExtraFields() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -153,7 +166,9 @@ public void readFrom_withExtraFields() { new EnrField(EnrField.ID, IdentitySchema.V4), new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), - new EnrField(EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey)), + new EnrField( + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey)), new EnrField("foo", Bytes.fromHexString("0x1234"))); nodeRecord.sign(privateKey); @@ -181,8 +196,10 @@ public void readFrom_withExtraFields() { @Test public void readFrom_invalidSignature() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f292"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f292")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -191,7 +208,8 @@ public void readFrom_invalidSignature() { new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); nodeRecord.set(EnrField.UDP, 1234); diff --git a/gradle/allowed-licenses.json b/gradle/allowed-licenses.json index 25270190806..f7a3d8be8d4 100644 --- a/gradle/allowed-licenses.json +++ b/gradle/allowed-licenses.json @@ -56,6 +56,11 @@ "moduleVersion": "1.0.3", "moduleName": "org.reactivestreams:reactive-streams" }, + { + "moduleLicense": "MIT-0", + "moduleVersion": "1.0.4", + "moduleName": "org.reactivestreams:reactive-streams" + }, { "moduleLicense": "Eclipse Public License - v 1.0", "moduleVersion": "4.13.2", diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 906d4a2ee27..eb7949b754a 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2402,15 +2402,15 @@ - - - + + + - - + + - - + + @@ -3359,44 +3359,44 @@ - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -5551,6 +5551,14 @@ + + + + + + + + @@ -5958,15 +5966,15 @@ - - - + + + - - + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 747a386cb38..27fd97fb42d 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -231,6 +231,6 @@ dependencyManagement { dependency 'org.yaml:snakeyaml:2.0' - dependency 'tech.pegasys.discovery:discovery:22.2.0' + dependency 'tech.pegasys.discovery:discovery:22.12.0' } } From 5a3ed755c341abb7b40db4478e73a53371330b8d Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Thu, 18 Jan 2024 07:46:49 +1000 Subject: [PATCH 19/56] Downgrade rocksdbjni to 8.3.2 following FOREST bug (#6419) Signed-off-by: Simon Dudley --- gradle/verification-metadata.xml | 10 +++++----- gradle/versions.gradle | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index eb7949b754a..673ac2df400 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -5567,12 +5567,12 @@ - - - + + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 27fd97fb42d..43fe774c463 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -204,7 +204,7 @@ dependencyManagement { dependency 'org.owasp.encoder:encoder:1.2.3' - dependency 'org.rocksdb:rocksdbjni:8.9.1' + dependency 'org.rocksdb:rocksdbjni:8.3.2' // 8.9.1 causes a bug with a FOREST canary dependencySet(group: 'org.slf4j', version:'2.0.10') { entry 'slf4j-api' From 2ba692e322e821023e13e224c9cc6f5f34042a2b Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Thu, 18 Jan 2024 10:01:28 +0100 Subject: [PATCH 20/56] Generate genesis root hash with the used data storage format (#6306) * add logic to use bonsai for genesis root hash calculation Signed-off-by: Karim Taam --- .../controller/BesuControllerBuilder.java | 4 +- .../besu/ethereum/chain/GenesisState.java | 59 +++++++++--- .../cache/NoOpCachedWorldStorageManager.java | 66 +++++++++++++ .../bonsai/trielog/NoOpTrieLogManager.java | 51 ++++++++++ .../bonsai/worldview/BonsaiWorldState.java | 2 +- .../common/GenesisWorldStateProvider.java | 89 ++++++++++++++++++ .../besu/ethereum/chain/GenesisStateTest.java | 67 +++++++++---- .../BonsaiReferenceTestWorldState.java | 93 ++----------------- 8 files changed, 311 insertions(+), 120 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 6bcfea3cad8..f2d3fce981a 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -591,7 +591,9 @@ public BesuController build() { prepForBuild(); final ProtocolSchedule protocolSchedule = createProtocolSchedule(); - final GenesisState genesisState = GenesisState.fromConfig(genesisConfig, protocolSchedule); + final GenesisState genesisState = + GenesisState.fromConfig( + dataStorageConfiguration.getDataStorageFormat(), genesisConfig, protocolSchedule); final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java index 87d45be8184..14def049f15 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.chain; import static java.util.Collections.emptyList; +import static org.hyperledger.besu.ethereum.trie.common.GenesisWorldStateProvider.createGenesisWorldState; import org.hyperledger.besu.config.GenesisAllocation; import org.hyperledger.besu.config.GenesisConfigFile; @@ -32,14 +33,10 @@ import org.hyperledger.besu.ethereum.core.Withdrawal; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.account.MutableAccount; -import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.log.LogsBloomFilter; import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; import java.util.HashMap; @@ -77,6 +74,21 @@ public static GenesisState fromJson(final String json, final ProtocolSchedule pr return fromConfig(GenesisConfigFile.fromConfig(json), protocolSchedule); } + /** + * Construct a {@link GenesisState} from a JSON string. + * + * @param dataStorageFormat A {@link DataStorageFormat} describing the storage format to use + * @param json A JSON string describing the genesis block + * @param protocolSchedule A protocol Schedule associated with + * @return A new {@link GenesisState}. + */ + public static GenesisState fromJson( + final DataStorageFormat dataStorageFormat, + final String json, + final ProtocolSchedule protocolSchedule) { + return fromConfig(dataStorageFormat, GenesisConfigFile.fromConfig(json), protocolSchedule); + } + /** * Construct a {@link GenesisState} from a JSON object. * @@ -86,10 +98,28 @@ public static GenesisState fromJson(final String json, final ProtocolSchedule pr */ public static GenesisState fromConfig( final GenesisConfigFile config, final ProtocolSchedule protocolSchedule) { + return fromConfig(DataStorageFormat.FOREST, config, protocolSchedule); + } + + /** + * Construct a {@link GenesisState} from a JSON object. + * + * @param dataStorageFormat A {@link DataStorageFormat} describing the storage format to use + * @param config A {@link GenesisConfigFile} describing the genesis block. + * @param protocolSchedule A protocol Schedule associated with + * @return A new {@link GenesisState}. + */ + public static GenesisState fromConfig( + final DataStorageFormat dataStorageFormat, + final GenesisConfigFile config, + final ProtocolSchedule protocolSchedule) { final List genesisAccounts = parseAllocations(config).toList(); final Block block = new Block( - buildHeader(config, calculateGenesisStateHash(genesisAccounts), protocolSchedule), + buildHeader( + config, + calculateGenesisStateHash(dataStorageFormat, genesisAccounts), + protocolSchedule), buildBody(config)); return new GenesisState(block, genesisAccounts); } @@ -133,15 +163,14 @@ private static void writeAccountsTo( target.persist(rootHeader); } - private static Hash calculateGenesisStateHash(final List genesisAccounts) { - final ForestWorldStateKeyValueStorage stateStorage = - new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); - final WorldStatePreimageKeyValueStorage preimageStorage = - new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); - final MutableWorldState worldState = - new ForestMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); - writeAccountsTo(worldState, genesisAccounts, null); - return worldState.rootHash(); + private static Hash calculateGenesisStateHash( + final DataStorageFormat dataStorageFormat, final List genesisAccounts) { + try (var worldState = createGenesisWorldState(dataStorageFormat)) { + writeAccountsTo(worldState, genesisAccounts, null); + return worldState.rootHash(); + } catch (Exception e) { + throw new RuntimeException(e); + } } private static BlockHeader buildHeader( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java new file mode 100644 index 00000000000..0359c464245 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java @@ -0,0 +1,66 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.bonsai.cache; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; + +import java.util.Optional; +import java.util.function.Function; + +public class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { + + public NoOpCachedWorldStorageManager( + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage) { + super(null, bonsaiWorldStateKeyValueStorage, new NoOpMetricsSystem()); + } + + @Override + public synchronized void addCachedLayer( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final BonsaiWorldState forWorldState) { + // no cache + } + + @Override + public boolean containWorldStateStorage(final Hash blockHash) { + return false; + } + + @Override + public Optional getWorldState(final Hash blockHash) { + return Optional.empty(); + } + + @Override + public Optional getNearestWorldState(final BlockHeader blockHeader) { + return Optional.empty(); + } + + @Override + public Optional getHeadWorldState( + final Function> hashBlockHeaderFunction) { + return Optional.empty(); + } + + @Override + public void reset() { + // world states are not re-used + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java new file mode 100644 index 00000000000..7cb024a259d --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java @@ -0,0 +1,51 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.bonsai.trielog; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.plugin.services.trielogs.TrieLog; + +import java.util.Optional; + +public class NoOpTrieLogManager extends TrieLogManager { + + public NoOpTrieLogManager() { + super(null, null, 0, null, TrieLogPruner.noOpTrieLogPruner()); + } + + @Override + public synchronized void saveTrieLog( + final BonsaiWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final BonsaiWorldState forWorldState) { + // notify trie log added observers, synchronously + TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); + trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); + } + + @Override + public long getMaxLayersToLoad() { + return 0; + } + + @Override + public Optional getTrieLogLayer(final Hash blockHash) { + return Optional.empty(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java index 548a3a1c414..4544a1c1de8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java @@ -92,7 +92,7 @@ public BonsaiWorldState( evmConfiguration); } - protected BonsaiWorldState( + public BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final CachedWorldStorageManager cachedWorldStorageManager, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java new file mode 100644 index 00000000000..ad1e920b77a --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java @@ -0,0 +1,89 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.common; + +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; +import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; + +import java.util.Objects; + +public class GenesisWorldStateProvider { + + /** + * Creates a Genesis world state based on the provided data storage format. + * + * @param dataStorageFormat the data storage format to use + * @return a mutable world state for the Genesis block + */ + public static MutableWorldState createGenesisWorldState( + final DataStorageFormat dataStorageFormat) { + if (Objects.requireNonNull(dataStorageFormat) == DataStorageFormat.BONSAI) { + return createGenesisBonsaiWorldState(); + } else { + return createGenesisForestWorldState(); + } + } + + /** + * Creates a Genesis world state using the Bonsai data storage format. + * + * @return a mutable world state for the Genesis block + */ + private static MutableWorldState createGenesisBonsaiWorldState() { + final CachedMerkleTrieLoader cachedMerkleTrieLoader = + new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = + new BonsaiWorldStateKeyValueStorage( + new KeyValueStorageProvider( + segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), + new InMemoryKeyValueStorage(), + new NoOpMetricsSystem()), + new NoOpMetricsSystem()); + return new BonsaiWorldState( + bonsaiWorldStateKeyValueStorage, + cachedMerkleTrieLoader, + new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), + new NoOpTrieLogManager(), + EvmConfiguration.DEFAULT); + } + + /** + * Creates a Genesis world state using the Forest data storage format. + * + * @return a mutable world state for the Genesis block + */ + private static MutableWorldState createGenesisForestWorldState() { + final ForestWorldStateKeyValueStorage stateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStatePreimageKeyValueStorage preimageStorage = + new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); + return new ForestMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java index 1a069542672..0a8fc12b628 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/GenesisStateTest.java @@ -24,14 +24,21 @@ import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.ProtocolScheduleFixture; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.account.Account; +import java.util.stream.Stream; + import com.google.common.base.Charsets; import com.google.common.io.Resources; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; import org.bouncycastle.util.encoders.Hex; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; public final class GenesisStateTest { @@ -46,10 +53,20 @@ public final class GenesisStateTest { private static final String EXPECTED_CODE = "0x608060405260043610610116577c01000000000000000000000000000000000000000000000000000000006000350463025e7c278114610158578063173825d91461019e57806320ea8d86146101d15780632f54bf6e146101fb5780633411c81c14610242578063547415251461027b5780637065cb48146102c1578063784547a7146102f45780638b51d13f1461031e5780639ace38c214610348578063a0e67e2b14610415578063a8abe69a1461047a578063b5dc40c3146104ba578063b77bf600146104e4578063ba51a6df146104f9578063c01a8c8414610523578063c64274741461054d578063d74f8edd14610615578063dc8452cd1461062a578063e20056e61461063f578063ee22610b1461067a575b60003411156101565760408051348152905133917fe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c919081900360200190a25b005b34801561016457600080fd5b506101826004803603602081101561017b57600080fd5b50356106a4565b60408051600160a060020a039092168252519081900360200190f35b3480156101aa57600080fd5b50610156600480360360208110156101c157600080fd5b5035600160a060020a03166106cc565b3480156101dd57600080fd5b50610156600480360360208110156101f457600080fd5b503561083c565b34801561020757600080fd5b5061022e6004803603602081101561021e57600080fd5b5035600160a060020a03166108f6565b604080519115158252519081900360200190f35b34801561024e57600080fd5b5061022e6004803603604081101561026557600080fd5b5080359060200135600160a060020a031661090b565b34801561028757600080fd5b506102af6004803603604081101561029e57600080fd5b50803515159060200135151561092b565b60408051918252519081900360200190f35b3480156102cd57600080fd5b50610156600480360360208110156102e457600080fd5b5035600160a060020a0316610997565b34801561030057600080fd5b5061022e6004803603602081101561031757600080fd5b5035610abc565b34801561032a57600080fd5b506102af6004803603602081101561034157600080fd5b5035610b43565b34801561035457600080fd5b506103726004803603602081101561036b57600080fd5b5035610bb2565b6040518085600160a060020a0316600160a060020a031681526020018481526020018060200183151515158152602001828103825284818151815260200191508051906020019080838360005b838110156103d75781810151838201526020016103bf565b50505050905090810190601f1680156104045780820380516001836020036101000a031916815260200191505b509550505050505060405180910390f35b34801561042157600080fd5b5061042a610c70565b60408051602080825283518183015283519192839290830191858101910280838360005b8381101561046657818101518382015260200161044e565b505050509050019250505060405180910390f35b34801561048657600080fd5b5061042a6004803603608081101561049d57600080fd5b508035906020810135906040810135151590606001351515610cd3565b3480156104c657600080fd5b5061042a600480360360208110156104dd57600080fd5b5035610e04565b3480156104f057600080fd5b506102af610f75565b34801561050557600080fd5b506101566004803603602081101561051c57600080fd5b5035610f7b565b34801561052f57600080fd5b506101566004803603602081101561054657600080fd5b5035610ffa565b34801561055957600080fd5b506102af6004803603606081101561057057600080fd5b600160a060020a03823516916020810135918101906060810160408201356401000000008111156105a057600080fd5b8201836020820111156105b257600080fd5b803590602001918460018302840111640100000000831117156105d457600080fd5b91908080601f0160208091040260200160405190810160405280939291908181526020018383808284376000920191909152509295506110c5945050505050565b34801561062157600080fd5b506102af6110e4565b34801561063657600080fd5b506102af6110e9565b34801561064b57600080fd5b506101566004803603604081101561066257600080fd5b50600160a060020a03813581169160200135166110ef565b34801561068657600080fd5b506101566004803603602081101561069d57600080fd5b5035611289565b60038054829081106106b257fe5b600091825260209091200154600160a060020a0316905081565b3330146106d857600080fd5b600160a060020a038116600090815260026020526040902054819060ff16151561070157600080fd5b600160a060020a0382166000908152600260205260408120805460ff191690555b600354600019018110156107d75782600160a060020a031660038281548110151561074957fe5b600091825260209091200154600160a060020a031614156107cf5760038054600019810190811061077657fe5b60009182526020909120015460038054600160a060020a03909216918390811061079c57fe5b9060005260206000200160006101000a815481600160a060020a030219169083600160a060020a031602179055506107d7565b600101610722565b506003805460001901906107eb9082611557565b5060035460045411156108045760035461080490610f7b565b604051600160a060020a038316907f8001553a916ef2f495d26a907cc54d96ed840d7bda71e73194bf5a9df7a76b9090600090a25050565b3360008181526002602052604090205460ff16151561085a57600080fd5b60008281526001602090815260408083203380855292529091205483919060ff16151561088657600080fd5b600084815260208190526040902060030154849060ff16156108a757600080fd5b6000858152600160209081526040808320338085529252808320805460ff191690555187927ff6a317157440607f36269043eb55f1287a5a19ba2216afeab88cd46cbcfb88e991a35050505050565b60026020526000908152604090205460ff1681565b600160209081526000928352604080842090915290825290205460ff1681565b6000805b60055481101561099057838015610958575060008181526020819052604090206003015460ff16155b8061097c575082801561097c575060008181526020819052604090206003015460ff165b15610988576001820191505b60010161092f565b5092915050565b3330146109a357600080fd5b600160a060020a038116600090815260026020526040902054819060ff16156109cb57600080fd5b81600160a060020a03811615156109e157600080fd5b600380549050600101600454603282111580156109fe5750818111155b8015610a0957508015155b8015610a1457508115155b1515610a1f57600080fd5b600160a060020a038516600081815260026020526040808220805460ff1916600190811790915560038054918201815583527fc2575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b01805473ffffffffffffffffffffffffffffffffffffffff191684179055517ff39e6e1eb0edcf53c221607b54b00cd28f3196fed0a24994dc308b8f611b682d9190a25050505050565b600080805b600354811015610b3b5760008481526001602052604081206003805491929184908110610aea57fe5b6000918252602080832090910154600160a060020a0316835282019290925260400190205460ff1615610b1e576001820191505b600454821415610b3357600192505050610b3e565b600101610ac1565b50505b919050565b6000805b600354811015610bac5760008381526001602052604081206003805491929184908110610b7057fe5b6000918252602080832090910154600160a060020a0316835282019290925260400190205460ff1615610ba4576001820191505b600101610b47565b50919050565b6000602081815291815260409081902080546001808301546002808501805487516101009582161595909502600019011691909104601f8101889004880284018801909652858352600160a060020a0390931695909491929190830182828015610c5d5780601f10610c3257610100808354040283529160200191610c5d565b820191906000526020600020905b815481529060010190602001808311610c4057829003601f168201915b5050506003909301549192505060ff1684565b60606003805480602002602001604051908101604052809291908181526020018280548015610cc857602002820191906000526020600020905b8154600160a060020a03168152600190910190602001808311610caa575b505050505090505b90565b606080600554604051908082528060200260200182016040528015610d02578160200160208202803883390190505b5090506000805b600554811015610d8457858015610d32575060008181526020819052604090206003015460ff16155b80610d565750848015610d56575060008181526020819052604090206003015460ff165b15610d7c57808383815181101515610d6a57fe5b60209081029091010152600191909101905b600101610d09565b878703604051908082528060200260200182016040528015610db0578160200160208202803883390190505b5093508790505b86811015610df9578281815181101515610dcd57fe5b9060200190602002015184898303815181101515610de757fe5b60209081029091010152600101610db7565b505050949350505050565b606080600380549050604051908082528060200260200182016040528015610e36578160200160208202803883390190505b5090506000805b600354811015610eee5760008581526001602052604081206003805491929184908110610e6657fe5b6000918252602080832090910154600160a060020a0316835282019290925260400190205460ff1615610ee6576003805482908110610ea157fe5b6000918252602090912001548351600160a060020a0390911690849084908110610ec757fe5b600160a060020a03909216602092830290910190910152600191909101905b600101610e3d565b81604051908082528060200260200182016040528015610f18578160200160208202803883390190505b509350600090505b81811015610f6d578281815181101515610f3657fe5b906020019060200201518482815181101515610f4e57fe5b600160a060020a03909216602092830290910190910152600101610f20565b505050919050565b60055481565b333014610f8757600080fd5b6003548160328211801590610f9c5750818111155b8015610fa757508015155b8015610fb257508115155b1515610fbd57600080fd5b60048390556040805184815290517fa3f1ee9126a074d9326c682f561767f710e927faa811f7a99829d49dc421797a9181900360200190a1505050565b3360008181526002602052604090205460ff16151561101857600080fd5b6000828152602081905260409020548290600160a060020a0316151561103d57600080fd5b60008381526001602090815260408083203380855292529091205484919060ff161561106857600080fd5b6000858152600160208181526040808420338086529252808420805460ff1916909317909255905187927f4a504a94899432a9846e1aa406dceb1bcfd538bb839071d49d1e5e23f5be30ef91a36110be85611289565b5050505050565b60006110d2848484611444565b90506110dd81610ffa565b9392505050565b603281565b60045481565b3330146110fb57600080fd5b600160a060020a038216600090815260026020526040902054829060ff16151561112457600080fd5b600160a060020a038216600090815260026020526040902054829060ff161561114c57600080fd5b82600160a060020a038116151561116257600080fd5b60005b6003548110156111ee5785600160a060020a031660038281548110151561118857fe5b600091825260209091200154600160a060020a031614156111e657846003828154811015156111b357fe5b9060005260206000200160006101000a815481600160a060020a030219169083600160a060020a031602179055506111ee565b600101611165565b50600160a060020a03808616600081815260026020526040808220805460ff1990811690915593881682528082208054909416600117909355915190917f8001553a916ef2f495d26a907cc54d96ed840d7bda71e73194bf5a9df7a76b9091a2604051600160a060020a038516907ff39e6e1eb0edcf53c221607b54b00cd28f3196fed0a24994dc308b8f611b682d90600090a25050505050565b3360008181526002602052604090205460ff1615156112a757600080fd5b60008281526001602090815260408083203380855292529091205483919060ff1615156112d357600080fd5b600084815260208190526040902060030154849060ff16156112f457600080fd5b6112fd85610abc565b156110be576000858152602081815260409182902060038101805460ff19166001908117909155815481830154600280850180548851601f6000199783161561010002979097019091169290920494850187900487028201870190975283815293956113cf95600160a060020a039093169491939283908301828280156113c55780601f1061139a576101008083540402835291602001916113c5565b820191906000526020600020905b8154815290600101906020018083116113a857829003601f168201915b5050505050611534565b156114045760405186907f33e13ecb54c3076d8e8bb8c2881800a4d972b792045ffae98fdf46df365fed7590600090a261143c565b60405186907f526441bb6c1aba3c9a4a6ca1d6545da9c2333c8c48343ef398eb858d72b7923690600090a260038101805460ff191690555b505050505050565b600083600160a060020a038116151561145c57600080fd5b60055460408051608081018252600160a060020a0388811682526020808301898152838501898152600060608601819052878152808452959095208451815473ffffffffffffffffffffffffffffffffffffffff1916941693909317835551600183015592518051949650919390926114dc926002850192910190611580565b50606091909101516003909101805460ff191691151591909117905560058054600101905560405182907fc0ba8fe4b176c1714197d43b9cc6bcf797a4a7461c5fe8d0ef6e184ae7601e5190600090a2509392505050565b6000806040516020840160008287838a8c6187965a03f198975050505050505050565b81548183558181111561157b5760008381526020902061157b9181019083016115fe565b505050565b828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f106115c157805160ff19168380011785556115ee565b828001600101855582156115ee579182015b828111156115ee5782518255916020019190600101906115d3565b506115fa9291506115fe565b5090565b610cd091905b808211156115fa576000815560010161160456fea165627a7a7230582070d3c680a2cf749f81772e7fffa2883f27a13c65fcfff32190d7585b0c6f0ce40029"; - @Test - public void createFromJsonWithAllocs() throws Exception { + static class GenesisStateTestArguments implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of( + Arguments.of(DataStorageFormat.BONSAI), Arguments.of(DataStorageFormat.FOREST)); + } + } + + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void createFromJsonWithAllocs(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString(GenesisStateTest.class.getResource("genesis1.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); @@ -74,10 +91,12 @@ public void createFromJsonWithAllocs() throws Exception { assertThat(second.getBalance().toLong()).isEqualTo(222222222); } - @Test - public void createFromJsonNoAllocs() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void createFromJsonNoAllocs(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString(GenesisStateTest.class.getResource("genesis2.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); @@ -89,10 +108,12 @@ public void createFromJsonNoAllocs() throws Exception { assertThat(header.getParentHash()).isEqualTo(Hash.ZERO); } - private void assertContractInvariants(final String sourceFile, final String blockHash) + private void assertContractInvariants( + final DataStorageFormat dataStorageFormat, final String sourceFile, final String blockHash) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString(GenesisStateTest.class.getResource(sourceFile), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); final BlockHeader header = genesisState.getBlock().getHeader(); @@ -113,16 +134,22 @@ private void assertContractInvariants(final String sourceFile, final String bloc "000000000000000000000000385ef55e292fa39cf5ffbad99f534294565519ba"); } - @Test - public void createFromJsonWithContract() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void createFromJsonWithContract(final DataStorageFormat dataStorageFormat) + throws Exception { assertContractInvariants( - "genesis3.json", "0xe7fd8db206dcaf066b7c97b8a42a0abc18653613560748557ab44868652a78b6"); + dataStorageFormat, + "genesis3.json", + "0xe7fd8db206dcaf066b7c97b8a42a0abc18653613560748557ab44868652a78b6"); } - @Test - public void createFromJsonWithNonce() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void createFromJsonWithNonce(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString( GenesisStateTest.class.getResource("genesisNonce.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); @@ -133,10 +160,12 @@ public void createFromJsonWithNonce() throws Exception { "0x36750291f1a8429aeb553a790dc2d149d04dbba0ca4cfc7fd5eb12d478117c9f")); } - @Test - public void encodeOlympicBlock() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void encodeOlympicBlock(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString( GenesisStateTest.class.getResource("genesis-olympic.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); @@ -152,10 +181,12 @@ private void assertStorageValue(final Account contract, final String key, final .isEqualTo(UInt256.fromHexString(value)); } - @Test - public void genesisFromShanghai() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void genesisFromShanghai(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString( GenesisStateTest.class.getResource("genesis_shanghai.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); @@ -200,10 +231,12 @@ public void genesisFromShanghai() throws Exception { assertThat(lastBalance).isEqualTo(Wei.fromHexString("0x123450000000000000000")); } - @Test - public void genesisFromCancun() throws Exception { + @ParameterizedTest + @ArgumentsSource(GenesisStateTestArguments.class) + public void genesisFromCancun(final DataStorageFormat dataStorageFormat) throws Exception { final GenesisState genesisState = GenesisState.fromJson( + dataStorageFormat, Resources.toString( GenesisStateTest.class.getResource("genesis_cancun.json"), Charsets.UTF_8), ProtocolScheduleFixture.MAINNET); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 26fd811bfae..034f8b9debf 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -20,22 +20,18 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.Map; -import java.util.Optional; -import java.util.function.Function; import java.util.stream.Stream; import com.fasterxml.jackson.annotation.JsonCreator; @@ -117,14 +113,14 @@ public static BonsaiReferenceTestWorldState create( final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = + new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider(), metricsSystem); + final BonsaiReferenceTestWorldStateStorage worldStateStorage = - new BonsaiReferenceTestWorldStateStorage( - new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), metricsSystem), - preImageProxy); + new BonsaiReferenceTestWorldStateStorage(bonsaiWorldStateKeyValueStorage, preImageProxy); final NoOpCachedWorldStorageManager noOpCachedWorldStorageManager = - new NoOpCachedWorldStorageManager(); + new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage); final BonsaiReferenceTestWorldState worldState = new BonsaiReferenceTestWorldState( @@ -149,81 +145,6 @@ public Stream streamAccounts(final Bytes32 startKeyHash, fina return this.refTestStorage.streamAccounts(this, startKeyHash, limit); } - static class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { - - public NoOpCachedWorldStorageManager() { - super( - null, - new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()), - new NoOpMetricsSystem()); - } - - @SuppressWarnings({"UnsynchronizedOverridesSynchronized", "squid:S3551"}) - @Override - public void addCachedLayer( - final BlockHeader blockHeader, - final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { - // reference test world states are not cached - } - - @Override - public boolean containWorldStateStorage(final Hash blockHash) { - return false; - } - - @Override - public Optional getWorldState(final Hash blockHash) { - return Optional.empty(); - } - - @Override - public Optional getNearestWorldState(final BlockHeader blockHeader) { - return Optional.empty(); - } - - @Override - public Optional getHeadWorldState( - final Function> hashBlockHeaderFunction) { - return Optional.empty(); - } - - @Override - public void reset() { - // reference test world states are not re-used - } - } - - static class NoOpTrieLogManager extends TrieLogManager { - - public NoOpTrieLogManager() { - super(null, null, 0, null, TrieLogPruner.noOpTrieLogPruner()); - } - - @SuppressWarnings({"UnsynchronizedOverridesSynchronized", "squid:S3551"}) - @Override - public void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, - final Hash forWorldStateRootHash, - final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { - // notify trie log added observers, synchronously - TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); - trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); - } - - @Override - public long getMaxLayersToLoad() { - return 0; - } - - @Override - public Optional getTrieLogLayer(final Hash blockHash) { - return Optional.empty(); - } - } - @Override protected Hash hashAndSavePreImage(final Bytes value) { // by default do not save has preImages From 2d7941f8b84eb311a42b52270daf32e0465429ca Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 18 Jan 2024 11:05:31 +0100 Subject: [PATCH 21/56] Promote block txs selection max time options to stable (#6423) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 6 +- .../org/hyperledger/besu/cli/BesuCommand.java | 8 +- .../converter/PositiveNumberConverter.java | 33 ++++++++ .../PositiveNumberConversionException.java | 30 +++++++ .../besu/cli/options/MiningOptions.java | 78 ++++++++----------- .../hyperledger/besu/cli/BesuCommandTest.java | 3 + .../besu/cli/options/MiningOptionsTest.java | 52 ++++++------- .../src/test/resources/everything_config.toml | 2 + .../txselection/BlockTransactionSelector.java | 2 +- .../AbstractBlockTransactionSelectorTest.java | 35 +++++---- ...FeeMarketBlockTransactionSelectorTest.java | 2 +- .../besu/ethereum/core/MiningParameters.java | 52 +++++++------ 12 files changed, 181 insertions(+), 122 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/converter/PositiveNumberConverter.java create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/converter/exception/PositiveNumberConversionException.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e09db4de540..4ab78beb7b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,18 +6,20 @@ - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) +- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. ### Deprecations ### Additions and Improvements - Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) - Disable transaction handling when the node is not in sync, to avoid unnecessary transaction validation work [#6302](https://github.com/hyperledger/besu/pull/6302) -- Introduce TransactionEvaluationContext to pass data between transaction selectors and plugin, during block creation [#6381](https://github.com/hyperledger/besu/pull/6381) +- Introduce TransactionEvaluationContext to pass data between transaction selectors and plugin, during block creation [#6381](https://github.com/hyperledger/besu/pull/6381) - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) -- Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) +- Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) - Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) - Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) - Upgrade `tech.pegasys.discovery:discovery` [#6414](https://github.com/hyperledger/besu/pull/6414) +- Options to tune the max allowed time that can be spent selecting transactions during block creation are now stable [#6423](https://github.com/hyperledger/besu/pull/6423) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index c51d03d4305..7f276c704c4 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2907,17 +2907,15 @@ private MiningParameters getMiningParameters() { ImmutableMiningParameters.builder().from(miningOptions.toDomainObject()); final var actualGenesisOptions = getActualGenesisConfigOptions(); if (actualGenesisOptions.isPoa()) { - miningParametersBuilder.unstable( - ImmutableMiningParameters.Unstable.builder() - .minBlockTime(getMinBlockTime(actualGenesisOptions)) - .build()); + miningParametersBuilder.genesisBlockPeriodSeconds( + getGenesisBlockPeriodSeconds(actualGenesisOptions)); } miningParameters = miningParametersBuilder.build(); } return miningParameters; } - private int getMinBlockTime(final GenesisConfigOptions genesisConfigOptions) { + private int getGenesisBlockPeriodSeconds(final GenesisConfigOptions genesisConfigOptions) { if (genesisConfigOptions.isClique()) { return genesisConfigOptions.getCliqueConfigOptions().getBlockPeriodSeconds(); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/converter/PositiveNumberConverter.java b/besu/src/main/java/org/hyperledger/besu/cli/converter/PositiveNumberConverter.java new file mode 100644 index 00000000000..2b45f35ffe5 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/converter/PositiveNumberConverter.java @@ -0,0 +1,33 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.converter; + +import org.hyperledger.besu.cli.converter.exception.PercentageConversionException; +import org.hyperledger.besu.util.number.PositiveNumber; + +import picocli.CommandLine; + +/** The PositiveNumber Cli type converter. */ +public class PositiveNumberConverter implements CommandLine.ITypeConverter { + + @Override + public PositiveNumber convert(final String value) throws PercentageConversionException { + try { + return PositiveNumber.fromString(value); + } catch (NullPointerException | IllegalArgumentException e) { + throw new PercentageConversionException(value); + } + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/converter/exception/PositiveNumberConversionException.java b/besu/src/main/java/org/hyperledger/besu/cli/converter/exception/PositiveNumberConversionException.java new file mode 100644 index 00000000000..c9f0d9e934b --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/converter/exception/PositiveNumberConversionException.java @@ -0,0 +1,30 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.converter.exception; + +import static java.lang.String.format; + +/** The custom PositiveNumber conversion exception. */ +public final class PositiveNumberConversionException extends Exception { + + /** + * Instantiates a new PositiveNumber conversion exception. + * + * @param value the invalid value to add in exception message + */ + public PositiveNumberConversionException(final String value) { + super(format("Invalid value: %s, should be a positive number >0.", value)); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java index 78e2032e4f6..55197b8e826 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java @@ -16,20 +16,20 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.hyperledger.besu.ethereum.core.MiningParameters.MutableInitValues.DEFAULT_EXTRA_DATA; import static org.hyperledger.besu.ethereum.core.MiningParameters.MutableInitValues.DEFAULT_MIN_BLOCK_OCCUPANCY_RATIO; import static org.hyperledger.besu.ethereum.core.MiningParameters.MutableInitValues.DEFAULT_MIN_PRIORITY_FEE_PER_GAS; import static org.hyperledger.besu.ethereum.core.MiningParameters.MutableInitValues.DEFAULT_MIN_TRANSACTION_GAS_PRICE; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_MAX_OMMERS_DEPTH; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POS_BLOCK_CREATION_MAX_TIME; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POS_BLOCK_CREATION_REPETITION_MIN_DURATION; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POW_JOB_TTL; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_REMOTE_SEALERS_LIMIT; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_REMOTE_SEALERS_TTL; -import org.hyperledger.besu.cli.converter.PercentageConverter; +import org.hyperledger.besu.cli.converter.PositiveNumberConverter; import org.hyperledger.besu.cli.util.CommandLineUtils; import org.hyperledger.besu.config.GenesisConfigOptions; import org.hyperledger.besu.datatypes.Address; @@ -37,7 +37,7 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningParameters; -import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.util.List; @@ -115,6 +115,24 @@ public class MiningOptions implements CLIOptions { + " If set, each block's gas limit will approach this setting over time.") private Long targetGasLimit = null; + @Option( + names = {"--block-txs-selection-max-time"}, + converter = PositiveNumberConverter.class, + description = + "Specifies the maximum time, in milliseconds, that could be spent selecting transactions to be included in the block." + + " Not compatible with PoA networks, see poa-block-txs-selection-max-time. (default: ${DEFAULT-VALUE})") + private PositiveNumber nonPoaBlockTxsSelectionMaxTime = + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; + + @Option( + names = {"--poa-block-txs-selection-max-time"}, + converter = PositiveNumberConverter.class, + description = + "Specifies the maximum time that could be spent selecting transactions to be included in the block, as a percentage of the fixed block time of the PoA network." + + " To be only used on PoA networks, for other networks see block-txs-selection-max-time." + + " (default: ${DEFAULT-VALUE})") + private PositiveNumber poaBlockTxsSelectionMaxTime = DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; + @CommandLine.ArgGroup(validate = false) private final Unstable unstableOptions = new Unstable(); @@ -168,25 +186,6 @@ static class Unstable { + " then it waits before next repetition. Must be positive and ≤ 2000 (default: ${DEFAULT-VALUE} milliseconds)") private Long posBlockCreationRepetitionMinDuration = DEFAULT_POS_BLOCK_CREATION_REPETITION_MIN_DURATION; - - @CommandLine.Option( - hidden = true, - names = {"--Xblock-txs-selection-max-time"}, - description = - "Specifies the maximum time, in milliseconds, that could be spent selecting transactions to be included in the block." - + " Not compatible with PoA networks, see Xpoa-block-txs-selection-max-time." - + " Must be positive and ≤ (default: ${DEFAULT-VALUE})") - private Long nonPoaBlockTxsSelectionMaxTime = DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; - - @CommandLine.Option( - hidden = true, - names = {"--Xpoa-block-txs-selection-max-time"}, - converter = PercentageConverter.class, - description = - "Specifies the maximum time that could be spent selecting transactions to be included in the block, as a percentage of the fixed block time of the PoA network." - + " To be only used on PoA networks, for other networks see Xblock-txs-selection-max-time." - + " (default: ${DEFAULT-VALUE})") - private Percentage poaBlockTxsSelectionMaxTime = DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; } private MiningOptions() {} @@ -270,26 +269,17 @@ public void validate( if (genesisConfigOptions.isPoa()) { CommandLineUtils.failIfOptionDoesntMeetRequirement( commandLine, - "--Xblock-txs-selection-max-time can't be used with PoA networks," - + " see Xpoa-block-txs-selection-max-time instead", + "--block-txs-selection-max-time can't be used with PoA networks," + + " see poa-block-txs-selection-max-time instead", false, - singletonList("--Xblock-txs-selection-max-time")); + singletonList("--block-txs-selection-max-time")); } else { CommandLineUtils.failIfOptionDoesntMeetRequirement( commandLine, - "--Xpoa-block-txs-selection-max-time can be only used with PoA networks," - + " see --Xblock-txs-selection-max-time instead", + "--poa-block-txs-selection-max-time can be only used with PoA networks," + + " see --block-txs-selection-max-time instead", false, - singletonList("--Xpoa-block-txs-selection-max-time")); - - if (unstableOptions.nonPoaBlockTxsSelectionMaxTime <= 0 - || unstableOptions.nonPoaBlockTxsSelectionMaxTime - > DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME) { - throw new ParameterException( - commandLine, - "--Xblock-txs-selection-max-time must be positive and ≤ " - + DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME); - } + singletonList("--poa-block-txs-selection-max-time")); } } @@ -303,6 +293,10 @@ static MiningOptions fromConfig(final MiningParameters miningParameters) { miningOptions.minTransactionGasPrice = miningParameters.getMinTransactionGasPrice(); miningOptions.minPriorityFeePerGas = miningParameters.getMinPriorityFeePerGas(); miningOptions.minBlockOccupancyRatio = miningParameters.getMinBlockOccupancyRatio(); + miningOptions.nonPoaBlockTxsSelectionMaxTime = + miningParameters.getNonPoaBlockTxsSelectionMaxTime(); + miningOptions.poaBlockTxsSelectionMaxTime = miningParameters.getPoaBlockTxsSelectionMaxTime(); + miningOptions.unstableOptions.remoteSealersLimit = miningParameters.getUnstable().getRemoteSealersLimit(); miningOptions.unstableOptions.remoteSealersTimeToLive = @@ -317,10 +311,6 @@ static MiningOptions fromConfig(final MiningParameters miningParameters) { miningParameters.getUnstable().getPosBlockCreationMaxTime(); miningOptions.unstableOptions.posBlockCreationRepetitionMinDuration = miningParameters.getUnstable().getPosBlockCreationRepetitionMinDuration(); - miningOptions.unstableOptions.nonPoaBlockTxsSelectionMaxTime = - miningParameters.getUnstable().getBlockTxsSelectionMaxTime(); - miningOptions.unstableOptions.poaBlockTxsSelectionMaxTime = - miningParameters.getUnstable().getPoaBlockTxsSelectionMaxTime(); miningParameters.getCoinbase().ifPresent(coinbase -> miningOptions.coinbase = coinbase); miningParameters.getTargetGasLimit().ifPresent(tgl -> miningOptions.targetGasLimit = tgl); @@ -350,6 +340,8 @@ public MiningParameters toDomainObject() { .isStratumMiningEnabled(iStratumMiningEnabled) .stratumNetworkInterface(stratumNetworkInterface) .stratumPort(stratumPort) + .nonPoaBlockTxsSelectionMaxTime(nonPoaBlockTxsSelectionMaxTime) + .poaBlockTxsSelectionMaxTime(poaBlockTxsSelectionMaxTime) .unstable( ImmutableMiningParameters.Unstable.builder() .remoteSealersLimit(unstableOptions.remoteSealersLimit) @@ -360,8 +352,6 @@ public MiningParameters toDomainObject() { .posBlockCreationMaxTime(unstableOptions.posBlockCreationMaxTime) .posBlockCreationRepetitionMinDuration( unstableOptions.posBlockCreationRepetitionMinDuration) - .nonPoaBlockTxsSelectionMaxTime(unstableOptions.nonPoaBlockTxsSelectionMaxTime) - .poaBlockTxsSelectionMaxTime(unstableOptions.poaBlockTxsSelectionMaxTime) .build()); return miningParametersBuilder.build(); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 26296556302..04d9f015530 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -95,6 +95,7 @@ import org.hyperledger.besu.plugin.services.rpc.PluginRpcRequest; import org.hyperledger.besu.util.number.Fraction; import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import org.hyperledger.besu.util.platform.PlatformDetector; import java.io.File; @@ -846,6 +847,8 @@ public void tomlThatConfiguresEverythingExceptPermissioningToml() throws IOExcep tomlResult.getDouble(tomlKey); } else if (Percentage.class.isAssignableFrom(optionSpec.type())) { tomlResult.getLong(tomlKey); + } else if (PositiveNumber.class.isAssignableFrom(optionSpec.type())) { + tomlResult.getLong(tomlKey); } else { tomlResult.getString(tomlKey); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java index 404c021e3b9..4b1fdddb53a 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java @@ -15,8 +15,8 @@ package org.hyperledger.besu.cli.options; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_POS_BLOCK_CREATION_MAX_TIME; import static org.mockito.Mockito.atMost; import static org.mockito.Mockito.verify; @@ -28,7 +28,7 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.Unstable; import org.hyperledger.besu.ethereum.core.MiningParameters; -import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.io.IOException; import java.nio.file.Path; @@ -315,35 +315,26 @@ public void posBlockCreationMaxTimeOutOfAllowedRange() { public void blockTxsSelectionMaxTimeDefaultValue() { internalTestSuccess( miningParams -> - assertThat(miningParams.getUnstable().getBlockTxsSelectionMaxTime()) + assertThat(miningParams.getNonPoaBlockTxsSelectionMaxTime()) .isEqualTo(DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME)); } @Test public void blockTxsSelectionMaxTimeOption() { internalTestSuccess( - miningParams -> - assertThat(miningParams.getUnstable().getBlockTxsSelectionMaxTime()).isEqualTo(1700L), - "--Xblock-txs-selection-max-time", + miningParams -> assertThat(miningParams.getBlockTxsSelectionMaxTime()).isEqualTo(1700L), + "--block-txs-selection-max-time", "1700"); } - @Test - public void blockTxsSelectionMaxTimeOutOfAllowedRange() { - internalTestFailure( - "--Xblock-txs-selection-max-time must be positive and ≤ 5000", - "--Xblock-txs-selection-max-time", - "6000"); - } - @Test public void blockTxsSelectionMaxTimeIncompatibleWithPoaNetworks() throws IOException { final Path genesisFileIBFT2 = createFakeGenesisFile(VALID_GENESIS_IBFT2_POST_LONDON); internalTestFailure( - "--Xblock-txs-selection-max-time can't be used with PoA networks, see Xpoa-block-txs-selection-max-time instead", + "--block-txs-selection-max-time can't be used with PoA networks, see poa-block-txs-selection-max-time instead", "--genesis-file", genesisFileIBFT2.toString(), - "--Xblock-txs-selection-max-time", + "--block-txs-selection-max-time", "2"); } @@ -351,7 +342,7 @@ public void blockTxsSelectionMaxTimeIncompatibleWithPoaNetworks() throws IOExcep public void poaBlockTxsSelectionMaxTimeDefaultValue() { internalTestSuccess( miningParams -> - assertThat(miningParams.getUnstable().getPoaBlockTxsSelectionMaxTime()) + assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) .isEqualTo(DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME)); } @@ -360,27 +351,32 @@ public void poaBlockTxsSelectionMaxTimeOption() throws IOException { final Path genesisFileIBFT2 = createFakeGenesisFile(VALID_GENESIS_IBFT2_POST_LONDON); internalTestSuccess( miningParams -> - assertThat(miningParams.getUnstable().getPoaBlockTxsSelectionMaxTime()) - .isEqualTo(Percentage.fromInt(80)), + assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) + .isEqualTo(PositiveNumber.fromInt(80)), "--genesis-file", genesisFileIBFT2.toString(), - "--Xpoa-block-txs-selection-max-time", + "--poa-block-txs-selection-max-time", "80"); } @Test - public void poaBlockTxsSelectionMaxTimeOutOfAllowedRange() { - internalTestFailure( - "Invalid value for option '--Xpoa-block-txs-selection-max-time': cannot convert '110' to Percentage", - "--Xpoa-block-txs-selection-max-time", - "110"); + public void poaBlockTxsSelectionMaxTimeOptionOver100Percent() throws IOException { + final Path genesisFileIBFT2 = createFakeGenesisFile(VALID_GENESIS_IBFT2_POST_LONDON); + internalTestSuccess( + miningParams -> + assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) + .isEqualTo(PositiveNumber.fromInt(200)), + "--genesis-file", + genesisFileIBFT2.toString(), + "--poa-block-txs-selection-max-time", + "200"); } @Test public void poaBlockTxsSelectionMaxTimeOnlyCompatibleWithPoaNetworks() { internalTestFailure( - "--Xpoa-block-txs-selection-max-time can be only used with PoA networks, see --Xblock-txs-selection-max-time instead", - "--Xpoa-block-txs-selection-max-time", + "--poa-block-txs-selection-max-time can be only used with PoA networks, see --block-txs-selection-max-time instead", + "--poa-block-txs-selection-max-time", "90"); } diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index c653f3f60fd..b8dae6c1889 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -142,6 +142,8 @@ min-priority-fee=0 min-block-occupancy-ratio=0.7 miner-stratum-host="0.0.0.0" miner-stratum-port=8008 +block-txs-selection-max-time=5000 +poa-block-txs-selection-max-time=75 Xminer-remote-sealers-limit=1000 Xminer-remote-sealers-hashrate-ttl=10 Xpos-block-creation-max-time=5 diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java index 944cf426d73..26ac79ef384 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java @@ -136,7 +136,7 @@ public BlockTransactionSelector( this.pluginTransactionSelector = pluginTransactionSelector; this.pluginOperationTracer = pluginTransactionSelector.getOperationTracer(); blockWorldStateUpdater = worldState.updater(); - blockTxsSelectionMaxTime = miningParameters.getUnstable().getBlockTxsSelectionMaxTime(); + blockTxsSelectionMaxTime = miningParameters.getBlockTxsSelectionMaxTime(); } private List createTransactionSelectors( diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java index 0b0b2cd7831..e593569ff28 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java @@ -17,7 +17,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; import static org.awaitility.Awaitility.await; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.BLOCK_SELECTION_TIMEOUT; import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.PRIORITY_FEE_PER_GAS_BELOW_CURRENT_MIN; import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.SELECTED; @@ -54,7 +54,6 @@ import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; -import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.Unstable; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.MutableWorldState; @@ -85,7 +84,7 @@ import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; import org.hyperledger.besu.plugin.services.txselection.TransactionEvaluationContext; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; -import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.math.BigInteger; import java.time.Instant; @@ -956,8 +955,8 @@ private void internalBlockSelectionTimeoutSimulation( final ProcessableBlockHeader blockHeader = createBlock(301_000); final Address miningBeneficiary = AddressHelpers.ofValue(1); - final int poaMinBlockTime = 1; - final long blockTxsSelectionMaxTime = 750; + final int poaGenesisBlockPeriod = 1; + final int blockTxsSelectionMaxTime = 750; final List transactionsToInject = new ArrayList<>(3); for (int i = 0; i < 2; i++) { @@ -987,9 +986,14 @@ private void internalBlockSelectionTimeoutSimulation( createBlockSelectorAndSetupTxPool( isPoa ? createMiningParameters( - Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, poaMinBlockTime, Percentage.fromInt(75)) + Wei.ZERO, + MIN_OCCUPANCY_100_PERCENT, + poaGenesisBlockPeriod, + PositiveNumber.fromInt(75)) : createMiningParameters( - Wei.ZERO, MIN_OCCUPANCY_100_PERCENT, blockTxsSelectionMaxTime), + Wei.ZERO, + MIN_OCCUPANCY_100_PERCENT, + PositiveNumber.fromInt(blockTxsSelectionMaxTime)), transactionProcessor, blockHeader, miningBeneficiary, @@ -1176,33 +1180,32 @@ private BlockHeader blockHeader(final long number) { } protected MiningParameters createMiningParameters( - final Wei minGasPrice, final double minBlockOccupancyRatio, final long txsSelectionMaxTime) { + final Wei minGasPrice, + final double minBlockOccupancyRatio, + final PositiveNumber txsSelectionMaxTime) { return ImmutableMiningParameters.builder() .mutableInitValues( MutableInitValues.builder() .minTransactionGasPrice(minGasPrice) .minBlockOccupancyRatio(minBlockOccupancyRatio) .build()) - .unstable(Unstable.builder().nonPoaBlockTxsSelectionMaxTime(txsSelectionMaxTime).build()) + .nonPoaBlockTxsSelectionMaxTime(txsSelectionMaxTime) .build(); } protected MiningParameters createMiningParameters( final Wei minGasPrice, final double minBlockOccupancyRatio, - final int minBlockTime, - final Percentage minBlockTimePercentage) { + final int genesisBlockPeriodSeconds, + final PositiveNumber minBlockTimePercentage) { return ImmutableMiningParameters.builder() .mutableInitValues( MutableInitValues.builder() .minTransactionGasPrice(minGasPrice) .minBlockOccupancyRatio(minBlockOccupancyRatio) .build()) - .unstable( - Unstable.builder() - .minBlockTime(minBlockTime) - .poaBlockTxsSelectionMaxTime(minBlockTimePercentage) - .build()) + .genesisBlockPeriodSeconds(genesisBlockPeriodSeconds) + .poaBlockTxsSelectionMaxTime(minBlockTimePercentage) .build(); } diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java index 63e0d54ecd7..eb1a98cbd76 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LondonFeeMarketBlockTransactionSelectorTest.java @@ -16,7 +16,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; -import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; +import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; import static org.mockito.Mockito.mock; import org.hyperledger.besu.config.GenesisConfigFile; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java index 32ac5ee926a..7f543f370a3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java @@ -16,7 +16,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.time.Duration; import java.util.Objects; @@ -32,6 +32,10 @@ @Value.Immutable @Value.Enclosing public abstract class MiningParameters { + public static final PositiveNumber DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME = + PositiveNumber.fromInt((int) Duration.ofSeconds(5).toMillis()); + public static final PositiveNumber DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME = + PositiveNumber.fromInt(75); public static final MiningParameters MINING_DISABLED = ImmutableMiningParameters.builder() .mutableInitValues( @@ -130,6 +134,28 @@ public int getStratumPort() { return 8008; } + @Value.Default + public PositiveNumber getNonPoaBlockTxsSelectionMaxTime() { + return DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; + } + + @Value.Default + public PositiveNumber getPoaBlockTxsSelectionMaxTime() { + return DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; + } + + public abstract OptionalInt getGenesisBlockPeriodSeconds(); + + @Value.Derived + public long getBlockTxsSelectionMaxTime() { + if (getGenesisBlockPeriodSeconds().isPresent()) { + return (TimeUnit.SECONDS.toMillis(getGenesisBlockPeriodSeconds().getAsInt()) + * getPoaBlockTxsSelectionMaxTime().getValue()) + / 100; + } + return getNonPoaBlockTxsSelectionMaxTime().getValue(); + } + @Value.Default protected MutableRuntimeValues getMutableRuntimeValues() { return new MutableRuntimeValues(getMutableInitValues()); @@ -266,8 +292,6 @@ public interface Unstable { int DEFAULT_MAX_OMMERS_DEPTH = 8; long DEFAULT_POS_BLOCK_CREATION_MAX_TIME = Duration.ofSeconds(12).toMillis(); long DEFAULT_POS_BLOCK_CREATION_REPETITION_MIN_DURATION = Duration.ofMillis(500).toMillis(); - long DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME = Duration.ofSeconds(5).toMillis(); - Percentage DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME = Percentage.fromInt(75); MiningParameters.Unstable DEFAULT = ImmutableMiningParameters.Unstable.builder().build(); @@ -305,27 +329,5 @@ default long getPosBlockCreationRepetitionMinDuration() { default String getStratumExtranonce() { return "080c"; } - - @Value.Default - default long getNonPoaBlockTxsSelectionMaxTime() { - return DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; - } - - @Value.Default - default Percentage getPoaBlockTxsSelectionMaxTime() { - return DEFAULT_POA_BLOCK_TXS_SELECTION_MAX_TIME; - } - - OptionalInt getMinBlockTime(); - - @Value.Derived - default long getBlockTxsSelectionMaxTime() { - if (getMinBlockTime().isPresent()) { - return (TimeUnit.SECONDS.toMillis(getMinBlockTime().getAsInt()) - * getPoaBlockTxsSelectionMaxTime().getValue()) - / 100; - } - return getNonPoaBlockTxsSelectionMaxTime(); - } } } From f81d5445f1032128ad3e3e06a56d7e32b5011222 Mon Sep 17 00:00:00 2001 From: Matt Nelson <85905982+non-fungible-nelson@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:02:58 -0700 Subject: [PATCH 22/56] Fix to increment/decrement gas-limit in block production (#6425) Signed-off-by: Matt Nelson <85905982+non-fungible-nelson@users.noreply.github.com> Co-authored-by: Stefan Pingel <16143240+pinges@users.noreply.github.com> Co-authored-by: Simon Dudley Co-authored-by: Sally MacFarlane Signed-off-by: Simon Dudley --- CHANGELOG.md | 1 + .../12_cancun_get_built_block.json | 4 +- .../AbstractGasLimitSpecification.java | 1 - .../FrontierTargetingGasLimitCalculator.java | 10 ++--- .../LondonTargetingGasLimitCalculator.java | 10 +---- .../TargetingGasLimitCalculatorTest.java | 43 ++++++++++++------- 6 files changed, 35 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ab78beb7b8..eecedac55ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Fix Besu Docker images with `openjdk-latest` tags since 23.10.3 using UID 1001 instead of 1000 for the `besu` user [#6360](https://github.com/hyperledger/besu/pull/6360) - Fluent EVM API definition for Tangerine Whistle had incorrect code size validation configured [#6382](https://github.com/hyperledger/besu/pull/6382) - Correct mining beneficiary for Clique networks in TraceServiceImpl [#6390](https://github.com/hyperledger/besu/pull/6390) +- Fix to gas limit delta calculations used in block production. Besu should now increment or decrement the block gas limit towards its target correctly (thanks @arbora) #6425 ### Download Links diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/cancun/test-cases/block-production/12_cancun_get_built_block.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/cancun/test-cases/block-production/12_cancun_get_built_block.json index 75300119246..8cbdd626427 100644 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/cancun/test-cases/block-production/12_cancun_get_built_block.json +++ b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/cancun/test-cases/block-production/12_cancun_get_built_block.json @@ -10,7 +10,7 @@ "stateRoot" : "0x8d9115d9211932d4a3a1f068fb8fe262b0b2ab0bfd74eaece1a572efe6336677", "logsBloom" : "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "prevRandao" : "0xc13da06dc53836ca0766057413b9683eb9a8773bbb8fcc5691e41c25b56dda1d", - "gasLimit" : "0x2ff3d8", + "gasLimit" : "0x2ffbd2", "gasUsed" : "0xf618", "timestamp" : "0x1236", "extraData" : "0x", @@ -70,7 +70,7 @@ "amount" : "0x64" } ], "blockNumber" : "0x1", - "blockHash" : "0xf1e35607932349e87f29e1053a4fb2666782e09fde21ded74c1f7e4a57d3fa2b", + "blockHash" : "0x736bdddc2eca36fe8ed4ed515e5d295a08d7eaddc0d0fda2a35408127eb890d0", "receiptsRoot" : "0x9af165447e5b3193e9ac8389418648ee6d6cb1d37459fe65cfc245fc358721bd", "blobGasUsed" : "0x60000" }, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractGasLimitSpecification.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractGasLimitSpecification.java index 70e80527b78..b1a21e54b4b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractGasLimitSpecification.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractGasLimitSpecification.java @@ -19,7 +19,6 @@ /** Specification for the block gasLimit. */ public abstract class AbstractGasLimitSpecification { - public static final long DEFAULT_MAX_CONSTANT_ADMUSTMENT_INCREMENT = 1024L; public static final long DEFAULT_MIN_GAS_LIMIT = 5000L; public static final long DEFAULT_MAX_GAS_LIMIT = Long.MAX_VALUE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/FrontierTargetingGasLimitCalculator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/FrontierTargetingGasLimitCalculator.java index 49295e39468..ed067731b5a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/FrontierTargetingGasLimitCalculator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/FrontierTargetingGasLimitCalculator.java @@ -23,16 +23,13 @@ public class FrontierTargetingGasLimitCalculator extends AbstractGasLimitSpecifi implements GasLimitCalculator { private static final Logger LOG = LoggerFactory.getLogger(FrontierTargetingGasLimitCalculator.class); - private final long maxConstantAdjustmentIncrement; public FrontierTargetingGasLimitCalculator() { - this(DEFAULT_MAX_CONSTANT_ADMUSTMENT_INCREMENT, DEFAULT_MIN_GAS_LIMIT, DEFAULT_MAX_GAS_LIMIT); + this(DEFAULT_MIN_GAS_LIMIT, DEFAULT_MAX_GAS_LIMIT); } - public FrontierTargetingGasLimitCalculator( - final long maxConstantAdjustmentIncrement, final long minGasLimit, final long maxGasLimit) { + public FrontierTargetingGasLimitCalculator(final long minGasLimit, final long maxGasLimit) { super(minGasLimit, maxGasLimit); - this.maxConstantAdjustmentIncrement = maxConstantAdjustmentIncrement; } @Override @@ -55,8 +52,7 @@ public long nextGasLimit( } private long adjustAmount(final long currentGasLimit) { - final long maxProportionalAdjustmentLimit = Math.max(deltaBound(currentGasLimit) - 1, 0); - return Math.min(maxConstantAdjustmentIncrement, maxProportionalAdjustmentLimit); + return Math.max(deltaBound(currentGasLimit) - 1, 0); } protected long safeAddAtMost(final long gasLimit) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/LondonTargetingGasLimitCalculator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/LondonTargetingGasLimitCalculator.java index 6e03a2523ea..3c86f770739 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/LondonTargetingGasLimitCalculator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/LondonTargetingGasLimitCalculator.java @@ -27,21 +27,15 @@ public class LondonTargetingGasLimitCalculator extends FrontierTargetingGasLimit public LondonTargetingGasLimitCalculator( final long londonForkBlock, final BaseFeeMarket feeMarket) { - this( - DEFAULT_MAX_CONSTANT_ADMUSTMENT_INCREMENT, - DEFAULT_MIN_GAS_LIMIT, - DEFAULT_MAX_GAS_LIMIT, - londonForkBlock, - feeMarket); + this(DEFAULT_MIN_GAS_LIMIT, DEFAULT_MAX_GAS_LIMIT, londonForkBlock, feeMarket); } public LondonTargetingGasLimitCalculator( - final long maxConstantAdjustmentIncrement, final long minGasLimit, final long maxGasLimit, final long londonForkBlock, final BaseFeeMarket feeMarket) { - super(maxConstantAdjustmentIncrement, minGasLimit, maxGasLimit); + super(minGasLimit, maxGasLimit); this.londonForkBlock = londonForkBlock; this.feeMarket = feeMarket; } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TargetingGasLimitCalculatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TargetingGasLimitCalculatorTest.java index b19d2310c5c..ef197143f2f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TargetingGasLimitCalculatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TargetingGasLimitCalculatorTest.java @@ -27,22 +27,6 @@ public class TargetingGasLimitCalculatorTest { private static final long ADJUSTMENT_FACTOR = 1024L; - @Test - public void verifyGasLimitIsIncreasedWithinLimits() { - FrontierTargetingGasLimitCalculator targetingGasLimitCalculator = - new FrontierTargetingGasLimitCalculator(); - assertThat(targetingGasLimitCalculator.nextGasLimit(8_000_000L, 10_000_000L, 1L)) - .isEqualTo(8_000_000L + ADJUSTMENT_FACTOR); - } - - @Test - public void verifyGasLimitIsDecreasedWithinLimits() { - FrontierTargetingGasLimitCalculator targetingGasLimitCalculator = - new FrontierTargetingGasLimitCalculator(); - assertThat(targetingGasLimitCalculator.nextGasLimit(12_000_000L, 10_000_000L, 1L)) - .isEqualTo(12_000_000L - ADJUSTMENT_FACTOR); - } - @Test public void verifyGasLimitReachesTarget() { final long target = 10_000_000L; @@ -55,6 +39,33 @@ public void verifyGasLimitReachesTarget() { .isEqualTo(target); } + @Test + public void verifyAdjustmentDeltas() { + assertDeltas(20000000L, 20019530L, 19980470L); + assertDeltas(40000000L, 40039061L, 39960939L); + } + + private void assertDeltas( + final long gasLimit, final long expectedIncrease, final long expectedDecrease) { + FrontierTargetingGasLimitCalculator targetingGasLimitCalculator = + new FrontierTargetingGasLimitCalculator(); + // increase + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, gasLimit * 2, 1L)) + .isEqualTo(expectedIncrease); + // decrease + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, 0, 1L)) + .isEqualTo(expectedDecrease); + // small decrease + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, gasLimit - 1, 1L)) + .isEqualTo(gasLimit - 1); + // small increase + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, gasLimit + 1, 1L)) + .isEqualTo(gasLimit + 1); + // no change + assertThat(targetingGasLimitCalculator.nextGasLimit(gasLimit, gasLimit, 1L)) + .isEqualTo(gasLimit); + } + @Test public void verifyMinGasLimit() { assertThat(AbstractGasLimitSpecification.isValidTargetGasLimit(DEFAULT_MIN_GAS_LIMIT - 1)) From 94d86afb40a6a59d86117c93a647719fd342991b Mon Sep 17 00:00:00 2001 From: Jason Frame Date: Fri, 19 Jan 2024 14:21:44 +1000 Subject: [PATCH 23/56] Bonsai keyvalue refactor (#6404) Refactor BonsaiWorldStateKeyValueStorage to use a FlatDbStrategyProvider Signed-off-by: Jason Frame --- .../storage/TrieLogSubCommand.java | 3 +- .../controller/BesuControllerBuilder.java | 3 +- .../storage/TrieLogHelperTest.java | 3 +- .../controller/BesuControllerBuilderTest.java | 17 ++- .../MergeBesuControllerBuilderTest.java | 4 +- .../QbftBesuControllerBuilderTest.java | 4 +- .../ethereum/storage/StorageProvider.java | 4 +- .../keyvalue/KeyValueStorageProvider.java | 8 +- .../trie/bonsai/BonsaiWorldStateProvider.java | 5 +- .../cache/CachedWorldStorageManager.java | 20 +-- .../cache/NoOpCachedWorldStorageManager.java | 3 +- ...nsaiSnapshotWorldStateKeyValueStorage.java | 18 +-- .../BonsaiWorldStateKeyValueStorage.java | 117 ++++++------------ .../storage/BonsaiWorldStateLayerStorage.java | 12 +- .../storage/flat/FlatDbStrategyProvider.java | 105 ++++++++++++++++ .../common/GenesisWorldStateProvider.java | 4 +- .../core/InMemoryKeyValueStorageProvider.java | 15 ++- .../BlockImportExceptionHandlingTest.java | 4 +- .../trie/bonsai/AbstractIsolationTests.java | 11 +- .../bonsai/BonsaiWorldStateArchiveTest.java | 25 ++-- .../bonsai/CachedMerkleTrieLoaderTest.java | 13 +- .../ethereum/trie/bonsai/LogRollingTests.java | 19 ++- .../ethereum/trie/bonsai/RollingImport.java | 4 +- .../BonsaiWorldStateKeyValueStorageTest.java | 8 +- .../flat/FlatDbStrategyProviderTest.java | 89 +++++++++++++ .../WorldStateDownloaderBenchmark.java | 5 +- .../FastWorldDownloadStateTest.java | 5 +- .../worldstate/PersistDataStepTest.java | 5 +- .../snapsync/AccountHealingTrackingTest.java | 5 +- .../sync/snapsync/PersistDataStepTest.java | 5 +- .../snapsync/SnapWorldDownloadStateTest.java | 5 +- .../eth/sync/snapsync/TaskGenerator.java | 5 +- ...ntFlatDatabaseHealingRangeRequestTest.java | 7 +- ...geFlatDatabaseHealingRangeRequestTest.java | 4 +- .../StorageTrieNodeHealingRequestTest.java | 4 +- .../BonsaiReferenceTestWorldState.java | 6 +- 36 files changed, 388 insertions(+), 186 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index bf75fd6eb9a..74e00197f59 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -139,8 +139,7 @@ private static TrieLogContext getTrieLogContext() { final StorageProvider storageProvider = besuController.getStorageProvider(); final BonsaiWorldStateKeyValueStorage rootWorldStateStorage = - (BonsaiWorldStateKeyValueStorage) - storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI); + (BonsaiWorldStateKeyValueStorage) storageProvider.createWorldStateStorage(config); final MutableBlockchain blockchain = besuController.getProtocolContext().getBlockchain(); return new TrieLogContext(config, rootWorldStateStorage, blockchain); } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index f2d3fce981a..f2ecbe85c86 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -598,7 +598,7 @@ public BesuController build() { final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); final WorldStateStorage worldStateStorage = - storageProvider.createWorldStateStorage(dataStorageConfiguration.getDataStorageFormat()); + storageProvider.createWorldStateStorage(dataStorageConfiguration); final BlockchainStorage blockchainStorage = storageProvider.createBlockchainStorage(protocolSchedule, variablesStorage); @@ -1088,7 +1088,6 @@ yield new BonsaiWorldStateProvider( blockchain, Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), cachedMerkleTrieLoader, - metricsSystem, besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), evmConfiguration, trieLogPruner); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 82659701b1c..0ba575a9ef1 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -75,7 +75,8 @@ public static void setup() throws IOException { blockHeader5 = new BlockHeaderTestFixture().number(5).buildHeader(); inMemoryWorldState = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); var updater = inMemoryWorldState.updater(); updater diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index dd9452e7806..9adf8da230f 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -131,7 +131,7 @@ public void setup() { when(synchronizerConfiguration.getBlockPropagationRange()).thenReturn(Range.closed(1L, 2L)); lenient() - .when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST)) + .when(storageProvider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG)) .thenReturn(worldStateStorage); lenient() .when(storageProvider.createWorldStatePreimageStorage()) @@ -166,6 +166,11 @@ BesuControllerBuilder visitWithMockConfigs(final BesuControllerBuilder builder) @Test public void shouldDisablePruningIfBonsaiIsEnabled() { + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad(DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .build(); BonsaiWorldState mockWorldState = mock(BonsaiWorldState.class, Answers.RETURNS_DEEP_STUBS); doReturn(worldStateArchive) .when(besuControllerBuilder) @@ -173,15 +178,9 @@ public void shouldDisablePruningIfBonsaiIsEnabled() { any(WorldStateStorage.class), any(Blockchain.class), any(CachedMerkleTrieLoader.class)); doReturn(mockWorldState).when(worldStateArchive).getMutable(); - when(storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI)) + when(storageProvider.createWorldStateStorage(dataStorageConfiguration)) .thenReturn(bonsaiWorldStateStorage); - besuControllerBuilder - .isPruningEnabled(true) - .dataStorageConfiguration( - ImmutableDataStorageConfiguration.builder() - .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .build()); + besuControllerBuilder.isPruningEnabled(true).dataStorageConfiguration(dataStorageConfiguration); besuControllerBuilder.build(); verify(storageProvider, never()) diff --git a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java index 4368e831381..9506a35f402 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java @@ -52,7 +52,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; @@ -145,7 +145,7 @@ public void setup() { .thenReturn(Range.closed(1L, 2L)); lenient() - .when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST)) + .when(storageProvider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG)) .thenReturn(worldStateStorage); lenient() .when(storageProvider.createWorldStatePreimageStorage()) diff --git a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java index 75fecf3f96e..fc1b2f55f3f 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java @@ -48,7 +48,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -114,7 +114,7 @@ public void setup() { new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), new MainnetBlockHeaderFunctions())); lenient() - .when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST)) + .when(storageProvider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG)) .thenReturn(worldStateStorage); lenient().when(worldStateStorage.isWorldStateAvailable(any(), any())).thenReturn(true); lenient().when(worldStateStorage.updater()).thenReturn(mock(WorldStateStorage.Updater.class)); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java index 6cc9741cb22..2f3cd3dff47 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java @@ -17,7 +17,7 @@ import org.hyperledger.besu.ethereum.chain.BlockchainStorage; import org.hyperledger.besu.ethereum.chain.VariablesStorage; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; @@ -34,7 +34,7 @@ public interface StorageProvider extends Closeable { BlockchainStorage createBlockchainStorage( ProtocolSchedule protocolSchedule, VariablesStorage variablesStorage); - WorldStateStorage createWorldStateStorage(DataStorageFormat dataStorageFormat); + WorldStateStorage createWorldStateStorage(DataStorageConfiguration dataStorageFormat); WorldStatePreimageStorage createWorldStatePreimageStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index dea6b932c32..44c15e81502 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; @@ -75,9 +76,10 @@ public BlockchainStorage createBlockchainStorage( } @Override - public WorldStateStorage createWorldStateStorage(final DataStorageFormat dataStorageFormat) { - if (dataStorageFormat.equals(DataStorageFormat.BONSAI)) { - return new BonsaiWorldStateKeyValueStorage(this, metricsSystem); + public WorldStateStorage createWorldStateStorage( + final DataStorageConfiguration dataStorageConfiguration) { + if (dataStorageConfiguration.getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { + return new BonsaiWorldStateKeyValueStorage(this, metricsSystem, dataStorageConfiguration); } else { return new ForestWorldStateKeyValueStorage( getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.WORLD_STATE)); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java index 4c783888c09..4f1b6d65bb4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java @@ -39,7 +39,6 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldState; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -73,13 +72,11 @@ public BonsaiWorldStateProvider( final Blockchain blockchain, final Optional maxLayersToLoad, final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final ObservableMetricsSystem metricsSystem, final BesuContext pluginContext, final EvmConfiguration evmConfiguration, final TrieLogPruner trieLogPruner) { - this.cachedWorldStorageManager = - new CachedWorldStorageManager(this, worldStateStorage, metricsSystem); + this.cachedWorldStorageManager = new CachedWorldStorageManager(this, worldStateStorage); // TODO: de-dup constructors this.trieLogManager = new TrieLogManager( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java index 8a09c136a5e..6ca38a27a92 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java @@ -22,7 +22,6 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; import java.util.ArrayList; import java.util.Comparator; @@ -41,7 +40,6 @@ public class CachedWorldStorageManager public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class); private final BonsaiWorldStateProvider archive; - private final ObservableMetricsSystem metricsSystem; private final EvmConfiguration evmConfiguration; private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; @@ -51,26 +49,18 @@ private CachedWorldStorageManager( final BonsaiWorldStateProvider archive, final BonsaiWorldStateKeyValueStorage worldStateStorage, final Map cachedWorldStatesByHash, - final ObservableMetricsSystem metricsSystem, final EvmConfiguration evmConfiguration) { worldStateStorage.subscribe(this); this.rootWorldStateStorage = worldStateStorage; this.cachedWorldStatesByHash = cachedWorldStatesByHash; this.archive = archive; - this.metricsSystem = metricsSystem; this.evmConfiguration = evmConfiguration; } public CachedWorldStorageManager( final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final ObservableMetricsSystem metricsSystem) { - this( - archive, - worldStateStorage, - new ConcurrentHashMap<>(), - metricsSystem, - EvmConfiguration.DEFAULT); + final BonsaiWorldStateKeyValueStorage worldStateStorage) { + this(archive, worldStateStorage, new ConcurrentHashMap<>(), EvmConfiguration.DEFAULT); } public synchronized void addCachedLayer( @@ -92,8 +82,7 @@ public synchronized void addCachedLayer( cachedBonsaiWorldView .get() .updateWorldStateStorage( - new BonsaiSnapshotWorldStateKeyValueStorage( - forWorldState.getWorldStateStorage(), metricsSystem)); + new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage())); } } else { LOG.atDebug() @@ -106,8 +95,7 @@ public synchronized void addCachedLayer( blockHeader.getHash(), new CachedBonsaiWorldView( blockHeader, - new BonsaiSnapshotWorldStateKeyValueStorage( - forWorldState.getWorldStateStorage(), metricsSystem))); + new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage()))); } else { // otherwise, add the layer to the cache cachedWorldStatesByHash.put( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java index 0359c464245..d86128b13f1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java @@ -18,7 +18,6 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.Optional; import java.util.function.Function; @@ -27,7 +26,7 @@ public class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { public NoOpCachedWorldStorageManager( final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage) { - super(null, bonsaiWorldStateKeyValueStorage, new NoOpMetricsSystem()); + super(null, bonsaiWorldStateKeyValueStorage); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index 69c74ec27ca..b9643cdc49b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -18,7 +18,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; @@ -43,26 +42,19 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey public BonsaiSnapshotWorldStateKeyValueStorage( final BonsaiWorldStateKeyValueStorage parentWorldStateStorage, final SnappedKeyValueStorage segmentedWorldStateStorage, - final KeyValueStorage trieLogStorage, - final ObservableMetricsSystem metricsSystem) { + final KeyValueStorage trieLogStorage) { super( - parentWorldStateStorage.flatDbMode, - parentWorldStateStorage.flatDbStrategy, - segmentedWorldStateStorage, - trieLogStorage, - metricsSystem); + parentWorldStateStorage.flatDbStrategyProvider, segmentedWorldStateStorage, trieLogStorage); this.parentWorldStateStorage = parentWorldStateStorage; this.subscribeParentId = parentWorldStateStorage.subscribe(this); } public BonsaiSnapshotWorldStateKeyValueStorage( - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final ObservableMetricsSystem metricsSystem) { + final BonsaiWorldStateKeyValueStorage worldStateStorage) { this( worldStateStorage, ((SnappableKeyValueStorage) worldStateStorage.composedWorldStateStorage).takeSnapshot(), - worldStateStorage.trieLogStorage, - metricsSystem); + worldStateStorage.trieLogStorage); } private boolean isClosedGet() { @@ -78,7 +70,7 @@ public BonsaiUpdater updater() { return new Updater( ((SnappedKeyValueStorage) composedWorldStateStorage).getSnapshotTransaction(), trieLogStorage.startTransaction(), - flatDbStrategy); + getFlatDbStrategy()); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 97384fd6c12..efe97f32c60 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -25,14 +25,14 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FullFlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.PartialFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategyProvider; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.account.AccountStorageEntry; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; @@ -64,17 +64,11 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC public static final byte[] WORLD_BLOCK_HASH_KEY = "worldBlockHash".getBytes(StandardCharsets.UTF_8); - // 0x666C61744462537461747573 - public static final byte[] FLAT_DB_MODE = "flatDbStatus".getBytes(StandardCharsets.UTF_8); - - protected FlatDbMode flatDbMode; - protected FlatDbStrategy flatDbStrategy; + protected final FlatDbStrategyProvider flatDbStrategyProvider; protected final SegmentedKeyValueStorage composedWorldStateStorage; protected final KeyValueStorage trieLogStorage; - protected final ObservableMetricsSystem metricsSystem; - private final AtomicBoolean shouldClose = new AtomicBoolean(false); protected final AtomicBoolean isClosed = new AtomicBoolean(false); @@ -82,62 +76,27 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC protected final Subscribers subscribers = Subscribers.create(); public BonsaiWorldStateKeyValueStorage( - final StorageProvider provider, final ObservableMetricsSystem metricsSystem) { + final StorageProvider provider, + final MetricsSystem metricsSystem, + final DataStorageConfiguration dataStorageConfiguration) { this.composedWorldStateStorage = provider.getStorageBySegmentIdentifiers( List.of( ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); this.trieLogStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); - this.metricsSystem = metricsSystem; - loadFlatDbStrategy(); + this.flatDbStrategyProvider = + new FlatDbStrategyProvider(metricsSystem, dataStorageConfiguration); + flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); } public BonsaiWorldStateKeyValueStorage( - final FlatDbMode flatDbMode, - final FlatDbStrategy flatDbStrategy, + final FlatDbStrategyProvider flatDbStrategyProvider, final SegmentedKeyValueStorage composedWorldStateStorage, - final KeyValueStorage trieLogStorage, - final ObservableMetricsSystem metricsSystem) { - this.flatDbMode = flatDbMode; - this.flatDbStrategy = flatDbStrategy; + final KeyValueStorage trieLogStorage) { + this.flatDbStrategyProvider = flatDbStrategyProvider; this.composedWorldStateStorage = composedWorldStateStorage; this.trieLogStorage = trieLogStorage; - this.metricsSystem = metricsSystem; - } - - private void loadFlatDbStrategy() { - // derive our flatdb strategy from db or default: - var newFlatDbMode = deriveFlatDbStrategy(); - - // if flatDbMode is not loaded or has changed, reload flatDbStrategy - if (this.flatDbMode == null || !this.flatDbMode.equals(newFlatDbMode)) { - this.flatDbMode = newFlatDbMode; - if (flatDbMode == FlatDbMode.FULL) { - this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem); - } else { - this.flatDbStrategy = new PartialFlatDbStrategy(metricsSystem); - } - } - } - - public FlatDbMode deriveFlatDbStrategy() { - var flatDbMode = - FlatDbMode.fromVersion( - composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE) - .map(Bytes::wrap) - .orElse(FlatDbMode.PARTIAL.getVersion())); - LOG.info("Bonsai flat db mode found {}", flatDbMode); - - return flatDbMode; - } - - public FlatDbStrategy getFlatDbStrategy() { - if (flatDbStrategy == null) { - loadFlatDbStrategy(); - } - return flatDbStrategy; } @Override @@ -147,7 +106,7 @@ public DataStorageFormat getDataStorageFormat() { @Override public FlatDbMode getFlatDbMode() { - return flatDbMode; + return flatDbStrategyProvider.getFlatDbMode(); } @Override @@ -155,12 +114,15 @@ public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); } else { - return getFlatDbStrategy().getFlatCode(codeHash, accountHash, composedWorldStateStorage); + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) + .getFlatCode(codeHash, accountHash, composedWorldStateStorage); } } public Optional getAccount(final Hash accountHash) { - return getFlatDbStrategy() + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) .getFlatAccount( this::getWorldStateRootHash, this::getAccountStateTrieNode, @@ -243,7 +205,8 @@ public Optional getStorageValueByStorageSlotKey( final Supplier> storageRootSupplier, final Hash accountHash, final StorageSlotKey storageSlotKey) { - return getFlatDbStrategy() + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) .getFlatStorageValueByStorageSlotKey( this::getWorldStateRootHash, storageRootSupplier, @@ -256,14 +219,16 @@ public Optional getStorageValueByStorageSlotKey( @Override public Map streamFlatAccounts( final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return getFlatDbStrategy() + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); } @Override public Map streamFlatStorages( final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return getFlatDbStrategy() + return flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) .streamStorageFlatDatabase( composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); } @@ -288,31 +253,23 @@ public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHas } public void upgradeToFullFlatDbMode() { - final SegmentedKeyValueStorageTransaction transaction = - composedWorldStateStorage.startTransaction(); - // TODO: consider ARCHIVE mode - transaction.put( - TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe()); - transaction.commit(); - loadFlatDbStrategy(); // force reload of flat db reader strategy + flatDbStrategyProvider.upgradeToFullFlatDbMode(composedWorldStateStorage); } public void downgradeToPartialFlatDbMode() { - final SegmentedKeyValueStorageTransaction transaction = - composedWorldStateStorage.startTransaction(); - transaction.put( - TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe()); - transaction.commit(); - loadFlatDbStrategy(); // force reload of flat db reader strategy + flatDbStrategyProvider.downgradeToPartialFlatDbMode(composedWorldStateStorage); } @Override public void clear() { subscribers.forEach(BonsaiStorageSubscriber::onClearStorage); - getFlatDbStrategy().clearAll(composedWorldStateStorage); + flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) + .clearAll(composedWorldStateStorage); composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); trieLogStorage.clear(); - loadFlatDbStrategy(); // force reload of flat db reader strategy + flatDbStrategyProvider.loadFlatDbStrategy( + composedWorldStateStorage); // force reload of flat db reader strategy } @Override @@ -324,7 +281,9 @@ public void clearTrieLog() { @Override public void clearFlatDatabase() { subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage); - getFlatDbStrategy().resetOnResync(composedWorldStateStorage); + flatDbStrategyProvider + .getFlatDbStrategy(composedWorldStateStorage) + .resetOnResync(composedWorldStateStorage); } @Override @@ -332,7 +291,7 @@ public BonsaiUpdater updater() { return new Updater( composedWorldStateStorage.startTransaction(), trieLogStorage.startTransaction(), - flatDbStrategy); + flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)); } @Override @@ -359,6 +318,10 @@ public void removeNodeAddedListener(final long id) { throw new RuntimeException("removeNodeAddedListener not available"); } + public FlatDbStrategy getFlatDbStrategy() { + return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); + } + public interface BonsaiUpdater extends WorldStateStorage.Updater { BonsaiUpdater removeCode(final Hash accountHash); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java index 0dd61b8d7a6..92d4f54fee3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -17,7 +17,6 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage; @@ -29,16 +28,14 @@ public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent this( new LayeredKeyValueStorage(parent.composedWorldStateStorage), parent.trieLogStorage, - parent, - parent.metricsSystem); + parent); } public BonsaiWorldStateLayerStorage( final SnappedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage, - final BonsaiWorldStateKeyValueStorage parent, - final ObservableMetricsSystem metricsSystem) { - super(parent, composedWorldStateStorage, trieLogStorage, metricsSystem); + final BonsaiWorldStateKeyValueStorage parent) { + super(parent, composedWorldStateStorage, trieLogStorage); } @Override @@ -51,7 +48,6 @@ public BonsaiWorldStateLayerStorage clone() { return new BonsaiWorldStateLayerStorage( ((LayeredKeyValueStorage) composedWorldStateStorage).clone(), trieLogStorage, - parentWorldStateStorage, - metricsSystem); + parentWorldStateStorage); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java new file mode 100644 index 00000000000..55f89fc663f --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java @@ -0,0 +1,105 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.nio.charset.StandardCharsets; + +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FlatDbStrategyProvider { + private static final Logger LOG = LoggerFactory.getLogger(FlatDbStrategyProvider.class); + + // 0x666C61744462537461747573 + public static final byte[] FLAT_DB_MODE = "flatDbStatus".getBytes(StandardCharsets.UTF_8); + private final MetricsSystem metricsSystem; + protected FlatDbMode flatDbMode; + protected FlatDbStrategy flatDbStrategy; + + public FlatDbStrategyProvider( + final MetricsSystem metricsSystem, final DataStorageConfiguration dataStorageConfiguration) { + this.metricsSystem = metricsSystem; + } + + public void loadFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) { + // derive our flatdb strategy from db or default: + var newFlatDbMode = deriveFlatDbStrategy(composedWorldStateStorage); + + // if flatDbMode is not loaded or has changed, reload flatDbStrategy + if (this.flatDbMode == null || !this.flatDbMode.equals(newFlatDbMode)) { + this.flatDbMode = newFlatDbMode; + if (flatDbMode == FlatDbMode.FULL) { + this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem); + } else { + this.flatDbStrategy = new PartialFlatDbStrategy(metricsSystem); + } + } + } + + private FlatDbMode deriveFlatDbStrategy( + final SegmentedKeyValueStorage composedWorldStateStorage) { + var flatDbMode = + FlatDbMode.fromVersion( + composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE) + .map(Bytes::wrap) + .orElse(FlatDbMode.PARTIAL.getVersion())); + LOG.info("Bonsai flat db mode found {}", flatDbMode); + + return flatDbMode; + } + + public FlatDbStrategy getFlatDbStrategy( + final SegmentedKeyValueStorage composedWorldStateStorage) { + if (flatDbStrategy == null) { + loadFlatDbStrategy(composedWorldStateStorage); + } + return flatDbStrategy; + } + + public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) { + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + // TODO: consider ARCHIVE mode + transaction.put( + TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe()); + transaction.commit(); + loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy + } + + public void downgradeToPartialFlatDbMode( + final SegmentedKeyValueStorage composedWorldStateStorage) { + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + transaction.put( + TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe()); + transaction.commit(); + loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy + } + + public FlatDbMode getFlatDbMode() { + return flatDbMode; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java index ad1e920b77a..6e61b55953a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -65,7 +66,8 @@ private static MutableWorldState createGenesisBonsaiWorldState() { segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), new InMemoryKeyValueStorage(), new NoOpMetricsSystem()), - new NoOpMetricsSystem()); + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); return new BonsaiWorldState( bonsaiWorldStateKeyValueStorage, cachedMerkleTrieLoader, diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 514c8821098..21f32441b27 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.core; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; + import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -32,7 +34,9 @@ import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -96,13 +100,18 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( new InMemoryKeyValueStorageProvider(); final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final DataStorageConfiguration bonsaiDataStorageConfig = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .unstable(DataStorageConfiguration.Unstable.DEFAULT) + .build(); return new BonsaiWorldStateProvider( (BonsaiWorldStateKeyValueStorage) - inMemoryKeyValueStorageProvider.createWorldStateStorage(DataStorageFormat.BONSAI), + inMemoryKeyValueStorageProvider.createWorldStateStorage(bonsaiDataStorageConfig), blockchain, Optional.empty(), cachedMerkleTrieLoader, - new NoOpMetricsSystem(), null, evmConfiguration, TrieLogPruner.noOpTrieLogPruner()); @@ -111,7 +120,7 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( public static MutableWorldState createInMemoryWorldState() { final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider(); return new ForestMutableWorldState( - provider.createWorldStateStorage(DataStorageFormat.FOREST), + provider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG), provider.createWorldStatePreimageStorage(), EvmConfiguration.DEFAULT); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java index f8ac492c93f..ecf17da9038 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java @@ -44,6 +44,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -80,7 +81,8 @@ class BlockImportExceptionHandlingTest { private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private final WorldStateStorage worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); private final WorldStateArchive worldStateArchive = // contains a BonsaiWorldState which we need to spy on. diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java index a77c51269d8..eba09987dce 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java @@ -68,7 +68,9 @@ import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.BesuConfiguration; @@ -147,14 +149,19 @@ public abstract class AbstractIsolationTests { public void createStorage() { bonsaiWorldStateStorage = (BonsaiWorldStateKeyValueStorage) - createKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.BONSAI); + createKeyValueStorageProvider() + .createWorldStateStorage( + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad( + DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .build()); archive = new BonsaiWorldStateProvider( bonsaiWorldStateStorage, blockchain, Optional.of(16L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), - new NoOpMetricsSystem(), null, EvmConfiguration.DEFAULT, TrieLogPruner.noOpTrieLogPruner()); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java index 6afde3c9400..7c1391029a9 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java @@ -44,6 +44,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; @@ -106,7 +107,8 @@ void testGetMutableReturnPersistedStateWhenNeeded() { new BonsaiWorldStateProvider( cachedWorldStorageManager, trieLogManager, - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), blockchain, new CachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); @@ -119,11 +121,11 @@ void testGetMutableReturnPersistedStateWhenNeeded() { void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { bonsaiWorldStateArchive = new BonsaiWorldStateProvider( - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), blockchain, Optional.of(512L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), - new NoOpMetricsSystem(), null, EvmConfiguration.DEFAULT, TrieLogPruner.noOpTrieLogPruner()); @@ -141,7 +143,8 @@ void testGetMutableWhenLoadLessThanLimitLayersBack() { new BonsaiWorldStateProvider( cachedWorldStorageManager, trieLogManager, - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), blockchain, new CachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); @@ -167,7 +170,8 @@ void testGetMutableWithStorageInconsistencyRollbackTheState() { .getTrieLogLayer(any(Hash.class)); var worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( @@ -193,7 +197,8 @@ void testGetMutableWithStorageInconsistencyRollbackTheState() { void testGetMutableWithStorageConsistencyNotRollbackTheState() { var worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( @@ -229,7 +234,8 @@ void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState() { .getTrieLogLayer(any(Hash.class)); var worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); bonsaiWorldStateArchive = spy( @@ -276,7 +282,10 @@ void testGetMutableWithRollbackNotOverrideTrieLogLayer() { new BonsaiWorldStateProvider( cachedWorldStorageManager, trieLogManager, - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + storageProvider, + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG), blockchain, new CachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java index 758d7088105..59dfacb5f60 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -48,7 +49,9 @@ class CachedMerkleTrieLoaderTest { private CachedMerkleTrieLoader merkleTrieLoader; private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private final BonsaiWorldStateKeyValueStorage inMemoryWorldState = - Mockito.spy(new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem())); + Mockito.spy( + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG)); final List
accounts = List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee")); @@ -71,7 +74,9 @@ void shouldAddAccountNodesInCacheDuringPreload() { final BonsaiWorldStateKeyValueStorage emptyStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); StoredMerklePatriciaTrie cachedTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> @@ -110,7 +115,9 @@ void shouldAddStorageNodesInCacheDuringPreload() { final List cachedSlots = new ArrayList<>(); final BonsaiWorldStateKeyValueStorage emptyStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); final StoredMerklePatriciaTrie cachedTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java index f2dad3bc6ea..26d0d1302b1 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.log.LogsBloomFilter; @@ -161,7 +162,8 @@ void simpleRollForwardTest() { final BonsaiWorldState worldState = new BonsaiWorldState( archive, - new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + provider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final WorldUpdater updater = worldState.updater(); @@ -174,7 +176,8 @@ void simpleRollForwardTest() { final BonsaiWorldState secondWorldState = new BonsaiWorldState( secondArchive, - new BonsaiWorldStateKeyValueStorage(secondProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + secondProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final BonsaiWorldStateUpdateAccumulator secondUpdater = (BonsaiWorldStateUpdateAccumulator) secondWorldState.updater(); @@ -205,7 +208,8 @@ void rollForwardTwice() { final BonsaiWorldState worldState = new BonsaiWorldState( archive, - new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + provider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final WorldUpdater updater = worldState.updater(); @@ -226,7 +230,8 @@ void rollForwardTwice() { final BonsaiWorldState secondWorldState = new BonsaiWorldState( secondArchive, - new BonsaiWorldStateKeyValueStorage(secondProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + secondProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final BonsaiWorldStateUpdateAccumulator secondUpdater = (BonsaiWorldStateUpdateAccumulator) secondWorldState.updater(); @@ -258,7 +263,8 @@ void rollBackOnce() { final BonsaiWorldState worldState = new BonsaiWorldState( archive, - new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + provider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final WorldUpdater updater = worldState.updater(); @@ -286,7 +292,8 @@ void rollBackOnce() { final BonsaiWorldState secondWorldState = new BonsaiWorldState( secondArchive, - new BonsaiWorldStateKeyValueStorage(secondProvider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + secondProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final WorldUpdater secondUpdater = secondWorldState.updater(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java index af230e436ea..301ddbc8ec6 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -56,7 +57,8 @@ public static void main(final String[] arg) throws IOException { final BonsaiWorldState bonsaiState = new BonsaiWorldState( archive, - new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + new BonsaiWorldStateKeyValueStorage( + provider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG), EvmConfiguration.DEFAULT); final SegmentedInMemoryKeyValueStorage worldStateStorage = (SegmentedInMemoryKeyValueStorage) diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index f6bb51197bb..239332d6a9a 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -36,6 +36,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.StorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -452,7 +453,9 @@ void isWorldStateAvailable_afterCallingSaveWorldstate(final FlatDbMode flatDbMod private BonsaiWorldStateKeyValueStorage emptyStorage() { return new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); } @Test @@ -487,6 +490,7 @@ private BonsaiWorldStateKeyValueStorage setupMockStorage( .thenReturn(mockTrieLogStorage); when(mockStorageProvider.getStorageBySegmentIdentifiers(any())) .thenReturn(mock(SegmentedKeyValueStorage.class)); - return new BonsaiWorldStateKeyValueStorage(mockStorageProvider, new NoOpMetricsSystem()); + return new BonsaiWorldStateKeyValueStorage( + mockStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java new file mode 100644 index 00000000000..8d2984879b0 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java @@ -0,0 +1,89 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; +import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class FlatDbStrategyProviderTest { + private final FlatDbStrategyProvider flatDbStrategyProvider = + new FlatDbStrategyProvider(new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + private final SegmentedKeyValueStorage composedWorldStateStorage = + new SegmentedInMemoryKeyValueStorage(List.of(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE)); + + @ParameterizedTest + @EnumSource(FlatDbMode.class) + void loadsFlatDbStrategyForStoredFlatDbMode(final FlatDbMode flatDbMode) { + updateFlatDbMode(flatDbMode); + + flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.getFlatDbMode()).isEqualTo(flatDbMode); + } + + @Test + void loadsPartialFlatDbStrategyWhenNoFlatDbModeStored() { + flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.getFlatDbMode()).isEqualTo(FlatDbMode.PARTIAL); + } + + @Test + void upgradesFlatDbStrategyToFullFlatDbMode() { + updateFlatDbMode(FlatDbMode.PARTIAL); + + flatDbStrategyProvider.upgradeToFullFlatDbMode(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.FULL); + assertThat(flatDbStrategyProvider.flatDbStrategy).isNotNull(); + assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)) + .isInstanceOf(FullFlatDbStrategy.class); + } + + @Test + void downgradesFlatDbStrategyToPartiallyFlatDbMode() { + updateFlatDbMode(FlatDbMode.FULL); + + flatDbStrategyProvider.downgradeToPartialFlatDbMode(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL); + assertThat(flatDbStrategyProvider.flatDbStrategy).isNotNull(); + assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)) + .isInstanceOf(PartialFlatDbStrategy.class); + } + + private void updateFlatDbMode(final FlatDbMode flatDbMode) { + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + transaction.put( + KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE, + FlatDbStrategyProvider.FLAT_DB_MODE, + flatDbMode.getVersion().toArrayUnsafe()); + transaction.commit(); + } +} diff --git a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java index 5f7cbcdc8df..71ba9b3ece0 100644 --- a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java +++ b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java @@ -38,7 +38,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -105,7 +105,8 @@ public void setUpUnchangedState() { final StorageProvider storageProvider = createKeyValueStorageProvider(tempDir, tempDir.resolve("database")); - worldStateStorage = storageProvider.createWorldStateStorage(DataStorageFormat.FOREST); + worldStateStorage = + storageProvider.createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG); pendingRequests = new InMemoryTasksPriorityQueues<>(); worldStateDownloader = diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index 91c770d47be..4b4c73f3f7e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -28,6 +28,7 @@ import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -80,7 +81,9 @@ public void setUp(final DataStorageFormat storageFormat) { if (storageFormat == DataStorageFormat.BONSAI) { worldStateStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); } else { worldStateStorage = new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java index e857648f7fc..7306cf30107 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.SimpleMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.services.tasks.Task; @@ -40,7 +40,8 @@ public class PersistDataStepTest { private final WorldStateStorage worldStateStorage = - new InMemoryKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.FOREST); + new InMemoryKeyValueStorageProvider() + .createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG); private final FastWorldDownloadState downloadState = mock(FastWorldDownloadState.class); private final Bytes rootNodeData = Bytes.of(1, 1, 1, 1); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index ead7a1ca61e..565441eaa31 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -34,6 +34,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -58,7 +59,9 @@ public class AccountHealingTrackingTest { private final List
accounts = List.of(Address.fromHexString("0xdeadbeef")); private final WorldStateStorage worldStateStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); private WorldStateProofProvider worldStateProofProvider; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java index 69047bc6e50..dedce974c49 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.services.tasks.Task; @@ -39,7 +39,8 @@ public class PersistDataStepTest { private final WorldStateStorage worldStateStorage = - new InMemoryKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.FOREST); + new InMemoryKeyValueStorageProvider() + .createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG); private final SnapSyncProcessState snapSyncState = mock(SnapSyncProcessState.class); private final SnapWorldDownloadState downloadState = mock(SnapWorldDownloadState.class); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index a8dc4a9b083..b275e60e273 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -108,7 +109,9 @@ public void setUp(final DataStorageFormat storageFormat) { if (storageFormat == DataStorageFormat.BONSAI) { worldStateStorage = new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + DataStorageConfiguration.DEFAULT_CONFIG); } else { worldStateStorage = new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java index 64bda37ae88..7d1a4f3db5d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java @@ -27,7 +27,7 @@ import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.services.tasks.Task; @@ -44,7 +44,8 @@ public class TaskGenerator { public static List> createAccountRequest(final boolean withData) { final WorldStateStorage worldStateStorage = - new InMemoryKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.FOREST); + new InMemoryKeyValueStorageProvider() + .createWorldStateStorage(DataStorageConfiguration.DEFAULT_CONFIG); final WorldStateProofProvider worldStateProofProvider = new WorldStateProofProvider(worldStateStorage); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index a61b5c8ea11..40db311c4f1 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -179,7 +180,8 @@ public void doNotPersistWhenProofIsValid() { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); final WorldStateStorage worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); final MerkleTrie accountStateTrie = TrieGenerator.generateTrie(worldStateStorage, 15); @@ -233,7 +235,8 @@ public void doHealAndPersistWhenProofIsInvalid() { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); final WorldStateStorage worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); final MerkleTrie accountStateTrie = TrieGenerator.generateTrie(worldStateStorage, 15); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java index 7fe3862573c..cfaf422db96 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java @@ -33,6 +33,7 @@ import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -78,7 +79,8 @@ class StorageFlatDatabaseHealingRangeRequestTest { public void setup() { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); proofProvider = new WorldStateProofProvider(worldStateStorage); trie = TrieGenerator.generateTrie( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java index 5295f4e3b5e..6f98a51ad16 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; @@ -74,7 +75,8 @@ public void setup(final DataStorageFormat storageFormat) { } else { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); } final MerkleTrie trie = TrieGenerator.generateTrie( diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 034f8b9debf..91c768a55cf 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -114,7 +115,10 @@ public static BonsaiReferenceTestWorldState create( new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = - new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider(), metricsSystem); + new BonsaiWorldStateKeyValueStorage( + new InMemoryKeyValueStorageProvider(), + metricsSystem, + DataStorageConfiguration.DEFAULT_CONFIG); final BonsaiReferenceTestWorldStateStorage worldStateStorage = new BonsaiReferenceTestWorldStateStorage(bonsaiWorldStateKeyValueStorage, preImageProxy); From e76bcf0babb5957b789587a20a5f024b2432464d Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Thu, 18 Jan 2024 23:24:12 -0700 Subject: [PATCH 24/56] EOF Spec Updates (#6431) Update a few EOF features based on updated spec * Add a prague reference test target * Run evmtool from prestate when no code specified * RETF and dangling immediate arg fixes Signed-off-by: Danno Ferrin --- .../java/org/hyperledger/besu/evmtool/EvmToolCommand.java | 3 +++ .../referencetests/ReferenceTestProtocolSchedules.java | 2 +- .../org/hyperledger/besu/evm/code/CodeV1Validation.java | 5 +++++ .../java/org/hyperledger/besu/evm/code/EOFLayout.java | 4 ++-- .../besu/evm/operation/AbstractCreateOperation.java | 1 - .../org/hyperledger/besu/evm/operation/RetFOperation.java | 2 +- .../besu/evm/processor/ContractCreationProcessor.java | 1 + .../hyperledger/besu/evm/tracing/StandardJsonTracer.java | 7 ++++--- .../java/org/hyperledger/besu/evm/code/EOFLayoutTest.java | 8 ++++---- 9 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java index 743c215ecbb..900fcca7074 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java @@ -371,6 +371,9 @@ public void run() { long txGas = gas - intrinsicGasCost - accessListCost; final EVM evm = protocolSpec.getEvm(); + if (codeBytes.isEmpty()) { + codeBytes = component.getWorldState().get(receiver).getCode(); + } Code code = evm.getCode(Hash.hash(codeBytes), codeBytes); if (!code.isValid()) { out.println(((CodeInvalid) code).getInvalidReason()); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java index 3c5d84567ae..31cdfaca167 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestProtocolSchedules.java @@ -79,7 +79,7 @@ public static ReferenceTestProtocolSchedules create(final StubGenesisConfigOptio createSchedule(genesisStub.clone().shanghaiTime(0).cancunTime(15000))); builder.put("Cancun", createSchedule(genesisStub.clone().cancunTime(0))); // TODO remove this after execution-test-specs finalize - builder.put("Shanghai+6780", createSchedule(genesisStub.clone().cancunTime(0))); + builder.put("Prague", createSchedule(genesisStub.clone().futureEipsTime(0))); builder.put("Future_EIPs", createSchedule(genesisStub.clone().futureEipsTime(0))); builder.put("Experimental_EIPs", createSchedule(genesisStub.clone().experimentalEipsTime(0))); return new ReferenceTestProtocolSchedules(builder.build()); diff --git a/evm/src/main/java/org/hyperledger/besu/evm/code/CodeV1Validation.java b/evm/src/main/java/org/hyperledger/besu/evm/code/CodeV1Validation.java index 6174a2734af..e16498c7834 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/code/CodeV1Validation.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/code/CodeV1Validation.java @@ -779,6 +779,11 @@ public static String validateStack(final int codeSectionToValidate, final EOFLay } currentPC += pcAdvance; + if (currentPC >= stackHeights.length) { + return String.format( + "Dangling immediate argument for opcode 0x%x at PC %d in code section %d.", + currentStackHeight, codeLength - pcAdvance, codeSectionToValidate); + } stackHeights[currentPC] = currentStackHeight; unusedBytes -= pcAdvance; } diff --git a/evm/src/main/java/org/hyperledger/besu/evm/code/EOFLayout.java b/evm/src/main/java/org/hyperledger/besu/evm/code/EOFLayout.java index f92022ca445..92016829a8d 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/code/EOFLayout.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/code/EOFLayout.java @@ -158,7 +158,7 @@ public static EOFLayout parseEOF(final Bytes container) { if (typeData[codeSectionCount - 1][2] == -1) { return invalidLayout(container, version, "Incomplete type section"); } - if (typeData[0][0] != 0 || typeData[0][1] != 0) { + if (typeData[0][0] != 0 || (typeData[0][1] & 0x7f) != 0) { return invalidLayout( container, version, "Code section does not have zero inputs and outputs"); } @@ -182,7 +182,7 @@ public static EOFLayout parseEOF(final Bytes container) { version, "Type data input stack too large - 0x" + Integer.toHexString(typeData[i][0])); } - if (typeData[i][1] > 0x7f) { + if (typeData[i][1] > 0x80) { return invalidLayout( container, version, diff --git a/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java b/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java index d695edf1c8c..e4fec2dbfb8 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java @@ -141,7 +141,6 @@ private void spawnChildMessage(final MessageFrame parent, final Code code, final final Wei value = Wei.wrap(parent.getStackItem(0)); final Address contractAddress = targetContractAddress(parent); - parent.addCreate(contractAddress); final long childGasStipend = gasCalculator().gasAvailableForChildCreate(parent.getRemainingGas()); diff --git a/evm/src/main/java/org/hyperledger/besu/evm/operation/RetFOperation.java b/evm/src/main/java/org/hyperledger/besu/evm/operation/RetFOperation.java index 327dac1cb81..3f40081bc06 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/operation/RetFOperation.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/operation/RetFOperation.java @@ -24,7 +24,7 @@ public class RetFOperation extends AbstractOperation { /** The Opcode. */ public static final int OPCODE = 0xe4; /** The Ret F success. */ - static final OperationResult retfSuccess = new OperationResult(4, null); + static final OperationResult retfSuccess = new OperationResult(3, null); /** * Instantiates a new Ret F operation. diff --git a/evm/src/main/java/org/hyperledger/besu/evm/processor/ContractCreationProcessor.java b/evm/src/main/java/org/hyperledger/besu/evm/processor/ContractCreationProcessor.java index 7acea893724..20daf3ab259 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/processor/ContractCreationProcessor.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/processor/ContractCreationProcessor.java @@ -122,6 +122,7 @@ public void start(final MessageFrame frame, final OperationTracer operationTrace operationTracer.traceAccountCreationResult( frame, Optional.of(ExceptionalHaltReason.INSUFFICIENT_GAS)); } else { + frame.addCreate(contractAddress); contract.incrementBalance(frame.getValue()); contract.setNonce(initialContractNonce); contract.clearStorage(); diff --git a/evm/src/main/java/org/hyperledger/besu/evm/tracing/StandardJsonTracer.java b/evm/src/main/java/org/hyperledger/besu/evm/tracing/StandardJsonTracer.java index 5d90107a69e..04254a718a1 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/tracing/StandardJsonTracer.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/tracing/StandardJsonTracer.java @@ -126,9 +126,7 @@ public void tracePreExecution(final MessageFrame messageFrame) { for (int i = messageFrame.stackSize() - 1; i >= 0; i--) { stack.add("\"" + shortBytes(messageFrame.getStackItem(i)) + "\""); } - pc = - messageFrame.getPC() - - messageFrame.getCode().getCodeSection(messageFrame.getSection()).getEntryPoint(); + pc = messageFrame.getPC() - messageFrame.getCode().getCodeSection(0).getEntryPoint(); section = messageFrame.getSection(); gas = shortNumber(messageFrame.getRemainingGas()); memorySize = messageFrame.memoryWordSize() * 32; @@ -171,6 +169,9 @@ public void tracePreExecution(final MessageFrame messageFrame) { public void tracePostExecution( final MessageFrame messageFrame, final Operation.OperationResult executeResult) { final Operation currentOp = messageFrame.getCurrentOperation(); + if (currentOp.isVirtualOperation()) { + return; + } final int opcode = currentOp.getOpcode(); final Bytes returnData = messageFrame.getReturnData(); diff --git a/evm/src/test/java/org/hyperledger/besu/evm/code/EOFLayoutTest.java b/evm/src/test/java/org/hyperledger/besu/evm/code/EOFLayoutTest.java index 3e8e17f2a65..a0f83ddd9e0 100644 --- a/evm/src/test/java/org/hyperledger/besu/evm/code/EOFLayoutTest.java +++ b/evm/src/test/java/org/hyperledger/besu/evm/code/EOFLayoutTest.java @@ -276,15 +276,15 @@ public static Collection typeSectionTests() { 1 }, { - "EF0001 010010 0200040001000200020002 030000 00 00000000 80000000 00010000 02030000 FE 5000 3000 8000", + "EF0001 010010 0200040001000200020002 030000 00 00000000 F0000000 00010000 02030000 FE 5000 3000 8000", "inputs too large", - "Type data input stack too large - 0x80", + "Type data input stack too large - 0xf0", 1 }, { - "EF0001 010010 0200040001000200020002 030000 00 00000000 01000000 00800000 02030000 FE 5000 3000 8000", + "EF0001 010010 0200040001000200020002 030000 00 00000000 01000000 00F00000 02030000 FE 5000 3000 8000", "outputs too large", - "Type data output stack too large - 0x80", + "Type data output stack too large - 0xf0", 1 }, { From 958a07237848fdd1bba7fe7675876c299b7f4a3d Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Fri, 19 Jan 2024 17:20:20 +1000 Subject: [PATCH 25/56] 24.1.2-SNAPSHOT (#6432) Signed-off-by: Sally MacFarlane --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 75ee1d04ee8..8fe780b083a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=24.1.1-SNAPSHOT +version=24.1.2-SNAPSHOT org.gradle.welcome=never # Set exports/opens flags required by Google Java Format and ErrorProne plugins. (JEP-396) From 925f4946b8cd62bbb047558c857e47b38a4957f1 Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Fri, 19 Jan 2024 10:34:02 +0000 Subject: [PATCH 26/56] Use `From` field in a `PING` packet when creating a peer table entry (#6225) * Use P2P 'from' host when parsing incoming P2P packets, if it is present Signed-off-by: Matthew Whitehead * Use UDP source address if PING 'from' address is 127.0.0.1 and add a unit test. Signed-off-by: Matthew Whitehead * Spotless Java, address PR comment Signed-off-by: Matthew Whitehead * Refactor handleIncomingPacket to allow for specific trace logs to show how selection is being done Signed-off-by: Matthew Whitehead * Add change log entry Signed-off-by: Matthew Whitehead * Refactor handleIncomingPacket Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead --- CHANGELOG.md | 1 + .../p2p/discovery/PeerDiscoveryAgent.java | 23 +++++++++++++++++- .../p2p/discovery/PeerDiscoveryAgentTest.java | 24 +++++++++++++++++++ .../discovery/PeerDiscoveryTestHelper.java | 8 +++++++ 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eecedac55ad..7f84aa88b75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3-hotfix/besu- ### Bug fixes - Fix Docker image name clash between Besu and evmtool [#6194](https://github.com/hyperledger/besu/pull/6194) - Fix `logIndex` in `eth_getTransactionReceipt` JSON RPC method [#6206](https://github.com/hyperledger/besu/pull/6206) +- Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) ### Download Links https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3/besu-23.10.3.zip / sha256 da7ef8a6ceb88d3e327cacddcdb32218d1750b464c14165a74068f6dc6e0871a diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java index 8af596018a1..2324630d831 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java @@ -282,8 +282,29 @@ protected void handleIncomingPacket(final Endpoint sourceEndpoint, final Packet .flatMap(Endpoint::getTcpPort) .orElse(udpPort); + // If the host is present in the P2P PING packet itself, use that as the endpoint. If the P2P + // PING packet specifies 127.0.0.1 (the default if a custom value is not specified with + // --p2p-host or via a suitable --nat-method) we ignore it in favour of the UDP source address. + // The likelihood is that the UDP source will be 127.0.0.1 anyway, but this reduces the chance + // of an unexpected change in behaviour as a result of + // https://github.com/hyperledger/besu/issues/6224 being fixed. + final String host = + packet + .getPacketData(PingPacketData.class) + .flatMap(PingPacketData::getFrom) + .map(Endpoint::getHost) + .filter(abc -> !abc.equals("127.0.0.1")) + .stream() + .peek( + h -> + LOG.trace( + "Using \"From\" endpoint {} specified in ping packet. Ignoring UDP source host {}", + h, + sourceEndpoint.getHost())) + .findFirst() + .orElseGet(sourceEndpoint::getHost); + // Notify the peer controller. - final String host = sourceEndpoint.getHost(); final DiscoveryPeer peer = DiscoveryPeer.fromEnode( EnodeURLImpl.builder() diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java index 97d167a26fb..240f4673eb5 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java @@ -244,6 +244,30 @@ public void neighborsPacketLimited() { } } + @Test + public void endpointHonoursCustomAdvertisedAddressInPingPacket() { + + // Start a peer with the default advertised host + final MockPeerDiscoveryAgent agent1 = helper.startDiscoveryAgent(); + + // Start another peer with its advertised host set to a custom value + final MockPeerDiscoveryAgent agent2 = helper.startDiscoveryAgent("192.168.0.1"); + + // Send a PING so we can exchange messages + Packet packet = helper.createPingPacket(agent2, agent1); + helper.sendMessageBetweenAgents(agent2, agent1, packet); + + // Agent 1's peers should have endpoints that match the custom advertised value... + agent1 + .streamDiscoveredPeers() + .forEach(peer -> assertThat(peer.getEndpoint().getHost()).isEqualTo("192.168.0.1")); + + // ...but agent 2's peers should have endpoints that match the default + agent2 + .streamDiscoveredPeers() + .forEach(peer -> assertThat(peer.getEndpoint().getHost()).isEqualTo("127.0.0.1")); + } + @Test public void shouldEvictPeerWhenPermissionsRevoked() { final PeerPermissionsDenylist denylist = PeerPermissionsDenylist.create(); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java index e51b6320a6e..464273243ad 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java @@ -165,6 +165,14 @@ public MockPeerDiscoveryAgent startDiscoveryAgent(final DiscoveryPeer... bootstr return startDiscoveryAgent(agentBuilder); } + public MockPeerDiscoveryAgent startDiscoveryAgent( + final String advertisedHost, final DiscoveryPeer... bootstrapPeers) { + final AgentBuilder agentBuilder = + agentBuilder().bootstrapPeers(bootstrapPeers).advertisedHost(advertisedHost); + + return startDiscoveryAgent(agentBuilder); + } + /** * Start a single discovery agent with the provided bootstrap peers. * From 98718ae270657cd26fb8f525eb9e9e506524c6c9 Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Fri, 19 Jan 2024 14:08:53 +0000 Subject: [PATCH 27/56] Only accept a address from a peer if it is a valid IP address (#6439) Signed-off-by: Matthew Whitehead --- .../besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java index 2324630d831..30272413d49 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java @@ -293,7 +293,9 @@ protected void handleIncomingPacket(final Endpoint sourceEndpoint, final Packet .getPacketData(PingPacketData.class) .flatMap(PingPacketData::getFrom) .map(Endpoint::getHost) - .filter(abc -> !abc.equals("127.0.0.1")) + .filter( + fromAddr -> + (!fromAddr.equals("127.0.0.1") && InetAddresses.isInetAddress(fromAddr))) .stream() .peek( h -> From 24718e391adf1bae94408560ce859b7185dd24ac Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Fri, 19 Jan 2024 16:52:48 +0000 Subject: [PATCH 28/56] Fix changelog after incorrect merge of 6225 (#6438) * Fix changelog after incorrect merge of 6225 Signed-off-by: Matthew Whitehead * Create 24.1.2-snapshot changelog entry Signed-off-by: Matthew Whitehead * Remove entry from previous version in changelog Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead --- CHANGELOG.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f84aa88b75..772efa5afaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog -## 24.1.1-SNAPSHOT +## 24.1.2-SNAPSHOT + +### Breaking Changes + +### Deprecations + +### Additions and Improvements + +### Bug fixes +- Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) + +### Download Links + +## 24.1.1 ### Breaking Changes - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) @@ -30,7 +43,6 @@ ### Download Links - ## 24.1.0 ### Breaking Changes @@ -87,7 +99,6 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3-hotfix/besu- ### Bug fixes - Fix Docker image name clash between Besu and evmtool [#6194](https://github.com/hyperledger/besu/pull/6194) - Fix `logIndex` in `eth_getTransactionReceipt` JSON RPC method [#6206](https://github.com/hyperledger/besu/pull/6206) -- Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) ### Download Links https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3/besu-23.10.3.zip / sha256 da7ef8a6ceb88d3e327cacddcdb32218d1750b464c14165a74068f6dc6e0871a From cfea3ab2fd590827669c1cfddc121fa9b244d4de Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 22 Jan 2024 11:11:10 +1000 Subject: [PATCH 29/56] Import export trie log (#6363) * Import and export trie log subcommands * change option name and fix descriptions Signed-off-by: Gabriel Fukushima Co-authored-by: Jason Frame --- .../options/stable/DataStorageOptions.java | 11 +- .../subcommands/storage/TrieLogHelper.java | 103 ++++++++-- .../storage/TrieLogSubCommand.java | 106 +++++++++- .../stable/DataStorageOptionsTest.java | 20 +- .../storage/TrieLogHelperTest.java | 192 +++++++++++++----- 5 files changed, 349 insertions(+), 83 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index e0b19735683..5b4cf43eb1e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -62,23 +62,28 @@ public class DataStorageOptions implements CLIOptions private final DataStorageOptions.Unstable unstableOptions = new Unstable(); static class Unstable { + private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = + "--Xbonsai-limit-trie-logs-enabled"; + private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD = + "--Xbonsai-trie-logs-retention-threshold"; + private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit"; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-pruning-enabled"}, + names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED}, description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})") private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-retention-threshold"}, + names = {BONSAI_TRIE_LOGS_RETENTION_THRESHOLD}, description = "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})") private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-pruning-limit"}, + names = {BONSAI_TRIE_LOG_PRUNING_LIMIT}, description = "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 52dbe559034..c4e924a8354 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -22,7 +22,11 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.File; @@ -32,6 +36,7 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.PrintWriter; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.IdentityHashMap; @@ -39,6 +44,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,16 +103,15 @@ private static void processTrieLogBatches( final String batchFileNameBase) { for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - + final String batchFileName = batchFileNameBase + "-" + batchNumber; final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); - final long lastBlockOfBatch = Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); - final List trieLogKeys = getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch); - saveTrieLogBatches(batchFileNameBase, rootWorldStateStorage, batchNumber, trieLogKeys); + LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); + saveTrieLogBatches(batchFileName, rootWorldStateStorage, trieLogKeys); } LOG.info("Clear trie logs..."); @@ -118,15 +123,12 @@ private static void processTrieLogBatches( } private static void saveTrieLogBatches( - final String batchFileNameBase, + final String batchFileName, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, final List trieLogKeys) { - LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); - try { - saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase); + saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchFileName); } catch (IOException e) { LOG.error("Error saving trie logs to file: {}", e.getMessage()); throw new RuntimeException(e); @@ -210,9 +212,8 @@ private static void recreateTrieLogs( final String batchFileNameBase) throws IOException { // process in chunk to avoid OOM - - IdentityHashMap trieLogsToRetain = - readTrieLogsFromFile(batchFileNameBase, batchNumber); + final String batchFileName = batchFileNameBase + "-" + batchNumber; + IdentityHashMap trieLogsToRetain = readTrieLogsFromFile(batchFileName); final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION; List keys = new ArrayList<>(trieLogsToRetain.keySet()); @@ -265,11 +266,10 @@ private static void validatePruneConfiguration(final DataStorageConfiguration co private static void saveTrieLogsInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, - final String batchFileNameBase) + final String batchFileName) throws IOException { - File file = new File(batchFileNameBase + "-" + batchNumber); + File file = new File(batchFileName); if (file.exists()) { LOG.error("File already exists, skipping file creation"); return; @@ -285,17 +285,14 @@ private static void saveTrieLogsInFile( } @SuppressWarnings("unchecked") - private static IdentityHashMap readTrieLogsFromFile( - final String batchFileNameBase, final long batchNumber) { + static IdentityHashMap readTrieLogsFromFile(final String batchFileName) { IdentityHashMap trieLogs; - try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber); + try (FileInputStream fis = new FileInputStream(batchFileName); ObjectInputStream ois = new ObjectInputStream(fis)) { trieLogs = (IdentityHashMap) ois.readObject(); - } catch (IOException | ClassNotFoundException e) { - LOG.error(e.getMessage()); throw new RuntimeException(e); } @@ -303,6 +300,52 @@ private static IdentityHashMap readTrieLogsFromFile( return trieLogs; } + private static void saveTrieLogsAsRlpInFile( + final List trieLogsKeys, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final String batchFileName) { + File file = new File(batchFileName); + if (file.exists()) { + LOG.error("File already exists, skipping file creation"); + return; + } + + final IdentityHashMap trieLogs = + getTrieLogs(trieLogsKeys, rootWorldStateStorage); + final Bytes rlp = + RLP.encode( + o -> + o.writeList( + trieLogs.entrySet(), (val, out) -> out.writeRaw(Bytes.wrap(val.getValue())))); + try { + Files.write(file.toPath(), rlp.toArrayUnsafe()); + } catch (IOException e) { + LOG.error(e.getMessage()); + throw new RuntimeException(e); + } + } + + static IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFileName) { + try { + final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName))); + final BytesValueRLPInput input = new BytesValueRLPInput(file, false); + + input.enterList(); + final IdentityHashMap trieLogs = new IdentityHashMap<>(); + while (!input.isEndOfCurrentList()) { + final Bytes trieLogBytes = input.currentListAsBytes(); + TrieLogLayer trieLogLayer = + TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(trieLogBytes), false)); + trieLogs.put(trieLogLayer.getBlockHash().toArrayUnsafe(), trieLogBytes.toArrayUnsafe()); + } + input.leaveList(); + + return trieLogs; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private static IdentityHashMap getTrieLogs( final List trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); @@ -357,5 +400,25 @@ static void printCount(final PrintWriter out, final TrieLogCount count) { count.total, count.canonicalCount, count.forkCount, count.orphanCount); } + static void importTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { + + var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString()); + + var updater = rootWorldStateStorage.updater(); + trieLog.forEach((key, value) -> updater.getTrieLogStorageTransaction().put(key, value)); + updater.getTrieLogStorageTransaction().commit(); + } + + static void exportTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final List trieLogHash, + final Path directoryPath) + throws IOException { + final String trieLogFile = directoryPath.toString(); + + saveTrieLogsAsRlpInFile(trieLogHash, rootWorldStateStorage, trieLogFile); + } + record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 74e00197f59..e624b5f3856 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; @@ -26,9 +27,11 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -43,7 +46,12 @@ description = "Manipulate trie logs", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, - subcommands = {TrieLogSubCommand.CountTrieLog.class, TrieLogSubCommand.PruneTrieLog.class}) + subcommands = { + TrieLogSubCommand.CountTrieLog.class, + TrieLogSubCommand.PruneTrieLog.class, + TrieLogSubCommand.ExportTrieLog.class, + TrieLogSubCommand.ImportTrieLog.class + }) public class TrieLogSubCommand implements Runnable { @SuppressWarnings("UnusedVariable") @@ -123,6 +131,102 @@ public void run() { } } + @Command( + name = "export", + description = "This command exports the trie log of a determined block to a binary file", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ExportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @CommandLine.Option( + names = "--trie-log-block-hash", + description = + "Comma separated list of hashes from the blocks you want to export the trie logs of", + split = " {0,1}, {0,1}", + arity = "1..*") + private List trieLogBlockHashList; + + @CommandLine.Option( + names = "--trie-log-file-path", + description = "The file you want to export the trie logs to", + arity = "1..1") + private Path trieLogFilePath = null; + + @Override + public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + + TrieLogContext context = getTrieLogContext(); + + final List listOfBlockHashes = + trieLogBlockHashList.stream().map(Hash::fromHexString).toList(); + + try { + TrieLogHelper.exportTrieLog( + context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Command( + name = "import", + description = "This command imports a trie log exported by another besu node", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ImportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @CommandLine.Option( + names = "--trie-log-file-path", + description = "The file you want to import the trie logs from", + arity = "1..1") + private Path trieLogFilePath = null; + + @Override + public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + + TrieLogContext context = getTrieLogContext(); + + TrieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath); + } + } + record TrieLogContext( DataStorageConfiguration config, BonsaiWorldStateKeyValueStorage rootWorldStateStorage, diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 437053afd56..2a63901975b 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -34,8 +34,8 @@ public void bonsaiTrieLogPruningLimitOption() { dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit()) .isEqualTo(1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-limit", "1"); } @@ -43,8 +43,8 @@ public void bonsaiTrieLogPruningLimitOption() { public void bonsaiTrieLogPruningLimitShouldBePositive() { internalTestFailure( "--Xbonsai-trie-log-pruning-limit=0 must be greater than 0", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-limit", "0"); } @@ -54,8 +54,8 @@ public void bonsaiTrieLogRetentionThresholdOption() { dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD + 1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "513"); } @@ -65,8 +65,8 @@ public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() { dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "512"); } @@ -74,8 +74,8 @@ public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() { public void bonsaiTrieLogRetentionThresholdShouldBeAboveMinimum() { internalTestFailure( "--Xbonsai-trie-log-retention-threshold minimum value is 512", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "511"); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 0ba575a9ef1..5d4ede7f32e 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.cli.subcommands.storage; +import static java.util.Collections.singletonList; import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -27,8 +28,11 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -36,11 +40,12 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -56,17 +61,14 @@ class TrieLogHelperTest { @Mock private MutableBlockchain blockchain; - @TempDir static Path dataDir; - - Path test; static BlockHeader blockHeader1; static BlockHeader blockHeader2; static BlockHeader blockHeader3; static BlockHeader blockHeader4; static BlockHeader blockHeader5; - @BeforeAll - public static void setup() throws IOException { + @BeforeEach + public void setup() throws IOException { blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader(); @@ -78,33 +80,33 @@ public static void setup() throws IOException { new BonsaiWorldStateKeyValueStorage( storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + createTrieLog(blockHeader1); + var updater = inMemoryWorldState.updater(); updater .getTrieLogStorageTransaction() - .put(blockHeader1.getHash().toArrayUnsafe(), Bytes.fromHexString("0x01").toArrayUnsafe()); + .put(blockHeader1.getHash().toArrayUnsafe(), createTrieLog(blockHeader1)); updater .getTrieLogStorageTransaction() - .put(blockHeader2.getHash().toArrayUnsafe(), Bytes.fromHexString("0x02").toArrayUnsafe()); + .put(blockHeader2.getHash().toArrayUnsafe(), createTrieLog(blockHeader2)); updater .getTrieLogStorageTransaction() - .put(blockHeader3.getHash().toArrayUnsafe(), Bytes.fromHexString("0x03").toArrayUnsafe()); + .put(blockHeader3.getHash().toArrayUnsafe(), createTrieLog(blockHeader3)); updater .getTrieLogStorageTransaction() - .put(blockHeader4.getHash().toArrayUnsafe(), Bytes.fromHexString("0x04").toArrayUnsafe()); + .put(blockHeader4.getHash().toArrayUnsafe(), createTrieLog(blockHeader4)); updater .getTrieLogStorageTransaction() - .put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); + .put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5)); updater.getTrieLogStorageTransaction().commit(); } - @BeforeEach - void createDirectory() throws IOException { - Files.createDirectories(dataDir.resolve("database")); - } - - @AfterEach - void deleteDirectory() throws IOException { - Files.deleteIfExists(dataDir.resolve("database")); + private static byte[] createTrieLog(final BlockHeader blockHeader) { + TrieLogLayer trieLogLayer = new TrieLogLayer(); + trieLogLayer.setBlockHash(blockHeader.getBlockHash()); + final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); + TrieLogFactoryImpl.writeTo(trieLogLayer, rlpLog); + return rlpLog.encoded().toArrayUnsafe(); } void mockBlockchainBase() { @@ -114,7 +116,8 @@ void mockBlockchainBase() { } @Test - public void prune() { + public void prune(final @TempDir Path dataDir) throws IOException { + Files.createDirectories(dataDir.resolve("database")); DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -134,14 +137,11 @@ public void prune() { // assert trie logs that will be pruned exist before prune call assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); @@ -151,18 +151,15 @@ public void prune() { // assert retained trie logs are in the DB assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); } @Test - public void cantPruneIfNoFinalizedIsFound() { + public void cantPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) @@ -184,7 +181,7 @@ public void cantPruneIfNoFinalizedIsFound() { } @Test - public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { + public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) @@ -205,7 +202,7 @@ public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { } @Test - public void cantPruneIfUserRequiredFurtherThanFinalized() { + public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -227,8 +224,7 @@ public void cantPruneIfUserRequiredFurtherThanFinalized() { } @Test - public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { - Files.delete(dataDir.resolve("database")); + public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -244,23 +240,121 @@ public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { assertThrows( RuntimeException.class, () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + TrieLogHelper.prune( + dataStorageConfiguration, + inMemoryWorldState, + blockchain, + dataDir.resolve("unknownPath"))); // assert all trie logs are still in the DB assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); + } + + @Test + public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() + .findFirst() + .get(); + + assertArrayEquals(trieLog.getKey(), blockHeader1.getHash().toArrayUnsafe()); + assertArrayEquals( + trieLog.getValue(), inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + } + + @Test + public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLogs = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() + .collect(Collectors.toMap(e -> Bytes.wrap(e.getKey()), Map.Entry::getValue)); + + assertArrayEquals( + trieLogs.get(blockHeader1.getHash()), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); + trieLogs.get(blockHeader2.getHash()), + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertArrayEquals( + trieLogs.get(blockHeader3.getHash()), + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); + } + + @Test + public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + } + + @Test + public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); + inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); } } From 921bc175c8af626c62ae4982f0ec09538bdc2322 Mon Sep 17 00:00:00 2001 From: Stefan Pingel <16143240+pinges@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:56:43 +1000 Subject: [PATCH 30/56] Make fork id the default and try to recover the DiscoveryPeer for incoming connections from the PeerTable (#5628) * make the request for the ENR the default and try to recover the DiscoveryPeer for incoming connections from the PeerTable Signed-off-by: Stefan Signed-off-by: stefan.pingel@consensys.net --- CHANGELOG.md | 4 +- .../options/unstable/NetworkingOptions.java | 6 +- .../cli/options/NetworkingOptionsTest.java | 2 +- .../besu/ethereum/eth/manager/EthPeers.java | 1 + .../eth/manager/EthProtocolManager.java | 124 +++++++++++------- .../p2p/config/DiscoveryConfiguration.java | 2 +- .../p2p/config/NetworkingConfiguration.java | 1 + .../p2p/discovery/PeerDiscoveryAgent.java | 8 +- .../discovery/VertxPeerDiscoveryAgent.java | 7 +- .../internal/PeerDiscoveryController.java | 55 ++------ .../p2p/discovery/internal/PeerTable.java | 14 +- .../p2p/network/DefaultP2PNetwork.java | 12 +- .../besu/ethereum/p2p/rlpx/RlpxAgent.java | 20 +-- .../netty/AbstractHandshakeHandler.java | 85 ++++++------ .../p2p/rlpx/connections/netty/DeFramer.java | 40 +++++- .../netty/HandshakeHandlerInbound.java | 7 +- .../netty/HandshakeHandlerOutbound.java | 7 +- .../netty/NettyConnectionInitializer.java | 12 +- .../netty/NettyTLSConnectionInitializer.java | 12 +- .../discovery/PeerDiscoveryTestHelper.java | 1 + .../internal/MockPeerDiscoveryAgent.java | 3 +- .../internal/PeerDiscoveryControllerTest.java | 46 +------ .../PeerDiscoveryTableRefreshTest.java | 2 - .../p2p/discovery/internal/PeerTableTest.java | 20 +-- .../RecursivePeerRefreshStateTest.java | 6 +- .../p2p/network/DefaultP2PNetworkTest.java | 26 +--- .../besu/ethereum/p2p/rlpx/RlpxAgentTest.java | 2 +- .../rlpx/connections/netty/DeFramerTest.java | 50 ++++++- .../NettyTLSConnectionInitializerTest.java | 5 +- 29 files changed, 308 insertions(+), 272 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 772efa5afaa..a3c84b37b9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,9 @@ - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) -- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. +- Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): + - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. +- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. ### Deprecations diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java index 0ad68e3627b..69e62edfc0e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java @@ -37,7 +37,7 @@ public class NetworkingOptions implements CLIOptions { private final String DNS_DISCOVERY_SERVER_OVERRIDE_FLAG = "--Xp2p-dns-discovery-server"; private final String DISCOVERY_PROTOCOL_V5_ENABLED = "--Xv5-discovery-enabled"; /** The constant FILTER_ON_ENR_FORK_ID. */ - public static final String FILTER_ON_ENR_FORK_ID = "--Xfilter-on-enr-fork-id"; + public static final String FILTER_ON_ENR_FORK_ID = "--filter-on-enr-fork-id"; @CommandLine.Option( names = INITIATE_CONNECTIONS_FREQUENCY_FLAG, @@ -76,9 +76,9 @@ public class NetworkingOptions implements CLIOptions { @CommandLine.Option( names = FILTER_ON_ENR_FORK_ID, hidden = true, - defaultValue = "false", + defaultValue = "true", description = "Whether to enable filtering of peers based on the ENR field ForkId)") - private final Boolean filterOnEnrForkId = false; + private final Boolean filterOnEnrForkId = NetworkingConfiguration.DEFAULT_FILTER_ON_ENR_FORK_ID; @CommandLine.Option( hidden = true, diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/NetworkingOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/NetworkingOptionsTest.java index 4b4601bb9ca..c96b9035e3f 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/NetworkingOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/NetworkingOptionsTest.java @@ -134,7 +134,7 @@ public void checkFilterByForkIdNotSet() { final NetworkingOptions options = cmd.getNetworkingOptions(); final NetworkingConfiguration networkingConfig = options.toDomainObject(); - assertThat(networkingConfig.getDiscovery().isFilterOnEnrForkIdEnabled()).isEqualTo(false); + assertThat(networkingConfig.getDiscovery().isFilterOnEnrForkIdEnabled()).isEqualTo(true); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); assertThat(commandOutput.toString(UTF_8)).isEmpty(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index e28b0a24ad9..c27b473d314 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -139,6 +139,7 @@ public EthPeers( "peer_limit", "The maximum number of peers this node allows to connect", () -> peerUpperBound); + connectedPeersCounter = metricsSystem.createCounter( BesuMetricCategory.PEERS, "connected_total", "Total number of peers connected"); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java index 6925df9d490..774d35cf185 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthProtocolManager.java @@ -110,7 +110,7 @@ public EthProtocolManager( this.blockBroadcaster = new BlockBroadcaster(ethContext); - supportedCapabilities = + this.supportedCapabilities = calculateCapabilities(synchronizerConfiguration, ethereumWireProtocolConfiguration); // Run validators @@ -252,11 +252,14 @@ public List getSupportedCapabilities() { @Override public void stop() { if (stopped.compareAndSet(false, true)) { - LOG.info("Stopping {} Subprotocol.", getSupportedProtocol()); + LOG.atInfo().setMessage("Stopping {} Subprotocol.").addArgument(getSupportedProtocol()).log(); scheduler.stop(); shutdown.countDown(); } else { - LOG.error("Attempted to stop already stopped {} Subprotocol.", getSupportedProtocol()); + LOG.atInfo() + .setMessage("Attempted to stop already stopped {} Subprotocol.") + .addArgument(this::getSupportedProtocol) + .log(); } } @@ -264,7 +267,10 @@ public void stop() { public void awaitStop() throws InterruptedException { shutdown.await(); scheduler.awaitStop(); - LOG.info("{} Subprotocol stopped.", getSupportedProtocol()); + LOG.atInfo() + .setMessage("{} Subprotocol stopped.") + .addArgument(this::getSupportedProtocol) + .log(); } @Override @@ -277,8 +283,10 @@ public void processMessage(final Capability cap, final Message message) { EthProtocolLogger.logProcessMessage(cap, code); final EthPeer ethPeer = ethPeers.peer(message.getConnection()); if (ethPeer == null) { - LOG.debug( - "Ignoring message received from unknown peer connection: {}", message.getConnection()); + LOG.atDebug() + .setMessage("Ignoring message received from unknown peer connection: {}") + .addArgument(message::getConnection) + .log(); return; } @@ -288,19 +296,24 @@ public void processMessage(final Capability cap, final Message message) { return; } else if (!ethPeer.statusHasBeenReceived()) { // Peers are required to send status messages before any other message type - LOG.debug( - "{} requires a Status ({}) message to be sent first. Instead, received message {} (BREACH_OF_PROTOCOL). Disconnecting from {}.", - this.getClass().getSimpleName(), - EthPV62.STATUS, - code, - ethPeer); + LOG.atDebug() + .setMessage( + "{} requires a Status ({}) message to be sent first. Instead, received message {} (BREACH_OF_PROTOCOL). Disconnecting from {}.") + .addArgument(() -> this.getClass().getSimpleName()) + .addArgument(EthPV62.STATUS) + .addArgument(code) + .addArgument(ethPeer::toString) + .log(); ethPeer.disconnect(DisconnectReason.BREACH_OF_PROTOCOL); return; } if (this.mergePeerFilter.isPresent()) { if (this.mergePeerFilter.get().disconnectIfGossipingBlocks(message, ethPeer)) { - LOG.debug("Post-merge disconnect: peer still gossiping blocks {}", ethPeer); + LOG.atDebug() + .setMessage("Post-merge disconnect: peer still gossiping blocks {}") + .addArgument(ethPeer::toString) + .log(); handleDisconnect(ethPeer.getConnection(), DisconnectReason.SUBPROTOCOL_TRIGGERED, false); return; } @@ -333,11 +346,12 @@ public void processMessage(final Capability cap, final Message message) { maybeResponseData = ethMessages.dispatch(ethMessage); } } catch (final RLPException e) { - LOG.debug( - "Received malformed message {} (BREACH_OF_PROTOCOL), disconnecting: {}", - messageData.getData(), - ethPeer, - e); + LOG.atDebug() + .setMessage("Received malformed message {} (BREACH_OF_PROTOCOL), disconnecting: {}, {}") + .addArgument(messageData::getData) + .addArgument(ethPeer::toString) + .addArgument(e::toString) + .log(); ethPeer.disconnect(DisconnectMessage.DisconnectReason.BREACH_OF_PROTOCOL); } @@ -368,23 +382,31 @@ public void handleNewConnection(final PeerConnection connection) { genesisHash, latestForkId); try { - LOG.trace("Sending status message to {} for connection {}.", peer.getId(), connection); + LOG.atTrace() + .setMessage("Sending status message to {} for connection {}.") + .addArgument(peer::getId) + .addArgument(connection::toString) + .log(); peer.send(status, getSupportedProtocol(), connection); peer.registerStatusSent(connection); } catch (final PeerNotConnected peerNotConnected) { // Nothing to do. } - LOG.trace("{}", ethPeers); + LOG.atTrace().setMessage("{}").addArgument(ethPeers::toString).log(); } @Override public boolean shouldConnect(final Peer peer, final boolean incoming) { - if (peer.getForkId().map(forkId -> forkIdManager.peerCheck(forkId)).orElse(true)) { - LOG.trace("ForkId OK or not available"); + if (peer.getForkId().map(forkIdManager::peerCheck).orElse(true)) { + LOG.atDebug() + .setMessage("ForkId OK or not available for peer {}") + .addArgument(peer::getId) + .log(); if (ethPeers.shouldConnect(peer, incoming)) { return true; } } + LOG.atDebug().setMessage("ForkId check failed for peer {}").addArgument(peer::getId).log(); return false; } @@ -397,11 +419,11 @@ public void handleDisconnect( LOG.atDebug() .setMessage("Disconnect - {} - {} - {}... - {} peers left") .addArgument(initiatedByPeer ? "Inbound" : "Outbound") - .addArgument(reason) - .addArgument(connection.getPeer().getId().slice(0, 8)) - .addArgument(ethPeers.peerCount()) + .addArgument(reason::toString) + .addArgument(() -> connection.getPeer().getId().slice(0, 8)) + .addArgument(ethPeers::peerCount) .log(); - LOG.trace("{}", ethPeers); + LOG.atTrace().setMessage("{}").addArgument(ethPeers::toString).log(); } } @@ -412,43 +434,41 @@ private void handleStatusMessage(final EthPeer peer, final Message message) { try { if (!status.networkId().equals(networkId)) { LOG.atDebug() - .setMessage("Mismatched network id: {}, EthPeer {}...") - .addArgument(status.networkId()) - .addArgument(peer.getShortNodeId()) - .log(); - LOG.atTrace() - .setMessage("Mismatched network id: {}, EthPeer {}") - .addArgument(status.networkId()) - .addArgument(peer) + .setMessage("Mismatched network id: {}, peer {}") + .addArgument(status::networkId) + .addArgument(() -> getPeerOrPeerId(peer)) .log(); peer.disconnect(DisconnectReason.SUBPROTOCOL_TRIGGERED); } else if (!forkIdManager.peerCheck(forkId) && status.protocolVersion() > 63) { - LOG.debug( - "{} has matching network id ({}), but non-matching fork id: {}", - peer, - networkId, - forkId); + LOG.atDebug() + .setMessage("{} has matching network id ({}), but non-matching fork id: {}") + .addArgument(() -> getPeerOrPeerId(peer)) + .addArgument(networkId::toString) + .addArgument(forkId) + .log(); peer.disconnect(DisconnectReason.SUBPROTOCOL_TRIGGERED); } else if (forkIdManager.peerCheck(status.genesisHash())) { - LOG.debug( - "{} has matching network id ({}), but non-matching genesis hash: {}", - peer, - networkId, - status.genesisHash()); + LOG.atDebug() + .setMessage("{} has matching network id ({}), but non-matching genesis hash: {}") + .addArgument(() -> getPeerOrPeerId(peer)) + .addArgument(networkId::toString) + .addArgument(status::genesisHash) + .log(); peer.disconnect(DisconnectReason.SUBPROTOCOL_TRIGGERED); } else if (mergePeerFilter.isPresent() && mergePeerFilter.get().disconnectIfPoW(status, peer)) { LOG.atDebug() .setMessage("Post-merge disconnect: peer still PoW {}") - .addArgument(peer.getShortNodeId()) + .addArgument(() -> getPeerOrPeerId(peer)) .log(); handleDisconnect(peer.getConnection(), DisconnectReason.SUBPROTOCOL_TRIGGERED, false); } else { - LOG.debug( - "Received status message from {}: {} with connection {}", - peer, - status, - message.getConnection()); + LOG.atDebug() + .setMessage("Received status message from {}: {} with connection {}") + .addArgument(peer::toString) + .addArgument(status::toString) + .addArgument(message::getConnection) + .log(); peer.registerStatusReceived( status.bestHash(), status.totalDifficulty(), @@ -467,6 +487,10 @@ private void handleStatusMessage(final EthPeer peer, final Message message) { } } + private Object getPeerOrPeerId(final EthPeer peer) { + return LOG.isTraceEnabled() ? peer : peer.getShortNodeId(); + } + @Override public void blockMined(final Block block) { // This assumes the block has already been included in the chain diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java index 036e592e1dd..86bb079a298 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java @@ -32,7 +32,7 @@ public class DiscoveryConfiguration { private List bootnodes = new ArrayList<>(); private String dnsDiscoveryURL; private boolean discoveryV5Enabled = false; - private boolean filterOnEnrForkId = false; + private boolean filterOnEnrForkId = NetworkingConfiguration.DEFAULT_FILTER_ON_ENR_FORK_ID; public static DiscoveryConfiguration create() { return new DiscoveryConfiguration(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java index 0de53cfd786..478e3617376 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/NetworkingConfiguration.java @@ -23,6 +23,7 @@ public class NetworkingConfiguration { public static final int DEFAULT_INITIATE_CONNECTIONS_FREQUENCY_SEC = 30; public static final int DEFAULT_CHECK_MAINTAINED_CONNECTIONS_FREQUENCY_SEC = 60; public static final int DEFAULT_PEER_LOWER_BOUND = 25; + public static final boolean DEFAULT_FILTER_ON_ENR_FORK_ID = true; private DiscoveryConfiguration discovery = new DiscoveryConfiguration(); private RlpxConfiguration rlpx = new RlpxConfiguration(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java index 30272413d49..de7d047180c 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerRequirement; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PingPacketData; import org.hyperledger.besu.ethereum.p2p.discovery.internal.TimerUtil; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; @@ -81,6 +82,7 @@ public abstract class PeerDiscoveryAgent { private final MetricsSystem metricsSystem; private final RlpxAgent rlpxAgent; private final ForkIdManager forkIdManager; + private final PeerTable peerTable; /* The peer controller, which takes care of the state machine of peers. */ protected Optional controller = Optional.empty(); @@ -109,7 +111,8 @@ protected PeerDiscoveryAgent( final MetricsSystem metricsSystem, final StorageProvider storageProvider, final ForkIdManager forkIdManager, - final RlpxAgent rlpxAgent) { + final RlpxAgent rlpxAgent, + final PeerTable peerTable) { this.metricsSystem = metricsSystem; checkArgument(nodeKey != null, "nodeKey cannot be null"); checkArgument(config != null, "provided configuration cannot be null"); @@ -130,6 +133,7 @@ protected PeerDiscoveryAgent( this.forkIdManager = forkIdManager; this.forkIdSupplier = () -> forkIdManager.getForkIdForChainHead().getForkIdAsBytesList(); this.rlpxAgent = rlpxAgent; + this.peerTable = peerTable; } protected abstract TimerUtil createTimer(); @@ -263,9 +267,9 @@ private PeerDiscoveryController createController(final DiscoveryPeer localNode) .peerRequirement(PeerRequirement.combine(peerRequirements)) .peerPermissions(peerPermissions) .metricsSystem(metricsSystem) - .forkIdManager(forkIdManager) .filterOnEnrForkId((config.isFilterOnEnrForkIdEnabled())) .rlpxAgent(rlpxAgent) + .peerTable(peerTable) .build(); } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java index 27a2be8beb3..ef098896a61 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController.AsyncExecutor; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.discovery.internal.TimerUtil; import org.hyperledger.besu.ethereum.p2p.discovery.internal.VertxTimerUtil; import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissions; @@ -73,7 +74,8 @@ public VertxPeerDiscoveryAgent( final MetricsSystem metricsSystem, final StorageProvider storageProvider, final ForkIdManager forkIdManager, - final RlpxAgent rlpxAgent) { + final RlpxAgent rlpxAgent, + final PeerTable peerTable) { super( nodeKey, config, @@ -82,7 +84,8 @@ public VertxPeerDiscoveryAgent( metricsSystem, storageProvider, forkIdManager, - rlpxAgent); + rlpxAgent, + peerTable); checkArgument(vertx != null, "vertx instance cannot be null"); this.vertx = vertx; diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java index ec829d20709..af3790def52 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java @@ -21,8 +21,6 @@ import static java.util.concurrent.TimeUnit.SECONDS; import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.ethereum.forkid.ForkId; -import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; import org.hyperledger.besu.ethereum.p2p.peers.Peer; @@ -129,7 +127,6 @@ public class PeerDiscoveryController { private final DiscoveryProtocolLogger discoveryProtocolLogger; private final LabelledMetric interactionCounter; private final LabelledMetric interactionRetryCounter; - private final ForkIdManager forkIdManager; private final boolean filterOnEnrForkId; private final RlpxAgent rlpxAgent; @@ -161,7 +158,6 @@ private PeerDiscoveryController( final PeerPermissions peerPermissions, final MetricsSystem metricsSystem, final Optional> maybeCacheForEnrRequests, - final ForkIdManager forkIdManager, final boolean filterOnEnrForkId, final RlpxAgent rlpxAgent) { this.timerUtil = timerUtil; @@ -197,11 +193,11 @@ private PeerDiscoveryController( "discovery_interaction_retry_count", "Total number of interaction retries performed", "type"); + this.cachedEnrRequests = maybeCacheForEnrRequests.orElse( CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(10, SECONDS).build()); - this.forkIdManager = forkIdManager; this.filterOnEnrForkId = filterOnEnrForkId; } @@ -314,6 +310,7 @@ public void onMessage(final Packet packet, final DiscoveryPeer sender) { } final DiscoveryPeer peer = resolvePeer(sender); + final Bytes peerId = peer.getId(); switch (packet.getType()) { case PING: if (peerPermissions.allowInboundBonding(peer)) { @@ -333,10 +330,10 @@ public void onMessage(final Packet packet, final DiscoveryPeer sender) { if (filterOnEnrForkId) { requestENR(peer); } - bondingPeers.invalidate(peer.getId()); + bondingPeers.invalidate(peerId); addToPeerTable(peer); recursivePeerRefreshState.onBondingComplete(peer); - Optional.ofNullable(cachedEnrRequests.getIfPresent(peer.getId())) + Optional.ofNullable(cachedEnrRequests.getIfPresent(peerId)) .ifPresent(cachedEnrRequest -> processEnrRequest(peer, cachedEnrRequest)); }); break; @@ -360,12 +357,12 @@ public void onMessage(final Packet packet, final DiscoveryPeer sender) { if (PeerDiscoveryStatus.BONDED.equals(peer.getStatus())) { processEnrRequest(peer, packet); } else if (PeerDiscoveryStatus.BONDING.equals(peer.getStatus())) { - LOG.trace("ENR_REQUEST cached for bonding peer Id: {}", peer.getId()); + LOG.trace("ENR_REQUEST cached for bonding peer Id: {}", peerId); // Due to UDP, it may happen that we receive the ENR_REQUEST just before the PONG. // Because peers want to send the ENR_REQUEST directly after the pong. // If this happens we don't want to ignore the request but process when bonded. // this cache allows to keep the request and to respond after having processed the PONG - cachedEnrRequests.put(peer.getId(), packet); + cachedEnrRequests.put(peerId, packet); } break; case ENR_RESPONSE: @@ -376,26 +373,6 @@ public void onMessage(final Packet packet, final DiscoveryPeer sender) { packet.getPacketData(ENRResponsePacketData.class); final NodeRecord enr = packetData.get().getEnr(); peer.setNodeRecord(enr); - - final Optional maybeForkId = peer.getForkId(); - if (maybeForkId.isPresent()) { - if (forkIdManager.peerCheck(maybeForkId.get())) { - connectOnRlpxLayer(peer); - LOG.debug( - "Peer {} PASSED fork id check. ForkId received: {}", - sender.getId(), - maybeForkId.get()); - } else { - LOG.debug( - "Peer {} FAILED fork id check. ForkId received: {}", - sender.getId(), - maybeForkId.get()); - } - } else { - // if the peer hasn't sent the ForkId try to connect to it anyways - connectOnRlpxLayer(peer); - LOG.debug("No fork id sent by peer: {}", peer.getId()); - } }); break; } @@ -431,9 +408,7 @@ private boolean addToPeerTable(final DiscoveryPeer peer) { if (peer.getStatus() != PeerDiscoveryStatus.BONDED) { peer.setStatus(PeerDiscoveryStatus.BONDED); - if (!filterOnEnrForkId) { - connectOnRlpxLayer(peer); - } + connectOnRlpxLayer(peer); } final PeerTable.AddResult result = peerTable.tryAdd(peer); @@ -560,8 +535,6 @@ void bond(final DiscoveryPeer peer) { */ @VisibleForTesting void requestENR(final DiscoveryPeer peer) { - peer.setStatus(PeerDiscoveryStatus.ENR_REQUESTED); - final Consumer action = interaction -> { final ENRRequestPacketData data = ENRRequestPacketData.create(); @@ -838,7 +811,6 @@ public static class Builder { private Cache cachedEnrRequests = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(10, SECONDS).build(); - private ForkIdManager forkIdManager; private RlpxAgent rlpxAgent; private Builder() {} @@ -846,10 +818,6 @@ private Builder() {} public PeerDiscoveryController build() { validate(); - if (peerTable == null) { - peerTable = new PeerTable(this.nodeKey.getPublicKey().getEncodedBytes(), 16); - } - return new PeerDiscoveryController( nodeKey, localPeer, @@ -864,7 +832,6 @@ public PeerDiscoveryController build() { peerPermissions, metricsSystem, Optional.of(cachedEnrRequests), - forkIdManager, filterOnEnrForkId, rlpxAgent); } @@ -875,8 +842,8 @@ private void validate() { validateRequiredDependency(timerUtil, "TimerUtil"); validateRequiredDependency(workerExecutor, "AsyncExecutor"); validateRequiredDependency(metricsSystem, "MetricsSystem"); - validateRequiredDependency(forkIdManager, "ForkIdManager"); validateRequiredDependency(rlpxAgent, "RlpxAgent"); + validateRequiredDependency(peerTable, "PeerTable"); } private void validateRequiredDependency(final Object object, final String name) { @@ -970,11 +937,5 @@ public Builder rlpxAgent(final RlpxAgent rlpxAgent) { this.rlpxAgent = rlpxAgent; return this; } - - public Builder forkIdManager(final ForkIdManager forkIdManager) { - checkNotNull(forkIdManager); - this.forkIdManager = forkIdManager; - return this; - } } } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java index e153acbfc26..f0e0be1fe2d 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTable.java @@ -56,26 +56,21 @@ public class PeerTable { * Builds a new peer table, where distance is calculated using the provided nodeId as a baseline. * * @param nodeId The ID of the node where this peer table is stored. - * @param bucketSize The maximum length of each k-bucket. */ - public PeerTable(final Bytes nodeId, final int bucketSize) { + public PeerTable(final Bytes nodeId) { this.keccak256 = Hash.keccak256(nodeId); this.table = Stream.generate(() -> new Bucket(DEFAULT_BUCKET_SIZE)) .limit(N_BUCKETS + 1) .toArray(Bucket[]::new); this.distanceCache = new ConcurrentHashMap<>(); - this.maxEntriesCnt = N_BUCKETS * bucketSize; + this.maxEntriesCnt = N_BUCKETS * DEFAULT_BUCKET_SIZE; // A bloom filter with 4096 expected insertions of 64-byte keys with a 0.1% false positive // probability yields a memory footprint of ~7.5kb. buildBloomFilter(); } - public PeerTable(final Bytes nodeId) { - this(nodeId, DEFAULT_BUCKET_SIZE); - } - /** * Returns the table's representation of a peer, if it exists. * @@ -83,11 +78,12 @@ public PeerTable(final Bytes nodeId) { * @return The stored representation. */ public Optional get(final PeerId peer) { - if (!idBloom.mightContain(peer.getId())) { + final Bytes peerId = peer.getId(); + if (!idBloom.mightContain(peerId)) { return Optional.empty(); } final int distance = distanceFrom(peer); - return table[distance].getAndTouch(peer.getId()); + return table[distance].getAndTouch(peerId); } /** diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java index ec65934b297..11352b38cc7 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryAgent; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; import org.hyperledger.besu.ethereum.p2p.discovery.VertxPeerDiscoveryAgent; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.DefaultPeerPrivileges; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; @@ -383,11 +384,12 @@ void checkMaintainedConnectionPeers() { @VisibleForTesting void attemptPeerConnections() { LOG.trace("Initiating connections to discovered peers."); - rlpxAgent.connect( + final Stream toTry = streamDiscoveredPeers() .filter(peer -> peer.getStatus() == PeerDiscoveryStatus.BONDED) .filter(peerDiscoveryAgent::checkForkId) - .sorted(Comparator.comparing(DiscoveryPeer::getLastAttemptedConnection))); + .sorted(Comparator.comparing(DiscoveryPeer::getLastAttemptedConnection)); + toTry.forEach(rlpxAgent::connect); } @Override @@ -511,6 +513,7 @@ public static class Builder { private Supplier> allConnectionsSupplier; private Supplier> allActiveConnectionsSupplier; private int peersLowerBound; + private PeerTable peerTable; public P2PNetwork build() { validate(); @@ -528,6 +531,7 @@ private P2PNetwork doBuild() { final MutableLocalNode localNode = MutableLocalNode.create(config.getRlpx().getClientId(), 5, supportedCapabilities); final PeerPrivileges peerPrivileges = new DefaultPeerPrivileges(maintainedPeers); + peerTable = new PeerTable(nodeKey.getPublicKey().getEncodedBytes()); rlpxAgent = rlpxAgent == null ? createRlpxAgent(localNode, peerPrivileges) : rlpxAgent; peerDiscoveryAgent = peerDiscoveryAgent == null ? createDiscoveryAgent() : peerDiscoveryAgent; @@ -572,7 +576,8 @@ private PeerDiscoveryAgent createDiscoveryAgent() { metricsSystem, storageProvider, forkIdManager, - rlpxAgent); + rlpxAgent, + peerTable); } private RlpxAgent createRlpxAgent( @@ -589,6 +594,7 @@ private RlpxAgent createRlpxAgent( .allConnectionsSupplier(allConnectionsSupplier) .allActiveConnectionsSupplier(allActiveConnectionsSupplier) .peersLowerBound(peersLowerBound) + .peerTable(peerTable) .build(); } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java index 98a1f60df37..4a8e227d3d5 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgent.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.peers.PeerPrivileges; @@ -162,13 +163,6 @@ public int getConnectionCount() { } } - public void connect(final Stream peerStream) { - if (!localNode.isReady()) { - return; - } - peerStream.forEach(this::connect); - } - public void disconnect(final Bytes peerId, final DisconnectReason reason) { try { allActiveConnectionsSupplier @@ -206,6 +200,7 @@ public CompletableFuture connect(final Peer peer) { + this.getClass().getSimpleName() + " has finished starting")); } + // Check peer is valid final EnodeURL enode = peer.getEnodeURL(); if (!enode.isListening()) { @@ -380,6 +375,7 @@ public static class Builder { private Supplier> allConnectionsSupplier; private Supplier> allActiveConnectionsSupplier; private int peersLowerBound; + private PeerTable peerTable; private Builder() {} @@ -399,12 +395,13 @@ public RlpxAgent build() { localNode, connectionEvents, metricsSystem, - p2pTLSConfiguration.get()); + p2pTLSConfiguration.get(), + peerTable); } else { LOG.debug("Using default NettyConnectionInitializer"); connectionInitializer = new NettyConnectionInitializer( - nodeKey, config, localNode, connectionEvents, metricsSystem); + nodeKey, config, localNode, connectionEvents, metricsSystem, peerTable); } } @@ -499,5 +496,10 @@ public Builder peersLowerBound(final int peersLowerBound) { this.peersLowerBound = peersLowerBound; return this; } + + public Builder peerTable(final PeerTable peerTable) { + this.peerTable = peerTable; + return this; + } } } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java index 003a6ab1d9d..80be0a673db 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/AbstractHandshakeHandler.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; @@ -60,6 +61,7 @@ abstract class AbstractHandshakeHandler extends SimpleChannelInboundHandler subProtocols, @@ -70,7 +72,8 @@ abstract class AbstractHandshakeHandler extends SimpleChannelInboundHandler { + if (ff.isSuccess()) { + LOG.trace("Successfully wrote hello message"); + } + }); + msg.retain(); + ctx.fireChannelRead(msg); } - - LOG.trace("Sending framed hello"); - - // Exchange keys done - final Framer framer = this.framerProvider.buildFramer(handshaker.secrets()); - - final ByteToMessageDecoder deFramer = - new DeFramer( - framer, - subProtocols, - localNode, - expectedPeer, - connectionEventDispatcher, - connectionFuture, - metricsSystem, - inboundInitiated); - - ctx.channel() - .pipeline() - .replace(this, "DeFramer", deFramer) - .addBefore("DeFramer", "validate", new ValidateFirstOutboundMessage(framer)); - - ctx.writeAndFlush(new OutboundMessage(null, HelloMessage.create(localNode.getPeerInfo()))) - .addListener( - ff -> { - if (ff.isSuccess()) { - LOG.trace("Successfully wrote hello message"); - } - }); - msg.retain(); - ctx.fireChannelRead(msg); } private void disconnect( diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java index b39a6b82193..c7c600d3fd3 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramer.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; +import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.network.exceptions.BreachOfProtocolException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.IncompatiblePeerException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.PeerChannelClosedException; @@ -70,6 +72,7 @@ final class DeFramer extends ByteToMessageDecoder { private final Optional expectedPeer; private final List subProtocols; private final boolean inboundInitiated; + private final PeerTable peerTable; private boolean hellosExchanged; private final LabelledMetric outboundMessagesCounter; @@ -81,7 +84,8 @@ final class DeFramer extends ByteToMessageDecoder { final PeerConnectionEventDispatcher connectionEventDispatcher, final CompletableFuture connectFuture, final MetricsSystem metricsSystem, - final boolean inboundInitiated) { + final boolean inboundInitiated, + final PeerTable peerTable) { this.framer = framer; this.subProtocols = subProtocols; this.localNode = localNode; @@ -89,6 +93,7 @@ final class DeFramer extends ByteToMessageDecoder { this.connectFuture = connectFuture; this.connectionEventDispatcher = connectionEventDispatcher; this.inboundInitiated = inboundInitiated; + this.peerTable = peerTable; this.outboundMessagesCounter = metricsSystem.createLabelledCounter( BesuMetricCategory.NETWORK, @@ -105,8 +110,11 @@ protected void decode(final ChannelHandlerContext ctx, final ByteBuf in, final L while ((message = framer.deframe(in)) != null) { if (hellosExchanged) { + out.add(message); + } else if (message.getCode() == WireMessageCodes.HELLO) { + hellosExchanged = true; // Decode first hello and use the payload to modify pipeline final PeerInfo peerInfo; @@ -129,13 +137,27 @@ protected void decode(final ChannelHandlerContext ctx, final ByteBuf in, final L subProtocols, localNode.getPeerInfo().getCapabilities(), peerInfo.getCapabilities()); - final Optional peer = expectedPeer.or(() -> createPeer(peerInfo, ctx)); - if (peer.isEmpty()) { - LOG.debug("Failed to create connection for peer {}", peerInfo); - connectFuture.completeExceptionally(new PeerChannelClosedException(peerInfo)); - ctx.close(); - return; + + Optional peer; + if (expectedPeer.isPresent()) { + peer = expectedPeer; + } else { + // This is an inbound "Hello" message. Create peer from information from the Hello message + peer = createPeer(peerInfo, ctx); + if (peer.isEmpty()) { + LOG.debug("Failed to create connection for peer {}", peerInfo); + connectFuture.completeExceptionally(new PeerChannelClosedException(peerInfo)); + ctx.close(); + return; + } + // If we can find the DiscoveryPeer for the peer in the PeerTable we use it, because + // it could contains additional information, like the fork id. + final Optional discoveryPeer = peerTable.get(peer.get()); + if (discoveryPeer.isPresent()) { + peer = Optional.of(discoveryPeer.get()); + } } + final PeerConnection connection = new NettyPeerConnection( ctx, @@ -176,7 +198,9 @@ protected void decode(final ChannelHandlerContext ctx, final ByteBuf in, final L capabilityMultiplexer, connection, connectionEventDispatcher, waitingForPong), new MessageFramer(capabilityMultiplexer, framer)); connectFuture.complete(connection); + } else if (message.getCode() == WireMessageCodes.DISCONNECT) { + final DisconnectMessage disconnectMessage = DisconnectMessage.readFrom(message); LOG.debug( "Peer {} disconnected before sending HELLO. Reason: {}", @@ -185,8 +209,10 @@ protected void decode(final ChannelHandlerContext ctx, final ByteBuf in, final L ctx.close(); connectFuture.completeExceptionally( new PeerDisconnectedException(disconnectMessage.getReason())); + } else { // Unexpected message - disconnect + LOG.debug( "Message received before HELLO's exchanged (BREACH_OF_PROTOCOL), disconnecting. Peer: {}, Code: {}, Data: {}", expectedPeer.map(Peer::getEnodeURLString).orElse("unknown"), diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java index 184cf5cf8c1..962de68f980 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerInbound.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty; import org.hyperledger.besu.cryptoservices.NodeKey; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnectionEventDispatcher; @@ -40,7 +41,8 @@ public HandshakeHandlerInbound( final PeerConnectionEventDispatcher connectionEventDispatcher, final MetricsSystem metricsSystem, final HandshakerProvider handshakerProvider, - final FramerProvider framerProvider) { + final FramerProvider framerProvider, + final PeerTable peerTable) { super( subProtocols, localNode, @@ -50,7 +52,8 @@ public HandshakeHandlerInbound( metricsSystem, handshakerProvider, framerProvider, - true); + true, + peerTable); handshaker.prepareResponder(nodeKey); } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java index 205b6f655cc..46e600d74b6 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/HandshakeHandlerOutbound.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.cryptoservices.NodeKey; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnection; @@ -50,7 +51,8 @@ public HandshakeHandlerOutbound( final PeerConnectionEventDispatcher connectionEventDispatcher, final MetricsSystem metricsSystem, final HandshakerProvider handshakerProvider, - final FramerProvider framerProvider) { + final FramerProvider framerProvider, + final PeerTable peerTable) { super( subProtocols, localNode, @@ -60,7 +62,8 @@ public HandshakeHandlerOutbound( metricsSystem, handshakerProvider, framerProvider, - false); + false, + peerTable); handshaker.prepareInitiator( nodeKey, SignatureAlgorithmFactory.getInstance().createPublicKey(peer.getId())); this.first = handshaker.firstMessage(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java index c20e511df9d..f386c59a384 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyConnectionInitializer.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.ConnectCallback; @@ -68,6 +69,7 @@ public class NettyConnectionInitializer private final PeerConnectionEventDispatcher eventDispatcher; private final MetricsSystem metricsSystem; private final Subscribers connectSubscribers = Subscribers.create(); + private final PeerTable peerTable; private ChannelFuture server; private final EventLoopGroup boss = new NioEventLoopGroup(1); @@ -80,12 +82,14 @@ public NettyConnectionInitializer( final RlpxConfiguration config, final LocalNode localNode, final PeerConnectionEventDispatcher eventDispatcher, - final MetricsSystem metricsSystem) { + final MetricsSystem metricsSystem, + final PeerTable peerTable) { this.nodeKey = nodeKey; this.config = config; this.localNode = localNode; this.eventDispatcher = eventDispatcher; this.metricsSystem = metricsSystem; + this.peerTable = peerTable; metricsSystem.createIntegerGauge( BesuMetricCategory.NETWORK, @@ -244,7 +248,8 @@ private HandshakeHandlerInbound inboundHandler( eventDispatcher, metricsSystem, this, - this); + this, + peerTable); } @Nonnull @@ -259,7 +264,8 @@ private HandshakeHandlerOutbound outboundHandler( eventDispatcher, metricsSystem, this, - this); + this, + peerTable); } @Nonnull diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java index 4e6010771fc..db41c1574c7 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializer.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.plain.PlainFramer; @@ -55,7 +56,8 @@ public NettyTLSConnectionInitializer( final LocalNode localNode, final PeerConnectionEventDispatcher eventDispatcher, final MetricsSystem metricsSystem, - final TLSConfiguration p2pTLSConfiguration) { + final TLSConfiguration p2pTLSConfiguration, + final PeerTable peerTable) { this( nodeKey, config, @@ -63,7 +65,8 @@ public NettyTLSConnectionInitializer( eventDispatcher, metricsSystem, defaultTlsContextFactorySupplier(p2pTLSConfiguration), - p2pTLSConfiguration.getClientHelloSniHeaderEnabled()); + p2pTLSConfiguration.getClientHelloSniHeaderEnabled(), + peerTable); } @VisibleForTesting @@ -74,8 +77,9 @@ public NettyTLSConnectionInitializer( final PeerConnectionEventDispatcher eventDispatcher, final MetricsSystem metricsSystem, final Supplier tlsContextFactorySupplier, - final Boolean clientHelloSniHeaderEnabled) { - super(nodeKey, config, localNode, eventDispatcher, metricsSystem); + final Boolean clientHelloSniHeaderEnabled, + final PeerTable peerTable) { + super(nodeKey, config, localNode, eventDispatcher, metricsSystem, peerTable); if (tlsContextFactorySupplier != null) { this.tlsContextFactorySupplier = Optional.of(Suppliers.memoize(tlsContextFactorySupplier::get)); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java index 464273243ad..ffc9fb1c300 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java @@ -295,6 +295,7 @@ public MockPeerDiscoveryAgent build() { config.setAdvertisedHost(advertisedHost); config.setBindPort(port); config.setActive(active); + config.setFilterOnEnrForkId(false); final ForkIdManager mockForkIdManager = mock(ForkIdManager.class); final ForkId forkId = new ForkId(Bytes.EMPTY, Bytes.EMPTY); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java index ea6e1593b83..88196f18b5d 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/MockPeerDiscoveryAgent.java @@ -63,7 +63,8 @@ public MockPeerDiscoveryAgent( new NoOpMetricsSystem(), new InMemoryKeyValueStorageProvider(), forkIdManager, - rlpxAgent); + rlpxAgent, + new PeerTable(nodeKey.getPublicKey().getEncodedBytes())); this.agentNetwork = agentNetwork; } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java index 182366ac03a..ca0d7c6430d 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryControllerTest.java @@ -35,8 +35,6 @@ import org.hyperledger.besu.crypto.SignatureAlgorithm; import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.ethereum.forkid.ForkId; -import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.Endpoint; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; @@ -1480,14 +1478,12 @@ public long read() { } @Test - public void shouldFiltersOnForkIdSuccess() { + public void forkIdShouldBeAvailableIfEnrPacketContainsForkId() { final List nodeKeys = PeerDiscoveryTestHelper.generateNodeKeys(1); final List peers = helper.createDiscoveryPeers(nodeKeys); - final ForkIdManager forkIdManager = mock(ForkIdManager.class); final DiscoveryPeer sender = peers.get(0); - final Packet enrPacket = prepareForForkIdCheck(forkIdManager, nodeKeys, sender, true); + final Packet enrPacket = prepareForForkIdCheck(nodeKeys, sender, true); - when(forkIdManager.peerCheck(any(ForkId.class))).thenReturn(true); controller.onMessage(enrPacket, sender); final Optional maybePeer = @@ -1501,35 +1497,12 @@ public void shouldFiltersOnForkIdSuccess() { verify(controller, times(1)).connectOnRlpxLayer(eq(maybePeer.get())); } - @Test - public void shouldFiltersOnForkIdFailure() { - final List nodeKeys = PeerDiscoveryTestHelper.generateNodeKeys(1); - final List peers = helper.createDiscoveryPeers(nodeKeys); - final ForkIdManager forkIdManager = mock(ForkIdManager.class); - final DiscoveryPeer sender = peers.get(0); - final Packet enrPacket = prepareForForkIdCheck(forkIdManager, nodeKeys, sender, true); - - when(forkIdManager.peerCheck(any(ForkId.class))).thenReturn(false); - controller.onMessage(enrPacket, sender); - - final Optional maybePeer = - controller - .streamDiscoveredPeers() - .filter(p -> p.getId().equals(sender.getId())) - .findFirst(); - - assertThat(maybePeer.isPresent()).isTrue(); - assertThat(maybePeer.get().getForkId().isPresent()).isTrue(); - verify(controller, never()).connectOnRlpxLayer(eq(maybePeer.get())); - } - @Test public void shouldStillCallConnectIfNoForkIdSent() { final List nodeKeys = PeerDiscoveryTestHelper.generateNodeKeys(1); final List peers = helper.createDiscoveryPeers(nodeKeys); final DiscoveryPeer sender = peers.get(0); - final Packet enrPacket = - prepareForForkIdCheck(mock(ForkIdManager.class), nodeKeys, sender, false); + final Packet enrPacket = prepareForForkIdCheck(nodeKeys, sender, false); controller.onMessage(enrPacket, sender); @@ -1546,10 +1519,7 @@ public void shouldStillCallConnectIfNoForkIdSent() { @NotNull private Packet prepareForForkIdCheck( - final ForkIdManager forkIdManager, - final List nodeKeys, - final DiscoveryPeer sender, - final boolean sendForkId) { + final List nodeKeys, final DiscoveryPeer sender, final boolean sendForkId) { final HashMap packetTypeBytesHashMap = new HashMap<>(); final OutboundMessageHandler outboundMessageHandler = (dp, pa) -> packetTypeBytesHashMap.put(pa.getType(), pa.getHash()); @@ -1573,7 +1543,6 @@ public long read() { .outboundMessageHandler(outboundMessageHandler) .enrCache(enrs) .filterOnForkId(true) - .forkIdManager(forkIdManager) .build(); // Mock the creation of the PING packet, so that we can control the hash, which gets validated @@ -1720,7 +1689,6 @@ static class ControllerBuilder { private Cache enrs = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(10, TimeUnit.SECONDS).build(); private boolean filterOnForkId = false; - private ForkIdManager forkIdManager; public static ControllerBuilder create() { return new ControllerBuilder(); @@ -1776,11 +1744,6 @@ public ControllerBuilder filterOnForkId(final boolean filterOnForkId) { return this; } - public ControllerBuilder forkIdManager(final ForkIdManager forkIdManager) { - this.forkIdManager = forkIdManager; - return this; - } - PeerDiscoveryController build() { checkNotNull(nodeKey); if (localPeer == null) { @@ -1803,7 +1766,6 @@ PeerDiscoveryController build() { .peerPermissions(peerPermissions) .metricsSystem(new NoOpMetricsSystem()) .cacheForEnrRequests(enrs) - .forkIdManager(forkIdManager == null ? mock(ForkIdManager.class) : forkIdManager) .filterOnEnrForkId(filterOnForkId) .rlpxAgent(mock(RlpxAgent.class)) .build()); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java index 949b318906b..6320c909622 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryTableRefreshTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.verify; import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.ethereum.forkid.ForkIdManager; import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus; import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryTestHelper; @@ -72,7 +71,6 @@ public void tableRefreshSingleNode() { .tableRefreshIntervalMs(0) .metricsSystem(new NoOpMetricsSystem()) .rlpxAgent(mock(RlpxAgent.class)) - .forkIdManager(mock(ForkIdManager.class)) .build()); controller.start(); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java index c0909a9b8b4..dff9d231658 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerTableTest.java @@ -43,7 +43,7 @@ public class PeerTableTest { @Test public void addPeer() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final List peers = helper.createDiscoveryPeers(5); for (final DiscoveryPeer peer : peers) { @@ -63,7 +63,7 @@ public void addSelf() { .ipAddress("127.0.0.1") .discoveryAndListeningPorts(12345) .build()); - final PeerTable table = new PeerTable(localPeer.getId(), 16); + final PeerTable table = new PeerTable(localPeer.getId()); final PeerTable.AddResult result = table.tryAdd(localPeer); assertThat(result.getOutcome()).isEqualTo(AddOutcome.SELF); @@ -72,7 +72,7 @@ public void addSelf() { @Test public void peerExists() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); assertThat(table.tryAdd(peer).getOutcome()).isEqualTo(AddOutcome.ADDED); @@ -87,7 +87,7 @@ public void peerExists() { @Test public void peerExists_withDifferentIp() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final Bytes peerId = SIGNATURE_ALGORITHM.get().generateKeyPair().getPublicKey().getEncodedBytes(); final DiscoveryPeer peer = @@ -107,7 +107,7 @@ public void peerExists_withDifferentIp() { @Test public void peerExists_withDifferentUdpPort() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final Bytes peerId = SIGNATURE_ALGORITHM.get().generateKeyPair().getPublicKey().getEncodedBytes(); final DiscoveryPeer peer = @@ -127,7 +127,7 @@ public void peerExists_withDifferentUdpPort() { @Test public void peerExists_withDifferentIdAndUdpPort() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final Bytes peerId = SIGNATURE_ALGORITHM.get().generateKeyPair().getPublicKey().getEncodedBytes(); final DiscoveryPeer peer = @@ -147,7 +147,7 @@ public void peerExists_withDifferentIdAndUdpPort() { @Test public void evictExistingPeerShouldEvict() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); table.tryAdd(peer); @@ -158,7 +158,7 @@ public void evictExistingPeerShouldEvict() { @Test public void evictPeerFromEmptyTableShouldNotEvict() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); final EvictResult evictResult = table.tryEvict(peer); @@ -167,7 +167,7 @@ public void evictPeerFromEmptyTableShouldNotEvict() { @Test public void evictAbsentPeerShouldNotEvict() { - final PeerTable table = new PeerTable(Peer.randomId(), 16); + final PeerTable table = new PeerTable(Peer.randomId()); final DiscoveryPeer peer = helper.createDiscoveryPeer(); final List otherPeers = helper.createDiscoveryPeers(5); otherPeers.forEach(table::tryAdd); @@ -179,7 +179,7 @@ public void evictAbsentPeerShouldNotEvict() { @Test public void evictSelfPeerShouldReturnSelfOutcome() { final DiscoveryPeer peer = helper.createDiscoveryPeer(); - final PeerTable table = new PeerTable(peer.getId(), 16); + final PeerTable table = new PeerTable(peer.getId()); final EvictResult evictResult = table.tryEvict(peer); assertThat(evictResult.getOutcome()).isEqualTo(EvictOutcome.SELF); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java index e79abb883dd..5d26f8cc6e9 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/RecursivePeerRefreshStateTest.java @@ -57,7 +57,7 @@ public class RecursivePeerRefreshStateTest { neighborFinder, timerUtil, localPeer, - new PeerTable(createId(999), 16), + new PeerTable(createId(999)), peerPermissions, 5, 100); @@ -180,7 +180,7 @@ public void shouldStopWhenMaximumNumberOfRoundsReached() { neighborFinder, timerUtil, localPeer, - new PeerTable(createId(999), 16), + new PeerTable(createId(999)), peerPermissions, 5, 1); @@ -466,7 +466,7 @@ public void shouldNotBondWithNonPermittedNode() { neighborFinder, timerUtil, localPeer, - new PeerTable(createId(999), 16), + new PeerTable(createId(999)), peerPermissions, 5, 100); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java index a54c9038b8e..68a4f76b582 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetworkTest.java @@ -55,7 +55,6 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; import java.util.stream.Stream; import io.vertx.core.Context; @@ -82,7 +81,7 @@ public final class DefaultP2PNetworkTest { @Mock PeerDiscoveryAgent discoveryAgent; @Mock RlpxAgent rlpxAgent; - @Captor private ArgumentCaptor> peerStreamCaptor; + @Captor private ArgumentCaptor peerCaptor; private final NetworkingConfiguration config = NetworkingConfiguration.create() @@ -276,12 +275,9 @@ public void attemptPeerConnections_bondedPeers() { final DefaultP2PNetwork network = network(); network.attemptPeerConnections(); - verify(rlpxAgent, times(1)).connect(peerStreamCaptor.capture()); + verify(rlpxAgent, times(1)).connect(peerCaptor.capture()); - final List capturedPeers = - peerStreamCaptor.getValue().collect(Collectors.toList()); - assertThat(capturedPeers.contains(discoPeer)).isTrue(); - assertThat(capturedPeers.size()).isEqualTo(1); + assertThat(peerCaptor.getValue()).isEqualTo(discoPeer); } @Test @@ -293,12 +289,7 @@ public void attemptPeerConnections_unbondedPeers() { final DefaultP2PNetwork network = network(); network.attemptPeerConnections(); - verify(rlpxAgent, times(1)).connect(peerStreamCaptor.capture()); - - final List capturedPeers = - peerStreamCaptor.getValue().collect(Collectors.toList()); - assertThat(capturedPeers.contains(discoPeer)).isFalse(); - assertThat(capturedPeers.size()).isEqualTo(0); + verify(rlpxAgent, times(0)).connect(any()); } @Test @@ -314,14 +305,7 @@ public void attemptPeerConnections_sortsPeersByLastContacted() { final DefaultP2PNetwork network = network(); network.attemptPeerConnections(); - verify(rlpxAgent, times(1)).connect(peerStreamCaptor.capture()); - - final List capturedPeers = - peerStreamCaptor.getValue().collect(Collectors.toList()); - assertThat(capturedPeers.size()).isEqualTo(3); - assertThat(capturedPeers.get(0)).isEqualTo(discoPeers.get(1)); - assertThat(capturedPeers.get(1)).isEqualTo(discoPeers.get(0)); - assertThat(capturedPeers.get(2)).isEqualTo(discoPeers.get(2)); + verify(rlpxAgent, times(3)).connect(any()); } @Test diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java index d2fdf8c76a6..c639579eee1 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/RlpxAgentTest.java @@ -296,7 +296,7 @@ public void connect_largeStreamOfPeers() { Stream.generate(PeerTestHelper::createPeer).limit(peerNo); agent = spy(agent); - agent.connect(peerStream); + peerStream.forEach(agent::connect); assertThat(agent.getMapOfCompletableFutures().size()).isEqualTo(peerNo); verify(agent, times(peerNo)).connect(any(Peer.class)); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java index f5030fd9ad6..952ebf7c2d5 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java @@ -24,6 +24,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.hyperledger.besu.ethereum.forkid.ForkId; +import org.hyperledger.besu.ethereum.p2p.discovery.DiscoveryPeer; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.network.exceptions.BreachOfProtocolException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.IncompatiblePeerException; import org.hyperledger.besu.ethereum.p2p.network.exceptions.PeerChannelClosedException; @@ -104,7 +107,7 @@ public class DeFramerTest { private final LocalNode localNode = LocalNode.create(clientId, p2pVersion, capabilities, localEnode); - private final DeFramer deFramer = createDeFramer(null); + private final DeFramer deFramer = createDeFramer(null, Optional.empty()); @BeforeEach @SuppressWarnings("unchecked") @@ -219,7 +222,7 @@ public void decode_handlesHelloFromPeerWithAdvertisedPortOf0() final Peer peer = createRemotePeer(); final PeerInfo remotePeerInfo = new PeerInfo(p2pVersion, clientId, capabilities, 0, peer.getId()); - final DeFramer deFramer = createDeFramer(null); + final DeFramer deFramer = createDeFramer(null, Optional.empty()); final HelloMessage helloMessage = HelloMessage.create(remotePeerInfo); final ByteBuf data = Unpooled.wrappedBuffer(helloMessage.getData().toArray()); @@ -260,6 +263,39 @@ public void decode_handlesHelloFromPeerWithAdvertisedPortOf0() assertThat(out.size()).isEqualTo(1); } + @Test + public void decode_duringHandshakeFindsPeerInPeerTable() + throws ExecutionException, InterruptedException { + final ChannelFuture future = NettyMocks.channelFuture(false); + when(channel.closeFuture()).thenReturn(future); + + final Peer peer = createRemotePeer(); + final PeerInfo remotePeerInfo = + new PeerInfo(p2pVersion, clientId, capabilities, 0, peer.getId()); + + final HelloMessage helloMessage = HelloMessage.create(remotePeerInfo); + final Bytes nodeId = helloMessage.getPeerInfo().getNodeId(); + final String enodeURLString = + "enode://" + nodeId.toString().substring(2) + "@" + "12.13.14.15:30303?discport=30301"; + final Optional discoveryPeer = + DiscoveryPeer.from(DefaultPeer.fromURI(enodeURLString)); + final ForkId forkId = new ForkId(Bytes.fromHexString("0x190a55ad"), 4L); + discoveryPeer.orElseThrow().setForkId(forkId); + final DeFramer deFramer = createDeFramer(null, discoveryPeer); + final ByteBuf data = Unpooled.wrappedBuffer(helloMessage.getData().toArray()); + when(framer.deframe(eq(data))) + .thenReturn(new RawMessage(helloMessage.getCode(), helloMessage.getData())) + .thenReturn(null); + final List out = new ArrayList<>(); + deFramer.decode(ctx, data, out); + + assertThat(connectFuture).isDone(); + assertThat(connectFuture).isNotCompletedExceptionally(); + final PeerConnection peerConnection = connectFuture.get(); + assertThat(peerConnection.getPeerInfo()).isEqualTo(remotePeerInfo); + assertThat(peerConnection.getPeer().getForkId().orElseThrow()).isEqualTo(forkId); + } + @Test public void decode_handlesUnexpectedPeerId() { final ChannelFuture future = NettyMocks.channelFuture(false); @@ -274,7 +310,7 @@ public void decode_handlesUnexpectedPeerId() { capabilities, peer.getEnodeURL().getListeningPortOrZero(), mismatchedId); - final DeFramer deFramer = createDeFramer(peer); + final DeFramer deFramer = createDeFramer(peer, Optional.empty()); final HelloMessage helloMessage = HelloMessage.create(remotePeerInfo); final ByteBuf data = Unpooled.wrappedBuffer(helloMessage.getData().toArray()); @@ -414,7 +450,10 @@ private PeerInfo createPeerInfo(final Peer forPeer) { forPeer.getId()); } - private DeFramer createDeFramer(final Peer expectedPeer) { + private DeFramer createDeFramer( + final Peer expectedPeer, final Optional peerInPeerTable) { + final PeerTable peerTable = new PeerTable(localNode.getPeerInfo().getNodeId()); + peerInPeerTable.ifPresent(peerTable::tryAdd); return new DeFramer( framer, Arrays.asList(MockSubProtocol.create("eth")), @@ -423,6 +462,7 @@ private DeFramer createDeFramer(final Peer expectedPeer) { connectionEventDispatcher, connectFuture, new NoOpMetricsSystem(), - true); + true, + peerTable); } } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java index d4637952b63..f7f3dbc5124 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/NettyTLSConnectionInitializerTest.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.p2p.config.RlpxConfiguration; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerTable; import org.hyperledger.besu.ethereum.p2p.peers.LocalNode; import org.hyperledger.besu.ethereum.p2p.peers.Peer; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.PeerConnectionEventDispatcher; @@ -44,6 +45,7 @@ import io.netty.handler.codec.compression.SnappyFrameEncoder; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; +import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -95,7 +97,8 @@ private NettyTLSConnectionInitializer createNettyTLSConnectionInitializer( eventDispatcher, new NoOpMetricsSystem(), () -> tlsContextFactory, - clientHelloSniHeaderEnabled); + clientHelloSniHeaderEnabled, + new PeerTable(Bytes.random(64))); } @Test From 1c1f53853449ed6308c722a9f7b553edc06eb13d Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Tue, 23 Jan 2024 08:42:57 +1100 Subject: [PATCH 31/56] Addition of Profile Configuration CLI Option (#6418) Signed-off-by: Gabriel-Trintinalia --- .../org/hyperledger/besu/cli/BesuCommand.java | 12 + .../cli/ConfigurationOverviewBuilder.java | 16 + .../besu/cli/DefaultCommandValues.java | 4 + .../besu/cli/config/ProfileName.java | 41 +++ .../subcommands/ValidateConfigSubCommand.java | 5 +- .../cli/util/AbstractConfigurationFinder.java | 125 +++++++ .../cli/util/CascadingDefaultProvider.java | 6 +- .../besu/cli/util/ConfigFileFinder.java | 100 ++++++ .../util/ConfigOptionSearchAndRunHandler.java | 67 ++-- .../besu/cli/util/ProfileFinder.java | 76 +++++ ... => TomlConfigurationDefaultProvider.java} | 95 ++++-- .../hyperledger/besu/cli/BesuCommandTest.java | 168 +--------- .../cli/CascadingDefaultProviderTest.java | 306 ++++++++++++++++++ .../besu/cli/CommandLineUtilsTest.java | 4 +- .../besu/cli/CommandTestAbstract.java | 5 + .../cli/ConfigurationOverviewBuilderTest.java | 8 + ...TomlConfigurationDefaultProviderTest.java} | 41 +-- .../ConfigOptionSearchAndRunHandlerTest.java | 5 +- .../src/test/resources/everything_config.toml | 2 +- besu/src/test/resources/partial_config.toml | 5 +- config/src/main/resources/profiles/dev.toml | 1 + 21 files changed, 818 insertions(+), 274 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/util/AbstractConfigurationFinder.java create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/util/ConfigFileFinder.java create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java rename besu/src/main/java/org/hyperledger/besu/cli/util/{TomlConfigFileDefaultProvider.java => TomlConfigurationDefaultProvider.java} (76%) create mode 100644 besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java rename besu/src/test/java/org/hyperledger/besu/cli/{TomlConfigFileDefaultProviderTest.java => TomlConfigurationDefaultProviderTest.java} (90%) create mode 100644 config/src/main/resources/profiles/dev.toml diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 7f276c704c4..0ad34a95218 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -46,6 +46,7 @@ import org.hyperledger.besu.chainimport.RlpBlockImporter; import org.hyperledger.besu.cli.config.EthNetworkConfig; import org.hyperledger.besu.cli.config.NetworkName; +import org.hyperledger.besu.cli.config.ProfileName; import org.hyperledger.besu.cli.converter.MetricCategoryConverter; import org.hyperledger.besu.cli.converter.PercentageConverter; import org.hyperledger.besu.cli.custom.CorsAllowedOriginsProperty; @@ -536,6 +537,13 @@ private InetAddress autoDiscoverDefaultIP() { + " (default: ${DEFAULT-VALUE})") private final NetworkName network = null; + @Option( + names = {PROFILE_OPTION_NAME}, + paramLabel = PROFILE_FORMAT_HELP, + description = + "Overwrite default settings. Possible values are ${COMPLETION-CANDIDATES}. (default: none)") + private final ProfileName profile = null; + @Option( names = {"--nat-method"}, description = @@ -3508,6 +3516,10 @@ private String generateConfigurationOverview() { builder.setNetwork(network.normalize()); } + if (profile != null) { + builder.setProfile(profile.toString()); + } + builder.setHasCustomGenesis(genesisFile != null); if (genesisFile != null) { builder.setCustomGenesis(genesisFile.getAbsolutePath()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index 4fd870e99ce..fe5488fbea4 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -41,6 +41,7 @@ public class ConfigurationOverviewBuilder { private String network; private BigInteger networkId; + private String profile; private boolean hasCustomGenesis; private String customGenesisFileName; private String dataStorage; @@ -88,6 +89,17 @@ public ConfigurationOverviewBuilder setNetworkId(final BigInteger networkId) { return this; } + /** + * Sets profile. + * + * @param profile the profile + * @return the profile + */ + public ConfigurationOverviewBuilder setProfile(final String profile) { + this.profile = profile; + return this; + } + /** * Sets whether a custom genesis has been specified. * @@ -290,6 +302,10 @@ public String build() { lines.add("Network Id: " + networkId); } + if (profile != null) { + lines.add("Profile: " + profile); + } + if (dataStorage != null) { lines.add("Data storage: " + dataStorage); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java b/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java index d8644c36de7..3a0db478160 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java @@ -54,6 +54,10 @@ public interface DefaultCommandValues { String MANDATORY_MODE_FORMAT_HELP = ""; /** The constant MANDATORY_NETWORK_FORMAT_HELP. */ String MANDATORY_NETWORK_FORMAT_HELP = ""; + /** The constant PROFILE_OPTION_NAME. */ + String PROFILE_OPTION_NAME = "--profile"; + /** The constant PROFILE_FORMAT_HELP. */ + String PROFILE_FORMAT_HELP = ""; /** The constant MANDATORY_NODE_ID_FORMAT_HELP. */ String MANDATORY_NODE_ID_FORMAT_HELP = ""; /** The constant PERMISSIONING_CONFIG_LOCATION. */ diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java new file mode 100644 index 00000000000..823dd9f142c --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java @@ -0,0 +1,41 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.config; + +/** Enum for profile names. Each profile corresponds to a configuration file. */ +public enum ProfileName { + /** The 'DEV' profile. Corresponds to the 'profiles/dev.toml' configuration file. */ + DEV("profiles/dev.toml"); + + private final String configFile; + + /** + * Constructs a new ProfileName. + * + * @param configFile the configuration file corresponding to the profile + */ + ProfileName(final String configFile) { + this.configFile = configFile; + } + + /** + * Gets the configuration file corresponding to the profile. + * + * @return the configuration file + */ + public String getConfigFile() { + return configFile; + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java index f34ac97470c..f17f1a09fc6 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java @@ -19,7 +19,7 @@ import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.cli.DefaultCommandValues; -import org.hyperledger.besu.cli.util.TomlConfigFileDefaultProvider; +import org.hyperledger.besu.cli.util.TomlConfigurationDefaultProvider; import org.hyperledger.besu.cli.util.VersionProvider; import java.io.PrintWriter; @@ -69,7 +69,8 @@ public ValidateConfigSubCommand(final CommandLine commandLine, final PrintWriter public void run() { checkNotNull(parentCommand); try { - new TomlConfigFileDefaultProvider(commandLine, dataPath.toFile()).loadConfigurationFromFile(); + TomlConfigurationDefaultProvider.fromFile(commandLine, dataPath.toFile()) + .loadConfigurationFromFile(); } catch (Exception e) { this.out.println(e); return; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/AbstractConfigurationFinder.java b/besu/src/main/java/org/hyperledger/besu/cli/util/AbstractConfigurationFinder.java new file mode 100644 index 00000000000..69e02887775 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/AbstractConfigurationFinder.java @@ -0,0 +1,125 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.util; + +import java.util.Map; +import java.util.Optional; + +import picocli.CommandLine; + +/** + * Abstract class for finding configuration resources. This class provides a common structure for + * classes that need to find configuration resources based on command line options and environment + * variables. + * + * @param the type of configuration resource this finder will return + */ +public abstract class AbstractConfigurationFinder { + + /** + * Returns the name of the configuration option. + * + * @return the name of the configuration option + */ + protected abstract String getConfigOptionName(); + + /** + * Returns the name of the environment variable for the configuration. + * + * @return the name of the environment variable for the configuration + */ + protected abstract String getConfigEnvName(); + + /** + * Finds the configuration resource based on command line options and environment variables. + * + * @param environment the environment variables + * @param parseResult the command line parse result + * @return an Optional containing the configuration resource, or an empty Optional if no + * configuration resource was found + */ + public Optional findConfiguration( + final Map environment, final CommandLine.ParseResult parseResult) { + final CommandLine commandLine = parseResult.commandSpec().commandLine(); + if (isConfigSpecifiedInBothSources(environment, parseResult)) { + throwExceptionForBothSourcesSpecified(environment, parseResult, commandLine); + } + if (parseResult.hasMatchedOption(getConfigOptionName())) { + return getFromOption(parseResult, commandLine); + } + if (environment.containsKey(getConfigEnvName())) { + return getFromEnvironment(environment, commandLine); + } + return Optional.empty(); + } + + /** + * Gets the configuration resource from the command line option. + * + * @param parseResult the command line parse result + * @param commandLine the command line + * @return an Optional containing the configuration resource, or an empty Optional if the + * configuration resource was not specified in the command line option + */ + protected abstract Optional getFromOption( + final CommandLine.ParseResult parseResult, final CommandLine commandLine); + + /** + * Gets the configuration resource from the environment variable. + * + * @param environment the environment variables + * @param commandLine the command line + * @return an Optional containing the configuration resource, or an empty Optional if the + * configuration resource was not specified in the environment variable + */ + protected abstract Optional getFromEnvironment( + final Map environment, final CommandLine commandLine); + + /** + * Checks if the configuration resource is specified in both command line options and environment + * variables. + * + * @param environment the environment variables + * @param parseResult the command line parse result + * @return true if the configuration resource is specified in both places, false otherwise + */ + public boolean isConfigSpecifiedInBothSources( + final Map environment, final CommandLine.ParseResult parseResult) { + return parseResult.hasMatchedOption(getConfigOptionName()) + && environment.containsKey(getConfigEnvName()); + } + + /** + * Throws an exception if the configuration resource is specified in both command line options and + * environment variables. + * + * @param environment the environment variables + * @param parseResult the command line parse result + * @param commandLine the command line + */ + public void throwExceptionForBothSourcesSpecified( + final Map environment, + final CommandLine.ParseResult parseResult, + final CommandLine commandLine) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + "Both %s=%s and %s %s specified. Please specify only one.", + getConfigEnvName(), + getConfigOptionName(), + environment.get(getConfigEnvName()), + parseResult.matchedOption(getConfigOptionName()).stringValues())); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/CascadingDefaultProvider.java b/besu/src/main/java/org/hyperledger/besu/cli/util/CascadingDefaultProvider.java index 489d61af95c..1e5d0bec6e7 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/CascadingDefaultProvider.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/CascadingDefaultProvider.java @@ -14,8 +14,6 @@ */ package org.hyperledger.besu.cli.util; -import static java.util.Arrays.asList; - import java.util.List; import picocli.CommandLine.IDefaultValueProvider; @@ -34,8 +32,8 @@ public class CascadingDefaultProvider implements IDefaultValueProvider { * * @param defaultValueProviders List of default value providers */ - public CascadingDefaultProvider(final IDefaultValueProvider... defaultValueProviders) { - this.defaultValueProviders = asList(defaultValueProviders); + public CascadingDefaultProvider(final List defaultValueProviders) { + this.defaultValueProviders = defaultValueProviders; } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigFileFinder.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigFileFinder.java new file mode 100644 index 00000000000..a17ef0b1a6a --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigFileFinder.java @@ -0,0 +1,100 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.util; + +import static org.hyperledger.besu.cli.DefaultCommandValues.CONFIG_FILE_OPTION_NAME; + +import java.io.File; +import java.util.Map; +import java.util.Optional; + +import picocli.CommandLine; + +/** + * Class for finding configuration files. This class extends the AbstractConfigurationFinder and + * provides methods for finding configuration files based on command line options and environment + * variables. + */ +public class ConfigFileFinder extends AbstractConfigurationFinder { + private static final String CONFIG_FILE_ENV_NAME = "BESU_CONFIG_FILE"; + + /** + * Returns the name of the configuration option. + * + * @return the name of the configuration option + */ + @Override + protected String getConfigOptionName() { + return CONFIG_FILE_OPTION_NAME; + } + + /** + * Returns the name of the environment variable for the configuration. + * + * @return the name of the environment variable for the configuration + */ + @Override + protected String getConfigEnvName() { + return CONFIG_FILE_ENV_NAME; + } + + /** + * Gets the configuration file from the command line option. + * + * @param parseResult the command line parse result + * @param commandLine the command line + * @return an Optional containing the configuration file, or an empty Optional if the + * configuration file was not specified in the command line option + */ + @Override + public Optional getFromOption( + final CommandLine.ParseResult parseResult, final CommandLine commandLine) { + final CommandLine.Model.OptionSpec configFileOption = + parseResult.matchedOption(CONFIG_FILE_OPTION_NAME); + try { + File file = configFileOption.getter().get(); + if (!file.exists()) { + throw new CommandLine.ParameterException( + commandLine, + String.format("Unable to read TOML configuration, file not found: %s", file)); + } + return Optional.of(file); + } catch (final Exception e) { + throw new CommandLine.ParameterException(commandLine, e.getMessage(), e); + } + } + + /** + * Gets the configuration file from the environment variable. + * + * @param environment the environment variables + * @param commandLine the command line + * @return an Optional containing the configuration file, or an empty Optional if the + * configuration file was not specified in the environment variable + */ + @Override + public Optional getFromEnvironment( + final Map environment, final CommandLine commandLine) { + final File toml = new File(environment.get(CONFIG_FILE_ENV_NAME)); + if (!toml.exists()) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + "TOML file %s specified in environment variable %s not found", + CONFIG_FILE_ENV_NAME, environment.get(CONFIG_FILE_ENV_NAME))); + } + return Optional.of(toml); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java index 5866d43cde6..52b67652822 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.cli.util; import java.io.File; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -25,7 +26,6 @@ import picocli.CommandLine.IDefaultValueProvider; import picocli.CommandLine.IExecutionStrategy; import picocli.CommandLine.IParameterExceptionHandler; -import picocli.CommandLine.Model.OptionSpec; import picocli.CommandLine.ParameterException; import picocli.CommandLine.ParseResult; @@ -54,8 +54,13 @@ public ConfigOptionSearchAndRunHandler( @Override public List handle(final ParseResult parseResult) throws ParameterException { final CommandLine commandLine = parseResult.commandSpec().commandLine(); - final Optional configFile = findConfigFile(parseResult, commandLine); - commandLine.setDefaultValueProvider(createDefaultValueProvider(commandLine, configFile)); + + commandLine.setDefaultValueProvider( + createDefaultValueProvider( + commandLine, + new ConfigFileFinder().findConfiguration(environment, parseResult), + new ProfileFinder().findConfiguration(environment, parseResult))); + commandLine.setExecutionStrategy(resultHandler); commandLine.setParameterExceptionHandler(parameterExceptionHandler); commandLine.execute(parseResult.originalArgs().toArray(new String[0])); @@ -63,38 +68,6 @@ public List handle(final ParseResult parseResult) throws ParameterExcept return new ArrayList<>(); } - private Optional findConfigFile( - final ParseResult parseResult, final CommandLine commandLine) { - if (parseResult.hasMatchedOption("--config-file") - && environment.containsKey("BESU_CONFIG_FILE")) { - throw new ParameterException( - commandLine, - String.format( - "TOML file specified using BESU_CONFIG_FILE=%s and --config-file %s", - environment.get("BESU_CONFIG_FILE"), - parseResult.matchedOption("--config-file").stringValues())); - } else if (parseResult.hasMatchedOption("--config-file")) { - final OptionSpec configFileOption = parseResult.matchedOption("--config-file"); - try { - return Optional.of(configFileOption.getter().get()); - } catch (final Exception e) { - throw new ParameterException(commandLine, e.getMessage(), e); - } - } else if (environment.containsKey("BESU_CONFIG_FILE")) { - final File toml = new File(environment.get("BESU_CONFIG_FILE")); - if (!toml.exists()) { - throw new ParameterException( - commandLine, - String.format( - "TOML file %s specified in environment variable BESU_CONFIG_FILE not found", - environment.get("BESU_CONFIG_FILE"))); - } - return Optional.of(toml); - } - - return Optional.empty(); - } - /** * Create default value provider default value provider. * @@ -104,14 +77,22 @@ private Optional findConfigFile( */ @VisibleForTesting IDefaultValueProvider createDefaultValueProvider( - final CommandLine commandLine, final Optional configFile) { - if (configFile.isPresent()) { - return new CascadingDefaultProvider( - new EnvironmentVariableDefaultProvider(environment), - new TomlConfigFileDefaultProvider(commandLine, configFile.get())); - } else { - return new EnvironmentVariableDefaultProvider(environment); - } + final CommandLine commandLine, + final Optional configFile, + final Optional profile) { + List providers = new ArrayList<>(); + providers.add(new EnvironmentVariableDefaultProvider(environment)); + + configFile.ifPresent( + config -> { + if (config.exists()) { + providers.add(TomlConfigurationDefaultProvider.fromFile(commandLine, config)); + } + }); + + profile.ifPresent( + p -> providers.add(TomlConfigurationDefaultProvider.fromInputStream(commandLine, p))); + return new CascadingDefaultProvider(providers); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java new file mode 100644 index 00000000000..8df7382db70 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java @@ -0,0 +1,76 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.util; + +import static org.hyperledger.besu.cli.DefaultCommandValues.PROFILE_OPTION_NAME; + +import org.hyperledger.besu.cli.config.ProfileName; + +import java.io.InputStream; +import java.util.Map; +import java.util.Optional; + +import picocli.CommandLine; + +/** + * Class for finding profile configurations. This class extends the AbstractConfigurationFinder and + * provides methods for finding profile configurations based on command line options and environment + * variables. Each profile corresponds to a TOML configuration file that contains settings for + * various options. The profile to use can be specified with the '--profile' command line option or + * the 'BESU_PROFILE' environment variable. + */ +public class ProfileFinder extends AbstractConfigurationFinder { + private static final String PROFILE_ENV_NAME = "BESU_PROFILE"; + + @Override + protected String getConfigOptionName() { + return PROFILE_OPTION_NAME; + } + + @Override + protected String getConfigEnvName() { + return PROFILE_ENV_NAME; + } + + @Override + public Optional getFromOption( + final CommandLine.ParseResult parseResult, final CommandLine commandLine) { + try { + return getProfile(parseResult.matchedOption(PROFILE_OPTION_NAME).getter().get(), commandLine); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public Optional getFromEnvironment( + final Map environment, final CommandLine commandLine) { + return getProfile(ProfileName.valueOf(environment.get(PROFILE_ENV_NAME)), commandLine); + } + + private static Optional getProfile( + final ProfileName profileName, final CommandLine commandLine) { + return Optional.of(getTomlFile(commandLine, profileName.getConfigFile())); + } + + private static InputStream getTomlFile(final CommandLine commandLine, final String file) { + InputStream resourceUrl = ProfileFinder.class.getClassLoader().getResourceAsStream(file); + if (resourceUrl == null) { + throw new CommandLine.ParameterException( + commandLine, String.format("TOML file %s not found", file)); + } + return resourceUrl; + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigFileDefaultProvider.java b/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java similarity index 76% rename from besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigFileDefaultProvider.java rename to besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java index c765b7f5dfc..4928d3f7ff6 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigFileDefaultProvider.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java @@ -19,7 +19,10 @@ import org.hyperledger.besu.util.number.Percentage; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; @@ -41,21 +44,52 @@ import picocli.CommandLine.ParameterException; /** The Toml config file default value provider used by PicoCli. */ -public class TomlConfigFileDefaultProvider implements IDefaultValueProvider { +public class TomlConfigurationDefaultProvider implements IDefaultValueProvider { private final CommandLine commandLine; - private final File configFile; + private final InputStream configurationInputStream; private TomlParseResult result; /** * Instantiates a new Toml config file default value provider. * * @param commandLine the command line - * @param configFile the config file + * @param configurationInputStream the input stream */ - public TomlConfigFileDefaultProvider(final CommandLine commandLine, final File configFile) { + private TomlConfigurationDefaultProvider( + final CommandLine commandLine, final InputStream configurationInputStream) { this.commandLine = commandLine; - this.configFile = configFile; + this.configurationInputStream = configurationInputStream; + } + + /** + * Creates a new TomlConfigurationDefaultProvider from a file. + * + * @param commandLine the command line + * @param configFile the configuration file + * @return a new TomlConfigurationDefaultProvider + * @throws ParameterException if the configuration file is not found + */ + public static TomlConfigurationDefaultProvider fromFile( + final CommandLine commandLine, final File configFile) { + try { + return new TomlConfigurationDefaultProvider(commandLine, new FileInputStream(configFile)); + } catch (final FileNotFoundException e) { + throw new ParameterException( + commandLine, "Unable to read TOML configuration, file not found."); + } + } + + /** + * Creates a new TomlConfigurationDefaultProvider from an input stream. + * + * @param commandLine the command line + * @param inputStream the input stream + * @return a new TomlConfigurationDefaultProvider + */ + public static TomlConfigurationDefaultProvider fromInputStream( + final CommandLine commandLine, final InputStream inputStream) { + return new TomlConfigurationDefaultProvider(commandLine, inputStream); } @Override @@ -70,35 +104,32 @@ public String defaultValue(final ArgSpec argSpec) { private String getConfigurationValue(final OptionSpec optionSpec) { // NOTE: This temporary fix is necessary to make certain options be treated as a multi-value. // This can be done automatically by picocli if the object implements Collection. - final boolean isArray = - getKeyName(optionSpec).map(keyName -> result.isArray(keyName)).orElse(false); - final String defaultValue; + final boolean isArray = getKeyName(optionSpec).map(result::isArray).orElse(false); - // Convert config values to the right string representation for default string value if (optionSpec.type().equals(Boolean.class) || optionSpec.type().equals(boolean.class)) { - defaultValue = getBooleanEntryAsString(optionSpec); + return getBooleanEntryAsString(optionSpec); } else if (optionSpec.isMultiValue() || isArray) { - defaultValue = getListEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Integer.class) || optionSpec.type().equals(int.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Long.class) || optionSpec.type().equals(long.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Wei.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(BigInteger.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Double.class) || optionSpec.type().equals(double.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Float.class) || optionSpec.type().equals(float.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Percentage.class)) { - defaultValue = getNumericEntryAsString(optionSpec); - } else if (optionSpec.type().equals(Fraction.class)) { - defaultValue = getNumericEntryAsString(optionSpec); + return getListEntryAsString(optionSpec); + } else if (isNumericType(optionSpec.type())) { + return getNumericEntryAsString(optionSpec); } else { // else will be treated as String - defaultValue = getEntryAsString(optionSpec); + return getEntryAsString(optionSpec); } - return defaultValue; + } + + private boolean isNumericType(final Class type) { + return type.equals(Integer.class) + || type.equals(int.class) + || type.equals(Long.class) + || type.equals(long.class) + || type.equals(Wei.class) + || type.equals(BigInteger.class) + || type.equals(Double.class) + || type.equals(double.class) + || type.equals(Float.class) + || type.equals(float.class) + || type.equals(Percentage.class) + || type.equals(Fraction.class); } private String getEntryAsString(final OptionSpec spec) { @@ -195,7 +226,8 @@ private String getNumericEntryAsString(final OptionSpec spec) { private void checkConfigurationValidity() { if (result == null || result.isEmpty()) throw new ParameterException( - commandLine, String.format("Unable to read TOML configuration file %s", configFile)); + commandLine, + String.format("Unable to read TOML configuration file %s", configurationInputStream)); } /** Load configuration from file. */ @@ -203,7 +235,7 @@ public void loadConfigurationFromFile() { if (result == null) { try { - final TomlParseResult result = Toml.parse(configFile.toPath()); + final TomlParseResult result = Toml.parse(configurationInputStream); if (result.hasErrors()) { final String errors = @@ -224,7 +256,6 @@ public void loadConfigurationFromFile() { commandLine, "Unable to read TOML configuration, file not found."); } } - checkConfigurationValidity(); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 04d9f015530..49e734b4647 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.cli; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.startsWith; @@ -33,7 +32,6 @@ import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.PERM; -import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.WEB3; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.GOERLI_BOOTSTRAP_NODES; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.GOERLI_DISCOVERY_URL; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_BOOTSTRAP_NODES; @@ -71,8 +69,6 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.authentication.JwtAlgorithm; import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration; -import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; -import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; @@ -123,7 +119,6 @@ import com.google.common.io.Resources; import io.vertx.core.json.JsonObject; import org.apache.commons.io.FileUtils; -import org.apache.commons.text.StringEscapeUtils; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.toml.Toml; import org.apache.tuweni.toml.TomlParseResult; @@ -306,17 +301,18 @@ public void callingWithConfigOptionButNonExistingFileShouldDisplayHelp() throws final Path tempConfigFilePath = createTempFile("an-invalid-file-name-without-extension", ""); parseCommand("--config-file", tempConfigFilePath.toString()); - final String expectedOutputStart = - "Unable to read TOML configuration file " + tempConfigFilePath; + final String expectedOutputStart = "Unable to read TOML configuration file"; assertThat(commandErrorOutput.toString(UTF_8)).startsWith(expectedOutputStart); assertThat(commandOutput.toString(UTF_8)).isEmpty(); } @Test public void callingWithConfigOptionButTomlFileNotFoundShouldDisplayHelp() { - parseCommand("--config-file", "./an-invalid-file-name-sdsd87sjhqoi34io23.toml"); + String invalidFile = "./an-invalid-file-name-sdsd87sjhqoi34io23.toml"; + parseCommand("--config-file", invalidFile); - final String expectedOutputStart = "Unable to read TOML configuration, file not found."; + final String expectedOutputStart = + String.format("Unable to read TOML configuration, file not found: %s", invalidFile); assertThat(commandErrorOutput.toString(UTF_8)).startsWith(expectedOutputStart); assertThat(commandOutput.toString(UTF_8)).isEmpty(); } @@ -364,81 +360,6 @@ public void callingWithConfigOptionButInvalidValueTomlFileShouldDisplayHelp() th assertThat(commandOutput.toString(UTF_8)).isEmpty(); } - @Test - public void overrideDefaultValuesIfKeyIsPresentInConfigFile(final @TempDir File dataFolder) - throws IOException { - final URL configFile = this.getClass().getResource("/complete_config.toml"); - final Path genesisFile = createFakeGenesisFile(GENESIS_VALID_JSON); - final String updatedConfig = - Resources.toString(configFile, UTF_8) - .replace("/opt/besu/genesis.json", escapeTomlString(genesisFile.toString())) - .replace( - "data-path=\"/opt/besu\"", - "data-path=\"" + escapeTomlString(dataFolder.getPath()) + "\""); - - final Path toml = createTempFile("toml", updatedConfig.getBytes(UTF_8)); - - final List expectedApis = asList(ETH.name(), WEB3.name()); - - final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); - jsonRpcConfiguration.setEnabled(false); - jsonRpcConfiguration.setHost("5.6.7.8"); - jsonRpcConfiguration.setPort(5678); - jsonRpcConfiguration.setCorsAllowedDomains(Collections.emptyList()); - jsonRpcConfiguration.setRpcApis(expectedApis); - jsonRpcConfiguration.setMaxActiveConnections(1000); - - final GraphQLConfiguration graphQLConfiguration = GraphQLConfiguration.createDefault(); - graphQLConfiguration.setEnabled(false); - graphQLConfiguration.setHost("6.7.8.9"); - graphQLConfiguration.setPort(6789); - - final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); - webSocketConfiguration.setEnabled(false); - webSocketConfiguration.setHost("9.10.11.12"); - webSocketConfiguration.setPort(9101); - webSocketConfiguration.setRpcApis(expectedApis); - - final MetricsConfiguration metricsConfiguration = - MetricsConfiguration.builder().enabled(false).host("8.6.7.5").port(309).build(); - - parseCommand("--config-file", toml.toString()); - - verify(mockRunnerBuilder).discovery(eq(false)); - verify(mockRunnerBuilder).ethNetworkConfig(ethNetworkConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).p2pAdvertisedHost(eq("1.2.3.4")); - verify(mockRunnerBuilder).p2pListenPort(eq(1234)); - verify(mockRunnerBuilder).jsonRpcConfiguration(eq(jsonRpcConfiguration)); - verify(mockRunnerBuilder).graphQLConfiguration(eq(graphQLConfiguration)); - verify(mockRunnerBuilder).webSocketConfiguration(eq(webSocketConfiguration)); - verify(mockRunnerBuilder).metricsConfiguration(eq(metricsConfiguration)); - verify(mockRunnerBuilder).build(); - - final List nodes = - asList( - EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567"), - EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567"), - EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567")); - assertThat(ethNetworkConfigArgumentCaptor.getValue().getBootNodes()).isEqualTo(nodes); - - final EthNetworkConfig networkConfig = - new EthNetworkConfig.Builder(EthNetworkConfig.getNetworkConfig(MAINNET)) - .setNetworkId(BigInteger.valueOf(42)) - .setGenesisConfig(encodeJsonGenesis(GENESIS_VALID_JSON)) - .setBootNodes(nodes) - .setDnsDiscoveryUrl(null) - .build(); - verify(mockControllerBuilder).dataDirectory(eq(dataFolder.toPath())); - verify(mockControllerBuilderFactory).fromEthNetworkConfig(eq(networkConfig), any(), any()); - verify(mockControllerBuilder).synchronizerConfiguration(syncConfigurationCaptor.capture()); - - assertThat(syncConfigurationCaptor.getValue().getSyncMode()).isEqualTo(SyncMode.FAST); - assertThat(syncConfigurationCaptor.getValue().getFastSyncMinimumPeerCount()).isEqualTo(13); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - @Test public void nodePermissionsSmartContractWithoutOptionMustError() { parseCommand("--permissions-nodes-contract-address"); @@ -861,81 +782,6 @@ public void tomlThatConfiguresEverythingExceptPermissioningToml() throws IOExcep .isEmpty(); } - @Test - public void noOverrideDefaultValuesIfKeyIsNotPresentInConfigFile() { - final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); - - parseCommand("--config-file", configFile); - final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); - - final GraphQLConfiguration graphQLConfiguration = GraphQLConfiguration.createDefault(); - - final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); - - final MetricsConfiguration metricsConfiguration = MetricsConfiguration.builder().build(); - - verify(mockRunnerBuilder).discovery(eq(true)); - verify(mockRunnerBuilder) - .ethNetworkConfig( - new EthNetworkConfig( - EthNetworkConfig.jsonConfig(MAINNET), - MAINNET.getNetworkId(), - MAINNET_BOOTSTRAP_NODES, - MAINNET_DISCOVERY_URL)); - verify(mockRunnerBuilder).p2pAdvertisedHost(eq("127.0.0.1")); - verify(mockRunnerBuilder).p2pListenPort(eq(30303)); - verify(mockRunnerBuilder).jsonRpcConfiguration(eq(jsonRpcConfiguration)); - verify(mockRunnerBuilder).graphQLConfiguration(eq(graphQLConfiguration)); - verify(mockRunnerBuilder).webSocketConfiguration(eq(webSocketConfiguration)); - verify(mockRunnerBuilder).metricsConfiguration(eq(metricsConfiguration)); - verify(mockRunnerBuilder).build(); - verify(mockControllerBuilder).build(); - verify(mockControllerBuilder).synchronizerConfiguration(syncConfigurationCaptor.capture()); - - final SynchronizerConfiguration syncConfig = syncConfigurationCaptor.getValue(); - assertThat(syncConfig.getSyncMode()).isEqualTo(SyncMode.FAST); - assertThat(syncConfig.getFastSyncMinimumPeerCount()).isEqualTo(5); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void envVariableOverridesValueFromConfigFile() { - final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); - final String expectedCoinbase = "0x0000000000000000000000000000000000000004"; - setEnvironmentVariable("BESU_MINER_COINBASE", expectedCoinbase); - parseCommand("--config-file", configFile); - - verify(mockControllerBuilder) - .miningParameters( - ImmutableMiningParameters.builder() - .mutableInitValues( - MutableInitValues.builder() - .coinbase(Address.fromHexString(expectedCoinbase)) - .build()) - .build()); - } - - @Test - public void cliOptionOverridesEnvVariableAndConfig() { - final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); - final String expectedCoinbase = "0x0000000000000000000000000000000000000006"; - setEnvironmentVariable("BESU_MINER_COINBASE", "0x0000000000000000000000000000000000000004"); - parseCommand("--config-file", configFile, "--miner-coinbase", expectedCoinbase); - - verify(mockControllerBuilder) - .miningParameters( - ImmutableMiningParameters.builder() - .mutableInitValues( - MutableInitValues.builder() - .coinbase(Address.fromHexString(expectedCoinbase)) - .build()) - .build()); - } - @Test public void nodekeyOptionMustBeUsed() throws Exception { final File file = new File("./specific/enclavePrivateKey"); @@ -4300,10 +4146,6 @@ public void canNotUseFlexiblePrivacyWhenPrivacyPluginEnabled() { "No Payload Provider has been provided. You must register one when enabling privacy plugin!"); } - private static String escapeTomlString(final String s) { - return StringEscapeUtils.escapeJava(s); - } - /** * Check logger calls * diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java new file mode 100644 index 00000000000..e9707f1dfde --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/cli/CascadingDefaultProviderTest.java @@ -0,0 +1,306 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Arrays.asList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.cli.config.NetworkName.DEV; +import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.WEB3; +import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_BOOTSTRAP_NODES; +import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_DISCOVERY_URL; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; + +import org.hyperledger.besu.cli.config.EthNetworkConfig; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; +import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration; +import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; +import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; +import org.hyperledger.besu.ethereum.eth.sync.SyncMode; +import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; +import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; +import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; +import org.hyperledger.besu.plugin.data.EnodeURL; + +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.net.URL; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; + +import com.google.common.io.Resources; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; + +public class CascadingDefaultProviderTest extends CommandTestAbstract { + private static final int GENESIS_CONFIG_TEST_CHAINID = 3141592; + private static final String VALID_NODE_ID = + "6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0"; + private static final JsonObject GENESIS_VALID_JSON = + (new JsonObject()) + .put("config", (new JsonObject()).put("chainId", GENESIS_CONFIG_TEST_CHAINID)); + + /** + * Test if the default values are overridden if the key is present in the configuration file. The + * test checks if the configuration file correctly overrides the default values for various + * settings, such as the JSON-RPC configuration, GraphQL configuration, WebSocket configuration, + * and metrics configuration. + */ + @Test + public void overrideDefaultValuesIfKeyIsPresentInConfigFile(final @TempDir File dataFolder) + throws IOException { + final URL configFile = this.getClass().getResource("/complete_config.toml"); + final Path genesisFile = createFakeGenesisFile(GENESIS_VALID_JSON); + final String updatedConfig = + Resources.toString(configFile, UTF_8) + .replace("/opt/besu/genesis.json", escapeTomlString(genesisFile.toString())) + .replace( + "data-path=\"/opt/besu\"", + "data-path=\"" + escapeTomlString(dataFolder.getPath()) + "\""); + + final Path toml = createTempFile("toml", updatedConfig.getBytes(UTF_8)); + + final List expectedApis = asList(ETH.name(), WEB3.name()); + + final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); + jsonRpcConfiguration.setEnabled(false); + jsonRpcConfiguration.setHost("5.6.7.8"); + jsonRpcConfiguration.setPort(5678); + jsonRpcConfiguration.setCorsAllowedDomains(Collections.emptyList()); + jsonRpcConfiguration.setRpcApis(expectedApis); + jsonRpcConfiguration.setMaxActiveConnections(1000); + + final GraphQLConfiguration graphQLConfiguration = GraphQLConfiguration.createDefault(); + graphQLConfiguration.setEnabled(false); + graphQLConfiguration.setHost("6.7.8.9"); + graphQLConfiguration.setPort(6789); + + final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); + webSocketConfiguration.setEnabled(false); + webSocketConfiguration.setHost("9.10.11.12"); + webSocketConfiguration.setPort(9101); + webSocketConfiguration.setRpcApis(expectedApis); + + final MetricsConfiguration metricsConfiguration = + MetricsConfiguration.builder().enabled(false).host("8.6.7.5").port(309).build(); + + parseCommand("--config-file", toml.toString()); + + verify(mockRunnerBuilder).discovery(eq(false)); + verify(mockRunnerBuilder).ethNetworkConfig(ethNetworkConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).p2pAdvertisedHost(eq("1.2.3.4")); + verify(mockRunnerBuilder).p2pListenPort(eq(1234)); + verify(mockRunnerBuilder).jsonRpcConfiguration(eq(jsonRpcConfiguration)); + verify(mockRunnerBuilder).graphQLConfiguration(eq(graphQLConfiguration)); + verify(mockRunnerBuilder).webSocketConfiguration(eq(webSocketConfiguration)); + verify(mockRunnerBuilder).metricsConfiguration(eq(metricsConfiguration)); + verify(mockRunnerBuilder).build(); + + final List nodes = + asList( + EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567"), + EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567"), + EnodeURLImpl.fromString("enode://" + VALID_NODE_ID + "@192.168.0.1:4567")); + assertThat(ethNetworkConfigArgumentCaptor.getValue().getBootNodes()).isEqualTo(nodes); + + final EthNetworkConfig networkConfig = + new EthNetworkConfig.Builder(EthNetworkConfig.getNetworkConfig(MAINNET)) + .setNetworkId(BigInteger.valueOf(42)) + .setGenesisConfig(encodeJsonGenesis(GENESIS_VALID_JSON)) + .setBootNodes(nodes) + .setDnsDiscoveryUrl(null) + .build(); + verify(mockControllerBuilder).dataDirectory(eq(dataFolder.toPath())); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(eq(networkConfig), any(), any()); + verify(mockControllerBuilder).synchronizerConfiguration(syncConfigurationCaptor.capture()); + + assertThat(syncConfigurationCaptor.getValue().getSyncMode()).isEqualTo(SyncMode.FAST); + assertThat(syncConfigurationCaptor.getValue().getFastSyncMinimumPeerCount()).isEqualTo(13); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + /** + * Test if the default values are not overridden if the key is not present in the configuration + * file. The test checks if the default values for various settings remain unchanged when the + * corresponding keys are not present in the configuration file. + */ + @Test + public void noOverrideDefaultValuesIfKeyIsNotPresentInConfigFile() { + final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); + + parseCommand("--config-file", configFile); + final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); + + final GraphQLConfiguration graphQLConfiguration = GraphQLConfiguration.createDefault(); + + final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); + + final MetricsConfiguration metricsConfiguration = MetricsConfiguration.builder().build(); + + verify(mockRunnerBuilder).discovery(eq(true)); + verify(mockRunnerBuilder) + .ethNetworkConfig( + new EthNetworkConfig( + EthNetworkConfig.jsonConfig(MAINNET), + MAINNET.getNetworkId(), + MAINNET_BOOTSTRAP_NODES, + MAINNET_DISCOVERY_URL)); + verify(mockRunnerBuilder).p2pAdvertisedHost(eq("127.0.0.1")); + verify(mockRunnerBuilder).p2pListenPort(eq(30303)); + verify(mockRunnerBuilder).jsonRpcConfiguration(eq(jsonRpcConfiguration)); + verify(mockRunnerBuilder).graphQLConfiguration(eq(graphQLConfiguration)); + verify(mockRunnerBuilder).webSocketConfiguration(eq(webSocketConfiguration)); + verify(mockRunnerBuilder).metricsConfiguration(eq(metricsConfiguration)); + verify(mockRunnerBuilder).build(); + verify(mockControllerBuilder).build(); + verify(mockControllerBuilder).synchronizerConfiguration(syncConfigurationCaptor.capture()); + + final SynchronizerConfiguration syncConfig = syncConfigurationCaptor.getValue(); + assertThat(syncConfig.getSyncMode()).isEqualTo(SyncMode.FAST); + assertThat(syncConfig.getFastSyncMinimumPeerCount()).isEqualTo(5); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + /** + * Test if the environment variable overrides the value from the configuration file. The test + * checks if the value of the miner's coinbase address set through an environment variable + * correctly overrides the value specified in the configuration file. + */ + @Test + public void envVariableOverridesValueFromConfigFile() { + final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); + final String expectedCoinbase = "0x0000000000000000000000000000000000000004"; + setEnvironmentVariable("BESU_MINER_COINBASE", expectedCoinbase); + parseCommand("--config-file", configFile); + + verify(mockControllerBuilder) + .miningParameters( + ImmutableMiningParameters.builder() + .mutableInitValues( + ImmutableMiningParameters.MutableInitValues.builder() + .coinbase(Address.fromHexString(expectedCoinbase)) + .build()) + .build()); + } + + /** + * Test if the command line option overrides the environment variable and configuration. The test + * checks if the value of the miner's coinbase address set through a command line option correctly + * overrides the value specified in the environment variable and the configuration file. + */ + @Test + public void cliOptionOverridesEnvVariableAndConfig() { + final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); + final String expectedCoinbase = "0x0000000000000000000000000000000000000006"; + setEnvironmentVariable("BESU_MINER_COINBASE", "0x0000000000000000000000000000000000000004"); + parseCommand("--config-file", configFile, "--miner-coinbase", expectedCoinbase); + + verify(mockControllerBuilder) + .miningParameters( + ImmutableMiningParameters.builder() + .mutableInitValues( + ImmutableMiningParameters.MutableInitValues.builder() + .coinbase(Address.fromHexString(expectedCoinbase)) + .build()) + .build()); + } + + /** + * Test if the profile option sets the correct defaults. The test checks if the 'dev' profile + * correctly sets the network ID to the expected value. + */ + @Test + public void profileOptionShouldSetCorrectDefaults() { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + parseCommand("--profile", "dev"); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); + verify(mockControllerBuilder).build(); + + final EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getNetworkId()).isEqualTo(DEV.getNetworkId()); + } + + /** + * Test if the command line option overrides the profile configuration. The test checks if the + * network ID set through a command line option correctly overrides the value specified in the + * 'dev' profile. + */ + @Test + public void cliOptionOverridesProfileConfiguration() { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + parseCommand("--profile", "dev", "--network", "MAINNET"); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); + verify(mockControllerBuilder).build(); + + final EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getNetworkId()).isEqualTo(MAINNET.getNetworkId()); + } + + /** + * Test if the configuration file overrides the profile configuration. The test checks if the + * network ID specified in the configuration file correctly overrides the value specified in the + * 'dev' profile. + */ + @Test + public void configFileOverridesProfileConfiguration() { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + final String configFile = this.getClass().getResource("/partial_config.toml").getFile(); + parseCommand("--profile", "dev", "--config-file", configFile); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); + verify(mockControllerBuilder).build(); + + final EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getNetworkId()).isEqualTo(MAINNET.getNetworkId()); + } + + /** + * Test if the environment variable overrides the profile configuration. The test checks if the + * network ID set through an environment variable correctly overrides the value specified in the + * 'dev' profile. + */ + @Test + public void environmentVariableOverridesProfileConfiguration() { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + setEnvironmentVariable("BESU_NETWORK", "MAINNET"); + parseCommand("--profile", "dev"); + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); + verify(mockControllerBuilder).build(); + + final EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getNetworkId()).isEqualTo(MAINNET.getNetworkId()); + } +} diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandLineUtilsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandLineUtilsTest.java index 726bf38965e..d8e8912eefa 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandLineUtilsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandLineUtilsTest.java @@ -22,7 +22,7 @@ import org.hyperledger.besu.cli.util.CommandLineUtils; import org.hyperledger.besu.cli.util.EnvironmentVariableDefaultProvider; -import org.hyperledger.besu.cli.util.TomlConfigFileDefaultProvider; +import org.hyperledger.besu.cli.util.TomlConfigurationDefaultProvider; import org.hyperledger.besu.util.StringUtils; import java.io.IOException; @@ -252,7 +252,7 @@ public void multipleMainOptionsToml() throws IOException { final AbstractTestCommand testCommand = new TestMultiCommandWithDeps(mockLogger); testCommand.commandLine.setDefaultValueProvider( - new TomlConfigFileDefaultProvider(testCommand.commandLine, toml.toFile())); + TomlConfigurationDefaultProvider.fromFile(testCommand.commandLine, toml.toFile())); testCommand.commandLine.parseWithHandlers(new RunLast(), defaultExceptionHandler()); verifyMultiOptionsConstraintLoggerCall( diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index e9d8d559641..0d2fb39427e 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -109,6 +109,7 @@ import io.vertx.core.Vertx; import io.vertx.core.VertxOptions; import io.vertx.core.json.JsonObject; +import org.apache.commons.text.StringEscapeUtils; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.awaitility.Awaitility; @@ -682,4 +683,8 @@ private enum TestType { PORT_CHECK, NO_PORT_CHECK } + + protected static String escapeTomlString(final String s) { + return StringEscapeUtils.escapeJava(s); + } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java index eee55c3975b..708101d1ab7 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java @@ -20,6 +20,7 @@ import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration.Implementation.SEQUENCED; import static org.mockito.Mockito.mock; +import org.hyperledger.besu.cli.config.ProfileName; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.math.BigInteger; @@ -209,4 +210,11 @@ void setWorldStateUpdateModeJournaled() { final String layeredTxPoolSelected = builder.build(); assertThat(layeredTxPoolSelected).contains("Using JOURNALED worldstate update mode"); } + + @Test + void setProfile() { + builder.setProfile(ProfileName.DEV.name()); + final String profileSelected = builder.build(); + assertThat(profileSelected).contains("Profile: DEV"); + } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigFileDefaultProviderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java similarity index 90% rename from besu/src/test/java/org/hyperledger/besu/cli/TomlConfigFileDefaultProviderTest.java rename to besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java index 5185a4133e3..0b47654f58c 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigFileDefaultProviderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java @@ -19,7 +19,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.when; -import org.hyperledger.besu.cli.util.TomlConfigFileDefaultProvider; +import org.hyperledger.besu.cli.util.TomlConfigurationDefaultProvider; import org.hyperledger.besu.datatypes.Wei; import java.io.BufferedWriter; @@ -42,7 +42,7 @@ import picocli.CommandLine.ParameterException; @ExtendWith(MockitoExtension.class) -public class TomlConfigFileDefaultProviderTest { +public class TomlConfigurationDefaultProviderTest { @Mock CommandLine mockCommandLine; @Mock CommandSpec mockCommandSpec; @@ -67,8 +67,8 @@ public void defaultValueForMatchingKey(final @TempDir Path temp) throws IOExcept fileWriter.write("a-longer-option='1234'"); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); // this option must be found in config assertThat( @@ -152,8 +152,8 @@ public void defaultValueForOptionMustMatchType(final @TempDir Path temp) throws fileWriter.write("a-double-value-option-int=1"); // should be able to parse int as double fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThat( providerUnderTest.defaultValue( @@ -221,16 +221,9 @@ public void defaultValueForOptionMustMatchType(final @TempDir Path temp) throws @Test public void configFileNotFoundMustThrow() { - final File nonExistingFile = new File("doesnt.exit"); - - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, nonExistingFile); - assertThatThrownBy( - () -> - providerUnderTest.defaultValue( - OptionSpec.builder("an-option").type(String.class).build())) + () -> TomlConfigurationDefaultProvider.fromFile(mockCommandLine, nonExistingFile)) .isInstanceOf(ParameterException.class) .hasMessage("Unable to read TOML configuration, file not found."); } @@ -240,8 +233,8 @@ public void invalidConfigMustThrow(final @TempDir Path temp) throws IOException final File tempConfigFile = Files.createTempFile("invalid", "toml").toFile(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThatThrownBy( () -> @@ -260,8 +253,8 @@ public void invalidConfigContentMustThrow(final @TempDir Path temp) throws IOExc fileWriter.write("an-invalid-syntax=======...."); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThatThrownBy( () -> @@ -286,8 +279,8 @@ public void unknownOptionMustThrow(final @TempDir Path temp) throws IOException fileWriter.write("invalid_option=true"); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThatThrownBy( () -> @@ -321,8 +314,8 @@ public void tomlTableHeadingsMustBeIgnored(final @TempDir Path temp) throws IOEx fileWriter.newLine(); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThat( providerUnderTest.defaultValue( @@ -361,8 +354,8 @@ public void tomlTableHeadingsMustNotSkipValidationOfUnknownOptions(final @TempDi fileWriter.newLine(); fileWriter.flush(); - final TomlConfigFileDefaultProvider providerUnderTest = - new TomlConfigFileDefaultProvider(mockCommandLine, tempConfigFile); + final TomlConfigurationDefaultProvider providerUnderTest = + TomlConfigurationDefaultProvider.fromFile(mockCommandLine, tempConfigFile); assertThatThrownBy( () -> diff --git a/besu/src/test/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandlerTest.java b/besu/src/test/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandlerTest.java index bd33703d9cc..fbc672dc0a1 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandlerTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandlerTest.java @@ -145,7 +145,7 @@ public void handleWithEnvironmentVariableOptionShouldRaiseExceptionIfNoFileParam public void shouldRetrieveConfigFromEnvironmentWhenConfigFileSpecified() throws Exception { final IDefaultValueProvider defaultValueProvider = configParsingHandler.createDefaultValueProvider( - mockCommandLine, Optional.of(new File("foo"))); + mockCommandLine, Optional.of(new File("foo")), Optional.empty()); final String value = defaultValueProvider.defaultValue(OptionSpec.builder("--logging").build()); assertThat(value).isEqualTo("ERROR"); } @@ -153,7 +153,8 @@ public void shouldRetrieveConfigFromEnvironmentWhenConfigFileSpecified() throws @Test public void shouldRetrieveConfigFromEnvironmentWhenConfigFileNotSpecified() throws Exception { final IDefaultValueProvider defaultValueProvider = - configParsingHandler.createDefaultValueProvider(mockCommandLine, Optional.empty()); + configParsingHandler.createDefaultValueProvider( + mockCommandLine, Optional.empty(), Optional.empty()); final String value = defaultValueProvider.defaultValue(OptionSpec.builder("--logging").build()); assertThat(value).isEqualTo("ERROR"); } diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index b8dae6c1889..b205fa6692e 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -16,7 +16,7 @@ node-private-key-file="./path/to/privateKey" pid-path="~/.pid" reorg-logging-threshold=0 static-nodes-file="~/besudata/static-nodes.json" - +profile="NONE" # Security Module plugin to use security-module="localfile" diff --git a/besu/src/test/resources/partial_config.toml b/besu/src/test/resources/partial_config.toml index f2d8eaa061f..1763bb6bdc6 100644 --- a/besu/src/test/resources/partial_config.toml +++ b/besu/src/test/resources/partial_config.toml @@ -1,4 +1,7 @@ # this is a valid partial TOML config file #mining -miner-coinbase="0x0000000000000000000000000000000000000002" \ No newline at end of file +miner-coinbase="0x0000000000000000000000000000000000000002" + +#network +network="mainnet" \ No newline at end of file diff --git a/config/src/main/resources/profiles/dev.toml b/config/src/main/resources/profiles/dev.toml new file mode 100644 index 00000000000..3c80770b4ef --- /dev/null +++ b/config/src/main/resources/profiles/dev.toml @@ -0,0 +1 @@ +network="DEV" \ No newline at end of file From 3fc3fb1225a1ebbd8a3a9dcd18071085d04e1f13 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 23 Jan 2024 09:10:58 +1000 Subject: [PATCH 32/56] Trie log prune async using TrieLogEvent (#6394) * Change TrieLogPruner to implement a trieLogObserver * Remove TrieLogPruner from TrieLogManager * Remove NoOpTrieLogPruner now it is conditionally added as an observer * Prune async using EthScheduler.servicesExecutor --------- Signed-off-by: Gabriel Fukushima Signed-off-by: Simon Dudley Co-authored-by: Simon Dudley --- .../controller/BesuControllerBuilder.java | 60 +++++++++------- .../trie/bonsai/BonsaiWorldStateProvider.java | 10 +-- .../bonsai/trielog/NoOpTrieLogManager.java | 2 +- .../trie/bonsai/trielog/TrieLogManager.java | 7 +- .../trie/bonsai/trielog/TrieLogPruner.java | 47 +++++-------- .../core/InMemoryKeyValueStorageProvider.java | 4 +- .../trie/bonsai/AbstractIsolationTests.java | 4 +- .../bonsai/BonsaiWorldStateArchiveTest.java | 4 +- .../bonsai/trielog/TrieLogManagerTests.java | 5 +- .../bonsai/trielog/TrieLogPrunerTest.java | 68 +++++++++++++++++-- .../ethereum/eth/manager/EthScheduler.java | 4 +- 11 files changed, 129 insertions(+), 86 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index f2ecbe85c86..3d7bcc4f312 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -83,6 +83,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.pruner.MarkSweepPruner; @@ -781,6 +782,15 @@ public BesuController build() { final JsonRpcMethods additionalJsonRpcMethodFactory = createAdditionalJsonRpcMethodFactory(protocolContext); + if (dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningEnabled() + && DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) { + final TrieLogManager trieLogManager = + ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager(); + final TrieLogPruner trieLogPruner = + createTrieLogPruner(worldStateStorage, blockchain, scheduler); + trieLogManager.subscribe(trieLogPruner); + } + final List closeables = new ArrayList<>(); closeables.add(protocolContext.getWorldStateArchive()); closeables.add(storageProvider); @@ -809,6 +819,26 @@ public BesuController build() { dataStorageConfiguration); } + private TrieLogPruner createTrieLogPruner( + final WorldStateStorage worldStateStorage, + final Blockchain blockchain, + final EthScheduler scheduler) { + final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get(); + final boolean isProofOfStake = genesisConfigOptions.getTerminalTotalDifficulty().isPresent(); + + final TrieLogPruner trieLogPruner = + new TrieLogPruner( + (BonsaiWorldStateKeyValueStorage) worldStateStorage, + blockchain, + scheduler::executeServiceTask, + dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(), + dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(), + isProofOfStake); + trieLogPruner.initialize(); + + return trieLogPruner; + } + /** * Create synchronizer synchronizer. * @@ -1069,29 +1099,13 @@ WorldStateArchive createWorldStateArchive( final Blockchain blockchain, final CachedMerkleTrieLoader cachedMerkleTrieLoader) { return switch (dataStorageConfiguration.getDataStorageFormat()) { - case BONSAI -> { - final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get(); - final boolean isProofOfStake = - genesisConfigOptions.getTerminalTotalDifficulty().isPresent(); - final TrieLogPruner trieLogPruner = - dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningEnabled() - ? new TrieLogPruner( - (BonsaiWorldStateKeyValueStorage) worldStateStorage, - blockchain, - dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(), - dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(), - isProofOfStake) - : TrieLogPruner.noOpTrieLogPruner(); - trieLogPruner.initialize(); - yield new BonsaiWorldStateProvider( - (BonsaiWorldStateKeyValueStorage) worldStateStorage, - blockchain, - Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), - cachedMerkleTrieLoader, - besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), - evmConfiguration, - trieLogPruner); - } + case BONSAI -> new BonsaiWorldStateProvider( + (BonsaiWorldStateKeyValueStorage) worldStateStorage, + blockchain, + Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), + cachedMerkleTrieLoader, + besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), + evmConfiguration); case FOREST -> { final WorldStatePreimageStorage preimageStorage = storageProvider.createWorldStatePreimageStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java index 4f1b6d65bb4..ad223b407dd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java @@ -31,7 +31,6 @@ import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; @@ -73,18 +72,13 @@ public BonsaiWorldStateProvider( final Optional maxLayersToLoad, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final BesuContext pluginContext, - final EvmConfiguration evmConfiguration, - final TrieLogPruner trieLogPruner) { + final EvmConfiguration evmConfiguration) { this.cachedWorldStorageManager = new CachedWorldStorageManager(this, worldStateStorage); // TODO: de-dup constructors this.trieLogManager = new TrieLogManager( - blockchain, - worldStateStorage, - maxLayersToLoad.orElse(RETAINED_LAYERS), - pluginContext, - trieLogPruner); + blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS), pluginContext); this.blockchain = blockchain; this.worldStateStorage = worldStateStorage; this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java index 7cb024a259d..4c9a520bd2f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java @@ -25,7 +25,7 @@ public class NoOpTrieLogManager extends TrieLogManager { public NoOpTrieLogManager() { - super(null, null, 0, null, TrieLogPruner.noOpTrieLogPruner()); + super(null, null, 0, null); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java index ffb4e0bedcf..5102b7a0b90 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java @@ -47,19 +47,16 @@ public class TrieLogManager { protected final Subscribers trieLogObservers = Subscribers.create(); protected final TrieLogFactory trieLogFactory; - private final TrieLogPruner trieLogPruner; public TrieLogManager( final Blockchain blockchain, final BonsaiWorldStateKeyValueStorage worldStateStorage, final long maxLayersToLoad, - final BesuContext pluginContext, - final TrieLogPruner trieLogPruner) { + final BesuContext pluginContext) { this.blockchain = blockchain; this.rootWorldStateStorage = worldStateStorage; this.maxLayersToLoad = maxLayersToLoad; this.trieLogFactory = setupTrieLogFactory(pluginContext); - this.trieLogPruner = trieLogPruner; } public synchronized void saveTrieLog( @@ -85,8 +82,6 @@ public synchronized void saveTrieLog( } finally { if (success) { stateUpdater.commit(); - trieLogPruner.addToPruneQueue(forBlockHeader.getNumber(), forBlockHeader.getBlockHash()); - trieLogPruner.pruneFromQueue(); } else { stateUpdater.rollback(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java index b1bf75818e9..b72796f2225 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java @@ -20,10 +20,12 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; import java.util.Comparator; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; import java.util.stream.Stream; import com.google.common.collect.ArrayListMultimap; @@ -33,7 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TrieLogPruner { +public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { private static final Logger LOG = LoggerFactory.getLogger(TrieLogPruner.class); @@ -41,6 +43,7 @@ public class TrieLogPruner { private final int loadingLimit; private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; private final Blockchain blockchain; + private final Consumer executeAsync; private final long numBlocksToRetain; private final boolean requireFinalizedBlock; @@ -50,11 +53,13 @@ public class TrieLogPruner { public TrieLogPruner( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Blockchain blockchain, + final Consumer executeAsync, final long numBlocksToRetain, final int pruningLimit, final boolean requireFinalizedBlock) { this.rootWorldStateStorage = rootWorldStateStorage; this.blockchain = blockchain; + this.executeAsync = executeAsync; this.numBlocksToRetain = numBlocksToRetain; this.pruningLimit = pruningLimit; this.loadingLimit = pruningLimit; // same as pruningLimit for now @@ -166,34 +171,18 @@ int pruneFromQueue() { return wasPruned.size(); } - public static TrieLogPruner noOpTrieLogPruner() { - return new NoOpTrieLogPruner(null, null, 0, 0); - } - - public static class NoOpTrieLogPruner extends TrieLogPruner { - private NoOpTrieLogPruner( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final Blockchain blockchain, - final long numBlocksToRetain, - final int pruningLimit) { - super(rootWorldStateStorage, blockchain, numBlocksToRetain, pruningLimit, true); - } - - @Override - public int initialize() { - // no-op - return -1; - } - - @Override - void addToPruneQueue(final long blockNumber, final Hash blockHash) { - // no-op - } - - @Override - int pruneFromQueue() { - // no-op - return -1; + @Override + public void onTrieLogAdded(final TrieLogEvent event) { + if (TrieLogEvent.Type.ADDED.equals(event.getType())) { + final Hash blockHash = event.layer().getBlockHash(); + final Optional blockNumber = event.layer().getBlockNumber(); + blockNumber.ifPresent( + blockNum -> + executeAsync.accept( + () -> { + addToPruneQueue(blockNum, blockHash); + pruneFromQueue(); + })); } } } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 21f32441b27..4a7f658548e 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -30,7 +30,6 @@ import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; @@ -113,8 +112,7 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( Optional.empty(), cachedMerkleTrieLoader, null, - evmConfiguration, - TrieLogPruner.noOpTrieLogPruner()); + evmConfiguration); } public static MutableWorldState createInMemoryWorldState() { diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java index eba09987dce..a9d36e29e21 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java @@ -67,7 +67,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; @@ -163,8 +162,7 @@ public void createStorage() { Optional.of(16L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), null, - EvmConfiguration.DEFAULT, - TrieLogPruner.noOpTrieLogPruner()); + EvmConfiguration.DEFAULT); var ws = archive.getMutable(); genesisState.writeStateTo(ws); protocolContext = new ProtocolContext(blockchain, archive, null, Optional.empty()); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java index 7c1391029a9..4857760900a 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateArchiveTest.java @@ -42,7 +42,6 @@ import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -127,8 +126,7 @@ storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFI Optional.of(512L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), null, - EvmConfiguration.DEFAULT, - TrieLogPruner.noOpTrieLogPruner()); + EvmConfiguration.DEFAULT); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(512).buildHeader(); when(blockchain.getChainHeadHeader()).thenReturn(chainHead); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java index a0065e2cb95..e5c716e413a 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.ethereum.trie.bonsai.trielog; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner.noOpTrieLogPruner; import static org.mockito.Mockito.spy; import org.hyperledger.besu.datatypes.Hash; @@ -57,9 +56,7 @@ class TrieLogManagerTests { @BeforeEach public void setup() { - trieLogManager = - new TrieLogManager( - blockchain, bonsaiWorldStateKeyValueStorage, 512, null, noOpTrieLogPruner()); + trieLogManager = new TrieLogManager(blockchain, bonsaiWorldStateKeyValueStorage, 512, null); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java index af5acbf18cf..ac84d35be40 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import java.util.Optional; +import java.util.function.Consumer; import java.util.stream.Stream; import org.apache.logging.log4j.Level; @@ -44,6 +45,7 @@ public class TrieLogPrunerTest { private BonsaiWorldStateKeyValueStorage worldState; private Blockchain blockchain; + private final Consumer executeAsync = Runnable::run; @SuppressWarnings("BannedMethod") @BeforeEach @@ -67,7 +69,8 @@ public void initialize_preloads_queue_and_prunes_orphaned_blocks() { when(blockchain.getBlockHeader(header2.getBlockHash())).thenReturn(Optional.empty()); // When - TrieLogPruner trieLogPruner = new TrieLogPruner(worldState, blockchain, 3, loadingLimit, false); + TrieLogPruner trieLogPruner = + new TrieLogPruner(worldState, blockchain, executeAsync, 3, loadingLimit, false); trieLogPruner.initialize(); // Then @@ -86,7 +89,8 @@ public void trieLogs_pruned_in_reverse_order_within_pruning_window() { when(worldState.pruneTrieLog(any(Hash.class))).thenReturn(true); // requireFinalizedBlock = false means this is not a PoS chain TrieLogPruner trieLogPruner = - new TrieLogPruner(worldState, blockchain, blocksToRetain, pruningWindowSize, false); + new TrieLogPruner( + worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, false); trieLogPruner.addToPruneQueue(0, key(0)); // older block outside prune window trieLogPruner.addToPruneQueue(1, key(1)); // block inside the prune window @@ -194,7 +198,8 @@ public void skip_pruning_when_finalized_block_required_but_not_present() { final int pruningWindowSize = (int) chainHeight; when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight); TrieLogPruner trieLogPruner = - new TrieLogPruner(worldState, blockchain, blocksToRetain, pruningWindowSize, true); + new TrieLogPruner( + worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true); trieLogPruner.addToPruneQueue(1, key(1)); trieLogPruner.addToPruneQueue(2, key(2)); @@ -228,6 +233,46 @@ public void do_not_count_trieLog_when_prune_fails_first_attempt() { assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(1); } + @Test + public void onTrieLogAdded_should_prune() { + // Given + final TriggerableConsumer triggerableConsumer = new TriggerableConsumer(); + TrieLogPruner trieLogPruner = + new TrieLogPruner(worldState, blockchain, triggerableConsumer, 0, 1, false); + assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0); + + final TrieLogLayer layer = new TrieLogLayer(); + layer.setBlockNumber(1L); + layer.setBlockHash(key(1)); + when(blockchain.getChainHeadBlockNumber()).thenReturn(1L); + + // When + trieLogPruner.onTrieLogAdded(new TrieLogAddedEvent(layer)); + verify(worldState, never()).pruneTrieLog(key(1)); + triggerableConsumer.run(); + + // Then + verify(worldState, times(1)).pruneTrieLog(key(1)); + } + + @Test + public void onTrieLogAdded_should_not_prune_when_no_blockNumber() { + // Given + TrieLogPruner trieLogPruner = + new TrieLogPruner(worldState, blockchain, executeAsync, 0, 1, false); + assertThat(trieLogPruner.pruneFromQueue()).isEqualTo(0); + + final TrieLogLayer layer = new TrieLogLayer(); + layer.setBlockHash(key(1)); + when(blockchain.getChainHeadBlockNumber()).thenReturn(1L); + + // When + trieLogPruner.onTrieLogAdded(new TrieLogAddedEvent(layer)); + + // Then + verify(worldState, never()).pruneTrieLog(key(1)); + } + private TrieLogPruner setupPrunerAndFinalizedBlock( final long configuredRetainHeight, final long finalizedBlockHeight) { final long chainHeight = 5; @@ -241,7 +286,8 @@ private TrieLogPruner setupPrunerAndFinalizedBlock( .thenReturn(Optional.of(finalizedHeader)); when(blockchain.getChainHeadBlockNumber()).thenReturn(chainHeight); TrieLogPruner trieLogPruner = - new TrieLogPruner(worldState, blockchain, blocksToRetain, pruningWindowSize, true); + new TrieLogPruner( + worldState, blockchain, executeAsync, blocksToRetain, pruningWindowSize, true); trieLogPruner.addToPruneQueue(1, key(1)); trieLogPruner.addToPruneQueue(2, key(2)); @@ -255,4 +301,18 @@ private TrieLogPruner setupPrunerAndFinalizedBlock( private Hash key(final int k) { return Hash.hash(Bytes.of(k)); } + + private static class TriggerableConsumer implements Consumer { + + private Runnable runnable; + + @Override + public void accept(final Runnable runnable) { + this.runnable = runnable; + } + + public void run() { + runnable.run(); + } + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java index 61a01bf54f3..dd0587b23ca 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthScheduler.java @@ -142,8 +142,8 @@ public void scheduleTxWorkerTask(final Runnable command) { txWorkerExecutor.execute(command); } - public CompletableFuture scheduleServiceTask(final Supplier task) { - return CompletableFuture.supplyAsync(task, servicesExecutor); + public void executeServiceTask(final Runnable command) { + servicesExecutor.execute(command); } public CompletableFuture scheduleServiceTask(final EthTask task) { From b90d0ed5211d54fee0fb12ea4c2dfb410f4cc27b Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Tue, 23 Jan 2024 16:11:30 +1000 Subject: [PATCH 33/56] Reuse --bonsai-historical-block-limit for limit trie logs feature (#6445) Renames and tidy-up Add change log for feature --------- Signed-off-by: Simon Dudley --- CHANGELOG.md | 2 + .../org/hyperledger/besu/cli/BesuCommand.java | 12 +- .../cli/ConfigurationOverviewBuilder.java | 38 +-- .../options/stable/DataStorageOptions.java | 79 ++--- .../subcommands/storage/TrieLogHelper.java | 73 ++--- .../storage/TrieLogSubCommand.java | 25 +- .../controller/BesuControllerBuilder.java | 6 +- .../cli/ConfigurationOverviewBuilderTest.java | 26 +- .../stable/DataStorageOptionsTest.java | 54 ++-- .../storage/TrieLogHelperTest.java | 280 +++++++++++------- .../worldstate/DataStorageConfiguration.java | 20 +- 11 files changed, 356 insertions(+), 259 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3c84b37b9d..5765a454270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ - Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) - Upgrade `tech.pegasys.discovery:discovery` [#6414](https://github.com/hyperledger/besu/pull/6414) - Options to tune the max allowed time that can be spent selecting transactions during block creation are now stable [#6423](https://github.com/hyperledger/besu/pull/6423) +- Introduce `--Xbonsai-limit-trie-logs-enabled` experimental feature which by default will only retain the latest 512 trie logs, saving about 3GB per week in database growth [#5390](https://github.com/hyperledger/besu/issues/5390) +- Introduce `besu storage x-trie-log prune` experimental offline subcommand which will prune all redundant trie logs except the latest 512 [#6303](https://github.com/hyperledger/besu/pull/6303) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 0ad34a95218..aeb518a2f6e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -3555,12 +3555,12 @@ private String generateConfigurationOverview() { builder.setHighSpecEnabled(); } - if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningEnabled()) { - builder.setTrieLogPruningEnabled(); - builder.setTrieLogRetentionThreshold( - dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogRetentionThreshold()); - builder.setTrieLogPruningLimit( - dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningLimit()); + if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiLimitTrieLogsEnabled()) { + builder.setLimitTrieLogsEnabled(); + builder.setTrieLogRetentionLimit( + dataStorageOptions.toDomainObject().getBonsaiMaxLayersToLoad()); + builder.setTrieLogsPruningWindowSize( + dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningWindowSize()); } builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index fe5488fbea4..bf03c675d8d 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -52,9 +52,9 @@ public class ConfigurationOverviewBuilder { private Collection engineApis; private String engineJwtFilePath; private boolean isHighSpec = false; - private boolean isTrieLogPruningEnabled = false; - private long trieLogRetentionThreshold = 0; - private Integer trieLogPruningLimit = null; + private boolean isBonsaiLimitTrieLogsEnabled = false; + private long trieLogRetentionLimit = 0; + private Integer trieLogsPruningWindowSize = null; private TransactionPoolConfiguration.Implementation txPoolImplementation; private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode; private Map environment; @@ -199,34 +199,34 @@ public ConfigurationOverviewBuilder setHighSpecEnabled() { } /** - * Sets trie log pruning enabled + * Sets limit trie logs enabled * * @return the builder */ - public ConfigurationOverviewBuilder setTrieLogPruningEnabled() { - isTrieLogPruningEnabled = true; + public ConfigurationOverviewBuilder setLimitTrieLogsEnabled() { + isBonsaiLimitTrieLogsEnabled = true; return this; } /** - * Sets trie log retention threshold + * Sets trie log retention limit * - * @param threshold the number of blocks to retain trie logs for + * @param limit the number of blocks to retain trie logs for * @return the builder */ - public ConfigurationOverviewBuilder setTrieLogRetentionThreshold(final long threshold) { - trieLogRetentionThreshold = threshold; + public ConfigurationOverviewBuilder setTrieLogRetentionLimit(final long limit) { + trieLogRetentionLimit = limit; return this; } /** - * Sets trie log pruning limit + * Sets trie logs pruning window size * - * @param limit the max number of blocks to load and prune trie logs for at startup + * @param size the max number of blocks to load and prune trie logs for at startup * @return the builder */ - public ConfigurationOverviewBuilder setTrieLogPruningLimit(final int limit) { - trieLogPruningLimit = limit; + public ConfigurationOverviewBuilder setTrieLogsPruningWindowSize(final int size) { + trieLogsPruningWindowSize = size; return this; } @@ -339,13 +339,13 @@ public String build() { lines.add("Using " + worldStateUpdateMode + " worldstate update mode"); - if (isTrieLogPruningEnabled) { + if (isBonsaiLimitTrieLogsEnabled) { final StringBuilder trieLogPruningString = new StringBuilder(); trieLogPruningString - .append("Trie log pruning enabled: retention: ") - .append(trieLogRetentionThreshold); - if (trieLogPruningLimit != null) { - trieLogPruningString.append("; prune limit: ").append(trieLogPruningLimit); + .append("Limit trie logs enabled: retention: ") + .append(trieLogRetentionLimit); + if (trieLogsPruningWindowSize != null) { + trieLogPruningString.append("; prune window: ").append(trieLogsPruningWindowSize); } lines.add(trieLogPruningString.toString()); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index 5b4cf43eb1e..a1f4b950bbb 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -17,10 +17,9 @@ package org.hyperledger.besu.cli.options.stable; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT; import org.hyperledger.besu.cli.options.CLIOptions; import org.hyperledger.besu.cli.util.CommandLineUtils; @@ -39,7 +38,8 @@ public class DataStorageOptions implements CLIOptions private static final String DATA_STORAGE_FORMAT = "--data-storage-format"; - private static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = + /** The maximum number of historical layers to load. */ + public static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = "--bonsai-historical-block-limit"; // Use Bonsai DB @@ -54,39 +54,37 @@ public class DataStorageOptions implements CLIOptions names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"}, paramLabel = "", description = - "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}).", + "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}). When using " + + Unstable.BONSAI_LIMIT_TRIE_LOGS_ENABLED + + " it will also be used as the number of layers of trie logs to retain.", arity = "1") private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; @CommandLine.ArgGroup(validate = false) private final DataStorageOptions.Unstable unstableOptions = new Unstable(); - static class Unstable { + /** The unstable options for data storage. */ + public static class Unstable { private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--Xbonsai-limit-trie-logs-enabled"; - private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD = - "--Xbonsai-trie-logs-retention-threshold"; - private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit"; - @CommandLine.Option( - hidden = true, - names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED}, - description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})") - private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; + /** The bonsai trie logs pruning window size. */ + public static final String BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = + "--Xbonsai-trie-logs-pruning-window-size"; @CommandLine.Option( hidden = true, - names = {BONSAI_TRIE_LOGS_RETENTION_THRESHOLD}, + names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED}, description = - "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})") - private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; + "Limit the number of trie logs that are retained. (default: ${DEFAULT-VALUE})") + private boolean bonsaiLimitTrieLogsEnabled = DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; @CommandLine.Option( hidden = true, - names = {BONSAI_TRIE_LOG_PRUNING_LIMIT}, + names = {BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE}, description = "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") - private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; + private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; } /** * Create data storage options. @@ -103,21 +101,31 @@ public static DataStorageOptions create() { * @param commandLine the full commandLine to check all the options specified by the user */ public void validate(final CommandLine commandLine) { - if (unstableOptions.bonsaiTrieLogPruningEnabled) { - if (unstableOptions.bonsaiTrieLogRetentionThreshold - < MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD) { + if (unstableOptions.bonsaiLimitTrieLogsEnabled) { + if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", + MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT)); + } + if (unstableOptions.bonsaiTrieLogPruningWindowSize <= 0) { throw new CommandLine.ParameterException( commandLine, String.format( - "--Xbonsai-trie-log-retention-threshold minimum value is %d", - MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD)); + Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than 0", + unstableOptions.bonsaiTrieLogPruningWindowSize)); } - if (unstableOptions.bonsaiTrieLogPruningLimit <= 0) { + if (unstableOptions.bonsaiTrieLogPruningWindowSize <= bonsaiMaxLayersToLoad) { throw new CommandLine.ParameterException( commandLine, String.format( - "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0", - unstableOptions.bonsaiTrieLogPruningLimit)); + Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + + "=%d must be greater than " + + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + + "=%d", + unstableOptions.bonsaiTrieLogPruningWindowSize, + bonsaiMaxLayersToLoad)); } } } @@ -126,12 +134,10 @@ static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject final DataStorageOptions dataStorageOptions = DataStorageOptions.create(); dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat(); dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad(); - dataStorageOptions.unstableOptions.bonsaiTrieLogPruningEnabled = - domainObject.getUnstable().getBonsaiTrieLogPruningEnabled(); - dataStorageOptions.unstableOptions.bonsaiTrieLogRetentionThreshold = - domainObject.getUnstable().getBonsaiTrieLogRetentionThreshold(); - dataStorageOptions.unstableOptions.bonsaiTrieLogPruningLimit = - domainObject.getUnstable().getBonsaiTrieLogPruningLimit(); + dataStorageOptions.unstableOptions.bonsaiLimitTrieLogsEnabled = + domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled(); + dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize = + domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize(); return dataStorageOptions; } @@ -143,9 +149,8 @@ public DataStorageConfiguration toDomainObject() { .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogPruningEnabled(unstableOptions.bonsaiTrieLogPruningEnabled) - .bonsaiTrieLogRetentionThreshold(unstableOptions.bonsaiTrieLogRetentionThreshold) - .bonsaiTrieLogPruningLimit(unstableOptions.bonsaiTrieLogPruningLimit) + .bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled) + .bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize) .build()) .build(); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index c4e924a8354..88d876cce2e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; +import org.hyperledger.besu.cli.options.stable.DataStorageOptions; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -44,6 +45,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; @@ -56,7 +58,7 @@ public class TrieLogHelper { private static final int ROCKSDB_MAX_INSERTS_PER_TRANSACTION = 1000; private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); - static void prune( + void prune( final DataStorageConfiguration config, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, @@ -66,7 +68,7 @@ static void prune( validatePruneConfiguration(config); - final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); + final long layersToRetain = config.getBonsaiMaxLayersToLoad(); final long chainHeight = blockchain.getChainHeadBlockNumber(); @@ -94,7 +96,7 @@ static void prune( } } - private static void processTrieLogBatches( + private void processTrieLogBatches( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, final long chainHeight, @@ -122,7 +124,7 @@ private static void processTrieLogBatches( } } - private static void saveTrieLogBatches( + private void saveTrieLogBatches( final String batchFileName, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final List trieLogKeys) { @@ -135,7 +137,7 @@ private static void saveTrieLogBatches( } } - private static void restoreTrieLogBatches( + private void restoreTrieLogBatches( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber, final String batchFileNameBase) { @@ -149,7 +151,7 @@ private static void restoreTrieLogBatches( } } - private static void deleteFiles(final String batchFileNameBase, final long numberOfBatches) { + private void deleteFiles(final String batchFileNameBase, final long numberOfBatches) { LOG.info("Deleting files..."); @@ -161,7 +163,7 @@ private static void deleteFiles(final String batchFileNameBase, final long numbe } } - private static List getTrieLogKeysForBlocks( + private List getTrieLogKeysForBlocks( final MutableBlockchain blockchain, final long firstBlockOfBatch, final long lastBlockOfBatch) { @@ -175,11 +177,11 @@ private static List getTrieLogKeysForBlocks( return trieLogKeys; } - private static long calculateNumberofBatches(final long layersToRetain) { + private long calculateNumberofBatches(final long layersToRetain) { return layersToRetain / BATCH_SIZE + ((layersToRetain % BATCH_SIZE == 0) ? 0 : 1); } - private static boolean validPruneRequirements( + private boolean validPruneRequirements( final MutableBlockchain blockchain, final long chainHeight, final long lastBlockNumberToRetainTrieLogsFor) { @@ -206,7 +208,7 @@ private static boolean validPruneRequirements( return true; } - private static void recreateTrieLogs( + private void recreateTrieLogs( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber, final String batchFileNameBase) @@ -222,7 +224,7 @@ private static void recreateTrieLogs( } } - private static void processTransactionChunk( + private void processTransactionChunk( final int startIndex, final int chunkSize, final List keys, @@ -242,28 +244,33 @@ private static void processTransactionChunk( updater.getTrieLogStorageTransaction().commit(); } - private static void validatePruneConfiguration(final DataStorageConfiguration config) { + @VisibleForTesting + void validatePruneConfiguration(final DataStorageConfiguration config) { checkArgument( - config.getUnstable().getBonsaiTrieLogRetentionThreshold() - >= config.getBonsaiMaxLayersToLoad(), + config.getBonsaiMaxLayersToLoad() + >= DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT, String.format( - "--Xbonsai-trie-log-retention-threshold minimum value is %d", - config.getBonsaiMaxLayersToLoad())); + DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", + DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT)); checkArgument( - config.getUnstable().getBonsaiTrieLogPruningLimit() > 0, + config.getUnstable().getBonsaiTrieLogPruningWindowSize() > 0, String.format( - "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0", - config.getUnstable().getBonsaiTrieLogPruningLimit())); + DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + + "=%d must be greater than 0", + config.getUnstable().getBonsaiTrieLogPruningWindowSize())); checkArgument( - config.getUnstable().getBonsaiTrieLogPruningLimit() - > config.getUnstable().getBonsaiTrieLogRetentionThreshold(), + config.getUnstable().getBonsaiTrieLogPruningWindowSize() + > config.getBonsaiMaxLayersToLoad(), String.format( - "--Xbonsai-trie-log-pruning-limit=%d must greater than --Xbonsai-trie-log-retention-threshold=%d", - config.getUnstable().getBonsaiTrieLogPruningLimit(), - config.getUnstable().getBonsaiTrieLogRetentionThreshold())); + DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + + "=%d must be greater than " + + DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + + "=%d", + config.getUnstable().getBonsaiTrieLogPruningWindowSize(), + config.getBonsaiMaxLayersToLoad())); } - private static void saveTrieLogsInFile( + private void saveTrieLogsInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final String batchFileName) @@ -285,7 +292,7 @@ private static void saveTrieLogsInFile( } @SuppressWarnings("unchecked") - static IdentityHashMap readTrieLogsFromFile(final String batchFileName) { + IdentityHashMap readTrieLogsFromFile(final String batchFileName) { IdentityHashMap trieLogs; try (FileInputStream fis = new FileInputStream(batchFileName); @@ -300,7 +307,7 @@ static IdentityHashMap readTrieLogsFromFile(final String batchFi return trieLogs; } - private static void saveTrieLogsAsRlpInFile( + private void saveTrieLogsAsRlpInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final String batchFileName) { @@ -325,7 +332,7 @@ private static void saveTrieLogsAsRlpInFile( } } - static IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFileName) { + IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFileName) { try { final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName))); final BytesValueRLPInput input = new BytesValueRLPInput(file, false); @@ -346,7 +353,7 @@ static IdentityHashMap readTrieLogsAsRlpFromFile(final String ba } } - private static IdentityHashMap getTrieLogs( + private IdentityHashMap getTrieLogs( final List trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); @@ -359,7 +366,7 @@ private static IdentityHashMap getTrieLogs( return trieLogsToRetain; } - static TrieLogCount getCount( + TrieLogCount getCount( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final int limit, final Blockchain blockchain) { @@ -394,13 +401,13 @@ static TrieLogCount getCount( return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); } - static void printCount(final PrintWriter out, final TrieLogCount count) { + void printCount(final PrintWriter out, final TrieLogCount count) { out.printf( "trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n", count.total, count.canonicalCount, count.forkCount, count.orphanCount); } - static void importTrieLog( + void importTrieLog( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString()); @@ -410,7 +417,7 @@ static void importTrieLog( updater.getTrieLogStorageTransaction().commit(); } - static void exportTrieLog( + void exportTrieLog( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final List trieLogHash, final Path directoryPath) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index e624b5f3856..a4a38737f91 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -89,14 +89,15 @@ static class CountTrieLog implements Runnable { @Override public void run() { - TrieLogContext context = getTrieLogContext(); + final TrieLogContext context = getTrieLogContext(); final PrintWriter out = spec.commandLine().getOut(); out.println("Counting trie logs..."); - TrieLogHelper.printCount( + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + trieLogHelper.printCount( out, - TrieLogHelper.getCount( + trieLogHelper.getCount( context.rootWorldStateStorage, Integer.MAX_VALUE, context.blockchain)); } } @@ -104,7 +105,7 @@ public void run() { @Command( name = "prune", description = - "This command prunes all trie log layers below the retention threshold, including orphaned trie logs.", + "This command prunes all trie log layers below the retention limit, including orphaned trie logs.", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) static class PruneTrieLog implements Runnable { @@ -119,11 +120,12 @@ static class PruneTrieLog implements Runnable { @Override public void run() { - TrieLogContext context = getTrieLogContext(); + final TrieLogContext context = getTrieLogContext(); final Path dataDirectoryPath = Paths.get( TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); - TrieLogHelper.prune( + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + trieLogHelper.prune( context.config(), context.rootWorldStateStorage(), context.blockchain(), @@ -146,6 +148,7 @@ static class ExportTrieLog implements Runnable { @CommandLine.Spec private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + @SuppressWarnings("unused") @CommandLine.Option( names = "--trie-log-block-hash", description = @@ -173,13 +176,15 @@ public void run() { .toString()); } - TrieLogContext context = getTrieLogContext(); + final TrieLogContext context = getTrieLogContext(); final List listOfBlockHashes = trieLogBlockHashList.stream().map(Hash::fromHexString).toList(); + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + try { - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath); } catch (IOException e) { throw new RuntimeException(e); @@ -222,8 +227,8 @@ public void run() { } TrieLogContext context = getTrieLogContext(); - - TrieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath); + final TrieLogHelper trieLogHelper = new TrieLogHelper(); + trieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath); } } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 3d7bcc4f312..8bc9deee8b9 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -782,7 +782,7 @@ public BesuController build() { final JsonRpcMethods additionalJsonRpcMethodFactory = createAdditionalJsonRpcMethodFactory(protocolContext); - if (dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningEnabled() + if (dataStorageConfiguration.getUnstable().getBonsaiLimitTrieLogsEnabled() && DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) { final TrieLogManager trieLogManager = ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager(); @@ -831,8 +831,8 @@ private TrieLogPruner createTrieLogPruner( (BonsaiWorldStateKeyValueStorage) worldStateStorage, blockchain, scheduler::executeServiceTask, - dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(), - dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(), + dataStorageConfiguration.getBonsaiMaxLayersToLoad(), + dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningWindowSize(), isProofOfStake); trieLogPruner.initialize(); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java index 708101d1ab7..55b1c95a7f2 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java @@ -152,21 +152,21 @@ void setHighSpecEnabled() { } @Test - void setTrieLogPruningEnabled() { - final String noTrieLogRetentionThresholdSet = builder.build(); - assertThat(noTrieLogRetentionThresholdSet).doesNotContain("Trie log pruning enabled"); - - builder.setTrieLogPruningEnabled(); - builder.setTrieLogRetentionThreshold(42); - String trieLogRetentionThresholdSet = builder.build(); - assertThat(trieLogRetentionThresholdSet) - .contains("Trie log pruning enabled") + void setBonsaiLimitTrieLogsEnabled() { + final String noTrieLogRetentionLimitSet = builder.build(); + assertThat(noTrieLogRetentionLimitSet).doesNotContain("Limit trie logs enabled"); + + builder.setLimitTrieLogsEnabled(); + builder.setTrieLogRetentionLimit(42); + String trieLogRetentionLimitSet = builder.build(); + assertThat(trieLogRetentionLimitSet) + .contains("Limit trie logs enabled") .contains("retention: 42"); - assertThat(trieLogRetentionThresholdSet).doesNotContain("prune limit"); + assertThat(trieLogRetentionLimitSet).doesNotContain("prune window"); - builder.setTrieLogPruningLimit(1000); - trieLogRetentionThresholdSet = builder.build(); - assertThat(trieLogRetentionThresholdSet).contains("prune limit: 1000"); + builder.setTrieLogsPruningWindowSize(1000); + trieLogRetentionLimitSet = builder.build(); + assertThat(trieLogRetentionLimitSet).contains("prune window: 1000"); } @Test diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 2a63901975b..8aed9ee2d70 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -16,7 +16,7 @@ package org.hyperledger.besu.cli.options.stable; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT; import org.hyperledger.besu.cli.options.AbstractCLIOptionsTest; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -32,50 +32,59 @@ public class DataStorageOptionsTest public void bonsaiTrieLogPruningLimitOption() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit()) - .isEqualTo(1), + assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningWindowSize()) + .isEqualTo(600), "--Xbonsai-limit-trie-logs-enabled", - "--Xbonsai-trie-logs-pruning-limit", - "1"); + "--Xbonsai-trie-logs-pruning-window-size", + "600"); } @Test - public void bonsaiTrieLogPruningLimitShouldBePositive() { + public void bonsaiTrieLogPruningWindowSizeShouldBePositive() { internalTestFailure( - "--Xbonsai-trie-log-pruning-limit=0 must be greater than 0", + "--Xbonsai-trie-logs-pruning-window-size=0 must be greater than 0", "--Xbonsai-limit-trie-logs-enabled", - "--Xbonsai-trie-logs-pruning-limit", + "--Xbonsai-trie-logs-pruning-window-size", "0"); } @Test - public void bonsaiTrieLogRetentionThresholdOption() { + public void bonsaiTrieLogPruningWindowSizeShouldBeAboveRetentionLimit() { + internalTestFailure( + "--Xbonsai-trie-logs-pruning-window-size=512 must be greater than --bonsai-historical-block-limit=512", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-window-size", + "512"); + } + + @Test + public void bonsaiTrieLogRetentionLimitOption() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) - .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD + 1), + assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()) + .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT + 1), "--Xbonsai-limit-trie-logs-enabled", - "--Xbonsai-trie-logs-retention-threshold", + "--bonsai-historical-block-limit", "513"); } @Test - public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() { + public void bonsaiTrieLogRetentionLimitOption_boundaryTest() { internalTestSuccess( dataStorageConfiguration -> - assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) - .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD), + assertThat(dataStorageConfiguration.getBonsaiMaxLayersToLoad()) + .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT), "--Xbonsai-limit-trie-logs-enabled", - "--Xbonsai-trie-logs-retention-threshold", + "--bonsai-historical-block-limit", "512"); } @Test - public void bonsaiTrieLogRetentionThresholdShouldBeAboveMinimum() { + public void bonsaiTrieLogRetentionLimitShouldBeAboveMinimum() { internalTestFailure( - "--Xbonsai-trie-log-retention-threshold minimum value is 512", + "--bonsai-historical-block-limit minimum value is 512", "--Xbonsai-limit-trie-logs-enabled", - "--Xbonsai-trie-logs-retention-threshold", + "--bonsai-historical-block-limit", "511"); } @@ -88,12 +97,11 @@ protected DataStorageConfiguration createDefaultDomainObject() { protected DataStorageConfiguration createCustomizedDomainObject() { return ImmutableDataStorageConfiguration.builder() .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(100L) + .bonsaiMaxLayersToLoad(513L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogPruningEnabled(true) - .bonsaiTrieLogRetentionThreshold(1000L) - .bonsaiTrieLogPruningLimit(20) + .bonsaiLimitTrieLogsEnabled(true) + .bonsaiTrieLogPruningWindowSize(514) .build()) .build(); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 5d4ede7f32e..ca856ba0fad 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -16,10 +16,9 @@ package org.hyperledger.besu.cli.subcommands.storage; import static java.util.Collections.singletonList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -37,6 +36,7 @@ import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -58,13 +58,19 @@ class TrieLogHelperTest { private static final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private static BonsaiWorldStateKeyValueStorage inMemoryWorldState; + private TrieLogHelper trieLogHelper; - @Mock private MutableBlockchain blockchain; + private static class NonValidatingTrieLogHelper extends TrieLogHelper { + @Override + void validatePruneConfiguration(final DataStorageConfiguration config) {} + } + @Mock private MutableBlockchain blockchain; static BlockHeader blockHeader1; static BlockHeader blockHeader2; static BlockHeader blockHeader3; static BlockHeader blockHeader4; + static BlockHeader blockHeader5; @BeforeEach @@ -99,6 +105,8 @@ public void setup() throws IOException { .getTrieLogStorageTransaction() .put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5)); updater.getTrieLogStorageTransaction().commit(); + + trieLogHelper = new NonValidatingTrieLogHelper(); } private static byte[] createTrieLog(final BlockHeader blockHeader) { @@ -122,12 +130,11 @@ public void prune(final @TempDir Path dataDir) throws IOException { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) + .bonsaiMaxLayersToLoad(3L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(3) - .build() - .withBonsaiTrieLogRetentionThreshold(3)) + .bonsaiLimitTrieLogsEnabled(true) + .build()) .build(); mockBlockchainBase(); @@ -136,73 +143,75 @@ public void prune(final @TempDir Path dataDir) throws IOException { when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3)); // assert trie logs that will be pruned exist before prune call - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader1)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader2)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader3)); - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); + trieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); // assert pruned trie logs are not in the DB - assertEquals(inMemoryWorldState.getTrieLog(blockHeader1.getHash()), Optional.empty()); - assertEquals(inMemoryWorldState.getTrieLog(blockHeader2.getHash()), Optional.empty()); + assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash())).isEqualTo(Optional.empty()); + assertThat(inMemoryWorldState.getTrieLog(blockHeader2.getHash())).isEqualTo(Optional.empty()); // assert retained trie logs are in the DB - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader3)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader4)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader5)); } @Test - public void cantPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) { + public void cannotPruneIfNoFinalizedIsFound() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) .bonsaiMaxLayersToLoad(2L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(2) - .build() - .withBonsaiTrieLogRetentionThreshold(2)) + .bonsaiLimitTrieLogsEnabled(true) + .build()) .build(); when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); when(blockchain.getFinalized()).thenReturn(Optional.empty()); - assertThrows( - RuntimeException.class, - () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + assertThatThrownBy( + () -> + trieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(RuntimeException.class) + .hasMessage("No finalized block present, can't safely run trie log prune"); } @Test - public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength(final @TempDir Path dataDir) { + public void cannotPruneIfUserRetainsMoreLayersThanExistingChainLength() { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) + .bonsaiMaxLayersToLoad(10L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(10) - .build() - .withBonsaiTrieLogRetentionThreshold(10)) + .bonsaiLimitTrieLogsEnabled(true) + .build()) .build(); when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); - assertThrows( - IllegalArgumentException.class, - () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + assertThatThrownBy( + () -> + trieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Trying to retain more trie logs than chain length (5), skipping pruning"); } @Test - public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) { + public void cannotPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -210,17 +219,84 @@ public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path data .bonsaiMaxLayersToLoad(2L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(2) - .build() - .withBonsaiTrieLogRetentionThreshold(2)) + .bonsaiLimitTrieLogsEnabled(true) + .build()) .build(); mockBlockchainBase(); - assertThrows( - IllegalArgumentException.class, - () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + assertThatThrownBy( + () -> + trieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Trying to prune more layers than the finalized block height, skipping pruning"); + } + + @Test + public void trieLogRetentionLimitShouldBeAboveMinimum() { + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(511L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .build()) + .build(); + + TrieLogHelper helper = new TrieLogHelper(); + assertThatThrownBy( + () -> + helper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(RuntimeException.class) + .hasMessage("--bonsai-historical-block-limit minimum value is 512"); + } + + @Test + public void trieLogPruningWindowSizeShouldBePositive() { + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(512L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .bonsaiTrieLogPruningWindowSize(0) + .build()) + .build(); + + TrieLogHelper helper = new TrieLogHelper(); + assertThatThrownBy( + () -> + helper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(RuntimeException.class) + .hasMessage("--Xbonsai-trie-logs-pruning-window-size=0 must be greater than 0"); + } + + @Test + public void trieLogPruningWindowSizeShouldBeAboveRetentionLimit() { + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(512L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .bonsaiTrieLogPruningWindowSize(512) + .build()) + .build(); + + TrieLogHelper helper = new TrieLogHelper(); + assertThatThrownBy( + () -> + helper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(RuntimeException.class) + .hasMessage( + "--Xbonsai-trie-logs-pruning-window-size=512 must be greater than --bonsai-historical-block-limit=512"); } @Test @@ -229,77 +305,81 @@ public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDi DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) - .bonsaiMaxLayersToLoad(2L) + .bonsaiMaxLayersToLoad(3L) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() - .bonsaiTrieLogRetentionThreshold(2) - .build() - .withBonsaiTrieLogRetentionThreshold(2)) + .bonsaiLimitTrieLogsEnabled(true) + .build()) .build(); - assertThrows( - RuntimeException.class, - () -> - TrieLogHelper.prune( - dataStorageConfiguration, - inMemoryWorldState, - blockchain, - dataDir.resolve("unknownPath"))); + mockBlockchainBase(); + when(blockchain.getBlockHeader(5)).thenReturn(Optional.of(blockHeader5)); + when(blockchain.getBlockHeader(4)).thenReturn(Optional.of(blockHeader4)); + when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3)); + + assertThatThrownBy( + () -> + trieLogHelper.prune( + dataStorageConfiguration, + inMemoryWorldState, + blockchain, + dataDir.resolve("unknownPath"))) + .isInstanceOf(RuntimeException.class) + .hasCauseExactlyInstanceOf(FileNotFoundException.class); // assert all trie logs are still in the DB - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); - assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader1)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader2)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader3)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader4)); + assertThat(inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get()) + .isEqualTo(createTrieLog(blockHeader5)); } @Test public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( inMemoryWorldState, singletonList(blockHeader1.getHash()), dataDir.resolve("trie-log-dump")); var trieLog = - TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + trieLogHelper + .readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) .entrySet() .stream() .findFirst() .get(); - assertArrayEquals(trieLog.getKey(), blockHeader1.getHash().toArrayUnsafe()); - assertArrayEquals( - trieLog.getValue(), inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertThat(trieLog.getKey()).isEqualTo(blockHeader1.getHash().toArrayUnsafe()); + assertThat(trieLog.getValue()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); } @Test public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( inMemoryWorldState, List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), dataDir.resolve("trie-log-dump")); var trieLogs = - TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + trieLogHelper + .readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) .entrySet() .stream() .collect(Collectors.toMap(e -> Bytes.wrap(e.getKey()), Map.Entry::getValue)); - assertArrayEquals( - trieLogs.get(blockHeader1.getHash()), - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); - assertArrayEquals( - trieLogs.get(blockHeader2.getHash()), - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); - assertArrayEquals( - trieLogs.get(blockHeader3.getHash()), - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); + assertThat(trieLogs.get(blockHeader1.getHash())) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertThat(trieLogs.get(blockHeader2.getHash())) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertThat(trieLogs.get(blockHeader3.getHash())) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); } @Test @@ -309,22 +389,21 @@ public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws new BonsaiWorldStateKeyValueStorage( tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( inMemoryWorldState, singletonList(blockHeader1.getHash()), dataDir.resolve("trie-log-dump")); var trieLog = - TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + trieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); var updater = inMemoryWorldState2.updater(); trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); updater.getTrieLogStorageTransaction().commit(); - assertArrayEquals( - inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertThat(inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); } @Test @@ -334,27 +413,24 @@ public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) th new BonsaiWorldStateKeyValueStorage( tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); - TrieLogHelper.exportTrieLog( + trieLogHelper.exportTrieLog( inMemoryWorldState, List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), dataDir.resolve("trie-log-dump")); var trieLog = - TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + trieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); var updater = inMemoryWorldState2.updater(); trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); updater.getTrieLogStorageTransaction().commit(); - assertArrayEquals( - inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); - assertArrayEquals( - inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get(), - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); - assertArrayEquals( - inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get(), - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); + assertThat(inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertThat(inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertThat(inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get()) + .isEqualTo(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java index a36b150337a..e8fc199b677 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -43,27 +43,21 @@ default Unstable getUnstable() { @Value.Immutable interface Unstable { - boolean DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED = false; - long DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD = 512L; - long MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; - int DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT = 30_000; + boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = false; + long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; + int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 30_000; DataStorageConfiguration.Unstable DEFAULT = ImmutableDataStorageConfiguration.Unstable.builder().build(); @Value.Default - default boolean getBonsaiTrieLogPruningEnabled() { - return DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; + default boolean getBonsaiLimitTrieLogsEnabled() { + return DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; } @Value.Default - default long getBonsaiTrieLogRetentionThreshold() { - return DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; - } - - @Value.Default - default int getBonsaiTrieLogPruningLimit() { - return DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; + default int getBonsaiTrieLogPruningWindowSize() { + return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; } } } From 928b34d95f98d8fbbfc0962d5214368cef39265b Mon Sep 17 00:00:00 2001 From: delehef Date: Wed, 24 Jan 2024 09:07:51 +0100 Subject: [PATCH 34/56] feat: add `OperationTracer.tracePrepareTransaction` (#6453) * feat: add `OperationTracer.tracePrepareTransaction` Signed-off-by: Franklin Delehelle --- CHANGELOG.md | 1 + .../besu/services/TraceServiceImplTest.java | 2 ++ .../ethereum/mainnet/MainnetTransactionProcessor.java | 2 ++ .../java/org/hyperledger/besu/evmtool/T8nExecutor.java | 1 + .../hyperledger/besu/evm/tracing/OperationTracer.java | 10 +++++++++- 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5765a454270..02abc8a0487 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Deprecations ### Additions and Improvements +- Add `OperationTracer.tracePrepareTransaction`, where the sender account has not yet been altered[#6453](https://github.com/hyperledger/besu/pull/6453) ### Bug fixes - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) diff --git a/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java b/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java index f7ae97cdf57..3df1e8e24d8 100644 --- a/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java +++ b/besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java @@ -115,6 +115,7 @@ void shouldRetrieveStateUpdatePostTracingForOneBlock() { .getTransactions() .forEach( tx -> { + verify(opTracer).tracePrepareTransaction(any(), eq(tx)); verify(opTracer).traceStartTransaction(any(), eq(tx)); verify(opTracer) .traceEndTransaction( @@ -162,6 +163,7 @@ void shouldRetrieveStateUpdatePostTracingForAllBlocks() { .getTransactions() .forEach( tx -> { + verify(opTracer).tracePrepareTransaction(any(), eq(tx)); verify(opTracer).traceStartTransaction(any(), eq(tx)); verify(opTracer) .traceEndTransaction( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java index 931388b3878..4ec3eea8605 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java @@ -293,6 +293,8 @@ public TransactionProcessingResult processTransaction( return TransactionProcessingResult.invalid(validationResult); } + operationTracer.tracePrepareTransaction(worldState, transaction); + final long previousNonce = sender.incrementNonce(); LOG.trace( "Incremented sender {} nonce ({} -> {})", diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java index 1dbdc63e92c..c3e831a25a1 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java @@ -271,6 +271,7 @@ static T8nResult runTest( final TransactionProcessingResult result; try { tracer = tracerManager.getManagedTracer(i, transaction.getHash()); + tracer.tracePrepareTransaction(worldStateUpdater, transaction); tracer.traceStartTransaction(worldStateUpdater, transaction); result = processor.processTransaction( diff --git a/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java b/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java index edaed8e2416..5bd6de591e5 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java @@ -68,7 +68,15 @@ default void traceAccountCreationResult( final MessageFrame frame, final Optional haltReason) {} /** - * Trace the start of a transaction. + * Trace the start of a transaction, before sender account alteration but after validation. + * + * @param worldView an immutable view of the execution context + * @param transaction the transaction which will be processed + */ + default void tracePrepareTransaction(final WorldView worldView, final Transaction transaction) {} + + /** + * Trace the start of a transaction, before execution but after sender account alteration. * * @param worldView an immutable view of the execution context * @param transaction the transaction which will be processed From f1c292142178e58178b9253c2ea944521ebbefcd Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Wed, 24 Jan 2024 09:55:32 +0100 Subject: [PATCH 35/56] use reference tests to verify trielog generation (#6415) use reference tests to validate trielog generation. Like that we will validate besu trielog generation for all of the tests we have in the references tests suite for past and future EIPs --------- Signed-off-by: Karim Taam --- .circleci/config.yml | 2 +- .../ethereum/trie/bonsai/BonsaiValue.java | 15 ++ .../bonsai/worldview/BonsaiWorldState.java | 6 +- .../BonsaiWorldStateUpdateAccumulator.java | 8 +- .../backwardsync/BackwardSyncContextTest.java | 4 +- .../backwardsync/ForwardSyncStepTest.java | 4 +- .../BonsaiReferenceTestUpdateAccumulator.java | 28 ++++ .../BonsaiReferenceTestWorldState.java | 137 +++++++++++++++++- ...ava => ForestReferenceTestWorldState.java} | 24 ++- .../ReferenceTestWorldState.java | 3 + .../vm/GeneralStateReferenceTestTools.java | 4 +- 11 files changed, 214 insertions(+), 21 deletions(-) rename ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/{DefaultReferenceTestWorldState.java => ForestReferenceTestWorldState.java} (67%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 065a0280d04..0e84ab22aaf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -204,7 +204,7 @@ jobs: at: ~/project - run: name: ReferenceTests - no_output_timeout: 30m + no_output_timeout: 40m command: | git submodule update --init --recursive ./gradlew --no-daemon referenceTest diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java index c4f8ab0efd0..c99da3fe24b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java @@ -42,6 +42,17 @@ public BonsaiValue(final T prior, final T updated, final boolean lastStepCleared this.clearedAtLeastOnce = lastStepCleared; } + public BonsaiValue( + final T prior, + final T updated, + final boolean lastStepCleared, + final boolean clearedAtLeastOnce) { + this.prior = prior; + this.updated = updated; + this.lastStepCleared = lastStepCleared; + this.clearedAtLeastOnce = clearedAtLeastOnce; + } + @Override public T getPrior() { return prior; @@ -117,4 +128,8 @@ public int hashCode() { .append(lastStepCleared) .toHashCode(); } + + public BonsaiValue copy() { + return new BonsaiValue(prior, updated, lastStepCleared, clearedAtLeastOnce); + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java index 4544a1c1de8..8abcd442cfd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java @@ -119,12 +119,12 @@ public BonsaiWorldState( } /** - * Having a protected method to override the accumulator solves the chicken-egg problem of needing - * a worldstate reference (this) when construction the Accumulator. + * Override the accumulator solves the chicken-egg problem of needing a worldstate reference + * (this) when construction the Accumulator. * * @param accumulator accumulator to use. */ - protected void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { + public void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { this.accumulator = accumulator; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 9f0dd07d3b1..d73c272829b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -60,13 +60,13 @@ public class BonsaiWorldStateUpdateAccumulator implements BonsaiWorldView, TrieLogAccumulator { private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class); - private final Consumer> accountPreloader; - private final Consumer storagePreloader; + protected final Consumer> accountPreloader; + protected final Consumer storagePreloader; private final AccountConsumingMap> accountsToUpdate; private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set
storageToClear = Collections.synchronizedSet(new HashSet<>()); - private final EvmConfiguration evmConfiguration; + protected final EvmConfiguration evmConfiguration; // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the @@ -74,7 +74,7 @@ public class BonsaiWorldStateUpdateAccumulator private final Map>> storageToUpdate = new ConcurrentHashMap<>(); - private boolean isAccumulatorStateChanged; + protected boolean isAccumulatorStateChanged; public BonsaiWorldStateUpdateAccumulator( final BonsaiWorldView world, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java index 3187f55ef9f..27c931bb786 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java @@ -48,7 +48,7 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; -import org.hyperledger.besu.ethereum.referencetests.DefaultReferenceTestWorldState; +import org.hyperledger.besu.ethereum.referencetests.ForestReferenceTestWorldState; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -154,7 +154,7 @@ public void setup() { new BlockProcessingOutputs( // use forest-based worldstate since it does not require // blockheader stateroot to match actual worldstate root - DefaultReferenceTestWorldState.create(Collections.emptyMap()), + ForestReferenceTestWorldState.create(Collections.emptyMap()), blockDataGenerator.receipts(block)))); }); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java index e38a9647b79..e1848493ddf 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java @@ -38,7 +38,7 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.referencetests.DefaultReferenceTestWorldState; +import org.hyperledger.besu.ethereum.referencetests.ForestReferenceTestWorldState; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.nio.charset.StandardCharsets; @@ -138,7 +138,7 @@ public void setup() { return new BlockProcessingResult( Optional.of( new BlockProcessingOutputs( - DefaultReferenceTestWorldState.create(Collections.emptyMap()), + ForestReferenceTestWorldState.create(Collections.emptyMap()), blockDataGenerator.receipts(block)))); }); } diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java index b56053f54af..8c1bd515522 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java @@ -23,7 +23,10 @@ import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; import org.hyperledger.besu.evm.internal.EvmConfiguration; +import java.util.concurrent.ConcurrentHashMap; + import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdateAccumulator { private final BonsaiPreImageProxy preImageProxy; @@ -43,4 +46,29 @@ protected Hash hashAndSavePreImage(final Bytes bytes) { // by default do not save hash preImages return preImageProxy.hashAndSavePreImage(bytes); } + + public BonsaiReferenceTestUpdateAccumulator createDetachedAccumulator() { + final BonsaiReferenceTestUpdateAccumulator copy = + new BonsaiReferenceTestUpdateAccumulator( + wrappedWorldView(), + accountPreloader, + storagePreloader, + preImageProxy, + evmConfiguration); + getAccountsToUpdate().forEach((k, v) -> copy.getAccountsToUpdate().put(k, v.copy())); + getCodeToUpdate().forEach((k, v) -> copy.getCodeToUpdate().put(k, v.copy())); + copy.getStorageToClear().addAll(getStorageToClear()); + getStorageToUpdate() + .forEach( + (k, v) -> { + StorageConsumingMap> newMap = + new StorageConsumingMap<>(k, new ConcurrentHashMap<>(), v.getConsumer()); + v.forEach((key, value) -> newMap.put(key, value.copy())); + copy.getStorageToUpdate().put(k, newMap); + }); + copy.updatedAccounts.putAll(updatedAccounts); + copy.deletedAccounts.addAll(deletedAccounts); + copy.isAccumulatorStateChanged = true; + return copy; + } } diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 91c768a55cf..e0a49fbf640 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -23,19 +23,25 @@ import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogAddedEvent; import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.Map; +import java.util.Optional; import java.util.stream.Stream; import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -98,6 +104,99 @@ protected void verifyWorldStateRoot(final Hash calculatedStateRoot, final BlockH // The test harness validates the root hash, no need to validate in-line for reference test } + @Override + public void processExtraStateStorageFormatValidation(final BlockHeader blockHeader) { + if (blockHeader != null) { + final Hash parentStateRoot = getWorldStateRootHash(); + final BonsaiReferenceTestUpdateAccumulator originalUpdater = + ((BonsaiReferenceTestUpdateAccumulator) updater()).createDetachedAccumulator(); + + // validate trielog generation with persisted state + validateStateRolling(parentStateRoot, originalUpdater, blockHeader, false); + // validate trielog generation with frozen state + validateStateRolling(parentStateRoot, originalUpdater, blockHeader, true); + } + } + + /** + * TrieLog is an important part of Bonsai, so it's important to verify the generation of the + * TrieLog by performing rollbacks and rollforwards. + * + * @param blockHeader header of the block to import + */ + private void validateStateRolling( + final Hash parentStateRoot, + final BonsaiReferenceTestUpdateAccumulator originalUpdater, + final BlockHeader blockHeader, + final boolean isFrozenState) { + // With Bonsai, a TrieLog is generated when the state is persisted. Therefore, we generate the + // TrieLog by triggering a state persist in order to closely match the real case scenario. + generateTrieLogFromState(blockHeader, originalUpdater, isFrozenState); + final TrieLog trieLogFromFrozenState = + trieLogManager + .getTrieLogLayer(blockHeader.getBlockHash()) + .orElseThrow(() -> new RuntimeException("trielog not found during test")); + // trying rollback rollfoward with frozen state + validateTrieLog(parentStateRoot, blockHeader, trieLogFromFrozenState); + } + + private void validateTrieLog( + final Hash parentStateRoot, final BlockHeader blockHeader, final TrieLog trieLog) { + + try (var bonsaiWorldState = createBonsaiWorldState(false)) { + BonsaiWorldStateUpdateAccumulator updaterForState = + (BonsaiWorldStateUpdateAccumulator) bonsaiWorldState.updater(); + updaterForState.rollForward(trieLog); + updaterForState.commit(); + bonsaiWorldState.persist(blockHeader); + Hash generatedRootHash = bonsaiWorldState.rootHash(); + if (!bonsaiWorldState.rootHash().equals(blockHeader.getStateRoot())) { + throw new RuntimeException( + "state root becomes invalid following a rollForward %s != %s" + .formatted(blockHeader.getStateRoot(), generatedRootHash)); + } + + updaterForState = (BonsaiWorldStateUpdateAccumulator) bonsaiWorldState.updater(); + updaterForState.rollBack(trieLog); + updaterForState.commit(); + bonsaiWorldState.persist(null); + generatedRootHash = bonsaiWorldState.rootHash(); + if (!bonsaiWorldState.rootHash().equals(parentStateRoot)) { + throw new RuntimeException( + "state root becomes invalid following a rollBackward %s != %s" + .formatted(parentStateRoot, generatedRootHash)); + } + } + } + + private void generateTrieLogFromState( + final BlockHeader blockHeader, + final BonsaiReferenceTestUpdateAccumulator originalUpdater, + final boolean isFrozen) { + // generate trielog + BonsaiReferenceTestUpdateAccumulator updaterForState = + originalUpdater.createDetachedAccumulator(); + try (var bonsaiWorldState = createBonsaiWorldState(isFrozen)) { + bonsaiWorldState.setAccumulator(updaterForState); + updaterForState.commit(); + bonsaiWorldState.persist(blockHeader); + } + } + + private BonsaiWorldState createBonsaiWorldState(final boolean isFrozen) { + BonsaiWorldState bonsaiWorldState = + new BonsaiWorldState( + new BonsaiWorldStateLayerStorage(worldStateStorage), + cachedMerkleTrieLoader, + cachedWorldStorageManager, + trieLogManager, + evmConfiguration); + if (isFrozen) { + bonsaiWorldState.freeze(); // freeze state + } + return bonsaiWorldState; + } + @JsonCreator public static BonsaiReferenceTestWorldState create( final Map accounts) { @@ -110,7 +209,7 @@ public static BonsaiReferenceTestWorldState create( final EvmConfiguration evmConfiguration) { final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); - final TrieLogManager trieLogManager = new NoOpTrieLogManager(); + final TrieLogManager trieLogManager = new ReferenceTestsInMemoryTrieLogManager(); final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); @@ -149,6 +248,40 @@ public Stream streamAccounts(final Bytes32 startKeyHash, fina return this.refTestStorage.streamAccounts(this, startKeyHash, limit); } + static class ReferenceTestsInMemoryTrieLogManager extends TrieLogManager { + + private final Cache trieLogCache = + CacheBuilder.newBuilder().maximumSize(5).build(); + + public ReferenceTestsInMemoryTrieLogManager() { + super(null, null, 0, null); + } + + @Override + public synchronized void saveTrieLog( + final BonsaiWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final BonsaiWorldState forWorldState) { + // notify trie log added observers, synchronously + TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); + trieLogCache.put(forBlockHeader.getHash(), trieLogFactory.serialize(trieLog)); + trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); + } + + @Override + public long getMaxLayersToLoad() { + return 0; + } + + @Override + public Optional getTrieLogLayer(final Hash blockHash) { + final byte[] trielog = trieLogCache.getIfPresent(blockHash); + trieLogCache.invalidate(blockHash); // remove trielog from the cache + return Optional.ofNullable(trieLogFactory.deserialize(trielog)); + } + } + @Override protected Hash hashAndSavePreImage(final Bytes value) { // by default do not save has preImages diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ForestReferenceTestWorldState.java similarity index 67% rename from ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java rename to ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ForestReferenceTestWorldState.java index d9ed8626760..9ab158a554c 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ForestReferenceTestWorldState.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.referencetests; import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; @@ -27,28 +28,41 @@ import com.fasterxml.jackson.annotation.JsonCreator; -public class DefaultReferenceTestWorldState extends ForestMutableWorldState +public class ForestReferenceTestWorldState extends ForestMutableWorldState implements ReferenceTestWorldState { - DefaultReferenceTestWorldState() { + ForestReferenceTestWorldState() { super( new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()), new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), EvmConfiguration.DEFAULT); } - public DefaultReferenceTestWorldState(final WorldState worldState) { + public ForestReferenceTestWorldState(final WorldState worldState) { super(worldState, EvmConfiguration.DEFAULT); } @Override public ReferenceTestWorldState copy() { - return new DefaultReferenceTestWorldState(this); + return new ForestReferenceTestWorldState(this); + } + + /** + * Executes additional validation checks that are specific to the storage format. + * + *

Depending on the storage format (e.g., Bonsai, etc.), this method performs additional checks + * to validate the state. This could include validating the TrieLog and rolling for Bonsai, or + * potentially other checks for other modes. This method is intended to be used before the state + * root has been validated, to ensure the integrity of other aspects of the state. + */ + @Override + public void processExtraStateStorageFormatValidation(final BlockHeader blockHeader) { + // nothing more to verify with forest } @JsonCreator public static ReferenceTestWorldState create(final Map accounts) { - final ReferenceTestWorldState worldState = new DefaultReferenceTestWorldState(); + final ReferenceTestWorldState worldState = new ForestReferenceTestWorldState(); final WorldUpdater updater = worldState.updater(); for (final Map.Entry entry : accounts.entrySet()) { diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java index 3d4db1edaab..ae446a7607c 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -90,6 +91,8 @@ static void insertAccount( ReferenceTestWorldState copy(); + void processExtraStateStorageFormatValidation(final BlockHeader blockHeader); + @JsonCreator static ReferenceTestWorldState create(final Map accounts) { // delegate to a Bonsai reference test world state: diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java index e0b0c7332e7..5314ff5cc81 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java @@ -28,7 +28,6 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; -import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; @@ -151,7 +150,7 @@ public static void executeTest(final GeneralStateTestCaseEipSpec spec) { return; } - final MutableWorldState worldState = initialWorldState.copy(); + final ReferenceTestWorldState worldState = initialWorldState.copy(); // Several of the GeneralStateTests check if the transaction could potentially // consume more gas than is left for the block it's attempted to be included in. // This check is performed within the `BlockImporter` rather than inside the @@ -193,6 +192,7 @@ public static void executeTest(final GeneralStateTestCaseEipSpec spec) { worldStateUpdater.deleteAccount(coinbase.getAddress()); } worldStateUpdater.commit(); + worldState.processExtraStateStorageFormatValidation(blockHeader); worldState.persist(blockHeader); // Check the world state root hash. From 666f795b634393abb818c9db1d8eeca329bec944 Mon Sep 17 00:00:00 2001 From: ahamlat Date: Wed, 24 Jan 2024 12:54:44 +0100 Subject: [PATCH 36/56] Improve the high spec flag (#6354) * Improve the high spec flag, limit it to few column families * Update changelog * spotless * Update the plugin API hash as one of the interfaces was changed\ * Fix failing unit tests Signed-off-by: Ameziane H Co-authored-by: Sally MacFarlane --- CHANGELOG.md | 3 + .../keyvalue/KeyValueSegmentIdentifier.java | 32 ++-- plugin-api/build.gradle | 2 +- .../services/storage/SegmentIdentifier.java | 8 + .../RocksDBColumnarKeyValueStorage.java | 139 ++++++++++-------- ...ksDBKeyValuePrivacyStorageFactoryTest.java | 2 +- .../RocksDBKeyValueStorageFactoryTest.java | 2 +- ...nDBRocksDBColumnarKeyValueStorageTest.java | 2 +- .../RocksDBColumnarKeyValueStorageTest.java | 55 +++++-- ...nDBRocksDBColumnarKeyValueStorageTest.java | 2 +- .../kvstore/InMemoryKeyValueStorage.java | 5 + 11 files changed, 164 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02abc8a0487..9730af0fd5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Additions and Improvements - Add `OperationTracer.tracePrepareTransaction`, where the sender account has not yet been altered[#6453](https://github.com/hyperledger/besu/pull/6453) +- Improve the high spec flag by limiting it to a few column families [#6354](https://github.com/hyperledger/besu/pull/6354) + ### Bug fixes - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) @@ -61,6 +63,7 @@ - Set Ethereum Classic mainnet activation block for Spiral network upgrade [#6267](https://github.com/hyperledger/besu/pull/6267) - Add custom genesis file name to config overview if specified [#6297](https://github.com/hyperledger/besu/pull/6297) - Update Gradle plugins and replace unmaintained License Gradle Plugin with the actively maintained Gradle License Report [#6275](https://github.com/hyperledger/besu/pull/6275) +- Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) ### Bug fixes - Hotfix for selfdestruct preimages on bonsai [#6359]((https://github.com/hyperledger/besu/pull/6359) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java index 9cce0230cdb..282c314157d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java @@ -16,18 +16,21 @@ import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import java.nio.charset.StandardCharsets; + import org.bouncycastle.util.Arrays; public enum KeyValueSegmentIdentifier implements SegmentIdentifier { - BLOCKCHAIN(new byte[] {1}, true), - WORLD_STATE(new byte[] {2}, new int[] {0, 1}), + DEFAULT("default".getBytes(StandardCharsets.UTF_8)), + BLOCKCHAIN(new byte[] {1}, true, true), + WORLD_STATE(new byte[] {2}, new int[] {0, 1}, false, true), PRIVATE_TRANSACTIONS(new byte[] {3}), PRIVATE_STATE(new byte[] {4}), PRUNING_STATE(new byte[] {5}, new int[] {0, 1}), - ACCOUNT_INFO_STATE(new byte[] {6}, new int[] {2}), + ACCOUNT_INFO_STATE(new byte[] {6}, new int[] {2}, false, true), CODE_STORAGE(new byte[] {7}, new int[] {2}), - ACCOUNT_STORAGE_STORAGE(new byte[] {8}, new int[] {2}), - TRIE_BRANCH_STORAGE(new byte[] {9}, new int[] {2}), + ACCOUNT_STORAGE_STORAGE(new byte[] {8}, new int[] {2}, false, true), + TRIE_BRANCH_STORAGE(new byte[] {9}, new int[] {2}, false, true), TRIE_LOG_STORAGE(new byte[] {10}, new int[] {2}), VARIABLES(new byte[] {11}), // formerly GOQUORUM_PRIVATE_WORLD_STATE @@ -45,24 +48,30 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier { private final byte[] id; private final int[] versionList; private final boolean containsStaticData; + private final boolean eligibleToHighSpecFlag; KeyValueSegmentIdentifier(final byte[] id) { this(id, new int[] {0, 1, 2}); } - KeyValueSegmentIdentifier(final byte[] id, final boolean containsStaticData) { - this(id, new int[] {0, 1, 2}, containsStaticData); + KeyValueSegmentIdentifier( + final byte[] id, final boolean containsStaticData, final boolean eligibleToHighSpecFlag) { + this(id, new int[] {0, 1, 2}, containsStaticData, eligibleToHighSpecFlag); } KeyValueSegmentIdentifier(final byte[] id, final int[] versionList) { - this(id, versionList, false); + this(id, versionList, false, false); } KeyValueSegmentIdentifier( - final byte[] id, final int[] versionList, final boolean containsStaticData) { + final byte[] id, + final int[] versionList, + final boolean containsStaticData, + final boolean eligibleToHighSpecFlag) { this.id = id; this.versionList = versionList; this.containsStaticData = containsStaticData; + this.eligibleToHighSpecFlag = eligibleToHighSpecFlag; } @Override @@ -80,6 +89,11 @@ public boolean containsStaticData() { return containsStaticData; } + @Override + public boolean isEligibleToHighSpecFlag() { + return eligibleToHighSpecFlag; + } + @Override public boolean includeInDatabaseVersion(final int version) { return Arrays.contains(versionList, version); diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 1dcfd00f7ee..c72ad934339 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'IGq+V3KaStHCRFkeK3KwPxJYKO4RX9YM1O4JYITk8S8=' + knownHash = 'ZsovOR0oPfomcLP4b+HjikWzM0Tx6sCwi68mf5qwZf4=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java index c535966876f..5db7fea5091 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java @@ -56,4 +56,12 @@ default boolean includeInDatabaseVersion(final int version) { * @return true if the segment contains only static data */ boolean containsStaticData(); + + /** + * This flag defines which segment is eligible for the high spec flag, so basically what column + * family is involved with high spec flag + * + * @return true if the segment is involved with the high spec flag + */ + boolean isEligibleToHighSpecFlag(); } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java index c6157217e3d..a7192cc3b51 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java @@ -70,18 +70,16 @@ public abstract class RocksDBColumnarKeyValueStorage implements SegmentedKeyValueStorage { private static final Logger LOG = LoggerFactory.getLogger(RocksDBColumnarKeyValueStorage.class); - static final String DEFAULT_COLUMN = "default"; private static final int ROCKSDB_FORMAT_VERSION = 5; private static final long ROCKSDB_BLOCK_SIZE = 32768; /** RocksDb blockcache size when using the high spec option */ protected static final long ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC = 1_073_741_824L; /** RocksDb memtable size when using the high spec option */ - protected static final long ROCKSDB_MEMTABLE_SIZE_HIGH_SPEC = 1_073_741_824L; + protected static final long ROCKSDB_MEMTABLE_SIZE_HIGH_SPEC = 536_870_912L; /** Max total size of all WAL file, after which a flush is triggered */ protected static final long WAL_MAX_TOTAL_SIZE = 1_073_741_824L; /** Expected size of a single WAL file, to determine how many WAL files to keep around */ protected static final long EXPECTED_WAL_FILE_SIZE = 67_108_864L; - /** RocksDb number of log files to keep on disk */ private static final long NUMBER_OF_LOG_FILES_TO_KEEP = 7; /** RocksDb Time to roll a log file (1 day = 3600 * 24 seconds) */ @@ -144,7 +142,6 @@ public RocksDBColumnarKeyValueStorage( this.rocksDBMetricsFactory = rocksDBMetricsFactory; try { - final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); trimmedSegments = new ArrayList<>(defaultSegments); final List existingColumnFamilies = RocksDB.listColumnFamilies(new Options(), configuration.getDatabaseDir().toString()); @@ -156,14 +153,9 @@ public RocksDBColumnarKeyValueStorage( .noneMatch(existed -> Arrays.equals(existed, ignorableSegment.getId()))) .forEach(trimmedSegments::remove); columnDescriptors = - trimmedSegments.stream().map(this::createColumnDescriptor).collect(Collectors.toList()); - columnDescriptors.add( - new ColumnFamilyDescriptor( - DEFAULT_COLUMN.getBytes(StandardCharsets.UTF_8), - columnFamilyOptions - .setTtl(0) - .setCompressionType(CompressionType.LZ4_COMPRESSION) - .setTableFormatConfig(createBlockBasedTableConfig(configuration)))); + trimmedSegments.stream() + .map(segment -> createColumnDescriptor(segment, configuration)) + .collect(Collectors.toList()); setGlobalOptions(configuration, stats); @@ -174,6 +166,80 @@ public RocksDBColumnarKeyValueStorage( } } + /** + * Create a Column Family Descriptor for a given segment It defines basically the different + * options to apply to the corresponding Column Family + * + * @param segment the segment identifier + * @param configuration RocksDB configuration + * @return a column family descriptor + */ + private ColumnFamilyDescriptor createColumnDescriptor( + final SegmentIdentifier segment, final RocksDBConfiguration configuration) { + + BlockBasedTableConfig basedTableConfig = createBlockBasedTableConfig(segment, configuration); + + final var options = + new ColumnFamilyOptions() + .setTtl(0) + .setCompressionType(CompressionType.LZ4_COMPRESSION) + .setTableFormatConfig(basedTableConfig); + + if (segment.containsStaticData()) { + options + .setEnableBlobFiles(true) + .setEnableBlobGarbageCollection(false) + .setMinBlobSize(100) + .setBlobCompressionType(CompressionType.LZ4_COMPRESSION); + } + + return new ColumnFamilyDescriptor(segment.getId(), options); + } + + /*** + * Create a Block Base Table configuration for each segment, depending on the configuration in place + * and the segment itself + * + * @param segment The segment related to the column family + * @param config RocksDB configuration + * @return Block Base Table configuration + */ + private BlockBasedTableConfig createBlockBasedTableConfig( + final SegmentIdentifier segment, final RocksDBConfiguration config) { + final LRUCache cache = + new LRUCache( + config.isHighSpec() && segment.isEligibleToHighSpecFlag() + ? ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC + : config.getCacheCapacity()); + return new BlockBasedTableConfig() + .setFormatVersion(ROCKSDB_FORMAT_VERSION) + .setBlockCache(cache) + .setFilterPolicy(new BloomFilter(10, false)) + .setPartitionFilters(true) + .setCacheIndexAndFilterBlocks(false) + .setBlockSize(ROCKSDB_BLOCK_SIZE); + } + + /*** + * Set Global options (DBOptions) + * + * @param configuration RocksDB configuration + * @param stats The statistics object + */ + private void setGlobalOptions(final RocksDBConfiguration configuration, final Statistics stats) { + options = new DBOptions(); + options + .setCreateIfMissing(true) + .setMaxOpenFiles(configuration.getMaxOpenFiles()) + .setStatistics(stats) + .setCreateMissingColumnFamilies(true) + .setLogFileTimeToRoll(TIME_TO_ROLL_LOG_FILE) + .setKeepLogFileNum(NUMBER_OF_LOG_FILES_TO_KEEP) + .setEnv(Env.getDefault().setBackgroundThreads(configuration.getBackgroundThreadCount())) + .setMaxTotalWalSize(WAL_MAX_TOTAL_SIZE) + .setRecycleLogFileNum(WAL_MAX_TOTAL_SIZE / EXPECTED_WAL_FILE_SIZE); + } + /** * Parse RocksDBException and wrap in StorageException * @@ -219,42 +285,6 @@ protected static StorageException parseRocksDBException( } } - private ColumnFamilyDescriptor createColumnDescriptor(final SegmentIdentifier segment) { - final var options = - new ColumnFamilyOptions() - .setTtl(0) - .setCompressionType(CompressionType.LZ4_COMPRESSION) - .setTableFormatConfig(createBlockBasedTableConfig(configuration)); - - if (segment.containsStaticData()) { - options - .setEnableBlobFiles(true) - .setEnableBlobGarbageCollection(false) - .setMinBlobSize(100) - .setBlobCompressionType(CompressionType.LZ4_COMPRESSION); - } - - return new ColumnFamilyDescriptor(segment.getId(), options); - } - - private void setGlobalOptions(final RocksDBConfiguration configuration, final Statistics stats) { - options = new DBOptions(); - options - .setCreateIfMissing(true) - .setMaxOpenFiles(configuration.getMaxOpenFiles()) - .setMaxTotalWalSize(WAL_MAX_TOTAL_SIZE) - .setRecycleLogFileNum(WAL_MAX_TOTAL_SIZE / EXPECTED_WAL_FILE_SIZE) - .setStatistics(stats) - .setCreateMissingColumnFamilies(true) - .setLogFileTimeToRoll(TIME_TO_ROLL_LOG_FILE) - .setKeepLogFileNum(NUMBER_OF_LOG_FILES_TO_KEEP) - .setEnv(Env.getDefault().setBackgroundThreads(configuration.getBackgroundThreadCount())); - - if (configuration.isHighSpec()) { - options.setDbWriteBufferSize(ROCKSDB_MEMTABLE_SIZE_HIGH_SPEC); - } - } - void initMetrics() { metrics = rocksDBMetricsFactory.create(metricsSystem, configuration, getDB(), stats); } @@ -287,19 +317,6 @@ void initColumnHandles() throws RocksDBException { })); } - BlockBasedTableConfig createBlockBasedTableConfig(final RocksDBConfiguration config) { - final LRUCache cache = - new LRUCache( - config.isHighSpec() ? ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC : config.getCacheCapacity()); - return new BlockBasedTableConfig() - .setFormatVersion(ROCKSDB_FORMAT_VERSION) - .setBlockCache(cache) - .setFilterPolicy(new BloomFilter(10, false)) - .setPartitionFilters(true) - .setCacheIndexAndFilterBlocks(false) - .setBlockSize(ROCKSDB_BLOCK_SIZE); - } - /** * Safe method to map segment identifier to column handle. * diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java index e0ed3f7b396..65d8ee7b58f 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java @@ -45,7 +45,7 @@ public class RocksDBKeyValuePrivacyStorageFactoryTest { @TempDir private Path temporaryFolder; private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final SegmentIdentifier segment = TestSegment.BAR; - private final List segments = List.of(segment); + private final List segments = List.of(TestSegment.DEFAULT, segment); @Test public void shouldDetectVersion1DatabaseIfNoMetadataFileFound() throws Exception { diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java index c3a8c32eb27..8f1e116b3a1 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java @@ -50,7 +50,7 @@ public class RocksDBKeyValueStorageFactoryTest { @TempDir public Path temporaryFolder; private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final SegmentIdentifier segment = TestSegment.FOO; - private final List segments = List.of(segment); + private final List segments = List.of(TestSegment.DEFAULT, segment); @Test public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticTransactionDBRocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticTransactionDBRocksDBColumnarKeyValueStorageTest.java index 7e8cef52cc2..bf838f9c4fd 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticTransactionDBRocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticTransactionDBRocksDBColumnarKeyValueStorageTest.java @@ -39,7 +39,7 @@ protected SegmentedKeyValueStorage createSegmentedStore() throws Exception { new RocksDBConfigurationBuilder() .databaseDir(Files.createTempDirectory("segmentedStore")) .build(), - Arrays.asList(TestSegment.FOO, TestSegment.BAR), + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), List.of(), new NoOpMetricsSystem(), RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java index 5462d239b91..80b9685b792 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java @@ -194,13 +194,17 @@ public void dbShouldIgnoreExperimentalSegmentsIfNotExisted(@TempDir final Path t SegmentedKeyValueStorage store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of(TestSegment.EXPERIMENTAL)); store.close(); // new db will be backward compatible with db without knowledge of experimental column family store = - createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); + createSegmentedStore( + testPath, + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), + List.of()); store.close(); } @@ -212,14 +216,18 @@ public void dbShouldNotIgnoreExperimentalSegmentsIfExisted(@TempDir final Path t SegmentedKeyValueStorage store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of()); store.close(); // new db will not be backward compatible with db without knowledge of experimental column // family try { - createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); + createSegmentedStore( + testPath, + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), + List.of()); fail("DB without knowledge of experimental column family should fail"); } catch (StorageException e) { assertThat(e.getMessage()).contains("Unhandled column families"); @@ -230,7 +238,8 @@ public void dbShouldNotIgnoreExperimentalSegmentsIfExisted(@TempDir final Path t store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of(TestSegment.EXPERIMENTAL)); store.close(); } @@ -242,27 +251,35 @@ public void dbWillBeBackwardIncompatibleAfterExperimentalSegmentsAreAdded( SegmentedKeyValueStorage store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of(TestSegment.EXPERIMENTAL)); store.close(); // new db will be backward compatible with db without knowledge of experimental column family store = - createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); + createSegmentedStore( + testPath, + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), + List.of()); store.close(); // Create new db without ignoring experimental colum family will add column to db store = createSegmentedStore( testPath, - Arrays.asList(TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), + Arrays.asList( + TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR, TestSegment.EXPERIMENTAL), List.of()); store.close(); // Now, the db will be backward incompatible with db without knowledge of experimental column // family try { - createSegmentedStore(testPath, Arrays.asList(TestSegment.FOO, TestSegment.BAR), List.of()); + createSegmentedStore( + testPath, + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), + List.of()); fail("DB without knowledge of experimental column family should fail"); } catch (StorageException e) { assertThat(e.getMessage()).contains("Unhandled column families"); @@ -293,7 +310,10 @@ public void createStoreMustCreateMetrics() throws Exception { final SegmentedKeyValueStorage store = createSegmentedStore( - folder, metricsSystemMock, List.of(TestSegment.FOO), List.of(TestSegment.EXPERIMENTAL)); + folder, + metricsSystemMock, + List.of(TestSegment.DEFAULT, TestSegment.FOO), + List.of(TestSegment.EXPERIMENTAL)); KeyValueStorage keyValueStorage = new SegmentedKeyValueStorageAdapter(TestSegment.FOO, store); @@ -343,24 +363,28 @@ public void createStoreMustCreateMetrics() throws Exception { } public enum TestSegment implements SegmentIdentifier { + DEFAULT("default".getBytes(StandardCharsets.UTF_8)), FOO(new byte[] {1}), BAR(new byte[] {2}), EXPERIMENTAL(new byte[] {3}), - STATIC_DATA(new byte[] {4}, true); + STATIC_DATA(new byte[] {4}, true, false); private final byte[] id; private final String nameAsUtf8; private final boolean containsStaticData; + private final boolean eligibleToHighSpecFlag; TestSegment(final byte[] id) { - this(id, false); + this(id, false, false); } - TestSegment(final byte[] id, final boolean containsStaticData) { + TestSegment( + final byte[] id, final boolean containsStaticData, final boolean eligibleToHighSpecFlag) { this.id = id; this.nameAsUtf8 = new String(id, StandardCharsets.UTF_8); this.containsStaticData = containsStaticData; + this.eligibleToHighSpecFlag = eligibleToHighSpecFlag; } @Override @@ -377,6 +401,11 @@ public byte[] getId() { public boolean containsStaticData() { return containsStaticData; } + + @Override + public boolean isEligibleToHighSpecFlag() { + return eligibleToHighSpecFlag; + } } protected abstract SegmentedKeyValueStorage createSegmentedStore() throws Exception; diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorageTest.java index bed64f58404..374f5534567 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/TransactionDBRocksDBColumnarKeyValueStorageTest.java @@ -36,7 +36,7 @@ public class TransactionDBRocksDBColumnarKeyValueStorageTest protected SegmentedKeyValueStorage createSegmentedStore() throws Exception { return new TransactionDBRocksDBColumnarKeyValueStorage( new RocksDBConfigurationBuilder().databaseDir(getTempSubFolder(folder)).build(), - Arrays.asList(TestSegment.FOO, TestSegment.BAR), + Arrays.asList(TestSegment.DEFAULT, TestSegment.FOO, TestSegment.BAR), List.of(), new NoOpMetricsSystem(), RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java index dbd62eaff9a..e3d82ca9dc4 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java @@ -50,6 +50,11 @@ public byte[] getId() { public boolean containsStaticData() { return false; } + + @Override + public boolean isEligibleToHighSpecFlag() { + return false; + } }; private static ConcurrentMap>> asSegmentMap( From 30958e636d923447cbbec75dfa7c73e702189c89 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Wed, 24 Jan 2024 11:11:47 -0800 Subject: [PATCH 37/56] Add broadcast address to PingPacketData from filter (#6456) * make list of ping packet data 'from' address which we ignore * add test coverage Signed-off-by: garyschulte --- .../p2p/discovery/PeerDiscoveryAgent.java | 58 +++++++++++-------- .../p2p/discovery/PeerDiscoveryAgentTest.java | 43 ++++++++++++++ 2 files changed, 78 insertions(+), 23 deletions(-) diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java index de7d047180c..a4af09481cd 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java @@ -74,6 +74,8 @@ public abstract class PeerDiscoveryAgent { // The devp2p specification says only accept packets up to 1280, but some // clients ignore that, so we add in a little extra padding. private static final int MAX_PACKET_SIZE_BYTES = 1600; + private static final List PING_PACKET_SOURCE_IGNORED = + List.of("127.0.0.1", "255.255.255.255"); protected final List bootstrapPeers; private final List peerRequirements = new CopyOnWriteArrayList<>(); @@ -286,29 +288,7 @@ protected void handleIncomingPacket(final Endpoint sourceEndpoint, final Packet .flatMap(Endpoint::getTcpPort) .orElse(udpPort); - // If the host is present in the P2P PING packet itself, use that as the endpoint. If the P2P - // PING packet specifies 127.0.0.1 (the default if a custom value is not specified with - // --p2p-host or via a suitable --nat-method) we ignore it in favour of the UDP source address. - // The likelihood is that the UDP source will be 127.0.0.1 anyway, but this reduces the chance - // of an unexpected change in behaviour as a result of - // https://github.com/hyperledger/besu/issues/6224 being fixed. - final String host = - packet - .getPacketData(PingPacketData.class) - .flatMap(PingPacketData::getFrom) - .map(Endpoint::getHost) - .filter( - fromAddr -> - (!fromAddr.equals("127.0.0.1") && InetAddresses.isInetAddress(fromAddr))) - .stream() - .peek( - h -> - LOG.trace( - "Using \"From\" endpoint {} specified in ping packet. Ignoring UDP source host {}", - h, - sourceEndpoint.getHost())) - .findFirst() - .orElseGet(sourceEndpoint::getHost); + final String host = deriveHost(sourceEndpoint, packet); // Notify the peer controller. final DiscoveryPeer peer = @@ -323,6 +303,38 @@ protected void handleIncomingPacket(final Endpoint sourceEndpoint, final Packet controller.ifPresent(c -> c.onMessage(packet, peer)); } + /** + * method to derive the host from the source endpoint and the P2P PING packet. If the host is + * present in the P2P PING packet itself, use that as the endpoint. If the P2P PING packet + * specifies 127.0.0.1 (the default if a custom value is not specified with --p2p-host or via a + * suitable --nat-method) we ignore it in favour of the UDP source address. Some implementations + * send 127.0.0.1 or 255.255.255.255 anyway, but this reduces the chance of an unexpected change + * in behaviour as a result of https://github.com/hyperledger/besu/issues/6224 being fixed. + * + * @param sourceEndpoint source endpoint of the packet + * @param packet P2P PING packet + * @return host address as string + */ + static String deriveHost(final Endpoint sourceEndpoint, final Packet packet) { + return packet + .getPacketData(PingPacketData.class) + .flatMap(PingPacketData::getFrom) + .map(Endpoint::getHost) + .filter( + fromAddr -> + (!PING_PACKET_SOURCE_IGNORED.contains(fromAddr) + && InetAddresses.isInetAddress(fromAddr))) + .stream() + .peek( + h -> + LOG.trace( + "Using \"From\" endpoint {} specified in ping packet. Ignoring UDP source host {}", + h, + sourceEndpoint.getHost())) + .findFirst() + .orElseGet(sourceEndpoint::getHost); + } + /** * Send a packet to the given recipient. * diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java index 240f4673eb5..aae83ffdc8c 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java @@ -36,6 +36,7 @@ import org.hyperledger.besu.ethereum.p2p.discovery.internal.NeighborsPacketData; import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import org.hyperledger.besu.ethereum.p2p.discovery.internal.PacketType; +import org.hyperledger.besu.ethereum.p2p.discovery.internal.PingPacketData; import org.hyperledger.besu.ethereum.p2p.peers.DefaultPeer; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; import org.hyperledger.besu.ethereum.p2p.peers.Peer; @@ -53,6 +54,7 @@ import com.google.common.base.Suppliers; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt64; import org.ethereum.beacon.discovery.schema.NodeRecord; import org.junit.jupiter.api.Test; @@ -834,6 +836,47 @@ public void shouldNotBeActiveWhenConfigIsFalse() { assertThat(agent.isActive()).isFalse(); } + @Test + public void assertHostCorrectlyRevertsOnIgnoredPacketFrom() { + final String sourceHost = "UDP_SOURCE_ORIGIN_HOST"; + final String localHost = "127.0.0.1"; + final String broadcastDefaultHost = "255.255.255.255"; + final String routableHost = "50.50.50.50"; + + Endpoint source = new Endpoint(sourceHost, 30303, Optional.empty()); + Endpoint endpointLocal = new Endpoint(localHost, 30303, Optional.empty()); + Endpoint endpointBroadcast = new Endpoint(broadcastDefaultHost, 30303, Optional.empty()); + Endpoint endpointRoutable = new Endpoint(routableHost, 30303, Optional.empty()); + + Packet mockLocal = + when(mock(Packet.class).getPacketData(any())) + .thenReturn( + Optional.of( + PingPacketData.create(Optional.of(endpointLocal), endpointLocal, UInt64.ONE))) + .getMock(); + Packet mockBroadcast = + when(mock(Packet.class).getPacketData(any())) + .thenReturn( + Optional.of( + PingPacketData.create( + Optional.of(endpointBroadcast), endpointLocal, UInt64.ONE))) + .getMock(); + Packet mockWellFormed = + when(mock(Packet.class).getPacketData(any())) + .thenReturn( + Optional.of( + PingPacketData.create( + Optional.of(endpointRoutable), endpointLocal, UInt64.ONE))) + .getMock(); + + // assert a pingpacketdata from address of 127.0.0.1 reverts to the udp source host + assertThat(PeerDiscoveryAgent.deriveHost(source, mockLocal)).isEqualTo(sourceHost); + // assert that 255.255.255.255 reverts to the udp source host + assertThat(PeerDiscoveryAgent.deriveHost(source, mockBroadcast)).isEqualTo(sourceHost); + // assert that a well-formed routable address in the ping packet data is used + assertThat(PeerDiscoveryAgent.deriveHost(source, mockWellFormed)).isEqualTo(routableHost); + } + protected void bondViaIncomingPing( final MockPeerDiscoveryAgent agent, final MockPeerDiscoveryAgent otherNode) { final Packet pingPacket = helper.createPingPacket(otherNode, agent); From 4ac32c6df9188ee550d60237348884bc273ad128 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Wed, 24 Jan 2024 11:53:00 -0800 Subject: [PATCH 38/56] silence dns query error warning, at least until we can demote the upstream log to DEBUG (#6458) Signed-off-by: garyschulte --- besu/src/main/resources/log4j2.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/besu/src/main/resources/log4j2.xml b/besu/src/main/resources/log4j2.xml index 6bf83a973f7..cb412bca716 100644 --- a/besu/src/main/resources/log4j2.xml +++ b/besu/src/main/resources/log4j2.xml @@ -42,6 +42,9 @@ + + + From ed1480ba336794e62a8f7a98507b438607ed1714 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Wed, 24 Jan 2024 13:13:00 -0800 Subject: [PATCH 39/56] ignore bws sync requests until initial sync is complete (#6455) * ignore bws sync requests until initial sync is complete * return failed Future when BWS is invoked but not ready Signed-off-by: garyschulte --- .../backwardsync/BackwardSyncContext.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java index 6a681bec288..aee56cda22a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java @@ -129,12 +129,16 @@ public synchronized CompletableFuture syncBackwardsUntil(final Hash newBlo backwardChain.addNewHash(newBlockHash); } - final Status status = getOrStartSyncSession(); - backwardChain - .getBlock(newBlockHash) - .ifPresent( - newTargetBlock -> status.updateTargetHeight(newTargetBlock.getHeader().getNumber())); - return status.currentFuture; + if (isReady()) { + final Status status = getOrStartSyncSession(); + backwardChain + .getBlock(newBlockHash) + .ifPresent( + newTargetBlock -> status.updateTargetHeight(newTargetBlock.getHeader().getNumber())); + return status.currentFuture; + } else { + return CompletableFuture.failedFuture(new Throwable("Backward sync is not ready")); + } } public synchronized CompletableFuture syncBackwardsUntil(final Block newPivot) { @@ -142,9 +146,13 @@ public synchronized CompletableFuture syncBackwardsUntil(final Block newPi backwardChain.appendTrustedBlock(newPivot); } - final Status status = getOrStartSyncSession(); - status.updateTargetHeight(newPivot.getHeader().getNumber()); - return status.currentFuture; + if (isReady()) { + final Status status = getOrStartSyncSession(); + status.updateTargetHeight(newPivot.getHeader().getNumber()); + return status.currentFuture; + } else { + return CompletableFuture.failedFuture(new Throwable("Backward sync is not ready")); + } } private Status getOrStartSyncSession() { From a1a5b20983aeab3648904da56e3d849ca43310cb Mon Sep 17 00:00:00 2001 From: Alyosha Karamazov Date: Thu, 25 Jan 2024 00:58:25 +0000 Subject: [PATCH 40/56] Enable limit on range Of JSON-RPC API trace_filter method (#5971) (#6446) * Enable limit on range of JSON-RPC API trace_filter method (#5971) Enable a limit on the range of blocks that can be supplied to the JSON-RPC trace_filter method. The limit has a default value and can be overridden with a command line option at start up. Signed-off-by: alyokaz Signed-off-by: Sally MacFarlane --------- Signed-off-by: alyokaz Signed-off-by: Sally MacFarlane Co-authored-by: Sally MacFarlane --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 9 ++- .../src/test/resources/everything_config.toml | 3 +- .../besu/ethereum/api/ApiConfiguration.java | 5 ++ .../jsonrpc/internal/methods/TraceFilter.java | 18 ++++- .../jsonrpc/methods/TraceJsonRpcMethods.java | 6 +- .../internal/methods/TraceFilterTest.java | 79 +++++++++++++++++++ 7 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9730af0fd5d..f966114ed92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 24.1.2-SNAPSHOT ### Breaking Changes +- The `trace-filter` method in JSON-RPC API now has a default block range limit of 1000, adjustable with `--rpc-max-trace-filter-range` [#6446](https://github.com/hyperledger/besu/pull/6446) ### Deprecations diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index aeb518a2f6e..9ee0e313166 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1267,6 +1267,12 @@ static class PermissionsOptionGroup { description = "Specifies the number of last blocks to cache (default: ${DEFAULT-VALUE})") private final Integer numberOfblocksToCache = 0; + @Option( + names = {"--rpc-max-trace-filter-range"}, + description = + "Specifies the maximum number of blocks for the trace_filter method. Must be >=0. 0 specifies no limit (default: $DEFAULT-VALUE)") + private final Long maxTraceFilterRange = 1000L; + @Mixin private P2PTLSConfigOptions p2pTLSConfigOptions; @Mixin private PkiBlockCreationOptions pkiBlockCreationOptions; @@ -2534,7 +2540,8 @@ private ApiConfiguration apiConfiguration() { .gasPriceMax(apiGasPriceMax) .maxLogsRange(rpcMaxLogsRange) .gasCap(rpcGasCap) - .isGasAndPriorityFeeLimitingEnabled(apiGasAndPriorityFeeLimitingEnabled); + .isGasAndPriorityFeeLimitingEnabled(apiGasAndPriorityFeeLimitingEnabled) + .maxTraceFilterRange(maxTraceFilterRange); if (apiGasAndPriorityFeeLimitingEnabled) { if (apiGasAndPriorityFeeLowerBoundCoefficient > apiGasAndPriorityFeeUpperBoundCoefficient) { throw new ParameterException( diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index b205fa6692e..3c6792b7b95 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -90,6 +90,7 @@ rpc-max-logs-range=100 json-pretty-print-enabled=false cache-last-blocks=512 rpc-gas-cap = 50000000 +rpc-max-trace-filter-range=100 # PRIVACY TLS privacy-tls-enabled=false @@ -232,4 +233,4 @@ Xp2p-tls-crl-file="none.file" Xp2p-tls-clienthello-sni=false #contracts -Xevm-jumpdest-cache-weight-kb=32000 \ No newline at end of file +Xevm-jumpdest-cache-weight-kb=32000 diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/ApiConfiguration.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/ApiConfiguration.java index e9f30fca336..42a7f8f62e0 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/ApiConfiguration.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/ApiConfiguration.java @@ -77,4 +77,9 @@ public Long getLowerBoundGasAndPriorityFeeCoefficient() { public Long getUpperBoundGasAndPriorityFeeCoefficient() { return DEFAULT_UPPER_BOUND_GAS_AND_PRIORITY_FEE_COEFFICIENT; } + + @Value.Default + public Long getMaxTraceFilterRange() { + return 1000L; + } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java index 8c94f785065..ff65feb8469 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilter.java @@ -24,8 +24,10 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.FlatTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.tracing.flat.RewardTraceGenerator; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; @@ -66,12 +68,15 @@ public class TraceFilter extends TraceBlock { private static final Logger LOG = LoggerFactory.getLogger(TraceFilter.class); + private final Long maxRange; public TraceFilter( final Supplier blockTracerSupplier, final ProtocolSchedule protocolSchedule, - final BlockchainQueries blockchainQueries) { + final BlockchainQueries blockchainQueries, + final Long maxRange) { super(protocolSchedule, blockchainQueries); + this.maxRange = maxRange; } @Override @@ -88,6 +93,17 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { final long toBlock = resolveBlockNumber(filterParameter.getToBlock()); LOG.trace("Received RPC rpcName={} fromBlock={} toBlock={}", getName(), fromBlock, toBlock); + if (maxRange > 0 && toBlock - fromBlock > maxRange) { + LOG.atDebug() + .setMessage("trace_filter request {} failed:") + .addArgument(requestContext.getRequest()) + .setCause( + new IllegalArgumentException(RpcErrorType.EXCEEDS_RPC_MAX_BLOCK_RANGE.getMessage())) + .log(); + return new JsonRpcErrorResponse( + requestContext.getRequest().getId(), RpcErrorType.EXCEEDS_RPC_MAX_BLOCK_RANGE); + } + final ObjectMapper mapper = new ObjectMapper(); final ArrayNodeWrapper resultArrayNode = new ArrayNodeWrapper( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java index 96faae9fd29..2333409b089 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/TraceJsonRpcMethods.java @@ -60,7 +60,11 @@ protected Map create() { new BlockReplay(protocolSchedule, blockchainQueries.getBlockchain()); return mapOf( new TraceReplayBlockTransactions(protocolSchedule, blockchainQueries), - new TraceFilter(() -> new BlockTracer(blockReplay), protocolSchedule, blockchainQueries), + new TraceFilter( + () -> new BlockTracer(blockReplay), + protocolSchedule, + blockchainQueries, + apiConfiguration.getMaxTraceFilterRange()), new TraceGet(() -> new BlockTracer(blockReplay), blockchainQueries, protocolSchedule), new TraceTransaction( () -> new BlockTracer(blockReplay), protocolSchedule, blockchainQueries), diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java new file mode 100644 index 00000000000..99db8851141 --- /dev/null +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceFilterTest.java @@ -0,0 +1,79 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.FilterParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; +import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; + +import java.util.function.Supplier; + +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class TraceFilterTest { + + private TraceFilter method; + + @Mock Supplier blockTracerSupplier; + @Mock ProtocolSchedule protocolSchedule; + @Mock BlockchainQueries blockchainQueries; + + @ParameterizedTest + @CsvSource({ + "0, 1001, 1000", "0, 5000, 1000", "1, 1002, 1000", "1, 6002, 1000", "1000, 3000, 1000", + "0, 501, 500", "0, 5000, 500", "1, 502, 500", "1, 6002, 500", "1000, 3000, 500" + }) + public void shouldFailIfParamsExceedMaxRange( + final long fromBlock, final long toBlock, final long maxFilterRange) { + final FilterParameter filterParameter = + new FilterParameter( + new BlockParameter(fromBlock), + new BlockParameter(toBlock), + null, + null, + null, + null, + null, + null, + null); + + JsonRpcRequestContext request = + new JsonRpcRequestContext( + new JsonRpcRequest("2.0", "trace_filter", new Object[] {filterParameter})); + + method = + new TraceFilter(blockTracerSupplier, protocolSchedule, blockchainQueries, maxFilterRange); + + final JsonRpcResponse response = method.response(request); + assertThat(response).isInstanceOf(JsonRpcErrorResponse.class); + + final JsonRpcErrorResponse errorResponse = (JsonRpcErrorResponse) response; + assertThat(errorResponse.getErrorType()).isEqualTo(RpcErrorType.EXCEEDS_RPC_MAX_BLOCK_RANGE); + } +} From 11b6c312720daa5a5ef79bfd9ea6563e6e93b957 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Thu, 25 Jan 2024 12:03:39 +1000 Subject: [PATCH 41/56] Snap sync downloader logging (#6403) * non-static logger for inheriting classes * refactor to some classes and log messages to make snap sync logs make sense * rename SnapsyncMetricsManage * rename FastImportBlocksStep -> ImportBlocksStep Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane --- ...ger.java => AbstractSyncTargetManager.java} | 6 +++--- .../eth/sync/PipelineChainDownloader.java | 4 ++-- .../CheckpointSyncChainDownloader.java | 6 +++--- .../sync/fastsync/FastSyncChainDownloader.java | 4 ++-- .../FastSyncDownloadPipelineFactory.java | 4 ++-- .../eth/sync/fastsync/FastSyncDownloader.java | 14 ++++++++------ ...rtBlocksStep.java => ImportBlocksStep.java} | 6 +++--- .../eth/sync/fastsync/PivotBlockRetriever.java | 3 +-- .../{FastSyncError.java => SyncError.java} | 2 +- ...stSyncException.java => SyncException.java} | 14 +++++++------- ...rgetManager.java => SyncTargetManager.java} | 8 ++++---- .../sync/fullsync/FullSyncTargetManager.java | 4 ++-- .../eth/sync/snapsync/SnapSyncDownloader.java | 5 ----- ...anager.java => SnapSyncMetricsManager.java} | 8 ++++---- .../sync/snapsync/SnapWorldDownloadState.java | 6 +++--- .../snapsync/SnapWorldStateDownloader.java | 6 +++--- .../request/AccountRangeDataRequest.java | 2 +- ...AccountFlatDatabaseHealingRangeRequest.java | 2 +- .../eth/sync/PipelineChainDownloaderTest.java | 2 +- ...tImportBlocksPercentageCalculationTest.java | 12 ++++++------ .../sync/fastsync/FastSyncDownloaderTest.java | 18 +++++++++--------- ...StepTest.java => ImportBlocksStepTest.java} | 6 +++--- .../sync/fastsync/PivotBlockRetrieverTest.java | 12 ++++++------ .../eth/sync/snapsync/PersistDataStepTest.java | 2 +- .../snapsync/SnapWorldDownloadStateTest.java | 2 +- ...untFlatDatabaseHealingRangeRequestTest.java | 4 ++-- 26 files changed, 79 insertions(+), 83 deletions(-) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/{SyncTargetManager.java => AbstractSyncTargetManager.java} (96%) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/{FastImportBlocksStep.java => ImportBlocksStep.java} (96%) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/{FastSyncError.java => SyncError.java} (96%) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/{FastSyncException.java => SyncException.java} (69%) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/{FastSyncTargetManager.java => SyncTargetManager.java} (96%) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/{SnapsyncMetricsManager.java => SnapSyncMetricsManager.java} (98%) rename ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/{FastImportBlocksStepTest.java => ImportBlocksStepTest.java} (97%) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncTargetManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/AbstractSyncTargetManager.java similarity index 96% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncTargetManager.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/AbstractSyncTargetManager.java index c231d13af35..9c6b9327e4b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncTargetManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/AbstractSyncTargetManager.java @@ -32,9 +32,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class SyncTargetManager { +public abstract class AbstractSyncTargetManager { - private static final Logger LOG = LoggerFactory.getLogger(SyncTargetManager.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractSyncTargetManager.class); private final SynchronizerConfiguration config; private final ProtocolSchedule protocolSchedule; @@ -42,7 +42,7 @@ public abstract class SyncTargetManager { private final EthContext ethContext; private final MetricsSystem metricsSystem; - protected SyncTargetManager( + protected AbstractSyncTargetManager( final SynchronizerConfiguration config, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java index d1c3d008830..5ae78ca280f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloader.java @@ -43,7 +43,7 @@ public class PipelineChainDownloader implements ChainDownloader { private static final Logger LOG = LoggerFactory.getLogger(PipelineChainDownloader.class); static final Duration PAUSE_AFTER_ERROR_DURATION = Duration.ofSeconds(2); private final SyncState syncState; - private final SyncTargetManager syncTargetManager; + private final AbstractSyncTargetManager syncTargetManager; private final DownloadPipelineFactory downloadPipelineFactory; private final EthScheduler scheduler; @@ -55,7 +55,7 @@ public class PipelineChainDownloader implements ChainDownloader { public PipelineChainDownloader( final SyncState syncState, - final SyncTargetManager syncTargetManager, + final AbstractSyncTargetManager syncTargetManager, final DownloadPipelineFactory downloadPipelineFactory, final EthScheduler scheduler, final MetricsSystem metricsSystem) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java index 854972c39fd..356ad0d078d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; -import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncTargetManager; +import org.hyperledger.besu.ethereum.eth.sync.fastsync.SyncTargetManager; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; @@ -38,8 +38,8 @@ public static ChainDownloader create( final MetricsSystem metricsSystem, final FastSyncState fastSyncState) { - final FastSyncTargetManager syncTargetManager = - new FastSyncTargetManager( + final SyncTargetManager syncTargetManager = + new SyncTargetManager( config, worldStateStorage, protocolSchedule, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java index c4034cb87a0..ae49cfdec29 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java @@ -38,8 +38,8 @@ public static ChainDownloader create( final MetricsSystem metricsSystem, final FastSyncState fastSyncState) { - final FastSyncTargetManager syncTargetManager = - new FastSyncTargetManager( + final SyncTargetManager syncTargetManager = + new SyncTargetManager( config, worldStateStorage, protocolSchedule, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java index 206b76c09a5..f7e42b32f65 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloadPipelineFactory.java @@ -136,8 +136,8 @@ public Pipeline createDownloadPipelineForSyncTarget(final SyncT new DownloadBodiesStep(protocolSchedule, ethContext, metricsSystem); final DownloadReceiptsStep downloadReceiptsStep = new DownloadReceiptsStep(ethContext, metricsSystem); - final FastImportBlocksStep importBlockStep = - new FastImportBlocksStep( + final ImportBlocksStep importBlockStep = + new ImportBlocksStep( protocolSchedule, protocolContext, attachedValidationPolicy, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index b73daa28301..d7c2707af08 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -44,7 +44,9 @@ public class FastSyncDownloader { private static final Duration FAST_SYNC_RETRY_DELAY = Duration.ofSeconds(5); - private static final Logger LOG = LoggerFactory.getLogger(FastSyncDownloader.class); + @SuppressWarnings("PrivateStaticFinalLoggers") + protected final Logger LOG = LoggerFactory.getLogger(getClass()); + private final WorldStateStorage worldStateStorage; private final WorldStateDownloader worldStateDownloader; private final TaskCollection taskCollection; @@ -75,7 +77,7 @@ public FastSyncDownloader( public CompletableFuture start() { if (!running.compareAndSet(false, true)) { - throw new IllegalStateException("FastSyncDownloader already running"); + throw new IllegalStateException("SyncDownloader already running"); } LOG.info("Starting sync"); return start(initialFastSyncState); @@ -107,7 +109,7 @@ public CompletableFuture findPivotBlock( protected CompletableFuture handleFailure(final Throwable error) { trailingPeerRequirements = Optional.empty(); Throwable rootCause = ExceptionUtils.rootCause(error); - if (rootCause instanceof FastSyncException) { + if (rootCause instanceof SyncException) { return CompletableFuture.failedFuture(error); } else if (rootCause instanceof StalledDownloadException) { LOG.debug("Stalled sync re-pivoting to newer block."); @@ -120,7 +122,7 @@ protected CompletableFuture handleFailure(final Throwable error) return start(FastSyncState.EMPTY_SYNC_STATE); } else { LOG.error( - "Encountered an unexpected error during fast sync. Restarting sync in " + "Encountered an unexpected error during sync. Restarting sync in " + FAST_SYNC_RETRY_DELAY.getSeconds() + " seconds.", error); @@ -132,7 +134,7 @@ protected CompletableFuture handleFailure(final Throwable error) public void stop() { synchronized (this) { if (running.compareAndSet(true, false)) { - LOG.info("Stopping fast sync"); + LOG.info("Stopping sync"); // Cancelling the world state download will also cause the chain download to be cancelled. worldStateDownloader.cancel(); } @@ -149,7 +151,7 @@ public void deleteFastSyncState() { MoreFiles.deleteRecursively(fastSyncDataDirectory, RecursiveDeleteOption.ALLOW_INSECURE); } } catch (final IOException e) { - LOG.error("Unable to clean up fast sync state", e); + LOG.error("Unable to clean up sync state", e); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStep.java similarity index 96% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStep.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStep.java index c2297faa199..bcf8f00307b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStep.java @@ -33,8 +33,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FastImportBlocksStep implements Consumer> { - private static final Logger LOG = LoggerFactory.getLogger(FastImportBlocksStep.class); +public class ImportBlocksStep implements Consumer> { + private static final Logger LOG = LoggerFactory.getLogger(ImportBlocksStep.class); private static final long PRINT_DELAY = TimeUnit.SECONDS.toMillis(30L); private final ProtocolSchedule protocolSchedule; @@ -46,7 +46,7 @@ public class FastImportBlocksStep implements Consumer> { private OptionalLong logStartBlock = OptionalLong.empty(); private final BlockHeader pivotHeader; - public FastImportBlocksStep( + public ImportBlocksStep( final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final ValidationPolicy headerValidationPolicy, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetriever.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetriever.java index e20bb16799c..6ddd7db9012 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetriever.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetriever.java @@ -156,8 +156,7 @@ private void handleContestedPivotBlock(final long contestedBlockNumber) { || pivotBlockNumber.get() <= BlockHeader.GENESIS_BLOCK_NUMBER) { LOG.info("Max retries reached, cancel pivot block download."); // Pivot block selection has failed - result.completeExceptionally( - new FastSyncException(FastSyncError.PIVOT_BLOCK_HEADER_MISMATCH)); + result.completeExceptionally(new SyncException(SyncError.PIVOT_BLOCK_HEADER_MISMATCH)); return; } else { LOG.info("Move pivot block back to {} and retry.", pivotBlockNumber); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncError.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncError.java similarity index 96% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncError.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncError.java index d8361733574..d1277ddfdc1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncError.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncError.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.eth.sync.fastsync; -public enum FastSyncError { +public enum SyncError { NO_PEERS_AVAILABLE, PIVOT_BLOCK_HEADER_MISMATCH, UNEXPECTED_ERROR diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncException.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncException.java similarity index 69% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncException.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncException.java index 79baf02c43c..72e928dc18b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncException.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncException.java @@ -14,21 +14,21 @@ */ package org.hyperledger.besu.ethereum.eth.sync.fastsync; -public class FastSyncException extends RuntimeException { +public class SyncException extends RuntimeException { - private final FastSyncError error; + private final SyncError error; - public FastSyncException(final FastSyncError error) { - super("Fast sync failed: " + error); + public SyncException(final SyncError error) { + super("Sync failed: " + error); this.error = error; } - public FastSyncError getError() { + public SyncError getError() { return error; } - public FastSyncException(final Throwable error) { + public SyncException(final Throwable error) { super(error); - this.error = FastSyncError.UNEXPECTED_ERROR; + this.error = SyncError.UNEXPECTED_ERROR; } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncTargetManager.java similarity index 96% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncTargetManager.java index 11dc8c1af61..d0e0a1f0fd0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/SyncTargetManager.java @@ -23,7 +23,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.EthPeers; -import org.hyperledger.besu.ethereum.eth.sync.SyncTargetManager; +import org.hyperledger.besu.ethereum.eth.sync.AbstractSyncTargetManager; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.tasks.RetryingGetHeaderFromPeerByNumberTask; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; @@ -39,8 +39,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FastSyncTargetManager extends SyncTargetManager { - private static final Logger LOG = LoggerFactory.getLogger(FastSyncTargetManager.class); +public class SyncTargetManager extends AbstractSyncTargetManager { + private static final Logger LOG = LoggerFactory.getLogger(SyncTargetManager.class); private final WorldStateStorage worldStateStorage; private final ProtocolSchedule protocolSchedule; @@ -53,7 +53,7 @@ public class FastSyncTargetManager extends SyncTargetManager { private final int logDebugRepeatDelay = 15; private final int logInfoRepeatDelay = 120; - public FastSyncTargetManager( + public SyncTargetManager( final SynchronizerConfiguration config, final WorldStateStorage worldStateStorage, final ProtocolSchedule protocolSchedule, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManager.java index 43c2cfa19c5..ee3d36a31ab 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fullsync/FullSyncTargetManager.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.eth.sync.SyncTargetManager; +import org.hyperledger.besu.ethereum.eth.sync.AbstractSyncTargetManager; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncTarget; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; @@ -34,7 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class FullSyncTargetManager extends SyncTargetManager { +class FullSyncTargetManager extends AbstractSyncTargetManager { private static final Logger LOG = LoggerFactory.getLogger(FullSyncTargetManager.class); private final ProtocolContext protocolContext; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java index ac409513156..32c2dc23c0b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java @@ -26,13 +26,8 @@ import java.nio.file.Path; import java.util.concurrent.CompletableFuture; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class SnapSyncDownloader extends FastSyncDownloader { - private static final Logger LOG = LoggerFactory.getLogger(SnapSyncDownloader.class); - public SnapSyncDownloader( final FastSyncActions fastSyncActions, final WorldStateStorage worldStateStorage, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapsyncMetricsManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java similarity index 98% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapsyncMetricsManager.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java index 25af2e4262e..da7607de7fd 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapsyncMetricsManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync; import static io.netty.util.internal.ObjectUtil.checkNonEmpty; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.HEAL_TRIE; +import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.HEAL_TRIE; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -37,9 +37,9 @@ import org.slf4j.LoggerFactory; /** Manages the metrics related to the SnapSync process. */ -public class SnapsyncMetricsManager { +public class SnapSyncMetricsManager { - private static final Logger LOG = LoggerFactory.getLogger(SnapsyncMetricsManager.class); + private static final Logger LOG = LoggerFactory.getLogger(SnapSyncMetricsManager.class); private static final long PRINT_DELAY = TimeUnit.MINUTES.toMillis(1); private final MetricsSystem metricsSystem; @@ -79,7 +79,7 @@ public class SnapsyncMetricsManager { private long lastNotifyTimestamp; - public SnapsyncMetricsManager(final MetricsSystem metricsSystem, final EthContext ethContext) { + public SnapSyncMetricsManager(final MetricsSystem metricsSystem, final EthContext ethContext) { this.metricsSystem = metricsSystem; this.ethContext = ethContext; percentageProgress = new AtomicReference<>(new BigDecimal(0)); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index dd7481cdd00..76d31562106 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -83,7 +83,7 @@ public class SnapWorldDownloadState extends WorldDownloadState private OptionalLong blockObserverId; // metrics around the snapsync - private final SnapsyncMetricsManager metricsManager; + private final SnapSyncMetricsManager metricsManager; public SnapWorldDownloadState( final WorldStateStorage worldStateStorage, @@ -93,7 +93,7 @@ public SnapWorldDownloadState( final InMemoryTasksPriorityQueues pendingRequests, final int maxRequestsWithoutProgress, final long minMillisBeforeStalling, - final SnapsyncMetricsManager metricsManager, + final SnapSyncMetricsManager metricsManager, final Clock clock) { super( worldStateStorage, @@ -400,7 +400,7 @@ public synchronized Task dequeueStorageFlatDatabaseHealingReque __ -> {}); } - public SnapsyncMetricsManager getMetricsManager() { + public SnapSyncMetricsManager getMetricsManager() { return metricsManager; } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index 313d5a3770f..5b2f51d87ba 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.DOWNLOAD; +import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.DOWNLOAD; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest.createAccountRangeDataRequest; import org.hyperledger.besu.datatypes.Hash; @@ -133,8 +133,8 @@ public CompletableFuture run( stateRoot, snapTaskCollection.size()); - final SnapsyncMetricsManager snapsyncMetricsManager = - new SnapsyncMetricsManager(metricsSystem, ethContext); + final SnapSyncMetricsManager snapsyncMetricsManager = + new SnapSyncMetricsManager(metricsSystem, ethContext); final SnapWorldDownloadState newDownloadState = new SnapWorldDownloadState( diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index ea5c25ec0de..acf8adc4aa9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -18,7 +18,7 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.findNewBeginElementInRange; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.ACCOUNT_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.DOWNLOAD; +import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.DOWNLOAD; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie.FlatDatabaseUpdater.noop; import org.hyperledger.besu.datatypes.Hash; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java index 7de77607450..526a0e9c58b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java @@ -16,7 +16,7 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.HEAL_FLAT; +import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.HEAL_FLAT; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloaderTest.java index babacfc5e0f..91d33a11145 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/PipelineChainDownloaderTest.java @@ -52,7 +52,7 @@ @ExtendWith(MockitoExtension.class) public class PipelineChainDownloaderTest { - @Mock private SyncTargetManager syncTargetManager; + @Mock private AbstractSyncTargetManager syncTargetManager; @Mock private DownloadPipelineFactory downloadPipelineFactory; @Mock private EthScheduler scheduler; @Mock private Pipeline downloadPipeline; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksPercentageCalculationTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksPercentageCalculationTest.java index a68194abbe7..ed5303897ae 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksPercentageCalculationTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksPercentageCalculationTest.java @@ -22,11 +22,11 @@ public class FastImportBlocksPercentageCalculationTest { @Test public void blocksPercent_calculations() { - assertThat(FastImportBlocksStep.getBlocksPercent(1, 1)).isEqualByComparingTo(100l); - assertThat(FastImportBlocksStep.getBlocksPercent(1, 100)).isEqualByComparingTo(1l); - assertThat(FastImportBlocksStep.getBlocksPercent(0, 100)).isEqualByComparingTo(0l); - assertThat(FastImportBlocksStep.getBlocksPercent(99, 0)).isEqualByComparingTo(0l); - assertThat(FastImportBlocksStep.getBlocksPercent(1, 1000)).isEqualByComparingTo(0l); - assertThat(FastImportBlocksStep.getBlocksPercent(1, 10000)).isEqualByComparingTo(0l); + assertThat(ImportBlocksStep.getBlocksPercent(1, 1)).isEqualByComparingTo(100l); + assertThat(ImportBlocksStep.getBlocksPercent(1, 100)).isEqualByComparingTo(1l); + assertThat(ImportBlocksStep.getBlocksPercent(0, 100)).isEqualByComparingTo(0l); + assertThat(ImportBlocksStep.getBlocksPercent(99, 0)).isEqualByComparingTo(0l); + assertThat(ImportBlocksStep.getBlocksPercent(1, 1000)).isEqualByComparingTo(0l); + assertThat(ImportBlocksStep.getBlocksPercent(1, 10000)).isEqualByComparingTo(0l); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java index 632fed6ede1..c05770e6d72 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java @@ -150,11 +150,11 @@ public void shouldResumeFastSync() { @Test public void shouldAbortIfSelectPivotBlockFails() { when(fastSyncActions.selectPivotBlock(FastSyncState.EMPTY_SYNC_STATE)) - .thenThrow(new FastSyncException(FastSyncError.UNEXPECTED_ERROR)); + .thenThrow(new SyncException(SyncError.UNEXPECTED_ERROR)); final CompletableFuture result = downloader.start(); - assertCompletedExceptionally(result, FastSyncError.UNEXPECTED_ERROR); + assertCompletedExceptionally(result, SyncError.UNEXPECTED_ERROR); verify(fastSyncActions).selectPivotBlock(FastSyncState.EMPTY_SYNC_STATE); verifyNoMoreInteractions(fastSyncActions); @@ -191,10 +191,10 @@ public void shouldAbortIfWorldStateDownloadFails() { assertThat(result).isNotDone(); - worldStateFuture.completeExceptionally(new FastSyncException(FastSyncError.NO_PEERS_AVAILABLE)); + worldStateFuture.completeExceptionally(new SyncException(SyncError.NO_PEERS_AVAILABLE)); verify(chainDownloader).cancel(); chainFuture.completeExceptionally(new CancellationException()); - assertCompletedExceptionally(result, FastSyncError.NO_PEERS_AVAILABLE); + assertCompletedExceptionally(result, SyncError.NO_PEERS_AVAILABLE); assertThat(chainFuture).isCancelled(); } @@ -229,8 +229,8 @@ public void shouldAbortIfChainDownloadFails() { assertThat(result).isNotDone(); - chainFuture.completeExceptionally(new FastSyncException(FastSyncError.NO_PEERS_AVAILABLE)); - assertCompletedExceptionally(result, FastSyncError.NO_PEERS_AVAILABLE); + chainFuture.completeExceptionally(new SyncException(SyncError.NO_PEERS_AVAILABLE)); + assertCompletedExceptionally(result, SyncError.NO_PEERS_AVAILABLE); assertThat(worldStateFuture).isCancelled(); } @@ -536,13 +536,13 @@ public void shouldNotHaveTrailingPeerRequirementsAfterDownloadCompletes() { } private void assertCompletedExceptionally( - final CompletableFuture future, final FastSyncError expectedError) { + final CompletableFuture future, final SyncError expectedError) { assertThat(future).isCompletedExceptionally(); future.exceptionally( actualError -> { assertThat(actualError) - .isInstanceOf(FastSyncException.class) - .extracting(ex -> ((FastSyncException) ex).getError()) + .isInstanceOf(SyncException.class) + .extracting(ex -> ((SyncException) ex).getError()) .isEqualTo(expectedError); return null; }); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStepTest.java similarity index 97% rename from ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStepTest.java rename to ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStepTest.java index 448608df715..70c9e10eba6 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastImportBlocksStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/ImportBlocksStepTest.java @@ -45,7 +45,7 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -public class FastImportBlocksStepTest { +public class ImportBlocksStepTest { @Mock private ProtocolSchedule protocolSchedule; @Mock private ProtocolSpec protocolSpec; @@ -56,7 +56,7 @@ public class FastImportBlocksStepTest { @Mock private BlockHeader pivotHeader; private final BlockDataGenerator gen = new BlockDataGenerator(); - private FastImportBlocksStep importBlocksStep; + private ImportBlocksStep importBlocksStep; @BeforeEach public void setUp() { @@ -66,7 +66,7 @@ public void setUp() { when(ommerValidationPolicy.getValidationModeForNextBlock()).thenReturn(LIGHT); importBlocksStep = - new FastImportBlocksStep( + new ImportBlocksStep( protocolSchedule, protocolContext, validationPolicy, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java index 7e539c8a596..723c5afe65f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotBlockRetrieverTest.java @@ -376,9 +376,9 @@ public void shouldRetryWhenPeersDisagreeOnPivot_exceedMaxRetries( assertThat(future).isCompletedExceptionally(); assertThatThrownBy(future::get) - .hasRootCauseInstanceOf(FastSyncException.class) - .extracting(e -> ((FastSyncException) ExceptionUtils.rootCause(e)).getError()) - .isEqualTo(FastSyncError.PIVOT_BLOCK_HEADER_MISMATCH); + .hasRootCauseInstanceOf(SyncException.class) + .extracting(e -> ((SyncException) ExceptionUtils.rootCause(e)).getError()) + .isEqualTo(SyncError.PIVOT_BLOCK_HEADER_MISMATCH); } @ParameterizedTest @@ -406,9 +406,9 @@ public void shouldRetryWhenPeersDisagreeOnPivot_pivotInvalidOnRetry( assertThat(future).isCompletedExceptionally(); assertThatThrownBy(future::get) - .hasRootCauseInstanceOf(FastSyncException.class) - .extracting(e -> ((FastSyncException) ExceptionUtils.rootCause(e)).getError()) - .isEqualTo(FastSyncError.PIVOT_BLOCK_HEADER_MISMATCH); + .hasRootCauseInstanceOf(SyncException.class) + .extracting(e -> ((SyncException) ExceptionUtils.rootCause(e)).getError()) + .isEqualTo(SyncError.PIVOT_BLOCK_HEADER_MISMATCH); } private Responder responderForFakeBlocks(final long... blockNumbers) { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java index dedce974c49..3b8bc7f2193 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java @@ -51,7 +51,7 @@ public class PersistDataStepTest { @BeforeEach public void setUp() { - when(downloadState.getMetricsManager()).thenReturn(mock(SnapsyncMetricsManager.class)); + when(downloadState.getMetricsManager()).thenReturn(mock(SnapSyncMetricsManager.class)); } @Test diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index b275e60e273..d10e2b21e29 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -82,7 +82,7 @@ public class SnapWorldDownloadStateTest { private final SnapSyncProcessState snapSyncState = mock(SnapSyncProcessState.class); private final SnapSyncStatePersistenceManager snapContext = mock(SnapSyncStatePersistenceManager.class); - private final SnapsyncMetricsManager metricsManager = mock(SnapsyncMetricsManager.class); + private final SnapSyncMetricsManager metricsManager = mock(SnapSyncMetricsManager.class); private final Blockchain blockchain = mock(Blockchain.class); private final DynamicPivotBlockSelector dynamicPivotBlockManager = mock(DynamicPivotBlockSelector.class); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index 40db311c4f1..ebd8c24a3d4 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -19,9 +19,9 @@ import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; +import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.storage.StorageProvider; @@ -67,7 +67,7 @@ public class AccountFlatDatabaseHealingRangeRequestTest { @BeforeEach public void setup() { Mockito.when(downloadState.getMetricsManager()) - .thenReturn(Mockito.mock(SnapsyncMetricsManager.class)); + .thenReturn(Mockito.mock(SnapSyncMetricsManager.class)); Mockito.when(downloadState.getAccountsHealingList()).thenReturn(new HashSet<>()); } From 317d2ac6f45b3a320a3d66c21265161c5d42c7fc Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Thu, 25 Jan 2024 14:07:13 +1100 Subject: [PATCH 42/56] [Refactor] - Extract websocket options to a new class (#6461) Signed-off-by: Gabriel-Trintinalia --- .../org/hyperledger/besu/cli/BesuCommand.java | 209 ++------------ .../options/stable/RpcWebsocketOptions.java | 266 ++++++++++++++++++ .../hyperledger/besu/cli/BesuCommandTest.java | 236 ---------------- .../besu/cli/CommandTestAbstract.java | 29 ++ .../cli/options/AbstractCLIOptionsTest.java | 27 -- .../cli/options/RpcWebsocketOptionsTest.java | 249 ++++++++++++++++ 6 files changed, 563 insertions(+), 453 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/options/stable/RpcWebsocketOptions.java create mode 100644 besu/src/test/java/org/hyperledger/besu/cli/options/RpcWebsocketOptionsTest.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 9ee0e313166..d30bb421470 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -31,7 +31,6 @@ import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.VALID_APIS; import static org.hyperledger.besu.ethereum.api.jsonrpc.authentication.EngineAuthService.EPHEMERAL_JWT_FILE; -import static org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration.DEFAULT_WEBSOCKET_PORT; import static org.hyperledger.besu.metrics.BesuMetricCategory.DEFAULT_METRIC_CATEGORIES; import static org.hyperledger.besu.metrics.MetricsProtocol.PROMETHEUS; import static org.hyperledger.besu.metrics.prometheus.MetricsConfiguration.DEFAULT_METRICS_PORT; @@ -61,6 +60,7 @@ import org.hyperledger.besu.cli.options.stable.LoggingLevelOption; import org.hyperledger.besu.cli.options.stable.NodePrivateKeyFileOption; import org.hyperledger.besu.cli.options.stable.P2PTLSConfigOptions; +import org.hyperledger.besu.cli.options.stable.RpcWebsocketOptions; import org.hyperledger.besu.cli.options.unstable.ChainPruningOptions; import org.hyperledger.besu.cli.options.unstable.DnsOptions; import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions; @@ -806,93 +806,7 @@ static class JsonRPCHttpOptionGroup { // JSON-RPC Websocket Options @CommandLine.ArgGroup(validate = false, heading = "@|bold JSON-RPC Websocket Options|@%n") - JsonRPCWebsocketOptionGroup jsonRPCWebsocketOptionGroup = new JsonRPCWebsocketOptionGroup(); - - static class JsonRPCWebsocketOptionGroup { - @Option( - names = {"--rpc-ws-authentication-jwt-algorithm"}, - description = - "Encryption algorithm used for Websockets JWT public key. Possible values are ${COMPLETION-CANDIDATES}" - + " (default: ${DEFAULT-VALUE})", - arity = "1") - private final JwtAlgorithm rpcWebsocketsAuthenticationAlgorithm = DEFAULT_JWT_ALGORITHM; - - @Option( - names = {"--rpc-ws-enabled"}, - description = "Set to start the JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") - private final Boolean isRpcWsEnabled = false; - - @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. - @Option( - names = {"--rpc-ws-host"}, - paramLabel = MANDATORY_HOST_FORMAT_HELP, - description = - "Host for JSON-RPC WebSocket service to listen on (default: ${DEFAULT-VALUE})", - arity = "1") - private String rpcWsHost; - - @Option( - names = {"--rpc-ws-port"}, - paramLabel = MANDATORY_PORT_FORMAT_HELP, - description = - "Port for JSON-RPC WebSocket service to listen on (default: ${DEFAULT-VALUE})", - arity = "1") - private final Integer rpcWsPort = DEFAULT_WEBSOCKET_PORT; - - @Option( - names = {"--rpc-ws-max-frame-size"}, - description = - "Maximum size in bytes for JSON-RPC WebSocket frames (default: ${DEFAULT-VALUE}). If this limit is exceeded, the websocket will be disconnected.", - arity = "1") - private final Integer rpcWsMaxFrameSize = DEFAULT_WS_MAX_FRAME_SIZE; - - @Option( - names = {"--rpc-ws-max-active-connections"}, - description = - "Maximum number of WebSocket connections allowed for JSON-RPC (default: ${DEFAULT-VALUE}). Once this limit is reached, incoming connections will be rejected.", - arity = "1") - private final Integer rpcWsMaxConnections = DEFAULT_WS_MAX_CONNECTIONS; - - @Option( - names = {"--rpc-ws-api", "--rpc-ws-apis"}, - paramLabel = "", - split = " {0,1}, {0,1}", - arity = "1..*", - description = - "Comma separated list of APIs to enable on JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") - private final List rpcWsApis = DEFAULT_RPC_APIS; - - @Option( - names = {"--rpc-ws-api-methods-no-auth", "--rpc-ws-api-method-no-auth"}, - paramLabel = "", - split = " {0,1}, {0,1}", - arity = "1..*", - description = - "Comma separated list of RPC methods to exclude from RPC authentication services, RPC WebSocket authentication must be enabled") - private final List rpcWsApiMethodsNoAuth = new ArrayList(); - - @Option( - names = {"--rpc-ws-authentication-enabled"}, - description = - "Require authentication for the JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") - private final Boolean isRpcWsAuthenticationEnabled = false; - - @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. - @CommandLine.Option( - names = {"--rpc-ws-authentication-credentials-file"}, - paramLabel = MANDATORY_FILE_FORMAT_HELP, - description = - "Storage file for JSON-RPC WebSocket authentication credentials (default: ${DEFAULT-VALUE})", - arity = "1") - private String rpcWsAuthenticationCredentialsFile = null; - - @CommandLine.Option( - names = {"--rpc-ws-authentication-jwt-public-key-file"}, - paramLabel = MANDATORY_FILE_FORMAT_HELP, - description = "JWT public key file for JSON-RPC WebSocket authentication", - arity = "1") - private final File rpcWsAuthenticationPublicKeyFile = null; - } + RpcWebsocketOptions rpcWebsocketOptions = new RpcWebsocketOptions(); // Privacy Options Group @CommandLine.ArgGroup(validate = false, heading = "@|bold Privacy Options|@%n") @@ -1820,6 +1734,7 @@ private void validateOptions() { validateDnsOptionsParams(); ensureValidPeerBoundParams(); validateRpcOptionsParams(); + validateRpcWsOptions(); validateChainDataPruningParams(); validatePostMergeCheckpointBlockRequirements(); validateTransactionPoolOptions(); @@ -1962,15 +1877,6 @@ private void validateRpcOptionsParams() { + invalidHttpApis.toString()); } - if (!jsonRPCWebsocketOptionGroup.rpcWsApis.stream().allMatch(configuredApis)) { - final List invalidWsApis = - new ArrayList(jsonRPCWebsocketOptionGroup.rpcWsApis); - invalidWsApis.removeAll(VALID_APIS); - throw new ParameterException( - this.commandLine, - "Invalid value for option '--rpc-ws-api': invalid entries found " + invalidWsApis); - } - final boolean validHttpApiMethods = jsonRPCHttpOptionGroup.rpcHttpApiMethodsNoAuth.stream() .allMatch(RpcMethod::rpcMethodExists); @@ -1980,16 +1886,15 @@ private void validateRpcOptionsParams() { this.commandLine, "Invalid value for option '--rpc-http-api-methods-no-auth', options must be valid RPC methods"); } + } - final boolean validWsApiMethods = - jsonRPCWebsocketOptionGroup.rpcWsApiMethodsNoAuth.stream() - .allMatch(RpcMethod::rpcMethodExists); - - if (!validWsApiMethods) { - throw new ParameterException( - this.commandLine, - "Invalid value for option '--rpc-ws-api-methods-no-auth', options must be valid RPC methods"); - } + private void validateRpcWsOptions() { + final Predicate configuredApis = + apiName -> + Arrays.stream(RpcApis.values()) + .anyMatch(builtInApi -> apiName.equals(builtInApi.name())) + || rpcEndpointServiceImpl.hasNamespace(apiName); + rpcWebsocketOptions.validate(logger, commandLine, configuredApis); } private void validateChainDataPruningParams() { @@ -2092,10 +1997,10 @@ private void configure() throws Exception { p2pTLSConfiguration = p2pTLSConfigOptions.p2pTLSConfiguration(commandLine); graphQLConfiguration = graphQLConfiguration(); webSocketConfiguration = - webSocketConfiguration( - jsonRPCWebsocketOptionGroup.rpcWsPort, - jsonRPCWebsocketOptionGroup.rpcWsApis, - hostsAllowlist); + rpcWebsocketOptions.webSocketConfiguration( + hostsAllowlist, + p2PDiscoveryOptionGroup.autoDiscoverDefaultIP().getHostAddress(), + unstableRPCOptions.getWsTimeoutSec()); jsonRpcIpcConfiguration = jsonRpcIpcConfiguration( unstableIpcOptions.isEnabled(), @@ -2464,71 +2369,6 @@ private boolean isRpcTlsConfigurationRequired() { return jsonRPCHttpOptionGroup.isRpcHttpEnabled && jsonRPCHttpOptionGroup.isRpcHttpTlsEnabled; } - private WebSocketConfiguration webSocketConfiguration( - final Integer listenPort, final List apiGroups, final List allowCallsFrom) { - - CommandLineUtils.checkOptionDependencies( - logger, - commandLine, - "--rpc-ws-enabled", - !jsonRPCWebsocketOptionGroup.isRpcWsEnabled, - asList( - "--rpc-ws-api", - "--rpc-ws-apis", - "--rpc-ws-api-method-no-auth", - "--rpc-ws-api-methods-no-auth", - "--rpc-ws-host", - "--rpc-ws-port", - "--rpc-ws-max-frame-size", - "--rpc-ws-max-active-connections", - "--rpc-ws-authentication-enabled", - "--rpc-ws-authentication-credentials-file", - "--rpc-ws-authentication-public-key-file", - "--rpc-ws-authentication-jwt-algorithm")); - - if (jsonRPCWebsocketOptionGroup.isRpcWsAuthenticationEnabled) { - CommandLineUtils.checkOptionDependencies( - logger, - commandLine, - "--rpc-ws-authentication-public-key-file", - jsonRPCWebsocketOptionGroup.rpcWsAuthenticationPublicKeyFile == null, - asList("--rpc-ws-authentication-jwt-algorithm")); - } - - if (jsonRPCWebsocketOptionGroup.isRpcWsAuthenticationEnabled - && rpcWsAuthenticationCredentialsFile() == null - && jsonRPCWebsocketOptionGroup.rpcWsAuthenticationPublicKeyFile == null) { - throw new ParameterException( - commandLine, - "Unable to authenticate JSON-RPC WebSocket endpoint without a supplied credentials file or authentication public key file"); - } - - final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); - webSocketConfiguration.setEnabled(jsonRPCWebsocketOptionGroup.isRpcWsEnabled); - webSocketConfiguration.setHost( - Strings.isNullOrEmpty(jsonRPCWebsocketOptionGroup.rpcWsHost) - ? p2PDiscoveryOptionGroup.autoDiscoverDefaultIP().getHostAddress() - : jsonRPCWebsocketOptionGroup.rpcWsHost); - webSocketConfiguration.setPort(listenPort); - webSocketConfiguration.setMaxFrameSize(jsonRPCWebsocketOptionGroup.rpcWsMaxFrameSize); - webSocketConfiguration.setMaxActiveConnections(jsonRPCWebsocketOptionGroup.rpcWsMaxConnections); - webSocketConfiguration.setRpcApis(apiGroups); - webSocketConfiguration.setRpcApisNoAuth( - jsonRPCWebsocketOptionGroup.rpcWsApiMethodsNoAuth.stream() - .distinct() - .collect(Collectors.toList())); - webSocketConfiguration.setAuthenticationEnabled( - jsonRPCWebsocketOptionGroup.isRpcWsAuthenticationEnabled); - webSocketConfiguration.setAuthenticationCredentialsFile(rpcWsAuthenticationCredentialsFile()); - webSocketConfiguration.setHostsAllowlist(allowCallsFrom); - webSocketConfiguration.setAuthenticationPublicKeyFile( - jsonRPCWebsocketOptionGroup.rpcWsAuthenticationPublicKeyFile); - webSocketConfiguration.setAuthenticationAlgorithm( - jsonRPCWebsocketOptionGroup.rpcWebsocketsAuthenticationAlgorithm); - webSocketConfiguration.setTimeoutSec(unstableRPCOptions.getWsTimeoutSec()); - return webSocketConfiguration; - } - private ApiConfiguration apiConfiguration() { checkApiOptionsDependencies(); var builder = @@ -2611,7 +2451,7 @@ public MetricsConfiguration metricsConfiguration() { private Optional permissioningConfiguration() throws Exception { if (!(localPermissionsEnabled() || contractPermissionsEnabled())) { if (jsonRPCHttpOptionGroup.rpcHttpApis.contains(RpcApis.PERM.name()) - || jsonRPCWebsocketOptionGroup.rpcWsApis.contains(RpcApis.PERM.name())) { + || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.PERM.name())) { logger.warn( "Permissions are disabled. Cannot enable PERM APIs when not using Permissions."); } @@ -2814,9 +2654,9 @@ private PrivacyParameters privacyParameters() { private boolean anyPrivacyApiEnabled() { return jsonRPCHttpOptionGroup.rpcHttpApis.contains(RpcApis.EEA.name()) - || jsonRPCWebsocketOptionGroup.rpcWsApis.contains(RpcApis.EEA.name()) + || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.EEA.name()) || jsonRPCHttpOptionGroup.rpcHttpApis.contains(RpcApis.PRIV.name()) - || jsonRPCWebsocketOptionGroup.rpcWsApis.contains(RpcApis.PRIV.name()); + || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.PRIV.name()); } private PrivacyKeyValueStorageProvider privacyKeyStorageProvider(final String name) { @@ -3191,15 +3031,6 @@ private String rpcHttpAuthenticationCredentialsFile() { return filename; } - private String rpcWsAuthenticationCredentialsFile() { - final String filename = jsonRPCWebsocketOptionGroup.rpcWsAuthenticationCredentialsFile; - - if (filename != null) { - RpcAuthFileValidator.validate(commandLine, filename, "WS"); - } - return filename; - } - private String getDefaultPermissioningFilePath() { return dataDir() + System.getProperty("file.separator") @@ -3337,9 +3168,7 @@ private List getEffectivePorts() { jsonRPCHttpOptionGroup.rpcHttpPort, jsonRPCHttpOptionGroup.isRpcHttpEnabled); addPortIfEnabled( - effectivePorts, - jsonRPCWebsocketOptionGroup.rpcWsPort, - jsonRPCWebsocketOptionGroup.isRpcWsEnabled); + effectivePorts, rpcWebsocketOptions.getRpcWsPort(), rpcWebsocketOptions.isRpcWsEnabled()); addPortIfEnabled(effectivePorts, engineRPCOptionGroup.engineRpcPort, isEngineApiEnabled()); addPortIfEnabled( effectivePorts, metricsOptionGroup.metricsPort, metricsOptionGroup.isMetricsEnabled); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/RpcWebsocketOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/RpcWebsocketOptions.java new file mode 100644 index 00000000000..cdc77a3f655 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/RpcWebsocketOptions.java @@ -0,0 +1,266 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.options.stable; + +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.VALID_APIS; +import static org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration.DEFAULT_WEBSOCKET_PORT; + +import org.hyperledger.besu.cli.DefaultCommandValues; +import org.hyperledger.besu.cli.custom.RpcAuthFileValidator; +import org.hyperledger.besu.cli.util.CommandLineUtils; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.authentication.JwtAlgorithm; +import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import picocli.CommandLine; + +/** This class represents the WebSocket options for the RPC. */ +public class RpcWebsocketOptions { + @CommandLine.Option( + names = {"--rpc-ws-authentication-jwt-algorithm"}, + description = + "Encryption algorithm used for Websockets JWT public key. Possible values are ${COMPLETION-CANDIDATES}" + + " (default: ${DEFAULT-VALUE})", + arity = "1") + private final JwtAlgorithm rpcWebsocketsAuthenticationAlgorithm = + DefaultCommandValues.DEFAULT_JWT_ALGORITHM; + + @CommandLine.Option( + names = {"--rpc-ws-enabled"}, + description = "Set to start the JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") + private final Boolean isRpcWsEnabled = false; + + @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. + @CommandLine.Option( + names = {"--rpc-ws-host"}, + paramLabel = DefaultCommandValues.MANDATORY_HOST_FORMAT_HELP, + description = "Host for JSON-RPC WebSocket service to listen on (default: ${DEFAULT-VALUE})", + arity = "1") + private String rpcWsHost; + + @CommandLine.Option( + names = {"--rpc-ws-port"}, + paramLabel = DefaultCommandValues.MANDATORY_PORT_FORMAT_HELP, + description = "Port for JSON-RPC WebSocket service to listen on (default: ${DEFAULT-VALUE})", + arity = "1") + private final Integer rpcWsPort = DEFAULT_WEBSOCKET_PORT; + + @CommandLine.Option( + names = {"--rpc-ws-max-frame-size"}, + description = + "Maximum size in bytes for JSON-RPC WebSocket frames (default: ${DEFAULT-VALUE}). If this limit is exceeded, the websocket will be disconnected.", + arity = "1") + private final Integer rpcWsMaxFrameSize = DefaultCommandValues.DEFAULT_WS_MAX_FRAME_SIZE; + + @CommandLine.Option( + names = {"--rpc-ws-max-active-connections"}, + description = + "Maximum number of WebSocket connections allowed for JSON-RPC (default: ${DEFAULT-VALUE}). Once this limit is reached, incoming connections will be rejected.", + arity = "1") + private final Integer rpcWsMaxConnections = DefaultCommandValues.DEFAULT_WS_MAX_CONNECTIONS; + + @CommandLine.Option( + names = {"--rpc-ws-api", "--rpc-ws-apis"}, + paramLabel = "", + split = " {0,1}, {0,1}", + arity = "1..*", + description = + "Comma separated list of APIs to enable on JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") + private final List rpcWsApis = DEFAULT_RPC_APIS; + + @CommandLine.Option( + names = {"--rpc-ws-api-methods-no-auth", "--rpc-ws-api-method-no-auth"}, + paramLabel = "", + split = " {0,1}, {0,1}", + arity = "1..*", + description = + "Comma separated list of RPC methods to exclude from RPC authentication services, RPC WebSocket authentication must be enabled") + private final List rpcWsApiMethodsNoAuth = new ArrayList(); + + @CommandLine.Option( + names = {"--rpc-ws-authentication-enabled"}, + description = + "Require authentication for the JSON-RPC WebSocket service (default: ${DEFAULT-VALUE})") + private final Boolean isRpcWsAuthenticationEnabled = false; + + @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. + @CommandLine.Option( + names = {"--rpc-ws-authentication-credentials-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = + "Storage file for JSON-RPC WebSocket authentication credentials (default: ${DEFAULT-VALUE})", + arity = "1") + private String rpcWsAuthenticationCredentialsFile = null; + + @CommandLine.Option( + names = {"--rpc-ws-authentication-jwt-public-key-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = "JWT public key file for JSON-RPC WebSocket authentication", + arity = "1") + private final File rpcWsAuthenticationPublicKeyFile = null; + + /** + * Validates the WebSocket options. + * + * @param logger Logger instance + * @param commandLine CommandLine instance + * @param configuredApis Predicate for configured APIs + */ + public void validate( + final Logger logger, final CommandLine commandLine, final Predicate configuredApis) { + checkOptionDependencies(logger, commandLine); + + if (!rpcWsApis.stream().allMatch(configuredApis)) { + final List invalidWsApis = new ArrayList<>(rpcWsApis); + invalidWsApis.removeAll(VALID_APIS); + throw new CommandLine.ParameterException( + commandLine, + "Invalid value for option '--rpc-ws-api': invalid entries found " + invalidWsApis); + } + + final boolean validWsApiMethods = + rpcWsApiMethodsNoAuth.stream().allMatch(RpcMethod::rpcMethodExists); + + if (!validWsApiMethods) { + throw new CommandLine.ParameterException( + commandLine, + "Invalid value for option '--rpc-ws-api-methods-no-auth', options must be valid RPC methods"); + } + + if (isRpcWsAuthenticationEnabled + && rpcWsAuthenticationCredentialsFile(commandLine) == null + && rpcWsAuthenticationPublicKeyFile == null) { + throw new CommandLine.ParameterException( + commandLine, + "Unable to authenticate JSON-RPC WebSocket endpoint without a supplied credentials file or authentication public key file"); + } + } + + /** + * Checks the dependencies of the WebSocket options. + * + * @param logger Logger instance + * @param commandLine CommandLine instance + */ + private void checkOptionDependencies(final Logger logger, final CommandLine commandLine) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-enabled", + !isRpcWsEnabled, + List.of( + "--rpc-ws-api", + "--rpc-ws-apis", + "--rpc-ws-api-method-no-auth", + "--rpc-ws-api-methods-no-auth", + "--rpc-ws-host", + "--rpc-ws-port", + "--rpc-ws-max-frame-size", + "--rpc-ws-max-active-connections", + "--rpc-ws-authentication-enabled", + "--rpc-ws-authentication-credentials-file", + "--rpc-ws-authentication-public-key-file", + "--rpc-ws-authentication-jwt-algorithm")); + + if (isRpcWsAuthenticationEnabled) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-ws-authentication-public-key-file", + rpcWsAuthenticationPublicKeyFile == null, + List.of("--rpc-ws-authentication-jwt-algorithm")); + } + } + + /** + * Creates a WebSocket configuration based on the WebSocket options. + * + * @param hostsAllowlist List of allowed hosts + * @param defaultHostAddress Default host address + * @param wsTimoutSec WebSocket timeout in seconds + * @return WebSocketConfiguration instance + */ + public WebSocketConfiguration webSocketConfiguration( + final List hostsAllowlist, final String defaultHostAddress, final Long wsTimoutSec) { + final WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); + webSocketConfiguration.setEnabled(isRpcWsEnabled); + webSocketConfiguration.setHost( + Strings.isNullOrEmpty(rpcWsHost) ? defaultHostAddress : rpcWsHost); + webSocketConfiguration.setPort(rpcWsPort); + webSocketConfiguration.setMaxFrameSize(rpcWsMaxFrameSize); + webSocketConfiguration.setMaxActiveConnections(rpcWsMaxConnections); + webSocketConfiguration.setRpcApis(rpcWsApis); + webSocketConfiguration.setRpcApisNoAuth( + rpcWsApiMethodsNoAuth.stream().distinct().collect(Collectors.toList())); + webSocketConfiguration.setAuthenticationEnabled(isRpcWsAuthenticationEnabled); + webSocketConfiguration.setAuthenticationCredentialsFile(rpcWsAuthenticationCredentialsFile); + webSocketConfiguration.setHostsAllowlist(hostsAllowlist); + webSocketConfiguration.setAuthenticationPublicKeyFile(rpcWsAuthenticationPublicKeyFile); + webSocketConfiguration.setAuthenticationAlgorithm(rpcWebsocketsAuthenticationAlgorithm); + webSocketConfiguration.setTimeoutSec(wsTimoutSec); + return webSocketConfiguration; + } + + /** + * Validates the authentication credentials file for the WebSocket. + * + * @param commandLine CommandLine instance + * @return Filename of the authentication credentials file + */ + private String rpcWsAuthenticationCredentialsFile(final CommandLine commandLine) { + final String filename = rpcWsAuthenticationCredentialsFile; + + if (filename != null) { + RpcAuthFileValidator.validate(commandLine, filename, "WS"); + } + return filename; + } + + /** + * Returns the list of APIs for the WebSocket. + * + * @return List of APIs + */ + public List getRpcWsApis() { + return rpcWsApis; + } + + /** + * Checks if the WebSocket service is enabled. + * + * @return Boolean indicating if the WebSocket service is enabled + */ + public Boolean isRpcWsEnabled() { + return isRpcWsEnabled; + } + + /** + * Returns the port for the WebSocket service. + * + * @return Port number + */ + public Integer getRpcWsPort() { + return rpcWsPort; + } +} diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 49e734b4647..938c25af9d9 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -27,7 +27,6 @@ import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; import static org.hyperledger.besu.cli.config.NetworkName.MORDOR; import static org.hyperledger.besu.cli.config.NetworkName.SEPOLIA; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ENGINE; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; @@ -2144,18 +2143,6 @@ public void rpcHttpNoAuthApiMethodsCannotBeInvalid() { "Invalid value for option '--rpc-http-api-methods-no-auth', options must be valid RPC methods"); } - @Test - public void rpcWsNoAuthApiMethodsCannotBeInvalid() { - parseCommand("--rpc-ws-enabled", "--rpc-ws-api-methods-no-auth", "invalid"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "Invalid value for option '--rpc-ws-api-methods-no-auth', options must be valid RPC methods"); - } - @Test public void rpcHttpOptionsRequiresServiceToBeEnabled() { parseCommand( @@ -2301,18 +2288,6 @@ public void rpcApisPropertyWithInvalidEntryMustDisplayError() { .contains("Invalid value for option '--rpc-http-api': invalid entries found [BOB]"); } - @Test - public void rpcWsApisPropertyWithInvalidEntryMustDisplayError() { - parseCommand("--rpc-ws-api", "ETH,BOB,TEST"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - - assertThat(commandErrorOutput.toString(UTF_8).trim()) - .contains("Invalid value for option '--rpc-ws-api': invalid entries found [BOB, TEST]"); - } - @Test public void rpcApisPropertyWithPluginNamespaceAreValid() { @@ -2394,35 +2369,6 @@ public void rpcHttpMaxActiveConnectionsPropertyMustBeUsed() { assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } - @Test - public void rpcWsMaxFrameSizePropertyMustBeUsed() { - final int maxFrameSize = 65535; - parseCommand("--rpc-ws-max-frame-size", String.valueOf(maxFrameSize)); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getMaxFrameSize()).isEqualTo(maxFrameSize); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsMaxActiveConnectionsPropertyMustBeUsed() { - final int maxConnections = 99; - parseCommand("--rpc-ws-max-active-connections", String.valueOf(maxConnections)); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getMaxActiveConnections()) - .isEqualTo(maxConnections); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - @Test public void rpcHttpTlsRequiresRpcHttpEnabled() { parseCommand("--rpc-http-tls-enabled"); @@ -3234,129 +3180,6 @@ public void rpcWsRpcEnabledPropertyDefaultIsFalse() { assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } - @Test - public void rpcWsRpcEnabledPropertyMustBeUsed() { - parseCommand("--rpc-ws-enabled"); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().isEnabled()).isTrue(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsOptionsRequiresServiceToBeEnabled() { - parseCommand( - "--rpc-ws-api", - "ETH,NET", - "--rpc-ws-host", - "0.0.0.0", - "--rpc-ws-port", - "1234", - "--rpc-ws-max-active-connections", - "77", - "--rpc-ws-max-frame-size", - "65535"); - - verifyOptionsConstraintLoggerCall( - "--rpc-ws-enabled", - "--rpc-ws-host", - "--rpc-ws-port", - "--rpc-ws-api", - "--rpc-ws-max-active-connections", - "--rpc-ws-max-frame-size"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsOptionsRequiresServiceToBeEnabledToml() throws IOException { - final Path toml = - createTempFile( - "toml", - "rpc-ws-api=[\"ETH\", \"NET\"]\n" - + "rpc-ws-host=\"0.0.0.0\"\n" - + "rpc-ws-port=1234\n" - + "rpc-ws-max-active-connections=77\n" - + "rpc-ws-max-frame-size=65535\n"); - - parseCommand("--config-file", toml.toString()); - - verifyOptionsConstraintLoggerCall( - "--rpc-ws-enabled", - "--rpc-ws-host", - "--rpc-ws-port", - "--rpc-ws-api", - "--rpc-ws-max-active-connections", - "--rpc-ws-max-frame-size"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsApiPropertyMustBeUsed() { - final TestBesuCommand command = parseCommand("--rpc-ws-enabled", "--rpc-ws-api", "ETH, NET"); - - assertThat(command).isNotNull(); - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getRpcApis()) - .containsExactlyInAnyOrder(ETH.name(), NET.name()); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsHostAndPortOptionMustBeUsed() { - final String host = "1.2.3.4"; - final int port = 1234; - parseCommand("--rpc-ws-enabled", "--rpc-ws-host", host, "--rpc-ws-port", String.valueOf(port)); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(wsRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsHostAndMayBeLocalhost() { - final String host = "localhost"; - parseCommand("--rpc-ws-enabled", "--rpc-ws-host", host); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcWsHostAndMayBeIPv6() { - final String host = "2600:DB8::8545"; - parseCommand("--rpc-ws-enabled", "--rpc-ws-host", host); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - @Test public void metricsEnabledPropertyDefaultIsFalse() { parseCommand(); @@ -4146,31 +3969,6 @@ public void canNotUseFlexiblePrivacyWhenPrivacyPluginEnabled() { "No Payload Provider has been provided. You must register one when enabling privacy plugin!"); } - /** - * Check logger calls - * - *

Here we check the calls to logger and not the result of the log line as we don't test the - * logger itself but the fact that we call it. - * - * @param dependentOptions the string representing the list of dependent options names - * @param mainOption the main option name - */ - private void verifyOptionsConstraintLoggerCall( - final String mainOption, final String... dependentOptions) { - verify(mockLogger, atLeast(1)) - .warn( - stringArgumentCaptor.capture(), - stringArgumentCaptor.capture(), - stringArgumentCaptor.capture()); - assertThat(stringArgumentCaptor.getAllValues().get(0)).isEqualTo(DEPENDENCY_WARNING_MSG); - - for (final String option : dependentOptions) { - assertThat(stringArgumentCaptor.getAllValues().get(1)).contains(option); - } - - assertThat(stringArgumentCaptor.getAllValues().get(2)).isEqualTo(mainOption); - } - /** * Check logger calls * @@ -4360,17 +4158,6 @@ public void httpAuthenticationAlgorithIsConfigured() { .isEqualTo(JwtAlgorithm.ES256); } - @Test - public void webSocketAuthenticationAlgorithIsConfigured() { - parseCommand("--rpc-ws-authentication-jwt-algorithm", "ES256"); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getAuthenticationAlgorithm()) - .isEqualTo(JwtAlgorithm.ES256); - } - @Test public void httpAuthenticationPublicKeyIsConfigured() throws IOException { final Path publicKey = Files.createTempFile("public_key", ""); @@ -4394,29 +4181,6 @@ public void httpAuthenticationWithoutRequiredConfiguredOptionsMustFail() { "Unable to authenticate JSON-RPC HTTP endpoint without a supplied credentials file or authentication public key file"); } - @Test - public void wsAuthenticationPublicKeyIsConfigured() throws IOException { - final Path publicKey = Files.createTempFile("public_key", ""); - parseCommand("--rpc-ws-authentication-jwt-public-key-file", publicKey.toString()); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().getAuthenticationPublicKeyFile().getPath()) - .isEqualTo(publicKey.toString()); - } - - @Test - public void wsAuthenticationWithoutRequiredConfiguredOptionsMustFail() { - parseCommand("--rpc-ws-enabled", "--rpc-ws-authentication-enabled"); - - verifyNoInteractions(mockRunnerBuilder); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "Unable to authenticate JSON-RPC WebSocket endpoint without a supplied credentials file or authentication public key file"); - } - @Test public void privHttpApisWithPrivacyDisabledLogsWarning() { parseCommand("--privacy-enabled=false", "--rpc-http-api", "PRIV", "--rpc-http-enabled"); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index 0d2fb39427e..2c5f68be8c5 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -15,14 +15,18 @@ package org.hyperledger.besu.cli; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.hyperledger.besu.Runner; @@ -687,4 +691,29 @@ private enum TestType { protected static String escapeTomlString(final String s) { return StringEscapeUtils.escapeJava(s); } + + /** + * Check logger calls + * + *

Here we check the calls to logger and not the result of the log line as we don't test the + * logger itself but the fact that we call it. + * + * @param dependentOptions the string representing the list of dependent options names + * @param mainOption the main option name + */ + protected void verifyOptionsConstraintLoggerCall( + final String mainOption, final String... dependentOptions) { + verify(mockLogger, atLeast(1)) + .warn( + stringArgumentCaptor.capture(), + stringArgumentCaptor.capture(), + stringArgumentCaptor.capture()); + assertThat(stringArgumentCaptor.getAllValues().get(0)).isEqualTo(DEPENDENCY_WARNING_MSG); + + for (final String option : dependentOptions) { + assertThat(stringArgumentCaptor.getAllValues().get(1)).contains(option); + } + + assertThat(stringArgumentCaptor.getAllValues().get(2)).isEqualTo(mainOption); + } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java index 35dcaa4baee..92b190d74bf 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java @@ -16,8 +16,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; -import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.verify; import org.hyperledger.besu.cli.CommandTestAbstract; @@ -125,29 +123,4 @@ protected void internalTestFailure(final String errorMsg, final String... args) assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).contains(errorMsg); } - - /** - * Check logger calls - * - *

Here we check the calls to logger and not the result of the log line as we don't test the - * logger itself but the fact that we call it. - * - * @param dependentOptions the string representing the list of dependent options names - * @param mainOption the main option name - */ - protected void verifyOptionsConstraintLoggerCall( - final String mainOption, final String... dependentOptions) { - verify(mockLogger, atLeast(1)) - .warn( - stringArgumentCaptor.capture(), - stringArgumentCaptor.capture(), - stringArgumentCaptor.capture()); - assertThat(stringArgumentCaptor.getAllValues().get(0)).isEqualTo(DEPENDENCY_WARNING_MSG); - - for (final String option : dependentOptions) { - assertThat(stringArgumentCaptor.getAllValues().get(1)).contains(option); - } - - assertThat(stringArgumentCaptor.getAllValues().get(2)).isEqualTo(mainOption); - } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/RpcWebsocketOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/RpcWebsocketOptionsTest.java new file mode 100644 index 00000000000..0aaf01622e7 --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/RpcWebsocketOptionsTest.java @@ -0,0 +1,249 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.options; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import org.hyperledger.besu.cli.CommandTestAbstract; +import org.hyperledger.besu.ethereum.api.jsonrpc.authentication.JwtAlgorithm; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class RpcWebsocketOptionsTest extends CommandTestAbstract { + + @Test + public void rpcWsApiPropertyMustBeUsed() { + final CommandTestAbstract.TestBesuCommand command = + parseCommand("--rpc-ws-enabled", "--rpc-ws-api", "ETH, NET"); + + assertThat(command).isNotNull(); + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().getRpcApis()) + .containsExactlyInAnyOrder(ETH.name(), NET.name()); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcWsHostAndPortOptionMustBeUsed() { + final String host = "1.2.3.4"; + final int port = 1234; + parseCommand("--rpc-ws-enabled", "--rpc-ws-host", host, "--rpc-ws-port", String.valueOf(port)); + + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + assertThat(wsRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcWsMaxFrameSizePropertyMustBeUsed() { + final int maxFrameSize = 65535; + parseCommand("--rpc-ws-max-frame-size", String.valueOf(maxFrameSize)); + + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().getMaxFrameSize()).isEqualTo(maxFrameSize); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcWsMaxActiveConnectionsPropertyMustBeUsed() { + final int maxConnections = 99; + parseCommand("--rpc-ws-max-active-connections", String.valueOf(maxConnections)); + + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().getMaxActiveConnections()) + .isEqualTo(maxConnections); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcWsRpcEnabledPropertyMustBeUsed() { + parseCommand("--rpc-ws-enabled"); + + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().isEnabled()).isTrue(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void webSocketAuthenticationAlgorithmIsConfigured() { + parseCommand("--rpc-ws-authentication-jwt-algorithm", "ES256"); + + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().getAuthenticationAlgorithm()) + .isEqualTo(JwtAlgorithm.ES256); + } + + @Test + public void wsAuthenticationPublicKeyIsConfigured() throws IOException { + final Path publicKey = Files.createTempFile("public_key", ""); + parseCommand("--rpc-ws-authentication-jwt-public-key-file", publicKey.toString()); + + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().getAuthenticationPublicKeyFile().getPath()) + .isEqualTo(publicKey.toString()); + } + + @Test + public void rpcWsHostAndMayBeLocalhost() { + final String host = "localhost"; + parseCommand("--rpc-ws-enabled", "--rpc-ws-host", host); + + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcWsHostAndMayBeIPv6() { + final String host = "2600:DB8::8545"; + parseCommand("--rpc-ws-enabled", "--rpc-ws-host", host); + + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcWsNoAuthApiMethodsCannotBeInvalid() { + parseCommand("--rpc-ws-enabled", "--rpc-ws-api-methods-no-auth", "invalid"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains( + "Invalid value for option '--rpc-ws-api-methods-no-auth', options must be valid RPC methods"); + } + + @Test + public void rpcWsApisPropertyWithInvalidEntryMustDisplayError() { + parseCommand("--rpc-ws-api", "ETH,BOB,TEST"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + + assertThat(commandErrorOutput.toString(UTF_8).trim()) + .contains("Invalid value for option '--rpc-ws-api': invalid entries found [BOB, TEST]"); + } + + @Test + public void rpcWsOptionsRequiresServiceToBeEnabled() { + parseCommand( + "--rpc-ws-api", + "ETH,NET", + "--rpc-ws-host", + "0.0.0.0", + "--rpc-ws-port", + "1234", + "--rpc-ws-max-active-connections", + "77", + "--rpc-ws-max-frame-size", + "65535"); + + verifyOptionsConstraintLoggerCall( + "--rpc-ws-enabled", + "--rpc-ws-host", + "--rpc-ws-port", + "--rpc-ws-api", + "--rpc-ws-max-active-connections", + "--rpc-ws-max-frame-size"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcWsOptionsRequiresServiceToBeEnabledToml() throws IOException { + final Path toml = + createTempFile( + "toml", + "rpc-ws-api=[\"ETH\", \"NET\"]\n" + + "rpc-ws-host=\"0.0.0.0\"\n" + + "rpc-ws-port=1234\n" + + "rpc-ws-max-active-connections=77\n" + + "rpc-ws-max-frame-size=65535\n"); + + parseCommand("--config-file", toml.toString()); + + verifyOptionsConstraintLoggerCall( + "--rpc-ws-enabled", + "--rpc-ws-host", + "--rpc-ws-port", + "--rpc-ws-api", + "--rpc-ws-max-active-connections", + "--rpc-ws-max-frame-size"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void wsAuthenticationWithoutRequiredConfiguredOptionsMustFail() { + parseCommand("--rpc-ws-enabled", "--rpc-ws-authentication-enabled"); + + verifyNoInteractions(mockRunnerBuilder); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains( + "Unable to authenticate JSON-RPC WebSocket endpoint without a supplied credentials file or authentication public key file"); + } +} From 6ac419bf0b18e2ca830ca81f00e6d23acb410a98 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Thu, 25 Jan 2024 17:01:23 +1000 Subject: [PATCH 43/56] Skip pruning if trie log count is less than retention limit (#6463) Rework error messaging in this case. Signed-off-by: Simon Dudley --- .../subcommands/storage/TrieLogHelper.java | 73 +++++++++++---- .../storage/TrieLogHelperTest.java | 93 ++++++++++++++++--- 2 files changed, 135 insertions(+), 31 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 88d876cce2e..22efd97c86d 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -16,7 +16,9 @@ package org.hyperledger.besu.cli.subcommands.storage; import static com.google.common.base.Preconditions.checkArgument; +import static org.hyperledger.besu.cli.options.stable.DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; import org.hyperledger.besu.cli.options.stable.DataStorageOptions; import org.hyperledger.besu.datatypes.Hash; @@ -74,11 +76,16 @@ void prune( final long lastBlockNumberToRetainTrieLogsFor = chainHeight - layersToRetain + 1; - if (!validPruneRequirements(blockchain, chainHeight, lastBlockNumberToRetainTrieLogsFor)) { + if (!validatePruneRequirements( + blockchain, + chainHeight, + lastBlockNumberToRetainTrieLogsFor, + rootWorldStateStorage, + layersToRetain)) { return; } - final long numberOfBatches = calculateNumberofBatches(layersToRetain); + final long numberOfBatches = calculateNumberOfBatches(layersToRetain); processTrieLogBatches( rootWorldStateStorage, @@ -88,11 +95,23 @@ void prune( numberOfBatches, batchFileNameBase); - if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { - deleteFiles(batchFileNameBase, numberOfBatches); - LOG.info("Prune ran successfully. Enjoy some disk space back! \uD83D\uDE80"); + // Should only be layersToRetain left but loading extra just in case of an unforeseen bug + final long countAfterPrune = + rootWorldStateStorage + .streamTrieLogKeys(layersToRetain + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE) + .count(); + if (countAfterPrune == layersToRetain) { + if (deleteFiles(batchFileNameBase, numberOfBatches)) { + LOG.info("Prune ran successfully. Enjoy some disk space back! \uD83D\uDE80"); + } else { + throw new IllegalStateException( + "There was an error deleting the trie log backup files. Please ensure besu is working before deleting them manually."); + } } else { - LOG.error("Prune failed. Re-run the subcommand to load the trie logs from file."); + throw new IllegalStateException( + String.format( + "Remaining trie logs (%d) did not match %s (%d). Trie logs backup files have not been deleted, it is safe to rerun the subcommand.", + countAfterPrune, BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, layersToRetain)); } } @@ -151,15 +170,21 @@ private void restoreTrieLogBatches( } } - private void deleteFiles(final String batchFileNameBase, final long numberOfBatches) { + private boolean deleteFiles(final String batchFileNameBase, final long numberOfBatches) { LOG.info("Deleting files..."); - for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - File file = new File(batchFileNameBase + "-" + batchNumber); - if (file.exists()) { - file.delete(); + try { + for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { + File file = new File(batchFileNameBase + "-" + batchNumber); + if (file.exists()) { + file.delete(); + } } + return true; + } catch (Exception e) { + LOG.error("Error deleting files", e); + return false; } } @@ -177,14 +202,17 @@ private List getTrieLogKeysForBlocks( return trieLogKeys; } - private long calculateNumberofBatches(final long layersToRetain) { + private long calculateNumberOfBatches(final long layersToRetain) { return layersToRetain / BATCH_SIZE + ((layersToRetain % BATCH_SIZE == 0) ? 0 : 1); } - private boolean validPruneRequirements( + private boolean validatePruneRequirements( final MutableBlockchain blockchain, final long chainHeight, - final long lastBlockNumberToRetainTrieLogsFor) { + final long lastBlockNumberToRetainTrieLogsFor, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final long layersToRetain) { + if (lastBlockNumberToRetainTrieLogsFor < 0) { throw new IllegalArgumentException( "Trying to retain more trie logs than chain length (" @@ -192,6 +220,19 @@ private boolean validPruneRequirements( + "), skipping pruning"); } + // Need to ensure we're loading at least layersToRetain if they exist + // plus extra threshold to account forks and orphans + final long clampedCountBeforePruning = + rootWorldStateStorage + .streamTrieLogKeys(layersToRetain + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE) + .count(); + if (clampedCountBeforePruning < layersToRetain) { + throw new IllegalArgumentException( + String.format( + "Trie log count (%d) is less than retention limit (%d), skipping pruning", + clampedCountBeforePruning, layersToRetain)); + } + final Optional finalizedBlockHash = blockchain.getFinalized(); if (finalizedBlockHash.isEmpty()) { @@ -250,7 +291,7 @@ void validatePruneConfiguration(final DataStorageConfiguration config) { config.getBonsaiMaxLayersToLoad() >= DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT, String.format( - DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + " minimum value is %d", DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT)); checkArgument( config.getUnstable().getBonsaiTrieLogPruningWindowSize() > 0, @@ -264,7 +305,7 @@ void validatePruneConfiguration(final DataStorageConfiguration config) { String.format( DataStorageOptions.Unstable.BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE + "=%d must be greater than " - + DataStorageOptions.BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + + BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD + "=%d", config.getUnstable().getBonsaiTrieLogPruningWindowSize(), config.getBonsaiMaxLayersToLoad())); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index ca856ba0fad..22a7c37523d 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -18,8 +18,10 @@ import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; @@ -44,6 +46,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.BeforeEach; @@ -58,7 +61,7 @@ class TrieLogHelperTest { private static final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private static BonsaiWorldStateKeyValueStorage inMemoryWorldState; - private TrieLogHelper trieLogHelper; + private TrieLogHelper nonValidatingTrieLogHelper; private static class NonValidatingTrieLogHelper extends TrieLogHelper { @Override @@ -106,7 +109,7 @@ public void setup() throws IOException { .put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5)); updater.getTrieLogStorageTransaction().commit(); - trieLogHelper = new NonValidatingTrieLogHelper(); + nonValidatingTrieLogHelper = new NonValidatingTrieLogHelper(); } private static byte[] createTrieLog(final BlockHeader blockHeader) { @@ -150,7 +153,8 @@ public void prune(final @TempDir Path dataDir) throws IOException { assertThat(inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()) .isEqualTo(createTrieLog(blockHeader3)); - trieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); // assert pruned trie logs are not in the DB assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash())).isEqualTo(Optional.empty()); @@ -182,7 +186,7 @@ public void cannotPruneIfNoFinalizedIsFound() { assertThatThrownBy( () -> - trieLogHelper.prune( + nonValidatingTrieLogHelper.prune( dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) .isInstanceOf(RuntimeException.class) .hasMessage("No finalized block present, can't safely run trie log prune"); @@ -204,7 +208,7 @@ public void cannotPruneIfUserRetainsMoreLayersThanExistingChainLength() { assertThatThrownBy( () -> - trieLogHelper.prune( + nonValidatingTrieLogHelper.prune( dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Trying to retain more trie logs than chain length (5), skipping pruning"); @@ -227,13 +231,70 @@ public void cannotPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path da assertThatThrownBy( () -> - trieLogHelper.prune( + nonValidatingTrieLogHelper.prune( dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)) .isInstanceOf(IllegalArgumentException.class) .hasMessage( "Trying to prune more layers than the finalized block height, skipping pruning"); } + @Test + public void skipPruningIfTrieLogCountIsLessThanMaxLayersToLoad() { + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(6L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .build()) + .build(); + + when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); + + assertThatThrownBy( + () -> + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldState, blockchain, Path.of(""))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Trie log count (5) is less than retention limit (6), skipping pruning"); + } + + @Test + public void mismatchInPrunedTrieLogCountShouldNotDeleteFiles(final @TempDir Path dataDir) + throws IOException { + Files.createDirectories(dataDir.resolve("database")); + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(3L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiLimitTrieLogsEnabled(true) + .build()) + .build(); + + mockBlockchainBase(); + when(blockchain.getBlockHeader(5)).thenReturn(Optional.of(blockHeader5)); + when(blockchain.getBlockHeader(4)).thenReturn(Optional.of(blockHeader4)); + when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3)); + + final BonsaiWorldStateKeyValueStorage inMemoryWorldStateSpy = spy(inMemoryWorldState); + // force a different value the second time the trie log count is called + when(inMemoryWorldStateSpy.streamTrieLogKeys(3L + DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE)) + .thenCallRealMethod() + .thenReturn(Stream.empty()); + assertThatThrownBy( + () -> + nonValidatingTrieLogHelper.prune( + dataStorageConfiguration, inMemoryWorldStateSpy, blockchain, dataDir)) + .isInstanceOf(RuntimeException.class) + .hasMessage( + "Remaining trie logs (0) did not match --bonsai-historical-block-limit (3). Trie logs backup files have not been deleted, it is safe to rerun the subcommand."); + } + @Test public void trieLogRetentionLimitShouldBeAboveMinimum() { @@ -319,7 +380,7 @@ public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDi assertThatThrownBy( () -> - trieLogHelper.prune( + nonValidatingTrieLogHelper.prune( dataStorageConfiguration, inMemoryWorldState, blockchain, @@ -342,13 +403,13 @@ public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDi @Test public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { - trieLogHelper.exportTrieLog( + nonValidatingTrieLogHelper.exportTrieLog( inMemoryWorldState, singletonList(blockHeader1.getHash()), dataDir.resolve("trie-log-dump")); var trieLog = - trieLogHelper + nonValidatingTrieLogHelper .readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) .entrySet() .stream() @@ -362,13 +423,13 @@ public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOE @Test public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { - trieLogHelper.exportTrieLog( + nonValidatingTrieLogHelper.exportTrieLog( inMemoryWorldState, List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), dataDir.resolve("trie-log-dump")); var trieLogs = - trieLogHelper + nonValidatingTrieLogHelper .readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) .entrySet() .stream() @@ -389,13 +450,14 @@ public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws new BonsaiWorldStateKeyValueStorage( tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); - trieLogHelper.exportTrieLog( + nonValidatingTrieLogHelper.exportTrieLog( inMemoryWorldState, singletonList(blockHeader1.getHash()), dataDir.resolve("trie-log-dump")); var trieLog = - trieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + nonValidatingTrieLogHelper.readTrieLogsAsRlpFromFile( + dataDir.resolve("trie-log-dump").toString()); var updater = inMemoryWorldState2.updater(); trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); @@ -413,13 +475,14 @@ public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) th new BonsaiWorldStateKeyValueStorage( tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); - trieLogHelper.exportTrieLog( + nonValidatingTrieLogHelper.exportTrieLog( inMemoryWorldState, List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), dataDir.resolve("trie-log-dump")); var trieLog = - trieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + nonValidatingTrieLogHelper.readTrieLogsAsRlpFromFile( + dataDir.resolve("trie-log-dump").toString()); var updater = inMemoryWorldState2.updater(); trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); From 16016ece0a2f8abe733e832c53f49c591549762a Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 25 Jan 2024 10:38:27 +0100 Subject: [PATCH 44/56] Fix `poa-block-txs-selection-max-time` option that was inadvertently reset to its default after being configured (#6444) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 31 +++++----- .../besu/cli/options/MiningOptions.java | 60 ++++++++++++------- .../TomlConfigurationDefaultProvider.java | 4 +- .../besu/cli/CommandTestAbstract.java | 27 ++++++++- .../cli/options/AbstractCLIOptionsTest.java | 17 ++++-- .../besu/cli/options/MiningOptionsTest.java | 27 +++++++-- .../cli/options/SynchronizerOptionsTest.java | 4 +- 8 files changed, 119 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f966114ed92..57416d07852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ### Bug fixes - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) +- Fix `poa-block-txs-selection-max-time` option that was inadvertently reset to its default after being configured [#6444](https://github.com/hyperledger/besu/pull/6444) ### Download Links diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index d30bb421470..b9a137cbc17 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -124,7 +124,6 @@ import org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration; import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration; import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; @@ -225,6 +224,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.TreeMap; import java.util.function.Function; @@ -2758,32 +2758,28 @@ private TransactionPoolConfiguration buildTransactionPoolConfiguration() { private MiningParameters getMiningParameters() { if (miningParameters == null) { - final var miningParametersBuilder = - ImmutableMiningParameters.builder().from(miningOptions.toDomainObject()); - final var actualGenesisOptions = getActualGenesisConfigOptions(); - if (actualGenesisOptions.isPoa()) { - miningParametersBuilder.genesisBlockPeriodSeconds( - getGenesisBlockPeriodSeconds(actualGenesisOptions)); - } - miningParameters = miningParametersBuilder.build(); + miningOptions.setGenesisBlockPeriodSeconds( + getGenesisBlockPeriodSeconds(getActualGenesisConfigOptions())); + miningParameters = miningOptions.toDomainObject(); } return miningParameters; } - private int getGenesisBlockPeriodSeconds(final GenesisConfigOptions genesisConfigOptions) { + private OptionalInt getGenesisBlockPeriodSeconds( + final GenesisConfigOptions genesisConfigOptions) { if (genesisConfigOptions.isClique()) { - return genesisConfigOptions.getCliqueConfigOptions().getBlockPeriodSeconds(); + return OptionalInt.of(genesisConfigOptions.getCliqueConfigOptions().getBlockPeriodSeconds()); } if (genesisConfigOptions.isIbft2()) { - return genesisConfigOptions.getBftConfigOptions().getBlockPeriodSeconds(); + return OptionalInt.of(genesisConfigOptions.getBftConfigOptions().getBlockPeriodSeconds()); } if (genesisConfigOptions.isQbft()) { - return genesisConfigOptions.getQbftConfigOptions().getBlockPeriodSeconds(); + return OptionalInt.of(genesisConfigOptions.getQbftConfigOptions().getBlockPeriodSeconds()); } - throw new IllegalArgumentException("Should only be called for a PoA network"); + return OptionalInt.empty(); } private boolean isPruningEnabled() { @@ -3246,7 +3242,12 @@ private Optional getEcCurveFromGenesisFile() { return genesisConfigOptions.getEcCurve(); } - private GenesisConfigOptions getActualGenesisConfigOptions() { + /** + * Return the genesis config options after applying any specified config overrides + * + * @return the genesis config options after applying any specified config overrides + */ + protected GenesisConfigOptions getActualGenesisConfigOptions() { return Optional.ofNullable(genesisConfigOptions) .orElseGet( () -> diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java index 55197b8e826..2e5a239f16d 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java @@ -40,6 +40,7 @@ import org.hyperledger.besu.util.number.PositiveNumber; import java.util.List; +import java.util.OptionalInt; import org.apache.tuweni.bytes.Bytes; import org.slf4j.Logger; @@ -188,6 +189,8 @@ static class Unstable { DEFAULT_POS_BLOCK_CREATION_REPETITION_MIN_DURATION; } + private OptionalInt maybeGenesisBlockPeriodSeconds; + private MiningOptions() {} /** @@ -199,6 +202,16 @@ public static MiningOptions create() { return new MiningOptions(); } + /** + * Set the optional genesis block period per seconds + * + * @param genesisBlockPeriodSeconds if the network is PoA then the block period in seconds + * specified in the genesis file, otherwise empty. + */ + public void setGenesisBlockPeriodSeconds(final OptionalInt genesisBlockPeriodSeconds) { + maybeGenesisBlockPeriodSeconds = genesisBlockPeriodSeconds; + } + /** * Validate that there are no inconsistencies in the specified options. For example that the * options are valid for the selected implementation. @@ -285,6 +298,7 @@ public void validate( static MiningOptions fromConfig(final MiningParameters miningParameters) { final MiningOptions miningOptions = MiningOptions.create(); + miningOptions.setGenesisBlockPeriodSeconds(miningParameters.getGenesisBlockPeriodSeconds()); miningOptions.isMiningEnabled = miningParameters.isMiningEnabled(); miningOptions.iStratumMiningEnabled = miningParameters.isStratumMiningEnabled(); miningOptions.stratumNetworkInterface = miningParameters.getStratumNetworkInterface(); @@ -319,6 +333,11 @@ static MiningOptions fromConfig(final MiningParameters miningParameters) { @Override public MiningParameters toDomainObject() { + if (maybeGenesisBlockPeriodSeconds == null) { + throw new IllegalStateException( + "genesisBlockPeriodSeconds must be set before using this object"); + } + final var updatableInitValuesBuilder = MutableInitValues.builder() .isMiningEnabled(isMiningEnabled) @@ -334,27 +353,26 @@ public MiningParameters toDomainObject() { updatableInitValuesBuilder.coinbase(coinbase); } - final var miningParametersBuilder = - ImmutableMiningParameters.builder() - .mutableInitValues(updatableInitValuesBuilder.build()) - .isStratumMiningEnabled(iStratumMiningEnabled) - .stratumNetworkInterface(stratumNetworkInterface) - .stratumPort(stratumPort) - .nonPoaBlockTxsSelectionMaxTime(nonPoaBlockTxsSelectionMaxTime) - .poaBlockTxsSelectionMaxTime(poaBlockTxsSelectionMaxTime) - .unstable( - ImmutableMiningParameters.Unstable.builder() - .remoteSealersLimit(unstableOptions.remoteSealersLimit) - .remoteSealersTimeToLive(unstableOptions.remoteSealersTimeToLive) - .powJobTimeToLive(unstableOptions.powJobTimeToLive) - .maxOmmerDepth(unstableOptions.maxOmmersDepth) - .stratumExtranonce(unstableOptions.stratumExtranonce) - .posBlockCreationMaxTime(unstableOptions.posBlockCreationMaxTime) - .posBlockCreationRepetitionMinDuration( - unstableOptions.posBlockCreationRepetitionMinDuration) - .build()); - - return miningParametersBuilder.build(); + return ImmutableMiningParameters.builder() + .genesisBlockPeriodSeconds(maybeGenesisBlockPeriodSeconds) + .mutableInitValues(updatableInitValuesBuilder.build()) + .isStratumMiningEnabled(iStratumMiningEnabled) + .stratumNetworkInterface(stratumNetworkInterface) + .stratumPort(stratumPort) + .nonPoaBlockTxsSelectionMaxTime(nonPoaBlockTxsSelectionMaxTime) + .poaBlockTxsSelectionMaxTime(poaBlockTxsSelectionMaxTime) + .unstable( + ImmutableMiningParameters.Unstable.builder() + .remoteSealersLimit(unstableOptions.remoteSealersLimit) + .remoteSealersTimeToLive(unstableOptions.remoteSealersTimeToLive) + .powJobTimeToLive(unstableOptions.powJobTimeToLive) + .maxOmmerDepth(unstableOptions.maxOmmersDepth) + .stratumExtranonce(unstableOptions.stratumExtranonce) + .posBlockCreationMaxTime(unstableOptions.posBlockCreationMaxTime) + .posBlockCreationRepetitionMinDuration( + unstableOptions.posBlockCreationRepetitionMinDuration) + .build()) + .build(); } @Override diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java b/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java index 4928d3f7ff6..a346e260d13 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.util.number.Fraction; import org.hyperledger.besu.util.number.Percentage; +import org.hyperledger.besu.util.number.PositiveNumber; import java.io.File; import java.io.FileInputStream; @@ -129,7 +130,8 @@ private boolean isNumericType(final Class type) { || type.equals(Float.class) || type.equals(float.class) || type.equals(Percentage.class) - || type.equals(Fraction.class); + || type.equals(Fraction.class) + || type.equals(PositiveNumber.class); } private String getEntryAsString(final OptionSpec spec) { diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index 2c5f68be8c5..1609758b966 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -44,6 +44,7 @@ import org.hyperledger.besu.cli.options.unstable.NetworkingOptions; import org.hyperledger.besu.cli.options.unstable.SynchronizerOptions; import org.hyperledger.besu.components.BesuComponent; +import org.hyperledger.besu.config.GenesisConfigOptions; import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfiguration; import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfigurationProvider; import org.hyperledger.besu.controller.BesuController; @@ -133,20 +134,37 @@ @ExtendWith(MockitoExtension.class) public abstract class CommandTestAbstract { private static final Logger TEST_LOGGER = LoggerFactory.getLogger(CommandTestAbstract.class); + + protected static final int POA_BLOCK_PERIOD_SECONDS = 5; protected static final JsonObject VALID_GENESIS_QBFT_POST_LONDON = (new JsonObject()) .put( "config", new JsonObject() .put("londonBlock", 0) - .put("qbft", new JsonObject().put("blockperiodseconds", 5))); + .put( + "qbft", + new JsonObject().put("blockperiodseconds", POA_BLOCK_PERIOD_SECONDS))); protected static final JsonObject VALID_GENESIS_IBFT2_POST_LONDON = (new JsonObject()) .put( "config", new JsonObject() .put("londonBlock", 0) - .put("ibft2", new JsonObject().put("blockperiodseconds", 5))); + .put( + "ibft2", + new JsonObject().put("blockperiodseconds", POA_BLOCK_PERIOD_SECONDS))); + + protected static final JsonObject VALID_GENESIS_CLIQUE_POST_LONDON = + (new JsonObject()) + .put( + "config", + new JsonObject() + .put("londonBlock", 0) + .put( + "clique", + new JsonObject().put("blockperiodseconds", POA_BLOCK_PERIOD_SECONDS))); + protected final PrintStream originalOut = System.out; protected final PrintStream originalErr = System.err; protected final ByteArrayOutputStream commandOutput = new ByteArrayOutputStream(); @@ -561,6 +579,11 @@ protected Vertx createVertx(final VertxOptions vertxOptions) { return vertx; } + @Override + public GenesisConfigOptions getActualGenesisConfigOptions() { + return super.getActualGenesisConfigOptions(); + } + public CommandSpec getSpec() { return spec; } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java index 92b190d74bf..21d8baf9ee9 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/AbstractCLIOptionsTest.java @@ -67,7 +67,10 @@ private void getCLIOptions(final D domainObject) { final TestBesuCommand cmd = parseCommand(cliOptions); final T optionsFromCommand = getOptionsFromBesuCommand(cmd); - assertThat(optionsFromCommand).usingRecursiveComparison().isEqualTo(options); + assertThat(optionsFromCommand) + .usingRecursiveComparison() + .ignoringFields(getNonOptionFields()) + .isEqualTo(options); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); @@ -82,10 +85,10 @@ public void defaultValues() { final T optionsFromCommand = getOptionsFromBesuCommand(cmd); // Check default values supplied by CLI match expected default values - final String[] fieldsToIgnore = getFieldsWithComputedDefaults().toArray(new String[0]); assertThat(optionsFromCommand) .usingRecursiveComparison() - .ignoringFields(fieldsToIgnore) + .ignoringFields(getFieldsWithComputedDefaults()) + .ignoringFields(getNonOptionFields()) .isEqualTo(defaultOptions); } @@ -93,8 +96,12 @@ public void defaultValues() { protected abstract D createCustomizedDomainObject(); - protected List getFieldsWithComputedDefaults() { - return Collections.emptyList(); + protected String[] getFieldsWithComputedDefaults() { + return new String[0]; + } + + protected String[] getNonOptionFields() { + return new String[0]; } protected List getFieldsToIgnore() { diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java index 4b1fdddb53a..74b696cdf9a 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/MiningOptionsTest.java @@ -32,7 +32,9 @@ import java.io.IOException; import java.nio.file.Path; +import java.time.Duration; import java.util.Optional; +import java.util.OptionalInt; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; @@ -361,13 +363,16 @@ public void poaBlockTxsSelectionMaxTimeOption() throws IOException { @Test public void poaBlockTxsSelectionMaxTimeOptionOver100Percent() throws IOException { - final Path genesisFileIBFT2 = createFakeGenesisFile(VALID_GENESIS_IBFT2_POST_LONDON); + final Path genesisFileClique = createFakeGenesisFile(VALID_GENESIS_CLIQUE_POST_LONDON); internalTestSuccess( - miningParams -> - assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) - .isEqualTo(PositiveNumber.fromInt(200)), + miningParams -> { + assertThat(miningParams.getPoaBlockTxsSelectionMaxTime()) + .isEqualTo(PositiveNumber.fromInt(200)); + assertThat(miningParams.getBlockTxsSelectionMaxTime()) + .isEqualTo(Duration.ofSeconds(POA_BLOCK_PERIOD_SECONDS * 2).toMillis()); + }, "--genesis-file", - genesisFileIBFT2.toString(), + genesisFileClique.toString(), "--poa-block-txs-selection-max-time", "200"); } @@ -407,6 +412,16 @@ protected MiningOptions optionsFromDomainObject(final MiningParameters domainObj @Override protected MiningOptions getOptionsFromBesuCommand(final TestBesuCommand besuCommand) { - return besuCommand.getMiningOptions(); + final var miningOptions = besuCommand.getMiningOptions(); + miningOptions.setGenesisBlockPeriodSeconds( + besuCommand.getActualGenesisConfigOptions().isPoa() + ? OptionalInt.of(POA_BLOCK_PERIOD_SECONDS) + : OptionalInt.empty()); + return miningOptions; + } + + @Override + protected String[] getNonOptionFields() { + return new String[] {"maybeGenesisBlockPeriodSeconds"}; } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java index 3b1a9e6839a..9585fadf0d8 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java @@ -82,8 +82,8 @@ protected SynchronizerConfiguration.Builder createCustomizedDomainObject() { } @Override - protected List getFieldsWithComputedDefaults() { - return Arrays.asList("maxTrailingPeers", "computationParallelism"); + protected String[] getFieldsWithComputedDefaults() { + return new String[] {"maxTrailingPeers", "computationParallelism"}; } @Override From efbd840747669f68e36a9181eecaaf5fc97e0d00 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 25 Jan 2024 14:26:03 +0100 Subject: [PATCH 45/56] Log blob count when importing a block via Engine API (#6466) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 2 +- .../engine/AbstractEngineNewPayload.java | 22 +++++++++---------- .../methods/engine/EngineNewPayloadV1.java | 5 ----- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57416d07852..02c839efddd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ ### Additions and Improvements - Add `OperationTracer.tracePrepareTransaction`, where the sender account has not yet been altered[#6453](https://github.com/hyperledger/besu/pull/6453) - Improve the high spec flag by limiting it to a few column families [#6354](https://github.com/hyperledger/besu/pull/6354) - +- Log blob count when importing a block via Engine API [#6466](https://github.com/hyperledger/besu/pull/6466) ### Bug fixes - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineNewPayload.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineNewPayload.java index a5b8acfc056..a9336497a1b 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineNewPayload.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineNewPayload.java @@ -234,9 +234,12 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext) return respondWithInvalid(reqId, blockParam, null, getInvalidBlockHashStatus(), errorMessage); } + final var blobTransactions = + transactions.stream().filter(transaction -> transaction.getType().supportsBlob()).toList(); + ValidationResult blobValidationResult = validateBlobs( - transactions, + blobTransactions, newBlockHeader, maybeParentHeader, maybeVersionedHashes, @@ -302,7 +305,8 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext) final BlockProcessingResult executionResult = mergeCoordinator.rememberBlock(block); if (executionResult.isSuccessful()) { - logImportedBlockInfo(block, (System.currentTimeMillis() - startTimeMs) / 1000.0); + logImportedBlockInfo( + block, blobTransactions.size(), (System.currentTimeMillis() - startTimeMs) / 1000.0); return respondWith(reqId, blockParam, newBlockHeader.getHash(), VALID); } else { if (executionResult.causedBy().isPresent()) { @@ -380,10 +384,6 @@ JsonRpcResponse respondWithInvalid( invalidStatus, latestValidHash, Optional.of(validationError))); } - protected boolean requireTerminalPoWBlockValidation() { - return false; - } - protected EngineStatus getInvalidBlockHashStatus() { return INVALID; } @@ -396,15 +396,12 @@ protected ValidationResult validateParameters( } protected ValidationResult validateBlobs( - final List transactions, + final List blobTransactions, final BlockHeader header, final Optional maybeParentHeader, final Optional> maybeVersionedHashes, final ProtocolSpec protocolSpec) { - var blobTransactions = - transactions.stream().filter(transaction -> transaction.getType().supportsBlob()).toList(); - final List transactionVersionedHashes = new ArrayList<>(); for (Transaction transaction : blobTransactions) { var versionedHashes = transaction.getVersionedHashes(); @@ -489,7 +486,7 @@ private Optional> extractVersionedHashes( .collect(Collectors.toList())); } - private void logImportedBlockInfo(final Block block, final double timeInS) { + private void logImportedBlockInfo(final Block block, final int blobCount, final double timeInS) { final StringBuilder message = new StringBuilder(); message.append("Imported #%,d / %d tx"); final List messageArgs = @@ -503,9 +500,10 @@ private void logImportedBlockInfo(final Block block, final double timeInS) { message.append(" / %d ds"); messageArgs.add(block.getBody().getDeposits().get().size()); } - message.append(" / base fee %s / %,d (%01.1f%%) gas / (%s) in %01.3fs. Peers: %d"); + message.append(" / %d blobs / base fee %s / %,d (%01.1f%%) gas / (%s) in %01.3fs. Peers: %d"); messageArgs.addAll( List.of( + blobCount, block.getHeader().getBaseFee().map(Wei::toHumanReadableString).orElse("N/A"), block.getHeader().getGasUsed(), (block.getHeader().getGasUsed() * 100.0) / block.getHeader().getGasLimit(), diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV1.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV1.java index d7019543c02..9675fb01195 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV1.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV1.java @@ -50,11 +50,6 @@ public String getName() { return RpcMethod.ENGINE_NEW_PAYLOAD_V1.getMethodName(); } - @Override - protected boolean requireTerminalPoWBlockValidation() { - return true; - } - @Override protected EngineStatus getInvalidBlockHashStatus() { return INVALID_BLOCK_HASH; From d721a3432b709908c13eafc941a1afc6a1dbf44a Mon Sep 17 00:00:00 2001 From: garyschulte Date: Thu, 25 Jan 2024 18:19:06 -0800 Subject: [PATCH 46/56] Fix 24.1.1 changelog (#6468) * fix changelog as it relates to 24.1.1 * add 24.1.1 SHAs Signed-off-by: garyschulte --- CHANGELOG.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02c839efddd..2ab5425cd19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Breaking Changes - The `trace-filter` method in JSON-RPC API now has a default block range limit of 1000, adjustable with `--rpc-max-trace-filter-range` [#6446](https://github.com/hyperledger/besu/pull/6446) +- Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): + - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. ### Deprecations @@ -11,6 +13,8 @@ - Add `OperationTracer.tracePrepareTransaction`, where the sender account has not yet been altered[#6453](https://github.com/hyperledger/besu/pull/6453) - Improve the high spec flag by limiting it to a few column families [#6354](https://github.com/hyperledger/besu/pull/6354) - Log blob count when importing a block via Engine API [#6466](https://github.com/hyperledger/besu/pull/6466) +- Introduce `--Xbonsai-limit-trie-logs-enabled` experimental feature which by default will only retain the latest 512 trie logs, saving about 3GB per week in database growth [#5390](https://github.com/hyperledger/besu/issues/5390) +- Introduce `besu storage x-trie-log prune` experimental offline subcommand which will prune all redundant trie logs except the latest 512 [#6303](https://github.com/hyperledger/besu/pull/6303) ### Bug fixes - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) @@ -24,9 +28,7 @@ - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) -- Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): - - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. -- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. +- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. [#6423](https://github.com/hyperledger/besu/pull/6423) ### Deprecations @@ -40,8 +42,6 @@ - Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) - Upgrade `tech.pegasys.discovery:discovery` [#6414](https://github.com/hyperledger/besu/pull/6414) - Options to tune the max allowed time that can be spent selecting transactions during block creation are now stable [#6423](https://github.com/hyperledger/besu/pull/6423) -- Introduce `--Xbonsai-limit-trie-logs-enabled` experimental feature which by default will only retain the latest 512 trie logs, saving about 3GB per week in database growth [#5390](https://github.com/hyperledger/besu/issues/5390) -- Introduce `besu storage x-trie-log prune` experimental offline subcommand which will prune all redundant trie logs except the latest 512 [#6303](https://github.com/hyperledger/besu/pull/6303) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) @@ -49,8 +49,12 @@ - Fluent EVM API definition for Tangerine Whistle had incorrect code size validation configured [#6382](https://github.com/hyperledger/besu/pull/6382) - Correct mining beneficiary for Clique networks in TraceServiceImpl [#6390](https://github.com/hyperledger/besu/pull/6390) - Fix to gas limit delta calculations used in block production. Besu should now increment or decrement the block gas limit towards its target correctly (thanks @arbora) #6425 +- Ensure Backward Sync waits for initial sync before starting a session [#6455](https://github.com/hyperledger/besu/issues/6455) +- Silence the noisy DNS query errors [#6458](https://github.com/hyperledger/besu/issues/6458) ### Download Links +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.zip / sha256 b6b64f939e0bb4937ce90fc647e0a7073ce3e359c10352b502059955070a60c6 +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.tar.gz / sha256 cfcae04c30769bf338b0740ac65870f9346d3469931bb46cdba3b2f65d311e7a ## 24.1.0 @@ -73,8 +77,8 @@ - mitigation for trielog failure [#6315]((https://github.com/hyperledger/besu/pull/6315) ### Download Links -https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.0/besu-24.1.0.zip / sha256 TBA -https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.0/besu-24.1.0.tar.gz / sha256 TBA +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.0/besu-24.1.0.zip / sha256 d36c8aeef70f0a516d4c26d3bc696c3e2a671e515c9e6e9475a31fe759e39f64 +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.0/besu-24.1.0.tar.gz / sha256 602b04c0729a7b17361d1f0b39f4ce6a2ebe47932165add666560fe594d9ca99 ## 23.10.3-hotfix From cd5159d495304d9a15ff76bc193eabeb3b8cfa21 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Mon, 29 Jan 2024 12:16:18 +1000 Subject: [PATCH 47/56] [CHANGELOG] updated download links and added contributor thanks (#6477) * updated dl links and added contributor thx Signed-off-by: Sally MacFarlane * added errata section with previous shas Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane --- CHANGELOG.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ab5425cd19..ba9a4e7a2c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 24.1.2-SNAPSHOT ### Breaking Changes -- The `trace-filter` method in JSON-RPC API now has a default block range limit of 1000, adjustable with `--rpc-max-trace-filter-range` [#6446](https://github.com/hyperledger/besu/pull/6446) +- The `trace-filter` method in JSON-RPC API now has a default block range limit of 1000, adjustable with `--rpc-max-trace-filter-range` (thanks @alyokaz) [#6446](https://github.com/hyperledger/besu/pull/6446) - Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. @@ -26,7 +26,7 @@ ### Breaking Changes - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) -- The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) +- The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 (thanks @h4l) [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) - The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. [#6423](https://github.com/hyperledger/besu/pull/6423) @@ -53,8 +53,13 @@ - Silence the noisy DNS query errors [#6458](https://github.com/hyperledger/besu/issues/6458) ### Download Links -https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.zip / sha256 b6b64f939e0bb4937ce90fc647e0a7073ce3e359c10352b502059955070a60c6 -https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.tar.gz / sha256 cfcae04c30769bf338b0740ac65870f9346d3469931bb46cdba3b2f65d311e7a +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.zip / sha256 e23c5b790180756964a70dcdd575ee2ed2c2efa79af00bce956d23bd2f7dc67c +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.tar.gz / sha256 4b0ddd5a25be2df5d2324bff935785eb63e4e3a5f421614ea690bacb5b9cb344 + +### Errata +Note, due to a CI race with the release job, the initial published version of 24.1.1 were overwritten by artifacts generated from the same sources, but differ in their embedded timestamps. The initial SHAs are noted here but are deprecated: +~~https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.zip / sha256 b6b64f939e0bb4937ce90fc647e0a7073ce3e359c10352b502059955070a60c6 +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.tar.gz / sha256 cfcae04c30769bf338b0740ac65870f9346d3469931bb46cdba3b2f65d311e7a~~ ## 24.1.0 From f915bfd7c96a284bec8986da7aa9b1acdb7234fd Mon Sep 17 00:00:00 2001 From: Stefan Pingel <16143240+pinges@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:52:22 +1000 Subject: [PATCH 48/56] remove repeated part of toString() (#6480) Signed-off-by: stefan.pingel@consensys.net --- .../org/hyperledger/besu/consensus/merge/ForkchoiceEvent.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/consensus/merge/ForkchoiceEvent.java b/ethereum/eth/src/main/java/org/hyperledger/besu/consensus/merge/ForkchoiceEvent.java index bc38cd38808..8207e13f759 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/consensus/merge/ForkchoiceEvent.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/consensus/merge/ForkchoiceEvent.java @@ -91,8 +91,6 @@ public String toString() { + safeBlockHash + ", finalizedBlockHash=" + finalizedBlockHash - + ", safeBlockHash=" - + safeBlockHash + '}'; } } From e55569cb9086d61dc642d3e04cfc98c7437272d2 Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Mon, 29 Jan 2024 14:28:49 +1100 Subject: [PATCH 49/56] [Refactor] - Extract JsonRpcHttpOptions to a new class (#6478) Signed-off-by: Gabriel-Trintinalia --- .../org/hyperledger/besu/cli/BesuCommand.java | 428 +----- .../options/stable/JsonRpcHttpOptions.java | 492 +++++++ .../hyperledger/besu/cli/BesuCommandTest.java | 1213 +++-------------- .../cli/options/JsonRpcHttpOptionsTest.java | 926 +++++++++++++ .../cli/options/RpcWebsocketOptionsTest.java | 13 + 5 files changed, 1605 insertions(+), 1467 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/options/stable/JsonRpcHttpOptions.java create mode 100644 besu/src/test/java/org/hyperledger/besu/cli/options/JsonRpcHttpOptionsTest.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index b9a137cbc17..68c3a6a5abc 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -26,10 +26,6 @@ import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; import static org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration.DEFAULT_GRAPHQL_HTTP_PORT; import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEFAULT_ENGINE_JSON_RPC_PORT; -import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEFAULT_JSON_RPC_PORT; -import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEFAULT_PRETTY_JSON_ENABLED; -import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; -import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.VALID_APIS; import static org.hyperledger.besu.ethereum.api.jsonrpc.authentication.EngineAuthService.EPHEMERAL_JWT_FILE; import static org.hyperledger.besu.metrics.BesuMetricCategory.DEFAULT_METRIC_CATEGORIES; import static org.hyperledger.besu.metrics.MetricsProtocol.PROMETHEUS; @@ -50,13 +46,13 @@ import org.hyperledger.besu.cli.converter.PercentageConverter; import org.hyperledger.besu.cli.custom.CorsAllowedOriginsProperty; import org.hyperledger.besu.cli.custom.JsonRPCAllowlistHostsProperty; -import org.hyperledger.besu.cli.custom.RpcAuthFileValidator; import org.hyperledger.besu.cli.error.BesuExecutionExceptionHandler; import org.hyperledger.besu.cli.error.BesuParameterExceptionHandler; import org.hyperledger.besu.cli.options.MiningOptions; import org.hyperledger.besu.cli.options.TransactionPoolOptions; import org.hyperledger.besu.cli.options.stable.DataStorageOptions; import org.hyperledger.besu.cli.options.stable.EthstatsOptions; +import org.hyperledger.besu.cli.options.stable.JsonRpcHttpOptions; import org.hyperledger.besu.cli.options.stable.LoggingLevelOption; import org.hyperledger.besu.cli.options.stable.NodePrivateKeyFileOption; import org.hyperledger.besu.cli.options.stable.P2PTLSConfigOptions; @@ -115,14 +111,10 @@ import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration; import org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis; -import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.authentication.JwtAlgorithm; import org.hyperledger.besu.ethereum.api.jsonrpc.ipc.JsonRpcIpcConfiguration; import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.api.tls.FileBasedPasswordProvider; -import org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration; -import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; @@ -212,8 +204,6 @@ import java.net.URI; import java.net.UnknownHostException; import java.nio.file.Path; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; @@ -231,8 +221,6 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLEngine; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; @@ -647,162 +635,7 @@ static class EngineRPCOptionGroup { // JSON-RPC HTTP Options @CommandLine.ArgGroup(validate = false, heading = "@|bold JSON-RPC HTTP Options|@%n") - JsonRPCHttpOptionGroup jsonRPCHttpOptionGroup = new JsonRPCHttpOptionGroup(); - - static class JsonRPCHttpOptionGroup { - @Option( - names = {"--rpc-http-enabled"}, - description = "Set to start the JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") - private final Boolean isRpcHttpEnabled = false; - - @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. - @Option( - names = {"--rpc-http-host"}, - paramLabel = MANDATORY_HOST_FORMAT_HELP, - description = "Host for JSON-RPC HTTP to listen on (default: ${DEFAULT-VALUE})", - arity = "1") - private String rpcHttpHost; - - @Option( - names = {"--rpc-http-port"}, - paramLabel = MANDATORY_PORT_FORMAT_HELP, - description = "Port for JSON-RPC HTTP to listen on (default: ${DEFAULT-VALUE})", - arity = "1") - private final Integer rpcHttpPort = DEFAULT_JSON_RPC_PORT; - - @Option( - names = {"--rpc-http-max-active-connections"}, - description = - "Maximum number of HTTP connections allowed for JSON-RPC (default: ${DEFAULT-VALUE}). Once this limit is reached, incoming connections will be rejected.", - arity = "1") - private final Integer rpcHttpMaxConnections = DEFAULT_HTTP_MAX_CONNECTIONS; - - // A list of origins URLs that are accepted by the JsonRpcHttpServer (CORS) - @Option( - names = {"--rpc-http-cors-origins"}, - description = "Comma separated origin domain URLs for CORS validation (default: none)") - private final CorsAllowedOriginsProperty rpcHttpCorsAllowedOrigins = - new CorsAllowedOriginsProperty(); - - @Option( - names = {"--rpc-http-api", "--rpc-http-apis"}, - paramLabel = "", - split = " {0,1}, {0,1}", - arity = "1..*", - description = - "Comma separated list of APIs to enable on JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") - private final List rpcHttpApis = DEFAULT_RPC_APIS; - - @Option( - names = {"--rpc-http-api-method-no-auth", "--rpc-http-api-methods-no-auth"}, - paramLabel = "", - split = " {0,1}, {0,1}", - arity = "1..*", - description = - "Comma separated list of API methods to exclude from RPC authentication services, RPC HTTP authentication must be enabled") - private final List rpcHttpApiMethodsNoAuth = new ArrayList(); - - @Option( - names = {"--rpc-http-authentication-enabled"}, - description = - "Require authentication for the JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") - private final Boolean isRpcHttpAuthenticationEnabled = false; - - @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. - @CommandLine.Option( - names = {"--rpc-http-authentication-credentials-file"}, - paramLabel = MANDATORY_FILE_FORMAT_HELP, - description = - "Storage file for JSON-RPC HTTP authentication credentials (default: ${DEFAULT-VALUE})", - arity = "1") - private String rpcHttpAuthenticationCredentialsFile = null; - - @CommandLine.Option( - names = {"--rpc-http-authentication-jwt-public-key-file"}, - paramLabel = MANDATORY_FILE_FORMAT_HELP, - description = "JWT public key file for JSON-RPC HTTP authentication", - arity = "1") - private final File rpcHttpAuthenticationPublicKeyFile = null; - - @Option( - names = {"--rpc-http-authentication-jwt-algorithm"}, - description = - "Encryption algorithm used for HTTP JWT public key. Possible values are ${COMPLETION-CANDIDATES}" - + " (default: ${DEFAULT-VALUE})", - arity = "1") - private final JwtAlgorithm rpcHttpAuthenticationAlgorithm = DEFAULT_JWT_ALGORITHM; - - @Option( - names = {"--rpc-http-tls-enabled"}, - description = "Enable TLS for the JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") - private final Boolean isRpcHttpTlsEnabled = false; - - @Option( - names = {"--rpc-http-tls-keystore-file"}, - paramLabel = MANDATORY_FILE_FORMAT_HELP, - description = - "Keystore (PKCS#12) containing key/certificate for the JSON-RPC HTTP service. Required if TLS is enabled.") - private final Path rpcHttpTlsKeyStoreFile = null; - - @Option( - names = {"--rpc-http-tls-keystore-password-file"}, - paramLabel = MANDATORY_FILE_FORMAT_HELP, - description = - "File containing password to unlock keystore for the JSON-RPC HTTP service. Required if TLS is enabled.") - private final Path rpcHttpTlsKeyStorePasswordFile = null; - - @Option( - names = {"--rpc-http-tls-client-auth-enabled"}, - description = - "Enable TLS client authentication for the JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") - private final Boolean isRpcHttpTlsClientAuthEnabled = false; - - @Option( - names = {"--rpc-http-tls-known-clients-file"}, - paramLabel = MANDATORY_FILE_FORMAT_HELP, - description = - "Path to file containing clients certificate common name and fingerprint for client authentication") - private final Path rpcHttpTlsKnownClientsFile = null; - - @Option( - names = {"--rpc-http-tls-ca-clients-enabled"}, - description = - "Enable to accept clients certificate signed by a valid CA for client authentication (default: ${DEFAULT-VALUE})") - private final Boolean isRpcHttpTlsCAClientsEnabled = false; - - @Option( - names = {"--rpc-http-tls-protocol", "--rpc-http-tls-protocols"}, - description = - "Comma separated list of TLS protocols to support (default: ${DEFAULT-VALUE})", - split = ",", - arity = "1..*") - private final List rpcHttpTlsProtocols = new ArrayList<>(DEFAULT_TLS_PROTOCOLS); - - @Option( - names = {"--rpc-http-tls-cipher-suite", "--rpc-http-tls-cipher-suites"}, - description = "Comma separated list of TLS cipher suites to support", - split = ",", - arity = "1..*") - private final List rpcHttpTlsCipherSuites = new ArrayList<>(); - - @CommandLine.Option( - names = {"--rpc-http-max-batch-size"}, - paramLabel = MANDATORY_INTEGER_FORMAT_HELP, - description = - "Specifies the maximum number of requests in a single RPC batch request via RPC. -1 specifies no limit (default: ${DEFAULT-VALUE})") - private final Integer rpcHttpMaxBatchSize = DEFAULT_HTTP_MAX_BATCH_SIZE; - - @CommandLine.Option( - names = {"--rpc-http-max-request-content-length"}, - paramLabel = MANDATORY_LONG_FORMAT_HELP, - description = "Specifies the maximum request content length. (default: ${DEFAULT-VALUE})") - private final Long rpcHttpMaxRequestContentLength = DEFAULT_MAX_REQUEST_CONTENT_LENGTH; - - @Option( - names = {"--json-pretty-print-enabled"}, - description = "Enable JSON pretty print format (default: ${DEFAULT-VALUE})") - private final Boolean prettyJsonEnabled = DEFAULT_PRETTY_JSON_ENABLED; - } + JsonRpcHttpOptions jsonRpcHttpOptions = new JsonRpcHttpOptions(); // JSON-RPC Websocket Options @CommandLine.ArgGroup(validate = false, heading = "@|bold JSON-RPC Websocket Options|@%n") @@ -1866,26 +1699,7 @@ private void validateRpcOptionsParams() { Arrays.stream(RpcApis.values()) .anyMatch(builtInApi -> apiName.equals(builtInApi.name())) || rpcEndpointServiceImpl.hasNamespace(apiName); - - if (!jsonRPCHttpOptionGroup.rpcHttpApis.stream().allMatch(configuredApis)) { - final List invalidHttpApis = - new ArrayList(jsonRPCHttpOptionGroup.rpcHttpApis); - invalidHttpApis.removeAll(VALID_APIS); - throw new ParameterException( - this.commandLine, - "Invalid value for option '--rpc-http-api': invalid entries found " - + invalidHttpApis.toString()); - } - - final boolean validHttpApiMethods = - jsonRPCHttpOptionGroup.rpcHttpApiMethodsNoAuth.stream() - .allMatch(RpcMethod::rpcMethodExists); - - if (!validHttpApiMethods) { - throw new ParameterException( - this.commandLine, - "Invalid value for option '--rpc-http-api-methods-no-auth', options must be valid RPC methods"); - } + jsonRpcHttpOptions.validate(logger, commandLine, configuredApis); } private void validateRpcWsOptions() { @@ -1987,8 +1801,10 @@ private void configure() throws Exception { ethNetworkConfig = updateNetworkConfig(network); jsonRpcConfiguration = - jsonRpcConfiguration( - jsonRPCHttpOptionGroup.rpcHttpPort, jsonRPCHttpOptionGroup.rpcHttpApis, hostsAllowlist); + jsonRpcHttpOptions.jsonRpcConfiguration( + hostsAllowlist, + p2PDiscoveryOptionGroup.autoDiscoverDefaultIP().getHostAddress(), + unstableRPCOptions.getHttpTimeoutSec()); if (isEngineApiEnabled()) { engineJsonRpcConfiguration = createEngineJsonRpcConfiguration( @@ -2160,9 +1976,15 @@ private GraphQLConfiguration graphQLConfiguration() { } private JsonRpcConfiguration createEngineJsonRpcConfiguration( - final Integer listenPort, final List allowCallsFrom) { + final Integer engineListenPort, final List allowCallsFrom) { + jsonRpcHttpOptions.checkDependencies(logger, commandLine); final JsonRpcConfiguration engineConfig = - jsonRpcConfiguration(listenPort, Arrays.asList("ENGINE", "ETH"), allowCallsFrom); + jsonRpcHttpOptions.jsonRpcConfiguration( + allowCallsFrom, + p2PDiscoveryOptionGroup.autoDiscoverDefaultIP().getHostAddress(), + unstableRPCOptions.getWsTimeoutSec()); + engineConfig.setPort(engineListenPort); + engineConfig.setRpcApis(Arrays.asList("ENGINE", "ETH")); engineConfig.setEnabled(isEngineApiEnabled()); if (!engineRPCOptionGroup.isEngineAuthDisabled) { engineConfig.setAuthenticationEnabled(true); @@ -2178,116 +2000,6 @@ private JsonRpcConfiguration createEngineJsonRpcConfiguration( return engineConfig; } - private JsonRpcConfiguration jsonRpcConfiguration( - final Integer listenPort, final List apiGroups, final List allowCallsFrom) { - checkRpcTlsClientAuthOptionsDependencies(); - checkRpcTlsOptionsDependencies(); - checkRpcHttpOptionsDependencies(); - - if (jsonRPCHttpOptionGroup.isRpcHttpAuthenticationEnabled) { - CommandLineUtils.checkOptionDependencies( - logger, - commandLine, - "--rpc-http-authentication-public-key-file", - jsonRPCHttpOptionGroup.rpcHttpAuthenticationPublicKeyFile == null, - asList("--rpc-http-authentication-jwt-algorithm")); - } - - if (jsonRPCHttpOptionGroup.isRpcHttpAuthenticationEnabled - && rpcHttpAuthenticationCredentialsFile() == null - && jsonRPCHttpOptionGroup.rpcHttpAuthenticationPublicKeyFile == null) { - throw new ParameterException( - commandLine, - "Unable to authenticate JSON-RPC HTTP endpoint without a supplied credentials file or authentication public key file"); - } - - final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); - jsonRpcConfiguration.setEnabled(jsonRPCHttpOptionGroup.isRpcHttpEnabled); - jsonRpcConfiguration.setHost( - Strings.isNullOrEmpty(jsonRPCHttpOptionGroup.rpcHttpHost) - ? p2PDiscoveryOptionGroup.autoDiscoverDefaultIP().getHostAddress() - : jsonRPCHttpOptionGroup.rpcHttpHost); - jsonRpcConfiguration.setPort(listenPort); - jsonRpcConfiguration.setMaxActiveConnections(jsonRPCHttpOptionGroup.rpcHttpMaxConnections); - jsonRpcConfiguration.setCorsAllowedDomains(jsonRPCHttpOptionGroup.rpcHttpCorsAllowedOrigins); - jsonRpcConfiguration.setRpcApis(apiGroups.stream().distinct().collect(Collectors.toList())); - jsonRpcConfiguration.setNoAuthRpcApis( - jsonRPCHttpOptionGroup.rpcHttpApiMethodsNoAuth.stream() - .distinct() - .collect(Collectors.toList())); - jsonRpcConfiguration.setHostsAllowlist(allowCallsFrom); - jsonRpcConfiguration.setAuthenticationEnabled( - jsonRPCHttpOptionGroup.isRpcHttpAuthenticationEnabled); - jsonRpcConfiguration.setAuthenticationCredentialsFile(rpcHttpAuthenticationCredentialsFile()); - jsonRpcConfiguration.setAuthenticationPublicKeyFile( - jsonRPCHttpOptionGroup.rpcHttpAuthenticationPublicKeyFile); - jsonRpcConfiguration.setAuthenticationAlgorithm( - jsonRPCHttpOptionGroup.rpcHttpAuthenticationAlgorithm); - jsonRpcConfiguration.setTlsConfiguration(rpcHttpTlsConfiguration()); - jsonRpcConfiguration.setHttpTimeoutSec(unstableRPCOptions.getHttpTimeoutSec()); - jsonRpcConfiguration.setMaxBatchSize(jsonRPCHttpOptionGroup.rpcHttpMaxBatchSize); - jsonRpcConfiguration.setMaxRequestContentLength( - jsonRPCHttpOptionGroup.rpcHttpMaxRequestContentLength); - jsonRpcConfiguration.setPrettyJsonEnabled(jsonRPCHttpOptionGroup.prettyJsonEnabled); - return jsonRpcConfiguration; - } - - private void checkRpcHttpOptionsDependencies() { - CommandLineUtils.checkOptionDependencies( - logger, - commandLine, - "--rpc-http-enabled", - !jsonRPCHttpOptionGroup.isRpcHttpEnabled, - asList( - "--rpc-http-api", - "--rpc-http-apis", - "--rpc-http-api-method-no-auth", - "--rpc-http-api-methods-no-auth", - "--rpc-http-cors-origins", - "--rpc-http-host", - "--rpc-http-port", - "--rpc-http-max-active-connections", - "--rpc-http-authentication-enabled", - "--rpc-http-authentication-credentials-file", - "--rpc-http-authentication-public-key-file", - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - "--rpc-http-tls-keystore-password-file", - "--rpc-http-tls-client-auth-enabled", - "--rpc-http-tls-known-clients-file", - "--rpc-http-tls-ca-clients-enabled", - "--rpc-http-authentication-jwt-algorithm", - "--rpc-http-tls-protocols", - "--rpc-http-tls-cipher-suite", - "--rpc-http-tls-cipher-suites")); - } - - private void checkRpcTlsOptionsDependencies() { - CommandLineUtils.checkOptionDependencies( - logger, - commandLine, - "--rpc-http-tls-enabled", - !jsonRPCHttpOptionGroup.isRpcHttpTlsEnabled, - asList( - "--rpc-http-tls-keystore-file", - "--rpc-http-tls-keystore-password-file", - "--rpc-http-tls-client-auth-enabled", - "--rpc-http-tls-known-clients-file", - "--rpc-http-tls-ca-clients-enabled", - "--rpc-http-tls-protocols", - "--rpc-http-tls-cipher-suite", - "--rpc-http-tls-cipher-suites")); - } - - private void checkRpcTlsClientAuthOptionsDependencies() { - CommandLineUtils.checkOptionDependencies( - logger, - commandLine, - "--rpc-http-tls-client-auth-enabled", - !jsonRPCHttpOptionGroup.isRpcHttpTlsClientAuthEnabled, - asList("--rpc-http-tls-known-clients-file", "--rpc-http-tls-ca-clients-enabled")); - } - private void checkPrivacyTlsOptionsDependencies() { CommandLineUtils.checkOptionDependencies( logger, @@ -2300,75 +2012,6 @@ private void checkPrivacyTlsOptionsDependencies() { "--privacy-tls-known-enclave-file")); } - private Optional rpcHttpTlsConfiguration() { - if (!isRpcTlsConfigurationRequired()) { - return Optional.empty(); - } - - if (jsonRPCHttpOptionGroup.rpcHttpTlsKeyStoreFile == null) { - throw new ParameterException( - commandLine, "Keystore file is required when TLS is enabled for JSON-RPC HTTP endpoint"); - } - - if (jsonRPCHttpOptionGroup.rpcHttpTlsKeyStorePasswordFile == null) { - throw new ParameterException( - commandLine, - "File containing password to unlock keystore is required when TLS is enabled for JSON-RPC HTTP endpoint"); - } - - if (jsonRPCHttpOptionGroup.isRpcHttpTlsClientAuthEnabled - && !jsonRPCHttpOptionGroup.isRpcHttpTlsCAClientsEnabled - && jsonRPCHttpOptionGroup.rpcHttpTlsKnownClientsFile == null) { - throw new ParameterException( - commandLine, - "Known-clients file must be specified or CA clients must be enabled when TLS client authentication is enabled for JSON-RPC HTTP endpoint"); - } - - jsonRPCHttpOptionGroup.rpcHttpTlsProtocols.retainAll(getJDKEnabledProtocols()); - if (jsonRPCHttpOptionGroup.rpcHttpTlsProtocols.isEmpty()) { - throw new ParameterException( - commandLine, - "No valid TLS protocols specified (the following protocols are enabled: " - + getJDKEnabledProtocols() - + ")"); - } - - for (final String cipherSuite : jsonRPCHttpOptionGroup.rpcHttpTlsCipherSuites) { - if (!getJDKEnabledCipherSuites().contains(cipherSuite)) { - throw new ParameterException( - commandLine, "Invalid TLS cipher suite specified " + cipherSuite); - } - } - - jsonRPCHttpOptionGroup.rpcHttpTlsCipherSuites.retainAll(getJDKEnabledCipherSuites()); - - return Optional.of( - TlsConfiguration.Builder.aTlsConfiguration() - .withKeyStorePath(jsonRPCHttpOptionGroup.rpcHttpTlsKeyStoreFile) - .withKeyStorePasswordSupplier( - new FileBasedPasswordProvider( - jsonRPCHttpOptionGroup.rpcHttpTlsKeyStorePasswordFile)) - .withClientAuthConfiguration(rpcHttpTlsClientAuthConfiguration()) - .withSecureTransportProtocols(jsonRPCHttpOptionGroup.rpcHttpTlsProtocols) - .withCipherSuites(jsonRPCHttpOptionGroup.rpcHttpTlsCipherSuites) - .build()); - } - - private TlsClientAuthConfiguration rpcHttpTlsClientAuthConfiguration() { - if (jsonRPCHttpOptionGroup.isRpcHttpTlsClientAuthEnabled) { - return TlsClientAuthConfiguration.Builder.aTlsClientAuthConfiguration() - .withKnownClientsFile(jsonRPCHttpOptionGroup.rpcHttpTlsKnownClientsFile) - .withCaClientsEnabled(jsonRPCHttpOptionGroup.isRpcHttpTlsCAClientsEnabled) - .build(); - } - - return null; - } - - private boolean isRpcTlsConfigurationRequired() { - return jsonRPCHttpOptionGroup.isRpcHttpEnabled && jsonRPCHttpOptionGroup.isRpcHttpTlsEnabled; - } - private ApiConfiguration apiConfiguration() { checkApiOptionsDependencies(); var builder = @@ -2450,7 +2093,7 @@ public MetricsConfiguration metricsConfiguration() { private Optional permissioningConfiguration() throws Exception { if (!(localPermissionsEnabled() || contractPermissionsEnabled())) { - if (jsonRPCHttpOptionGroup.rpcHttpApis.contains(RpcApis.PERM.name()) + if (jsonRpcHttpOptions.getRpcHttpApis().contains(RpcApis.PERM.name()) || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.PERM.name())) { logger.warn( "Permissions are disabled. Cannot enable PERM APIs when not using Permissions."); @@ -2653,9 +2296,9 @@ private PrivacyParameters privacyParameters() { } private boolean anyPrivacyApiEnabled() { - return jsonRPCHttpOptionGroup.rpcHttpApis.contains(RpcApis.EEA.name()) + return jsonRpcHttpOptions.getRpcHttpApis().contains(RpcApis.EEA.name()) || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.EEA.name()) - || jsonRPCHttpOptionGroup.rpcHttpApis.contains(RpcApis.PRIV.name()) + || jsonRpcHttpOptions.getRpcHttpApis().contains(RpcApis.PRIV.name()) || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.PRIV.name()); } @@ -3018,15 +2661,6 @@ private File resolveNodePrivateKeyFile(final File nodePrivateKeyFile) { .orElseGet(() -> KeyPairUtil.getDefaultKeyFile(dataDir())); } - private String rpcHttpAuthenticationCredentialsFile() { - final String filename = jsonRPCHttpOptionGroup.rpcHttpAuthenticationCredentialsFile; - - if (filename != null) { - RpcAuthFileValidator.validate(commandLine, filename, "HTTP"); - } - return filename; - } - private String getDefaultPermissioningFilePath() { return dataDir() + System.getProperty("file.separator") @@ -3160,9 +2794,7 @@ private List getEffectivePorts() { graphQlOptionGroup.graphQLHttpPort, graphQlOptionGroup.isGraphQLHttpEnabled); addPortIfEnabled( - effectivePorts, - jsonRPCHttpOptionGroup.rpcHttpPort, - jsonRPCHttpOptionGroup.isRpcHttpEnabled); + effectivePorts, jsonRpcHttpOptions.getRpcHttpPort(), jsonRpcHttpOptions.isRpcHttpEnabled()); addPortIfEnabled( effectivePorts, rpcWebsocketOptions.getRpcWsPort(), rpcWebsocketOptions.isRpcWsEnabled()); addPortIfEnabled(effectivePorts, engineRPCOptionGroup.engineRpcPort, isEngineApiEnabled()); @@ -3310,28 +2942,6 @@ private boolean isEngineApiEnabled() { return engineRPCOptionGroup.overrideEngineRpcEnabled || isMergeEnabled(); } - private static List getJDKEnabledCipherSuites() { - try { - final SSLContext context = SSLContext.getInstance("TLS"); - context.init(null, null, null); - final SSLEngine engine = context.createSSLEngine(); - return Arrays.asList(engine.getEnabledCipherSuites()); - } catch (final KeyManagementException | NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - - private static List getJDKEnabledProtocols() { - try { - final SSLContext context = SSLContext.getInstance("TLS"); - context.init(null, null, null); - final SSLEngine engine = context.createSSLEngine(); - return Arrays.asList(engine.getEnabledProtocols()); - } catch (final KeyManagementException | NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - private SyncMode getDefaultSyncModeIfNotSet() { return Optional.ofNullable(syncMode) .orElse( diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/JsonRpcHttpOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/JsonRpcHttpOptions.java new file mode 100644 index 00000000000..51b6a59f1cb --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/JsonRpcHttpOptions.java @@ -0,0 +1,492 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.options.stable; + +import static java.util.Arrays.asList; +import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEFAULT_JSON_RPC_PORT; +import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEFAULT_PRETTY_JSON_ENABLED; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.VALID_APIS; + +import org.hyperledger.besu.cli.DefaultCommandValues; +import org.hyperledger.besu.cli.custom.CorsAllowedOriginsProperty; +import org.hyperledger.besu.cli.custom.RpcAuthFileValidator; +import org.hyperledger.besu.cli.util.CommandLineUtils; +import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.authentication.JwtAlgorithm; +import org.hyperledger.besu.ethereum.api.tls.FileBasedPasswordProvider; +import org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration; +import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration; + +import java.io.File; +import java.nio.file.Path; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; + +import com.google.common.base.Strings; +import org.slf4j.Logger; +import picocli.CommandLine; + +/** + * Handles configuration options for the JSON-RPC HTTP service, including validation and creation of + * a JSON-RPC configuration. + */ +public class JsonRpcHttpOptions { + @CommandLine.Option( + names = {"--rpc-http-enabled"}, + description = "Set to start the JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") + private final Boolean isRpcHttpEnabled = false; + + @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. + @CommandLine.Option( + names = {"--rpc-http-host"}, + paramLabel = DefaultCommandValues.MANDATORY_HOST_FORMAT_HELP, + description = "Host for JSON-RPC HTTP to listen on (default: ${DEFAULT-VALUE})", + arity = "1") + private String rpcHttpHost; + + @CommandLine.Option( + names = {"--rpc-http-port"}, + paramLabel = DefaultCommandValues.MANDATORY_PORT_FORMAT_HELP, + description = "Port for JSON-RPC HTTP to listen on (default: ${DEFAULT-VALUE})", + arity = "1") + private final Integer rpcHttpPort = DEFAULT_JSON_RPC_PORT; + + @CommandLine.Option( + names = {"--rpc-http-max-active-connections"}, + description = + "Maximum number of HTTP connections allowed for JSON-RPC (default: ${DEFAULT-VALUE}). Once this limit is reached, incoming connections will be rejected.", + arity = "1") + private final Integer rpcHttpMaxConnections = DefaultCommandValues.DEFAULT_HTTP_MAX_CONNECTIONS; + + // A list of origins URLs that are accepted by the JsonRpcHttpServer (CORS) + @CommandLine.Option( + names = {"--rpc-http-cors-origins"}, + description = "Comma separated origin domain URLs for CORS validation (default: none)") + private final CorsAllowedOriginsProperty rpcHttpCorsAllowedOrigins = + new CorsAllowedOriginsProperty(); + + @CommandLine.Option( + names = {"--rpc-http-api", "--rpc-http-apis"}, + paramLabel = "", + split = " {0,1}, {0,1}", + arity = "1..*", + description = + "Comma separated list of APIs to enable on JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") + private final List rpcHttpApis = DEFAULT_RPC_APIS; + + @CommandLine.Option( + names = {"--rpc-http-api-method-no-auth", "--rpc-http-api-methods-no-auth"}, + paramLabel = "", + split = " {0,1}, {0,1}", + arity = "1..*", + description = + "Comma separated list of API methods to exclude from RPC authentication services, RPC HTTP authentication must be enabled") + private final List rpcHttpApiMethodsNoAuth = new ArrayList(); + + @CommandLine.Option( + names = {"--rpc-http-authentication-enabled"}, + description = + "Require authentication for the JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") + private final Boolean isRpcHttpAuthenticationEnabled = false; + + @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. + @CommandLine.Option( + names = {"--rpc-http-authentication-credentials-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = + "Storage file for JSON-RPC HTTP authentication credentials (default: ${DEFAULT-VALUE})", + arity = "1") + private String rpcHttpAuthenticationCredentialsFile = null; + + @CommandLine.Option( + names = {"--rpc-http-authentication-jwt-public-key-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = "JWT public key file for JSON-RPC HTTP authentication", + arity = "1") + private final File rpcHttpAuthenticationPublicKeyFile = null; + + @CommandLine.Option( + names = {"--rpc-http-authentication-jwt-algorithm"}, + description = + "Encryption algorithm used for HTTP JWT public key. Possible values are ${COMPLETION-CANDIDATES}" + + " (default: ${DEFAULT-VALUE})", + arity = "1") + private final JwtAlgorithm rpcHttpAuthenticationAlgorithm = + DefaultCommandValues.DEFAULT_JWT_ALGORITHM; + + @CommandLine.Option( + names = {"--rpc-http-tls-enabled"}, + description = "Enable TLS for the JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") + private final Boolean isRpcHttpTlsEnabled = false; + + @CommandLine.Option( + names = {"--rpc-http-tls-keystore-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = + "Keystore (PKCS#12) containing key/certificate for the JSON-RPC HTTP service. Required if TLS is enabled.") + private final Path rpcHttpTlsKeyStoreFile = null; + + @CommandLine.Option( + names = {"--rpc-http-tls-keystore-password-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = + "File containing password to unlock keystore for the JSON-RPC HTTP service. Required if TLS is enabled.") + private final Path rpcHttpTlsKeyStorePasswordFile = null; + + @CommandLine.Option( + names = {"--rpc-http-tls-client-auth-enabled"}, + description = + "Enable TLS client authentication for the JSON-RPC HTTP service (default: ${DEFAULT-VALUE})") + private final Boolean isRpcHttpTlsClientAuthEnabled = false; + + @CommandLine.Option( + names = {"--rpc-http-tls-known-clients-file"}, + paramLabel = DefaultCommandValues.MANDATORY_FILE_FORMAT_HELP, + description = + "Path to file containing clients certificate common name and fingerprint for client authentication") + private final Path rpcHttpTlsKnownClientsFile = null; + + @CommandLine.Option( + names = {"--rpc-http-tls-ca-clients-enabled"}, + description = + "Enable to accept clients certificate signed by a valid CA for client authentication (default: ${DEFAULT-VALUE})") + private final Boolean isRpcHttpTlsCAClientsEnabled = false; + + @CommandLine.Option( + names = {"--rpc-http-tls-protocol", "--rpc-http-tls-protocols"}, + description = "Comma separated list of TLS protocols to support (default: ${DEFAULT-VALUE})", + split = ",", + arity = "1..*") + private final List rpcHttpTlsProtocols = + new ArrayList<>(DefaultCommandValues.DEFAULT_TLS_PROTOCOLS); + + @CommandLine.Option( + names = {"--rpc-http-tls-cipher-suite", "--rpc-http-tls-cipher-suites"}, + description = "Comma separated list of TLS cipher suites to support", + split = ",", + arity = "1..*") + private final List rpcHttpTlsCipherSuites = new ArrayList<>(); + + @CommandLine.Option( + names = {"--rpc-http-max-batch-size"}, + paramLabel = DefaultCommandValues.MANDATORY_INTEGER_FORMAT_HELP, + description = + "Specifies the maximum number of requests in a single RPC batch request via RPC. -1 specifies no limit (default: ${DEFAULT-VALUE})") + private final Integer rpcHttpMaxBatchSize = DefaultCommandValues.DEFAULT_HTTP_MAX_BATCH_SIZE; + + @CommandLine.Option( + names = {"--rpc-http-max-request-content-length"}, + paramLabel = DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP, + description = "Specifies the maximum request content length. (default: ${DEFAULT-VALUE})") + private final Long rpcHttpMaxRequestContentLength = + DefaultCommandValues.DEFAULT_MAX_REQUEST_CONTENT_LENGTH; + + @CommandLine.Option( + names = {"--json-pretty-print-enabled"}, + description = "Enable JSON pretty print format (default: ${DEFAULT-VALUE})") + private final Boolean prettyJsonEnabled = DEFAULT_PRETTY_JSON_ENABLED; + + /** + * Validates the Rpc Http options. + * + * @param logger Logger instance + * @param commandLine CommandLine instance + * @param configuredApis Predicate for configured APIs + */ + public void validate( + final Logger logger, final CommandLine commandLine, final Predicate configuredApis) { + + if (!rpcHttpApis.stream().allMatch(configuredApis)) { + final List invalidHttpApis = new ArrayList<>(rpcHttpApis); + invalidHttpApis.removeAll(VALID_APIS); + throw new CommandLine.ParameterException( + commandLine, + "Invalid value for option '--rpc-http-api': invalid entries found " + invalidHttpApis); + } + + final boolean validHttpApiMethods = + rpcHttpApiMethodsNoAuth.stream().allMatch(RpcMethod::rpcMethodExists); + + if (!validHttpApiMethods) { + throw new CommandLine.ParameterException( + commandLine, + "Invalid value for option '--rpc-http-api-methods-no-auth', options must be valid RPC methods"); + } + + if (isRpcHttpAuthenticationEnabled) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-http-authentication-public-key-file", + rpcHttpAuthenticationPublicKeyFile == null, + List.of("--rpc-http-authentication-jwt-algorithm")); + } + + if (isRpcHttpAuthenticationEnabled + && rpcHttpAuthenticationCredentialsFile(commandLine) == null + && rpcHttpAuthenticationPublicKeyFile == null) { + throw new CommandLine.ParameterException( + commandLine, + "Unable to authenticate JSON-RPC HTTP endpoint without a supplied credentials file or authentication public key file"); + } + + checkDependencies(logger, commandLine); + + if (isRpcTlsConfigurationRequired()) { + validateTls(commandLine); + } + } + + /** + * Creates a JsonRpcConfiguration based on the provided options. + * + * @param hostsAllowlist List of hosts allowed + * @param defaultHostAddress Default host address + * @param timoutSec timeout in seconds + * @return A JsonRpcConfiguration instance + */ + public JsonRpcConfiguration jsonRpcConfiguration( + final List hostsAllowlist, final String defaultHostAddress, final Long timoutSec) { + + final JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); + jsonRpcConfiguration.setEnabled(isRpcHttpEnabled); + jsonRpcConfiguration.setHost( + Strings.isNullOrEmpty(rpcHttpHost) ? defaultHostAddress : rpcHttpHost); + jsonRpcConfiguration.setPort(rpcHttpPort); + jsonRpcConfiguration.setMaxActiveConnections(rpcHttpMaxConnections); + jsonRpcConfiguration.setCorsAllowedDomains(rpcHttpCorsAllowedOrigins); + jsonRpcConfiguration.setRpcApis(rpcHttpApis.stream().distinct().collect(Collectors.toList())); + jsonRpcConfiguration.setNoAuthRpcApis( + rpcHttpApiMethodsNoAuth.stream().distinct().collect(Collectors.toList())); + jsonRpcConfiguration.setHostsAllowlist(hostsAllowlist); + jsonRpcConfiguration.setAuthenticationEnabled(isRpcHttpAuthenticationEnabled); + jsonRpcConfiguration.setAuthenticationCredentialsFile(rpcHttpAuthenticationCredentialsFile); + jsonRpcConfiguration.setAuthenticationPublicKeyFile(rpcHttpAuthenticationPublicKeyFile); + jsonRpcConfiguration.setAuthenticationAlgorithm(rpcHttpAuthenticationAlgorithm); + jsonRpcConfiguration.setTlsConfiguration(rpcHttpTlsConfiguration()); + jsonRpcConfiguration.setHttpTimeoutSec(timoutSec); + jsonRpcConfiguration.setMaxBatchSize(rpcHttpMaxBatchSize); + jsonRpcConfiguration.setMaxRequestContentLength(rpcHttpMaxRequestContentLength); + jsonRpcConfiguration.setPrettyJsonEnabled(prettyJsonEnabled); + return jsonRpcConfiguration; + } + + /** + * Checks dependencies between options. + * + * @param logger Logger instance + * @param commandLine CommandLine instance + */ + public void checkDependencies(final Logger logger, final CommandLine commandLine) { + checkRpcTlsClientAuthOptionsDependencies(logger, commandLine); + checkRpcTlsOptionsDependencies(logger, commandLine); + checkRpcHttpOptionsDependencies(logger, commandLine); + } + + private void checkRpcTlsClientAuthOptionsDependencies( + final Logger logger, final CommandLine commandLine) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-http-tls-client-auth-enabled", + !isRpcHttpTlsClientAuthEnabled, + asList("--rpc-http-tls-known-clients-file", "--rpc-http-tls-ca-clients-enabled")); + } + + private void checkRpcTlsOptionsDependencies(final Logger logger, final CommandLine commandLine) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-http-tls-enabled", + !isRpcHttpTlsEnabled, + asList( + "--rpc-http-tls-keystore-file", + "--rpc-http-tls-keystore-password-file", + "--rpc-http-tls-client-auth-enabled", + "--rpc-http-tls-known-clients-file", + "--rpc-http-tls-ca-clients-enabled", + "--rpc-http-tls-protocols", + "--rpc-http-tls-cipher-suite", + "--rpc-http-tls-cipher-suites")); + } + + private void checkRpcHttpOptionsDependencies(final Logger logger, final CommandLine commandLine) { + CommandLineUtils.checkOptionDependencies( + logger, + commandLine, + "--rpc-http-enabled", + !isRpcHttpEnabled, + asList( + "--rpc-http-api", + "--rpc-http-apis", + "--rpc-http-api-method-no-auth", + "--rpc-http-api-methods-no-auth", + "--rpc-http-cors-origins", + "--rpc-http-host", + "--rpc-http-port", + "--rpc-http-max-active-connections", + "--rpc-http-authentication-enabled", + "--rpc-http-authentication-credentials-file", + "--rpc-http-authentication-public-key-file", + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + "--rpc-http-tls-keystore-password-file", + "--rpc-http-tls-client-auth-enabled", + "--rpc-http-tls-known-clients-file", + "--rpc-http-tls-ca-clients-enabled", + "--rpc-http-authentication-jwt-algorithm", + "--rpc-http-tls-protocols", + "--rpc-http-tls-cipher-suite", + "--rpc-http-tls-cipher-suites")); + } + + private void validateTls(final CommandLine commandLine) { + if (rpcHttpTlsKeyStoreFile == null) { + throw new CommandLine.ParameterException( + commandLine, "Keystore file is required when TLS is enabled for JSON-RPC HTTP endpoint"); + } + + if (rpcHttpTlsKeyStorePasswordFile == null) { + throw new CommandLine.ParameterException( + commandLine, + "File containing password to unlock keystore is required when TLS is enabled for JSON-RPC HTTP endpoint"); + } + + if (isRpcHttpTlsClientAuthEnabled + && !isRpcHttpTlsCAClientsEnabled + && rpcHttpTlsKnownClientsFile == null) { + throw new CommandLine.ParameterException( + commandLine, + "Known-clients file must be specified or CA clients must be enabled when TLS client authentication is enabled for JSON-RPC HTTP endpoint"); + } + + rpcHttpTlsProtocols.retainAll(getJDKEnabledProtocols()); + if (rpcHttpTlsProtocols.isEmpty()) { + throw new CommandLine.ParameterException( + commandLine, + "No valid TLS protocols specified (the following protocols are enabled: " + + getJDKEnabledProtocols() + + ")"); + } + + for (final String cipherSuite : rpcHttpTlsCipherSuites) { + if (!getJDKEnabledCipherSuites().contains(cipherSuite)) { + throw new CommandLine.ParameterException( + commandLine, "Invalid TLS cipher suite specified " + cipherSuite); + } + } + } + + private Optional rpcHttpTlsConfiguration() { + if (!isRpcTlsConfigurationRequired()) { + return Optional.empty(); + } + + rpcHttpTlsCipherSuites.retainAll(getJDKEnabledCipherSuites()); + + return Optional.of( + TlsConfiguration.Builder.aTlsConfiguration() + .withKeyStorePath(rpcHttpTlsKeyStoreFile) + .withKeyStorePasswordSupplier( + new FileBasedPasswordProvider(rpcHttpTlsKeyStorePasswordFile)) + .withClientAuthConfiguration(rpcHttpTlsClientAuthConfiguration()) + .withSecureTransportProtocols(rpcHttpTlsProtocols) + .withCipherSuites(rpcHttpTlsCipherSuites) + .build()); + } + + private boolean isRpcTlsConfigurationRequired() { + return isRpcHttpEnabled && isRpcHttpTlsEnabled; + } + + private TlsClientAuthConfiguration rpcHttpTlsClientAuthConfiguration() { + if (isRpcHttpTlsClientAuthEnabled) { + return TlsClientAuthConfiguration.Builder.aTlsClientAuthConfiguration() + .withKnownClientsFile(rpcHttpTlsKnownClientsFile) + .withCaClientsEnabled(isRpcHttpTlsCAClientsEnabled) + .build(); + } + + return null; + } + + private static List getJDKEnabledCipherSuites() { + try { + final SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, null, null); + final SSLEngine engine = context.createSSLEngine(); + return Arrays.asList(engine.getEnabledCipherSuites()); + } catch (final KeyManagementException | NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + private static List getJDKEnabledProtocols() { + try { + final SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, null, null); + final SSLEngine engine = context.createSSLEngine(); + return Arrays.asList(engine.getEnabledProtocols()); + } catch (final KeyManagementException | NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + private String rpcHttpAuthenticationCredentialsFile(final CommandLine commandLine) { + final String filename = rpcHttpAuthenticationCredentialsFile; + + if (filename != null) { + RpcAuthFileValidator.validate(commandLine, filename, "HTTP"); + } + return filename; + } + + /** + * Returns the list of APIs enabled for RPC over HTTP. + * + * @return A list of APIs + */ + public List getRpcHttpApis() { + return rpcHttpApis; + } + + /** + * Returns the port for RPC over HTTP. + * + * @return The port number + */ + public Integer getRpcHttpPort() { + return rpcHttpPort; + } + + /** + * Checks if RPC over HTTP is enabled. + * + * @return true if enabled, false otherwise + */ + public Boolean isRpcHttpEnabled() { + return isRpcHttpEnabled; + } +} diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 938c25af9d9..c69d8861096 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -28,9 +28,6 @@ import static org.hyperledger.besu.cli.config.NetworkName.MORDOR; import static org.hyperledger.besu.cli.config.NetworkName.SEPOLIA; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ENGINE; -import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; -import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; -import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.PERM; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.GOERLI_BOOTSTRAP_NODES; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.GOERLI_DISCOVERY_URL; import static org.hyperledger.besu.ethereum.p2p.config.DefaultDiscoveryConfiguration.MAINNET_BOOTSTRAP_NODES; @@ -45,7 +42,6 @@ import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -64,10 +60,7 @@ import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; import org.hyperledger.besu.ethereum.api.handlers.TimeoutOptions; import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration; -import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; -import org.hyperledger.besu.ethereum.api.jsonrpc.authentication.JwtAlgorithm; import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; -import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; @@ -87,7 +80,6 @@ import org.hyperledger.besu.pki.config.PkiKeyStoreConfiguration; import org.hyperledger.besu.plugin.data.EnodeURL; import org.hyperledger.besu.plugin.services.privacy.PrivateMarkerTransactionFactory; -import org.hyperledger.besu.plugin.services.rpc.PluginRpcRequest; import org.hyperledger.besu.util.number.Fraction; import org.hyperledger.besu.util.number.Percentage; import org.hyperledger.besu.util.number.PositiveNumber; @@ -96,7 +88,6 @@ import java.io.File; import java.io.IOException; import java.math.BigInteger; -import java.net.ServerSocket; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -110,7 +101,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -1545,37 +1535,6 @@ public void p2pPeerUpperBound_without_p2pPeerLowerBound_shouldSetLowerBoundEqual verify(mockRunnerBuilder).build(); } - @Test - public void rpcHttpMaxBatchSizeOptionMustBeUsed() { - final int rpcHttpMaxBatchSize = 1; - parseCommand("--rpc-http-max-batch-size", Integer.toString(rpcHttpMaxBatchSize)); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getMaxBatchSize()) - .isEqualTo(rpcHttpMaxBatchSize); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpMaxRequestContentLengthOptionMustBeUsed() { - final int rpcHttpMaxRequestContentLength = 1; - parseCommand( - "--rpc-http-max-request-content-length", Long.toString(rpcHttpMaxRequestContentLength)); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getMaxRequestContentLength()) - .isEqualTo(rpcHttpMaxRequestContentLength); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - @Test public void maxpeersSet_p2pPeerLowerBoundSet() { @@ -1941,1046 +1900,267 @@ public void natManagerPodNameCannotBeUsedWithNatNoneMethod() { @Test public void natMethodFallbackEnabledPropertyIsCorrectlyUpdatedWithKubernetes() { - parseCommand("--nat-method", "KUBERNETES", "--Xnat-method-fallback-enabled", "false"); - verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(false)); - parseCommand("--nat-method", "KUBERNETES", "--Xnat-method-fallback-enabled", "true"); - verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(true)); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void natMethodFallbackEnabledPropertyIsCorrectlyUpdatedWithDocker() { - - parseCommand("--nat-method", "DOCKER", "--Xnat-method-fallback-enabled", "false"); - verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(false)); - parseCommand("--nat-method", "DOCKER", "--Xnat-method-fallback-enabled", "true"); - verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(true)); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void natMethodFallbackEnabledPropertyIsCorrectlyUpdatedWithUpnp() { - - parseCommand("--nat-method", "UPNP", "--Xnat-method-fallback-enabled", "false"); - verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(false)); - parseCommand("--nat-method", "UPNP", "--Xnat-method-fallback-enabled", "true"); - verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(true)); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void natMethodFallbackEnabledCannotBeUsedWithAutoMethod() { - parseCommand("--nat-method", "AUTO", "--Xnat-method-fallback-enabled", "false"); - Mockito.verifyNoInteractions(mockRunnerBuilder); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "The `--Xnat-method-fallback-enabled` parameter cannot be used in AUTO mode. Either remove --Xnat-method-fallback-enabled or select another mode (via --nat--method=XXXX)"); - } - - @Test - public void rpcHttpEnabledPropertyDefaultIsFalse() { - parseCommand(); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().isEnabled()).isFalse(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpEnabledPropertyMustBeUsed() { - parseCommand("--rpc-http-enabled"); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().isEnabled()).isTrue(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void graphQLHttpEnabledPropertyDefaultIsFalse() { - parseCommand(); - - verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(graphQLConfigArgumentCaptor.getValue().isEnabled()).isFalse(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void graphQLHttpEnabledPropertyMustBeUsed() { - parseCommand("--graphql-http-enabled"); - - verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(graphQLConfigArgumentCaptor.getValue().isEnabled()).isTrue(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcApisPropertyMustBeUsed() { - parseCommand("--rpc-http-api", "ETH,NET,PERM", "--rpc-http-enabled"); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - verify(mockLogger) - .warn("Permissions are disabled. Cannot enable PERM APIs when not using Permissions."); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getRpcApis()) - .containsExactlyInAnyOrder(ETH.name(), NET.name(), PERM.name()); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcApisSupportsEngine() { - parseCommand("--rpc-http-api", "ENGINE", "--rpc-http-enabled"); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getRpcApis()) - .containsExactlyInAnyOrder(ENGINE.name()); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcApisPropertyIgnoresDuplicatesAndMustBeUsed() { - parseCommand("--rpc-http-api", "ETH,NET,NET", "--rpc-http-enabled"); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getRpcApis()) - .containsExactlyInAnyOrder(ETH.name(), NET.name()); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcApiNoAuthMethodsIgnoresDuplicatesAndMustBeUsed() { - parseCommand( - "--rpc-http-api-methods-no-auth", - "admin_peers, admin_peers, eth_getWork", - "--rpc-http-enabled"); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getNoAuthRpcApis()) - .containsExactlyInAnyOrder( - RpcMethod.ADMIN_PEERS.getMethodName(), RpcMethod.ETH_GET_WORK.getMethodName()); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void engineApiAuthOptions() { - // TODO: once we have mainnet TTD, we can remove the TTD override parameter here - // https://github.com/hyperledger/besu/issues/3874 - parseCommand( - "--override-genesis-config", - "terminalTotalDifficulty=1337", - "--rpc-http-enabled", - "--engine-jwt-secret", - "/tmp/fakeKey.hex"); - verify(mockRunnerBuilder).engineJsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - assertThat(jsonRpcConfigArgumentCaptor.getValue().isAuthenticationEnabled()).isTrue(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void engineApiDisableAuthOptions() { - // TODO: once we have mainnet TTD, we can remove the TTD override parameter here - // https://github.com/hyperledger/besu/issues/3874 - parseCommand( - "--override-genesis-config", - "terminalTotalDifficulty=1337", - "--rpc-http-enabled", - "--engine-jwt-disabled", - "--engine-jwt-secret", - "/tmp/fakeKey.hex"); - verify(mockRunnerBuilder).engineJsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - assertThat(jsonRpcConfigArgumentCaptor.getValue().isAuthenticationEnabled()).isFalse(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpNoAuthApiMethodsCannotBeInvalid() { - parseCommand("--rpc-http-enabled", "--rpc-http-api-method-no-auth", "invalid"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "Invalid value for option '--rpc-http-api-methods-no-auth', options must be valid RPC methods"); - } - - @Test - public void rpcHttpOptionsRequiresServiceToBeEnabled() { - parseCommand( - "--rpc-http-api", - "ETH,NET", - "--rpc-http-host", - "0.0.0.0", - "--rpc-http-port", - "1234", - "--rpc-http-cors-origins", - "all", - "--rpc-http-max-active-connections", - "88"); - - verifyOptionsConstraintLoggerCall( - "--rpc-http-enabled", - "--rpc-http-host", - "--rpc-http-port", - "--rpc-http-cors-origins", - "--rpc-http-api", - "--rpc-http-max-active-connections"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpOptionsRequiresServiceToBeEnabledToml() throws IOException { - final Path toml = - createTempFile( - "toml", - "rpc-http-api=[\"ETH\",\"NET\"]\n" - + "rpc-http-host=\"0.0.0.0\"\n" - + "rpc-http-port=1234\n" - + "rpc-http-cors-origins=[\"all\"]\n" - + "rpc-http-max-active-connections=88"); - - parseCommand("--config-file", toml.toString()); - - verifyOptionsConstraintLoggerCall( - "--rpc-http-enabled", - "--rpc-http-host", - "--rpc-http-port", - "--rpc-http-cors-origins", - "--rpc-http-api", - "--rpc-http-max-active-connections"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void privacyTlsOptionsRequiresTlsToBeEnabled() { - when(storageService.getByName("rocksdb-privacy")) - .thenReturn(Optional.of(rocksDBSPrivacyStorageFactory)); - final URL configFile = this.getClass().getResource("/orion_publickey.pub"); - final String coinbaseStr = String.format("%040x", 1); - - parseCommand( - "--privacy-enabled", - "--miner-enabled", - "--miner-coinbase=" + coinbaseStr, - "--min-gas-price", - "0", - "--privacy-url", - ENCLAVE_URI, - "--privacy-public-key-file", - configFile.getPath(), - "--privacy-tls-keystore-file", - "/Users/me/key"); - - verifyOptionsConstraintLoggerCall("--privacy-tls-enabled", "--privacy-tls-keystore-file"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void privacyTlsOptionsRequiresTlsToBeEnabledToml() throws IOException { - when(storageService.getByName("rocksdb-privacy")) - .thenReturn(Optional.of(rocksDBSPrivacyStorageFactory)); - final URL configFile = this.getClass().getResource("/orion_publickey.pub"); - final String coinbaseStr = String.format("%040x", 1); - - final Path toml = - createTempFile( - "toml", - "privacy-enabled=true\n" - + "miner-enabled=true\n" - + "miner-coinbase=\"" - + coinbaseStr - + "\"\n" - + "min-gas-price=0\n" - + "privacy-url=\"" - + ENCLAVE_URI - + "\"\n" - + "privacy-public-key-file=\"" - + configFile.getPath() - + "\"\n" - + "privacy-tls-keystore-file=\"/Users/me/key\""); - - parseCommand("--config-file", toml.toString()); - - verifyOptionsConstraintLoggerCall("--privacy-tls-enabled", "--privacy-tls-keystore-file"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void privacyTlsOptionsRequiresPrivacyToBeEnabled() { - parseCommand("--privacy-tls-enabled", "--privacy-tls-keystore-file", "/Users/me/key"); - - verifyOptionsConstraintLoggerCall("--privacy-enabled", "--privacy-tls-enabled"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void privacyTlsOptionsRequiresPrivacyToBeEnabledToml() throws IOException { - final Path toml = - createTempFile( - "toml", "privacy-tls-enabled=true\n" + "privacy-tls-keystore-file=\"/Users/me/key\""); - - parseCommand("--config-file", toml.toString()); - - verifyOptionsConstraintLoggerCall("--privacy-enabled", "--privacy-tls-enabled"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcApisPropertyWithInvalidEntryMustDisplayError() { - parseCommand("--rpc-http-api", "BOB"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - // PicoCLI uses longest option name for message when option has multiple names, so here plural. - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Invalid value for option '--rpc-http-api': invalid entries found [BOB]"); - } - - @Test - public void rpcApisPropertyWithPluginNamespaceAreValid() { - - rpcEndpointServiceImpl.registerRPCEndpoint( - "bob", "method", (Function) request -> "nothing"); - - parseCommand("--rpc-http-api", "BOB"); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getRpcApis()) - .containsExactlyInAnyOrder("BOB"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpHostAndPortOptionsMustBeUsed() { - - final String host = "1.2.3.4"; - final int port = 1234; - parseCommand( - "--rpc-http-enabled", "--rpc-http-host", host, "--rpc-http-port", String.valueOf(port)); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpHostMayBeLocalhost() { - - final String host = "localhost"; - parseCommand("--rpc-http-enabled", "--rpc-http-host", host); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpHostMayBeIPv6() { - - final String host = "2600:DB8::8545"; - parseCommand("--rpc-http-enabled", "--rpc-http-host", host); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpMaxActiveConnectionsPropertyMustBeUsed() { - final int maxConnections = 99; - parseCommand("--rpc-http-max-active-connections", String.valueOf(maxConnections)); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getMaxActiveConnections()) - .isEqualTo(maxConnections); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpTlsRequiresRpcHttpEnabled() { - parseCommand("--rpc-http-tls-enabled"); - - verifyOptionsConstraintLoggerCall("--rpc-http-enabled", "--rpc-http-tls-enabled"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpTlsRequiresRpcHttpEnabledToml() throws IOException { - final Path toml = createTempFile("toml", "rpc-http-tls-enabled=true\n"); - - parseCommand("--config-file", toml.toString()); - - verifyOptionsConstraintLoggerCall("--rpc-http-enabled", "--rpc-http-tls-enabled"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpTlsWithoutKeystoreReportsError() { - parseCommand("--rpc-http-enabled", "--rpc-http-tls-enabled"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Keystore file is required when TLS is enabled for JSON-RPC HTTP endpoint"); - } - - @Test - public void rpcHttpTlsWithoutPasswordfileReportsError() { - parseCommand( - "--rpc-http-enabled", - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - "/tmp/test.p12"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "File containing password to unlock keystore is required when TLS is enabled for JSON-RPC HTTP endpoint"); - } - - @Test - public void rpcHttpTlsKeystoreAndPasswordMustBeUsed() { - final String host = "1.2.3.4"; - final int port = 1234; - final String keystoreFile = "/tmp/test.p12"; - final String keystorePasswordFile = "/tmp/test.txt"; - - parseCommand( - "--rpc-http-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - keystoreFile, - "--rpc-http-tls-keystore-password-file", - keystorePasswordFile); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - final Optional tlsConfiguration = - jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); - assertThat(tlsConfiguration.isPresent()).isTrue(); - assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); - assertThat(tlsConfiguration.get().getClientAuthConfiguration().isEmpty()).isTrue(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpTlsClientAuthWithoutKnownFileReportsError() { - final String host = "1.2.3.4"; - final int port = 1234; - final String keystoreFile = "/tmp/test.p12"; - final String keystorePasswordFile = "/tmp/test.txt"; - parseCommand( - "--rpc-http-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - keystoreFile, - "--rpc-http-tls-keystore-password-file", - keystorePasswordFile, - "--rpc-http-tls-client-auth-enabled"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "Known-clients file must be specified or CA clients must be enabled when TLS client authentication is enabled for JSON-RPC HTTP endpoint"); - } - - @Test - public void rpcHttpTlsClientAuthWithKnownClientFile() { - final String host = "1.2.3.4"; - final int port = 1234; - final String keystoreFile = "/tmp/test.p12"; - final String keystorePasswordFile = "/tmp/test.txt"; - final String knownClientFile = "/tmp/knownClientFile"; - parseCommand( - "--rpc-http-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - keystoreFile, - "--rpc-http-tls-keystore-password-file", - keystorePasswordFile, - "--rpc-http-tls-client-auth-enabled", - "--rpc-http-tls-known-clients-file", - knownClientFile); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - final Optional tlsConfiguration = - jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); - assertThat(tlsConfiguration.isPresent()).isTrue(); - assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); - assertThat(tlsConfiguration.get().getClientAuthConfiguration().isPresent()).isTrue(); - assertThat( - tlsConfiguration.get().getClientAuthConfiguration().get().getKnownClientsFile().get()) - .isEqualTo(Path.of(knownClientFile)); - assertThat(tlsConfiguration.get().getClientAuthConfiguration().get().isCaClientsEnabled()) - .isFalse(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpTlsClientAuthWithCAClient() { - final String host = "1.2.3.4"; - final int port = 1234; - final String keystoreFile = "/tmp/test.p12"; - final String keystorePasswordFile = "/tmp/test.txt"; - parseCommand( - "--rpc-http-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - keystoreFile, - "--rpc-http-tls-keystore-password-file", - keystorePasswordFile, - "--rpc-http-tls-client-auth-enabled", - "--rpc-http-tls-ca-clients-enabled"); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - final Optional tlsConfiguration = - jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); - assertThat(tlsConfiguration.isPresent()).isTrue(); - assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); - assertThat(tlsConfiguration.get().getClientAuthConfiguration().isPresent()).isTrue(); - assertThat( - tlsConfiguration - .get() - .getClientAuthConfiguration() - .get() - .getKnownClientsFile() - .isEmpty()) - .isTrue(); - assertThat(tlsConfiguration.get().getClientAuthConfiguration().get().isCaClientsEnabled()) - .isTrue(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpTlsClientAuthWithCAClientAndKnownClientFile() { - final String host = "1.2.3.4"; - final int port = 1234; - final String keystoreFile = "/tmp/test.p12"; - final String keystorePasswordFile = "/tmp/test.txt"; - final String knownClientFile = "/tmp/knownClientFile"; - parseCommand( - "--rpc-http-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - keystoreFile, - "--rpc-http-tls-keystore-password-file", - keystorePasswordFile, - "--rpc-http-tls-client-auth-enabled", - "--rpc-http-tls-ca-clients-enabled", - "--rpc-http-tls-known-clients-file", - knownClientFile); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - final Optional tlsConfiguration = - jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); - assertThat(tlsConfiguration.isPresent()).isTrue(); - assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); - assertThat(tlsConfiguration.get().getClientAuthConfiguration().isPresent()).isTrue(); - assertThat( - tlsConfiguration.get().getClientAuthConfiguration().get().getKnownClientsFile().get()) - .isEqualTo(Path.of(knownClientFile)); - assertThat(tlsConfiguration.get().getClientAuthConfiguration().get().isCaClientsEnabled()) - .isTrue(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpTlsCheckDefaultProtocolsAndCipherSuites() { - final String host = "1.2.3.4"; - final int port = 1234; - final String keystoreFile = "/tmp/test.p12"; - final String keystorePasswordFile = "/tmp/test.txt"; - - parseCommand( - "--rpc-http-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - keystoreFile, - "--rpc-http-tls-keystore-password-file", - keystorePasswordFile); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - final Optional tlsConfiguration = - jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); - assertThat(tlsConfiguration).isPresent(); - assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); - assertThat(tlsConfiguration.get().getClientAuthConfiguration()).isEmpty(); - assertThat(tlsConfiguration.get().getCipherSuites().get()).isEmpty(); - assertThat(tlsConfiguration.get().getSecureTransportProtocols().get()) - .containsExactly("TLSv1.3", "TLSv1.2"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpTlsCheckInvalidProtocols() { - final String host = "1.2.3.4"; - final int port = 1234; - final String keystoreFile = "/tmp/test.p12"; - final String keystorePasswordFile = "/tmp/test.txt"; - final String protocol = "TLsv1.4"; - - parseCommand( - "--rpc-http-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - keystoreFile, - "--rpc-http-tls-keystore-password-file", - keystorePasswordFile, - "--rpc-http-tls-protocols", - protocol); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).contains("No valid TLS protocols specified"); - } - - @Test - public void rpcHttpTlsCheckInvalidCipherSuites() { - final String host = "1.2.3.4"; - final int port = 1234; - final String keystoreFile = "/tmp/test.p12"; - final String keystorePasswordFile = "/tmp/test.txt"; - final String cipherSuites = "Invalid"; - - parseCommand( - "--rpc-http-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - keystoreFile, - "--rpc-http-tls-keystore-password-file", - keystorePasswordFile, - "--rpc-http-tls-cipher-suites", - cipherSuites); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Invalid TLS cipher suite specified " + cipherSuites); - } - - @Test - public void rpcHttpTlsCheckValidProtocolsAndCipherSuites() { - final String host = "1.2.3.4"; - final int port = 1234; - final String keystoreFile = "/tmp/test.p12"; - final String keystorePasswordFile = "/tmp/test.txt"; - final String protocols = "TLSv1.3,TLSv1.2"; - final String cipherSuites = - "TLS_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"; - - parseCommand( - "--rpc-http-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-enabled", - "--rpc-http-tls-keystore-file", - keystoreFile, - "--rpc-http-tls-keystore-password-file", - keystorePasswordFile, - "--rpc-http-tls-protocols", - protocols, - "--rpc-http-tls-cipher-suites", - cipherSuites); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - final Optional tlsConfiguration = - jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); - assertThat(tlsConfiguration).isPresent(); - assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); - assertThat(tlsConfiguration.get().getClientAuthConfiguration()).isEmpty(); - assertThat(tlsConfiguration.get().getCipherSuites().get()) - .containsExactlyInAnyOrder( - "TLS_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); - assertThat(tlsConfiguration.get().getSecureTransportProtocols().get()) - .containsExactlyInAnyOrder("TLSv1.2", "TLSv1.3"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void rpcHttpTlsWarnIfCipherSuitesSpecifiedWithoutTls() { - final String host = "1.2.3.4"; - final int port = 1234; - final String cipherSuites = "Invalid"; - - parseCommand( - "--rpc-http-enabled", - "--engine-rpc-enabled", - "--rpc-http-host", - host, - "--rpc-http-port", - String.valueOf(port), - "--rpc-http-tls-cipher-suite", - cipherSuites); - verify( - mockLogger, - times(2)) // this is verified for both the full suite of apis, and the engine group. - .warn( - "{} has been ignored because {} was not defined on the command line.", - "--rpc-http-tls-cipher-suite", - "--rpc-http-tls-enabled"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void graphQLHttpHostAndPortOptionsMustBeUsed() { - - final String host = "1.2.3.4"; - final int port = 1234; - parseCommand( - "--graphql-http-enabled", - "--graphql-http-host", - host, - "--graphql-http-port", - String.valueOf(port)); - - verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(graphQLConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); - assertThat(graphQLConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + parseCommand("--nat-method", "KUBERNETES", "--Xnat-method-fallback-enabled", "false"); + verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(false)); + parseCommand("--nat-method", "KUBERNETES", "--Xnat-method-fallback-enabled", "true"); + verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(true)); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void graphQLHttpHostMayBeLocalhost() { - - final String host = "localhost"; - parseCommand("--graphql-http-enabled", "--graphql-http-host", host); - - verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); + public void natMethodFallbackEnabledPropertyIsCorrectlyUpdatedWithDocker() { - assertThat(graphQLConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + parseCommand("--nat-method", "DOCKER", "--Xnat-method-fallback-enabled", "false"); + verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(false)); + parseCommand("--nat-method", "DOCKER", "--Xnat-method-fallback-enabled", "true"); + verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(true)); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void graphQLHttpHostMayBeIPv6() { - - final String host = "2600:DB8::8545"; - parseCommand("--graphql-http-enabled", "--graphql-http-host", host); - - verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); + public void natMethodFallbackEnabledPropertyIsCorrectlyUpdatedWithUpnp() { - assertThat(graphQLConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + parseCommand("--nat-method", "UPNP", "--Xnat-method-fallback-enabled", "false"); + verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(false)); + parseCommand("--nat-method", "UPNP", "--Xnat-method-fallback-enabled", "true"); + verify(mockRunnerBuilder).natMethodFallbackEnabled(eq(true)); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsTwoDomainsMustBuildListWithBothDomains() { - final String[] origins = {"http://domain1.com", "https://domain2.com"}; - parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", String.join(",", origins)); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains().toArray()) - .isEqualTo(origins); - + public void natMethodFallbackEnabledCannotBeUsedWithAutoMethod() { + parseCommand("--nat-method", "AUTO", "--Xnat-method-fallback-enabled", "false"); + Mockito.verifyNoInteractions(mockRunnerBuilder); assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains( + "The `--Xnat-method-fallback-enabled` parameter cannot be used in AUTO mode. Either remove --Xnat-method-fallback-enabled or select another mode (via --nat--method=XXXX)"); } @Test - public void rpcHttpCorsOriginsDoubleCommaFilteredOut() { - final String[] origins = {"http://domain1.com", "https://domain2.com"}; - parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", String.join(",,", origins)); + public void rpcHttpEnabledPropertyDefaultIsFalse() { + parseCommand(); verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); verify(mockRunnerBuilder).build(); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains().toArray()) - .isEqualTo(origins); + assertThat(jsonRpcConfigArgumentCaptor.getValue().isEnabled()).isFalse(); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsWithWildcardMustBuildListWithWildcard() { - final String[] origins = {"*"}; - parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", String.join(",", origins)); + public void graphQLHttpEnabledPropertyDefaultIsFalse() { + parseCommand(); - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); verify(mockRunnerBuilder).build(); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains().toArray()) - .isEqualTo(origins); + assertThat(graphQLConfigArgumentCaptor.getValue().isEnabled()).isFalse(); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsWithAllMustBuildListWithWildcard() { - parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", "all"); + public void graphQLHttpEnabledPropertyMustBeUsed() { + parseCommand("--graphql-http-enabled"); - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); verify(mockRunnerBuilder).build(); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains()).containsExactly("*"); + assertThat(graphQLConfigArgumentCaptor.getValue().isEnabled()).isTrue(); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsWithNoneMustBuildEmptyList() { - final String[] origins = {"none"}; - parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", String.join(",", origins)); + public void rpcApisSupportsEngine() { + parseCommand("--rpc-http-api", "ENGINE", "--rpc-http-enabled"); verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); verify(mockRunnerBuilder).build(); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains()).isEmpty(); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getRpcApis()) + .containsExactlyInAnyOrder(ENGINE.name()); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsNoneWithAnotherDomainMustFail() { - final String[] origins = {"http://domain1.com", "none"}; - parseCommand("--rpc-http-cors-origins", String.join(",", origins)); - - Mockito.verifyNoInteractions(mockRunnerBuilder); + public void engineApiAuthOptions() { + // TODO: once we have mainnet TTD, we can remove the TTD override parameter here + // https://github.com/hyperledger/besu/issues/3874 + parseCommand( + "--override-genesis-config", + "terminalTotalDifficulty=1337", + "--rpc-http-enabled", + "--engine-jwt-secret", + "/tmp/fakeKey.hex"); + verify(mockRunnerBuilder).engineJsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + assertThat(jsonRpcConfigArgumentCaptor.getValue().isAuthenticationEnabled()).isTrue(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + @Test + public void engineApiDisableAuthOptions() { + // TODO: once we have mainnet TTD, we can remove the TTD override parameter here + // https://github.com/hyperledger/besu/issues/3874 + parseCommand( + "--override-genesis-config", + "terminalTotalDifficulty=1337", + "--rpc-http-enabled", + "--engine-jwt-disabled", + "--engine-jwt-secret", + "/tmp/fakeKey.hex"); + verify(mockRunnerBuilder).engineJsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + assertThat(jsonRpcConfigArgumentCaptor.getValue().isAuthenticationEnabled()).isFalse(); assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Value 'none' can't be used with other domains"); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsNoneWithAnotherDomainMustFailNoneFirst() { - final String[] origins = {"none", "http://domain1.com"}; - parseCommand("--rpc-http-cors-origins", String.join(",", origins)); + public void privacyTlsOptionsRequiresTlsToBeEnabled() { + when(storageService.getByName("rocksdb-privacy")) + .thenReturn(Optional.of(rocksDBSPrivacyStorageFactory)); + final URL configFile = this.getClass().getResource("/orion_publickey.pub"); + final String coinbaseStr = String.format("%040x", 1); - Mockito.verifyNoInteractions(mockRunnerBuilder); + parseCommand( + "--privacy-enabled", + "--miner-enabled", + "--miner-coinbase=" + coinbaseStr, + "--min-gas-price", + "0", + "--privacy-url", + ENCLAVE_URI, + "--privacy-public-key-file", + configFile.getPath(), + "--privacy-tls-keystore-file", + "/Users/me/key"); + + verifyOptionsConstraintLoggerCall("--privacy-tls-enabled", "--privacy-tls-keystore-file"); assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Value 'none' can't be used with other domains"); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsAllWithAnotherDomainMustFail() { - parseCommand("--rpc-http-cors-origins=http://domain1.com,all"); + public void privacyTlsOptionsRequiresTlsToBeEnabledToml() throws IOException { + when(storageService.getByName("rocksdb-privacy")) + .thenReturn(Optional.of(rocksDBSPrivacyStorageFactory)); + final URL configFile = this.getClass().getResource("/orion_publickey.pub"); + final String coinbaseStr = String.format("%040x", 1); - Mockito.verifyNoInteractions(mockRunnerBuilder); + final Path toml = + createTempFile( + "toml", + "privacy-enabled=true\n" + + "miner-enabled=true\n" + + "miner-coinbase=\"" + + coinbaseStr + + "\"\n" + + "min-gas-price=0\n" + + "privacy-url=\"" + + ENCLAVE_URI + + "\"\n" + + "privacy-public-key-file=\"" + + configFile.getPath() + + "\"\n" + + "privacy-tls-keystore-file=\"/Users/me/key\""); + + parseCommand("--config-file", toml.toString()); + + verifyOptionsConstraintLoggerCall("--privacy-tls-enabled", "--privacy-tls-keystore-file"); assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Values '*' or 'all' can't be used with other domains"); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsAllWithAnotherDomainMustFailAsFlags() { - parseCommand("--rpc-http-cors-origins=http://domain1.com", "--rpc-http-cors-origins=all"); + public void privacyTlsOptionsRequiresPrivacyToBeEnabled() { + parseCommand("--privacy-tls-enabled", "--privacy-tls-keystore-file", "/Users/me/key"); - Mockito.verifyNoInteractions(mockRunnerBuilder); + verifyOptionsConstraintLoggerCall("--privacy-enabled", "--privacy-tls-enabled"); assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Values '*' or 'all' can't be used with other domains"); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsWildcardWithAnotherDomainMustFail() { - parseCommand("--rpc-http-cors-origins=http://domain1.com,*"); + public void privacyTlsOptionsRequiresPrivacyToBeEnabledToml() throws IOException { + final Path toml = + createTempFile( + "toml", "privacy-tls-enabled=true\n" + "privacy-tls-keystore-file=\"/Users/me/key\""); - Mockito.verifyNoInteractions(mockRunnerBuilder); + parseCommand("--config-file", toml.toString()); + + verifyOptionsConstraintLoggerCall("--privacy-enabled", "--privacy-tls-enabled"); assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Values '*' or 'all' can't be used with other domains"); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsWildcardWithAnotherDomainMustFailAsFlags() { - parseCommand("--rpc-http-cors-origins=http://domain1.com", "--rpc-http-cors-origins=*"); + public void graphQLHttpHostAndPortOptionsMustBeUsed() { - Mockito.verifyNoInteractions(mockRunnerBuilder); + final String host = "1.2.3.4"; + final int port = 1234; + parseCommand( + "--graphql-http-enabled", + "--graphql-http-host", + host, + "--graphql-http-port", + String.valueOf(port)); + + verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(graphQLConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + assertThat(graphQLConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Values '*' or 'all' can't be used with other domains"); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsInvalidRegexShouldFail() { - final String[] origins = {"**"}; - parseCommand("--rpc-http-cors-origins", String.join(",", origins)); + public void graphQLHttpHostMayBeLocalhost() { - Mockito.verifyNoInteractions(mockRunnerBuilder); + final String host = "localhost"; + parseCommand("--graphql-http-enabled", "--graphql-http-host", host); + + verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(graphQLConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Domain values result in invalid regex pattern"); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } @Test - public void rpcHttpCorsOriginsEmptyValueFails() { - parseCommand("--rpc-http-cors-origins="); + public void graphQLHttpHostMayBeIPv6() { - Mockito.verifyNoInteractions(mockRunnerBuilder); + final String host = "2600:DB8::8545"; + parseCommand("--graphql-http-enabled", "--graphql-http-host", host); + + verify(mockRunnerBuilder).graphQLConfiguration(graphQLConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(graphQLConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Domain cannot be empty string or null string."); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } /** test deprecated CLI option * */ @@ -3167,19 +2347,6 @@ public void rpcHttpHostAllowlistEmptyValueFails() { .contains("Hostname cannot be empty string or null string."); } - @Test - public void rpcWsRpcEnabledPropertyDefaultIsFalse() { - parseCommand(); - - verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(wsRpcConfigArgumentCaptor.getValue().isEnabled()).isFalse(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - @Test public void metricsEnabledPropertyDefaultIsFalse() { parseCommand(); @@ -4147,40 +3314,6 @@ public void requiredBlocksMultipleBlocksTwoArgs() { .containsEntry(block2, Hash.fromHexStringLenient(hash2)); } - @Test - public void httpAuthenticationAlgorithIsConfigured() { - parseCommand("--rpc-http-authentication-jwt-algorithm", "ES256"); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getAuthenticationAlgorithm()) - .isEqualTo(JwtAlgorithm.ES256); - } - - @Test - public void httpAuthenticationPublicKeyIsConfigured() throws IOException { - final Path publicKey = Files.createTempFile("public_key", ""); - parseCommand("--rpc-http-authentication-jwt-public-key-file", publicKey.toString()); - - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - assertThat(jsonRpcConfigArgumentCaptor.getValue().getAuthenticationPublicKeyFile().getPath()) - .isEqualTo(publicKey.toString()); - } - - @Test - public void httpAuthenticationWithoutRequiredConfiguredOptionsMustFail() { - parseCommand("--rpc-http-enabled", "--rpc-http-authentication-enabled"); - - verifyNoInteractions(mockRunnerBuilder); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "Unable to authenticate JSON-RPC HTTP endpoint without a supplied credentials file or authentication public key file"); - } - @Test public void privHttpApisWithPrivacyDisabledLogsWarning() { parseCommand("--privacy-enabled=false", "--rpc-http-api", "PRIV", "--rpc-http-enabled"); @@ -4412,29 +3545,6 @@ public void assertThatCompatibilityEth64ForkIdIsPresentInHelpMessage() { assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } - @Test - public void assertThatCheckPortClashAcceptsAsExpected() throws Exception { - // use WS port for HTTP - final int port = 8546; - parseCommand("--rpc-http-enabled", "--rpc-http-port", String.valueOf(port)); - verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void assertThatCheckPortClashRejectsAsExpected() throws Exception { - // use WS port for HTTP - final int port = 8546; - parseCommand("--rpc-http-enabled", "--rpc-http-port", String.valueOf(port), "--rpc-ws-enabled"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "Port number '8546' has been specified multiple times. Please review the supplied configuration."); - } - @Test public void assertThatCheckPortClashRejectsAsExpectedForEngineApi() throws Exception { // use WS port for HTTP @@ -4638,19 +3748,6 @@ public void logWarnIfFastSyncMinPeersUsedWithFullSync() { verify(mockLogger).warn("--sync-min-peers is ignored in FULL sync-mode"); } - @Test - public void portInUseReportsError() throws IOException { - final ServerSocket serverSocket = new ServerSocket(8545); - - parseCommandWithPortCheck("--rpc-http-enabled"); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Port(s) '[8545]' already in use. Check for other processes using the port(s)."); - - serverSocket.close(); - } - @Test public void presentRequiredOptionShouldPass() { parseCommandWithRequiredOption("--accept-terms-and-conditions", "true"); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/JsonRpcHttpOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/JsonRpcHttpOptionsTest.java new file mode 100644 index 00000000000..83da0246036 --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/JsonRpcHttpOptionsTest.java @@ -0,0 +1,926 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.options; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; +import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.PERM; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import org.hyperledger.besu.cli.CommandTestAbstract; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.authentication.JwtAlgorithm; +import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration; +import org.hyperledger.besu.plugin.services.rpc.PluginRpcRequest; + +import java.io.IOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.function.Function; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class JsonRpcHttpOptionsTest extends CommandTestAbstract { + + @Test + public void rpcHttpEnabledPropertyMustBeUsed() { + parseCommand("--rpc-http-enabled"); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().isEnabled()).isTrue(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcApisPropertyMustBeUsed() { + parseCommand("--rpc-http-api", "ETH,NET,PERM", "--rpc-http-enabled"); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + verify(mockLogger) + .warn("Permissions are disabled. Cannot enable PERM APIs when not using Permissions."); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getRpcApis()) + .containsExactlyInAnyOrder(ETH.name(), NET.name(), PERM.name()); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcApisPropertyIgnoresDuplicatesAndMustBeUsed() { + parseCommand("--rpc-http-api", "ETH,NET,NET", "--rpc-http-enabled"); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getRpcApis()) + .containsExactlyInAnyOrder(ETH.name(), NET.name()); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcApiNoAuthMethodsIgnoresDuplicatesAndMustBeUsed() { + parseCommand( + "--rpc-http-api-methods-no-auth", + "admin_peers, admin_peers, eth_getWork", + "--rpc-http-enabled"); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getNoAuthRpcApis()) + .containsExactlyInAnyOrder( + RpcMethod.ADMIN_PEERS.getMethodName(), RpcMethod.ETH_GET_WORK.getMethodName()); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpNoAuthApiMethodsCannotBeInvalid() { + parseCommand("--rpc-http-enabled", "--rpc-http-api-method-no-auth", "invalid"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains( + "Invalid value for option '--rpc-http-api-methods-no-auth', options must be valid RPC methods"); + } + + @Test + public void rpcHttpOptionsRequiresServiceToBeEnabled() { + parseCommand( + "--rpc-http-api", + "ETH,NET", + "--rpc-http-host", + "0.0.0.0", + "--rpc-http-port", + "1234", + "--rpc-http-cors-origins", + "all", + "--rpc-http-max-active-connections", + "88"); + + verifyOptionsConstraintLoggerCall( + "--rpc-http-enabled", + "--rpc-http-host", + "--rpc-http-port", + "--rpc-http-cors-origins", + "--rpc-http-api", + "--rpc-http-max-active-connections"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpOptionsRequiresServiceToBeEnabledToml() throws IOException { + final Path toml = + createTempFile( + "toml", + "rpc-http-api=[\"ETH\",\"NET\"]\n" + + "rpc-http-host=\"0.0.0.0\"\n" + + "rpc-http-port=1234\n" + + "rpc-http-cors-origins=[\"all\"]\n" + + "rpc-http-max-active-connections=88"); + + parseCommand("--config-file", toml.toString()); + + verifyOptionsConstraintLoggerCall( + "--rpc-http-enabled", + "--rpc-http-host", + "--rpc-http-port", + "--rpc-http-cors-origins", + "--rpc-http-api", + "--rpc-http-max-active-connections"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpHostAndPortOptionsMustBeUsed() { + + final String host = "1.2.3.4"; + final int port = 1234; + parseCommand( + "--rpc-http-enabled", "--rpc-http-host", host, "--rpc-http-port", String.valueOf(port)); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpHostMayBeLocalhost() { + + final String host = "localhost"; + parseCommand("--rpc-http-enabled", "--rpc-http-host", host); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpHostMayBeIPv6() { + + final String host = "2600:DB8::8545"; + parseCommand("--rpc-http-enabled", "--rpc-http-host", host); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpMaxActiveConnectionsPropertyMustBeUsed() { + final int maxConnections = 99; + parseCommand("--rpc-http-max-active-connections", String.valueOf(maxConnections)); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getMaxActiveConnections()) + .isEqualTo(maxConnections); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpTlsRequiresRpcHttpEnabled() { + parseCommand("--rpc-http-tls-enabled"); + + verifyOptionsConstraintLoggerCall("--rpc-http-enabled", "--rpc-http-tls-enabled"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpTlsRequiresRpcHttpEnabledToml() throws IOException { + final Path toml = createTempFile("toml", "rpc-http-tls-enabled=true\n"); + + parseCommand("--config-file", toml.toString()); + + verifyOptionsConstraintLoggerCall("--rpc-http-enabled", "--rpc-http-tls-enabled"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpTlsWithoutKeystoreReportsError() { + parseCommand("--rpc-http-enabled", "--rpc-http-tls-enabled"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Keystore file is required when TLS is enabled for JSON-RPC HTTP endpoint"); + } + + @Test + public void rpcHttpTlsWithoutPasswordfileReportsError() { + parseCommand( + "--rpc-http-enabled", + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + "/tmp/test.p12"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains( + "File containing password to unlock keystore is required when TLS is enabled for JSON-RPC HTTP endpoint"); + } + + @Test + public void rpcHttpTlsKeystoreAndPasswordMustBeUsed() { + final String host = "1.2.3.4"; + final int port = 1234; + final String keystoreFile = "/tmp/test.p12"; + final String keystorePasswordFile = "/tmp/test.txt"; + + parseCommand( + "--rpc-http-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + keystoreFile, + "--rpc-http-tls-keystore-password-file", + keystorePasswordFile); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + final Optional tlsConfiguration = + jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); + assertThat(tlsConfiguration.isPresent()).isTrue(); + assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); + assertThat(tlsConfiguration.get().getClientAuthConfiguration().isEmpty()).isTrue(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpTlsClientAuthWithoutKnownFileReportsError() { + final String host = "1.2.3.4"; + final int port = 1234; + final String keystoreFile = "/tmp/test.p12"; + final String keystorePasswordFile = "/tmp/test.txt"; + parseCommand( + "--rpc-http-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + keystoreFile, + "--rpc-http-tls-keystore-password-file", + keystorePasswordFile, + "--rpc-http-tls-client-auth-enabled"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains( + "Known-clients file must be specified or CA clients must be enabled when TLS client authentication is enabled for JSON-RPC HTTP endpoint"); + } + + @Test + public void rpcHttpTlsClientAuthWithKnownClientFile() { + final String host = "1.2.3.4"; + final int port = 1234; + final String keystoreFile = "/tmp/test.p12"; + final String keystorePasswordFile = "/tmp/test.txt"; + final String knownClientFile = "/tmp/knownClientFile"; + parseCommand( + "--rpc-http-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + keystoreFile, + "--rpc-http-tls-keystore-password-file", + keystorePasswordFile, + "--rpc-http-tls-client-auth-enabled", + "--rpc-http-tls-known-clients-file", + knownClientFile); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + final Optional tlsConfiguration = + jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); + assertThat(tlsConfiguration.isPresent()).isTrue(); + assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); + assertThat(tlsConfiguration.get().getClientAuthConfiguration().isPresent()).isTrue(); + assertThat( + tlsConfiguration.get().getClientAuthConfiguration().get().getKnownClientsFile().get()) + .isEqualTo(Path.of(knownClientFile)); + assertThat(tlsConfiguration.get().getClientAuthConfiguration().get().isCaClientsEnabled()) + .isFalse(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpTlsClientAuthWithCAClient() { + final String host = "1.2.3.4"; + final int port = 1234; + final String keystoreFile = "/tmp/test.p12"; + final String keystorePasswordFile = "/tmp/test.txt"; + parseCommand( + "--rpc-http-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + keystoreFile, + "--rpc-http-tls-keystore-password-file", + keystorePasswordFile, + "--rpc-http-tls-client-auth-enabled", + "--rpc-http-tls-ca-clients-enabled"); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + final Optional tlsConfiguration = + jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); + assertThat(tlsConfiguration.isPresent()).isTrue(); + assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); + assertThat(tlsConfiguration.get().getClientAuthConfiguration().isPresent()).isTrue(); + assertThat( + tlsConfiguration + .get() + .getClientAuthConfiguration() + .get() + .getKnownClientsFile() + .isEmpty()) + .isTrue(); + assertThat(tlsConfiguration.get().getClientAuthConfiguration().get().isCaClientsEnabled()) + .isTrue(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpTlsClientAuthWithCAClientAndKnownClientFile() { + final String host = "1.2.3.4"; + final int port = 1234; + final String keystoreFile = "/tmp/test.p12"; + final String keystorePasswordFile = "/tmp/test.txt"; + final String knownClientFile = "/tmp/knownClientFile"; + parseCommand( + "--rpc-http-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + keystoreFile, + "--rpc-http-tls-keystore-password-file", + keystorePasswordFile, + "--rpc-http-tls-client-auth-enabled", + "--rpc-http-tls-ca-clients-enabled", + "--rpc-http-tls-known-clients-file", + knownClientFile); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + final Optional tlsConfiguration = + jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); + assertThat(tlsConfiguration.isPresent()).isTrue(); + assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); + assertThat(tlsConfiguration.get().getClientAuthConfiguration().isPresent()).isTrue(); + assertThat( + tlsConfiguration.get().getClientAuthConfiguration().get().getKnownClientsFile().get()) + .isEqualTo(Path.of(knownClientFile)); + assertThat(tlsConfiguration.get().getClientAuthConfiguration().get().isCaClientsEnabled()) + .isTrue(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpTlsCheckDefaultProtocolsAndCipherSuites() { + final String host = "1.2.3.4"; + final int port = 1234; + final String keystoreFile = "/tmp/test.p12"; + final String keystorePasswordFile = "/tmp/test.txt"; + + parseCommand( + "--rpc-http-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + keystoreFile, + "--rpc-http-tls-keystore-password-file", + keystorePasswordFile); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + final Optional tlsConfiguration = + jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); + assertThat(tlsConfiguration).isPresent(); + assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); + assertThat(tlsConfiguration.get().getClientAuthConfiguration()).isEmpty(); + assertThat(tlsConfiguration.get().getCipherSuites().get()).isEmpty(); + assertThat(tlsConfiguration.get().getSecureTransportProtocols().get()) + .containsExactly("TLSv1.3", "TLSv1.2"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpTlsCheckInvalidProtocols() { + final String host = "1.2.3.4"; + final int port = 1234; + final String keystoreFile = "/tmp/test.p12"; + final String keystorePasswordFile = "/tmp/test.txt"; + final String protocol = "TLsv1.4"; + + parseCommand( + "--rpc-http-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + keystoreFile, + "--rpc-http-tls-keystore-password-file", + keystorePasswordFile, + "--rpc-http-tls-protocols", + protocol); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).contains("No valid TLS protocols specified"); + } + + @Test + public void rpcHttpTlsCheckInvalidCipherSuites() { + final String host = "1.2.3.4"; + final int port = 1234; + final String keystoreFile = "/tmp/test.p12"; + final String keystorePasswordFile = "/tmp/test.txt"; + final String cipherSuites = "Invalid"; + + parseCommand( + "--rpc-http-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + keystoreFile, + "--rpc-http-tls-keystore-password-file", + keystorePasswordFile, + "--rpc-http-tls-cipher-suites", + cipherSuites); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Invalid TLS cipher suite specified " + cipherSuites); + } + + @Test + public void rpcHttpTlsCheckValidProtocolsAndCipherSuites() { + final String host = "1.2.3.4"; + final int port = 1234; + final String keystoreFile = "/tmp/test.p12"; + final String keystorePasswordFile = "/tmp/test.txt"; + final String protocols = "TLSv1.3,TLSv1.2"; + final String cipherSuites = + "TLS_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"; + + parseCommand( + "--rpc-http-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-enabled", + "--rpc-http-tls-keystore-file", + keystoreFile, + "--rpc-http-tls-keystore-password-file", + keystorePasswordFile, + "--rpc-http-tls-protocols", + protocols, + "--rpc-http-tls-cipher-suites", + cipherSuites); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo(host); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + final Optional tlsConfiguration = + jsonRpcConfigArgumentCaptor.getValue().getTlsConfiguration(); + assertThat(tlsConfiguration).isPresent(); + assertThat(tlsConfiguration.get().getKeyStorePath()).isEqualTo(Path.of(keystoreFile)); + assertThat(tlsConfiguration.get().getClientAuthConfiguration()).isEmpty(); + assertThat(tlsConfiguration.get().getCipherSuites().get()) + .containsExactlyInAnyOrder( + "TLS_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + assertThat(tlsConfiguration.get().getSecureTransportProtocols().get()) + .containsExactlyInAnyOrder("TLSv1.2", "TLSv1.3"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpTlsWarnIfCipherSuitesSpecifiedWithoutTls() { + final String host = "1.2.3.4"; + final int port = 1234; + final String cipherSuites = "Invalid"; + + parseCommand( + "--rpc-http-enabled", + "--engine-rpc-enabled", + "--rpc-http-host", + host, + "--rpc-http-port", + String.valueOf(port), + "--rpc-http-tls-cipher-suite", + cipherSuites); + verify( + mockLogger, + times(2)) // this is verified for both the full suite of apis, and the engine group. + .warn( + "{} has been ignored because {} was not defined on the command line.", + "--rpc-http-tls-cipher-suite", + "--rpc-http-tls-enabled"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpCorsOriginsTwoDomainsMustBuildListWithBothDomains() { + final String[] origins = {"http://domain1.com", "https://domain2.com"}; + parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", String.join(",", origins)); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains().toArray()) + .isEqualTo(origins); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpCorsOriginsDoubleCommaFilteredOut() { + final String[] origins = {"http://domain1.com", "https://domain2.com"}; + parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", String.join(",,", origins)); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains().toArray()) + .isEqualTo(origins); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpCorsOriginsWithWildcardMustBuildListWithWildcard() { + final String[] origins = {"*"}; + parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", String.join(",", origins)); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains().toArray()) + .isEqualTo(origins); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpCorsOriginsWithAllMustBuildListWithWildcard() { + parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", "all"); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains()).containsExactly("*"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpCorsOriginsWithNoneMustBuildEmptyList() { + final String[] origins = {"none"}; + parseCommand("--rpc-http-enabled", "--rpc-http-cors-origins", String.join(",", origins)); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getCorsAllowedDomains()).isEmpty(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpCorsOriginsNoneWithAnotherDomainMustFail() { + final String[] origins = {"http://domain1.com", "none"}; + parseCommand("--rpc-http-cors-origins", String.join(",", origins)); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Value 'none' can't be used with other domains"); + } + + @Test + public void rpcHttpCorsOriginsNoneWithAnotherDomainMustFailNoneFirst() { + final String[] origins = {"none", "http://domain1.com"}; + parseCommand("--rpc-http-cors-origins", String.join(",", origins)); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Value 'none' can't be used with other domains"); + } + + @Test + public void rpcHttpCorsOriginsAllWithAnotherDomainMustFail() { + parseCommand("--rpc-http-cors-origins=http://domain1.com,all"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Values '*' or 'all' can't be used with other domains"); + } + + @Test + public void rpcHttpCorsOriginsAllWithAnotherDomainMustFailAsFlags() { + parseCommand("--rpc-http-cors-origins=http://domain1.com", "--rpc-http-cors-origins=all"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Values '*' or 'all' can't be used with other domains"); + } + + @Test + public void rpcHttpCorsOriginsWildcardWithAnotherDomainMustFail() { + parseCommand("--rpc-http-cors-origins=http://domain1.com,*"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Values '*' or 'all' can't be used with other domains"); + } + + @Test + public void rpcHttpCorsOriginsWildcardWithAnotherDomainMustFailAsFlags() { + parseCommand("--rpc-http-cors-origins=http://domain1.com", "--rpc-http-cors-origins=*"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Values '*' or 'all' can't be used with other domains"); + } + + @Test + public void rpcHttpCorsOriginsInvalidRegexShouldFail() { + final String[] origins = {"**"}; + parseCommand("--rpc-http-cors-origins", String.join(",", origins)); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Domain values result in invalid regex pattern"); + } + + @Test + public void rpcHttpCorsOriginsEmptyValueFails() { + parseCommand("--rpc-http-cors-origins="); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Domain cannot be empty string or null string."); + } + + @Test + public void rpcApisPropertyWithInvalidEntryMustDisplayError() { + parseCommand("--rpc-http-api", "BOB"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + // PicoCLI uses longest option name for message when option has multiple names, so here plural. + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Invalid value for option '--rpc-http-api': invalid entries found [BOB]"); + } + + @Test + public void rpcApisPropertyWithPluginNamespaceAreValid() { + + rpcEndpointServiceImpl.registerRPCEndpoint( + "bob", "method", (Function) request -> "nothing"); + + parseCommand("--rpc-http-api", "BOB"); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getRpcApis()) + .containsExactlyInAnyOrder("BOB"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpMaxRequestContentLengthOptionMustBeUsed() { + final int rpcHttpMaxRequestContentLength = 1; + parseCommand( + "--rpc-http-max-request-content-length", Long.toString(rpcHttpMaxRequestContentLength)); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getMaxRequestContentLength()) + .isEqualTo(rpcHttpMaxRequestContentLength); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void rpcHttpMaxBatchSizeOptionMustBeUsed() { + final int rpcHttpMaxBatchSize = 1; + parseCommand("--rpc-http-max-batch-size", Integer.toString(rpcHttpMaxBatchSize)); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getMaxBatchSize()) + .isEqualTo(rpcHttpMaxBatchSize); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void portInUseReportsError() throws IOException { + final ServerSocket serverSocket = new ServerSocket(8545); + + parseCommandWithPortCheck("--rpc-http-enabled"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Port(s) '[8545]' already in use. Check for other processes using the port(s)."); + + serverSocket.close(); + } + + @Test + public void assertThatCheckPortClashRejectsAsExpected() throws Exception { + // use WS port for HTTP + final int port = 8546; + parseCommand("--rpc-http-enabled", "--rpc-http-port", String.valueOf(port), "--rpc-ws-enabled"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains( + "Port number '8546' has been specified multiple times. Please review the supplied configuration."); + } + + @Test + public void assertThatCheckPortClashAcceptsAsExpected() throws Exception { + // use WS port for HTTP + final int port = 8546; + parseCommand("--rpc-http-enabled", "--rpc-http-port", String.valueOf(port)); + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void httpAuthenticationWithoutRequiredConfiguredOptionsMustFail() { + parseCommand("--rpc-http-enabled", "--rpc-http-authentication-enabled"); + + verifyNoInteractions(mockRunnerBuilder); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains( + "Unable to authenticate JSON-RPC HTTP endpoint without a supplied credentials file or authentication public key file"); + } + + @Test + public void httpAuthenticationAlgorithIsConfigured() { + parseCommand("--rpc-http-authentication-jwt-algorithm", "ES256"); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getAuthenticationAlgorithm()) + .isEqualTo(JwtAlgorithm.ES256); + } + + @Test + public void httpAuthenticationPublicKeyIsConfigured() throws IOException { + final Path publicKey = Files.createTempFile("public_key", ""); + parseCommand("--rpc-http-authentication-jwt-public-key-file", publicKey.toString()); + + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(jsonRpcConfigArgumentCaptor.getValue().getAuthenticationPublicKeyFile().getPath()) + .isEqualTo(publicKey.toString()); + } +} diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/RpcWebsocketOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/RpcWebsocketOptionsTest.java index 0aaf01622e7..0583afd7cc0 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/RpcWebsocketOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/RpcWebsocketOptionsTest.java @@ -246,4 +246,17 @@ public void wsAuthenticationWithoutRequiredConfiguredOptionsMustFail() { .contains( "Unable to authenticate JSON-RPC WebSocket endpoint without a supplied credentials file or authentication public key file"); } + + @Test + public void rpcWsRpcEnabledPropertyDefaultIsFalse() { + parseCommand(); + + verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + assertThat(wsRpcConfigArgumentCaptor.getValue().isEnabled()).isFalse(); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } } From 0f91a674ca4e24bd6decf61c61a433876c4affe6 Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Mon, 29 Jan 2024 04:51:42 +0000 Subject: [PATCH 50/56] Write a DEBUG log entry to make it clear of a BFT node is a validator or not (#6470) Signed-off-by: Matthew Whitehead Co-authored-by: Sally MacFarlane --- .../common/bft/statemachine/BftFinalState.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/statemachine/BftFinalState.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/statemachine/BftFinalState.java index 13f6959779b..0a65acc8652 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/statemachine/BftFinalState.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/statemachine/BftFinalState.java @@ -28,8 +28,14 @@ import java.time.Clock; import java.util.Collection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** This is the full data set, or context, required for many of the aspects of BFT workflows. */ public class BftFinalState { + + private static final Logger LOG = LoggerFactory.getLogger(BftFinalState.class); + private final ValidatorProvider validatorProvider; private final NodeKey nodeKey; private final Address localAddress; @@ -126,7 +132,9 @@ public boolean isLocalNodeProposerForRound(final ConsensusRoundIdentifier roundI * @return the boolean */ public boolean isLocalNodeValidator() { - return getValidators().contains(localAddress); + final boolean isValidator = getValidators().contains(localAddress); + LOG.debug(isValidator ? "Local node is a validator" : "Local node is a non-validator"); + return isValidator; } /** From f125aed72f2d522ef5c052bfce9faf95ad429b3b Mon Sep 17 00:00:00 2001 From: Thabokani <149070269+Thabokani@users.noreply.github.com> Date: Mon, 29 Jan 2024 16:00:25 +0800 Subject: [PATCH 51/56] Fix typos (#6481) * Fix typos Signed-off-by: Thabokani <149070269+Thabokani@users.noreply.github.com> * update plugin-api/build.gradle Signed-off-by: Thabokani <149070269+Thabokani@users.noreply.github.com> --------- Signed-off-by: Thabokani <149070269+Thabokani@users.noreply.github.com> --- .../java/org/hyperledger/besu/collections/undo/UndoScalar.java | 2 +- plugin-api/build.gradle | 2 +- .../besu/plugin/services/trielogs/TrieLogAccumulator.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/evm/src/main/java/org/hyperledger/besu/collections/undo/UndoScalar.java b/evm/src/main/java/org/hyperledger/besu/collections/undo/UndoScalar.java index 0e3356c379b..55e3a84e0b6 100644 --- a/evm/src/main/java/org/hyperledger/besu/collections/undo/UndoScalar.java +++ b/evm/src/main/java/org/hyperledger/besu/collections/undo/UndoScalar.java @@ -61,7 +61,7 @@ public long lastUpdate() { } /** - * Has this scalar had any change since the inital value + * Has this scalar had any change since the initial value * * @return true if there are any changes to undo */ diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index c72ad934339..01ed8833dad 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'ZsovOR0oPfomcLP4b+HjikWzM0Tx6sCwi68mf5qwZf4=' + knownHash = 'VpNy2KuAtEUc9hPguNivbjwy2YM3vIF444RCREJojqY=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogAccumulator.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogAccumulator.java index 6984ca48a1e..384a327166d 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogAccumulator.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/trielogs/TrieLogAccumulator.java @@ -23,7 +23,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; -/** Accumulator interface tor provding trie updates for creating TrieLogs. */ +/** Accumulator interface for providing trie updates for creating TrieLogs. */ public interface TrieLogAccumulator { /** From 98ab26df2f50823c473fd6d63b7d603dd6681051 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Mon, 29 Jan 2024 08:41:31 -0800 Subject: [PATCH 52/56] filter empty ipv4 and ipv6 ping packet source addresses (#6474) * add empty ipv4 and ipv6 to the filtered ping packet source addresses Signed-off-by: garyschulte --- .../p2p/discovery/PeerDiscoveryAgent.java | 2 +- .../p2p/discovery/PeerDiscoveryAgentTest.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java index a4af09481cd..5d05cc6531d 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java @@ -75,7 +75,7 @@ public abstract class PeerDiscoveryAgent { // clients ignore that, so we add in a little extra padding. private static final int MAX_PACKET_SIZE_BYTES = 1600; private static final List PING_PACKET_SOURCE_IGNORED = - List.of("127.0.0.1", "255.255.255.255"); + List.of("127.0.0.1", "255.255.255.255", "0.0.0.0", "::", "0:0:0:0:0:0:0:0"); protected final List bootstrapPeers; private final List peerRequirements = new CopyOnWriteArrayList<>(); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java index aae83ffdc8c..269eb92b084 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java @@ -839,15 +839,31 @@ public void shouldNotBeActiveWhenConfigIsFalse() { @Test public void assertHostCorrectlyRevertsOnIgnoredPacketFrom() { final String sourceHost = "UDP_SOURCE_ORIGIN_HOST"; + final String emptyIPv4Host = "0.0.0.0"; + final String emptyIPv6Host = "::"; final String localHost = "127.0.0.1"; final String broadcastDefaultHost = "255.255.255.255"; final String routableHost = "50.50.50.50"; Endpoint source = new Endpoint(sourceHost, 30303, Optional.empty()); + Endpoint emptyIPv4 = new Endpoint(emptyIPv4Host, 30303, Optional.empty()); + Endpoint emptyIPv6 = new Endpoint(emptyIPv6Host, 30303, Optional.empty()); Endpoint endpointLocal = new Endpoint(localHost, 30303, Optional.empty()); Endpoint endpointBroadcast = new Endpoint(broadcastDefaultHost, 30303, Optional.empty()); Endpoint endpointRoutable = new Endpoint(routableHost, 30303, Optional.empty()); + Packet mockEmptyIPv4 = + when(mock(Packet.class).getPacketData(any())) + .thenReturn( + Optional.of( + PingPacketData.create(Optional.of(emptyIPv4), endpointLocal, UInt64.ONE))) + .getMock(); + Packet mockEmptyIPv6 = + when(mock(Packet.class).getPacketData(any())) + .thenReturn( + Optional.of( + PingPacketData.create(Optional.of(emptyIPv6), endpointLocal, UInt64.ONE))) + .getMock(); Packet mockLocal = when(mock(Packet.class).getPacketData(any())) .thenReturn( @@ -869,6 +885,10 @@ public void assertHostCorrectlyRevertsOnIgnoredPacketFrom() { Optional.of(endpointRoutable), endpointLocal, UInt64.ONE))) .getMock(); + // assert a pingpacketdata with empty ipv4 address reverts to the udp source host + assertThat(PeerDiscoveryAgent.deriveHost(source, mockEmptyIPv4)).isEqualTo(sourceHost); + // assert a pingpacketdata with empty ipv6 address reverts to the udp source host + assertThat(PeerDiscoveryAgent.deriveHost(source, mockEmptyIPv6)).isEqualTo(sourceHost); // assert a pingpacketdata from address of 127.0.0.1 reverts to the udp source host assertThat(PeerDiscoveryAgent.deriveHost(source, mockLocal)).isEqualTo(sourceHost); // assert that 255.255.255.255 reverts to the udp source host From 9084bde9d8df82a4818dc96678cdbe89b526f7c8 Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Tue, 30 Jan 2024 12:08:06 +1100 Subject: [PATCH 53/56] Move permission options to itw own class (#6490) Signed-off-by: Gabriel-Trintinalia --- .../org/hyperledger/besu/cli/BesuCommand.java | 176 +------ .../options/stable/PermissionsOptions.java | 206 ++++++++ .../hyperledger/besu/cli/BesuCommandTest.java | 412 ---------------- .../cli/options/PermissionsOptionsTest.java | 453 ++++++++++++++++++ 4 files changed, 671 insertions(+), 576 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/options/stable/PermissionsOptions.java create mode 100644 besu/src/test/java/org/hyperledger/besu/cli/options/PermissionsOptionsTest.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 68c3a6a5abc..8e59b4c23dd 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -56,6 +56,7 @@ import org.hyperledger.besu.cli.options.stable.LoggingLevelOption; import org.hyperledger.besu.cli.options.stable.NodePrivateKeyFileOption; import org.hyperledger.besu.cli.options.stable.P2PTLSConfigOptions; +import org.hyperledger.besu.cli.options.stable.PermissionsOptions; import org.hyperledger.besu.cli.options.stable.RpcWebsocketOptions; import org.hyperledger.besu.cli.options.unstable.ChainPruningOptions; import org.hyperledger.besu.cli.options.unstable.DnsOptions; @@ -130,8 +131,6 @@ import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import org.hyperledger.besu.ethereum.permissioning.LocalPermissioningConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; -import org.hyperledger.besu.ethereum.permissioning.PermissioningConfigurationBuilder; -import org.hyperledger.besu.ethereum.permissioning.SmartContractPermissioningConfiguration; import org.hyperledger.besu.ethereum.privacy.storage.keyvalue.PrivacyKeyValueStorageProvider; import org.hyperledger.besu.ethereum.privacy.storage.keyvalue.PrivacyKeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.storage.StorageProvider; @@ -825,62 +824,7 @@ static class MetricsOptionGroup { // Permission Option Group @CommandLine.ArgGroup(validate = false, heading = "@|bold Permissions Options|@%n") - PermissionsOptionGroup permissionsOptionGroup = new PermissionsOptionGroup(); - - static class PermissionsOptionGroup { - @Option( - names = {"--permissions-nodes-config-file-enabled"}, - description = "Enable node level permissions (default: ${DEFAULT-VALUE})") - private final Boolean permissionsNodesEnabled = false; - - @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. - @CommandLine.Option( - names = {"--permissions-nodes-config-file"}, - description = - "Node permissioning config TOML file (default: a file named \"permissions_config.toml\" in the Besu data folder)") - private String nodePermissionsConfigFile = null; - - @Option( - names = {"--permissions-accounts-config-file-enabled"}, - description = "Enable account level permissions (default: ${DEFAULT-VALUE})") - private final Boolean permissionsAccountsEnabled = false; - - @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. - @CommandLine.Option( - names = {"--permissions-accounts-config-file"}, - description = - "Account permissioning config TOML file (default: a file named \"permissions_config.toml\" in the Besu data folder)") - private String accountPermissionsConfigFile = null; - - @Option( - names = {"--permissions-nodes-contract-address"}, - description = "Address of the node permissioning smart contract", - arity = "1") - private final Address permissionsNodesContractAddress = null; - - @Option( - names = {"--permissions-nodes-contract-version"}, - description = "Version of the EEA Node Permissioning interface (default: ${DEFAULT-VALUE})") - private final Integer permissionsNodesContractVersion = 1; - - @Option( - names = {"--permissions-nodes-contract-enabled"}, - description = - "Enable node level permissions via smart contract (default: ${DEFAULT-VALUE})") - private final Boolean permissionsNodesContractEnabled = false; - - @Option( - names = {"--permissions-accounts-contract-address"}, - description = "Address of the account permissioning smart contract", - arity = "1") - private final Address permissionsAccountsContractAddress = null; - - @Option( - names = {"--permissions-accounts-contract-enabled"}, - description = - "Enable account level permissions via smart contract (default: ${DEFAULT-VALUE})") - private final Boolean permissionsAccountsContractEnabled = false; - } + PermissionsOptions permissionsOptions = new PermissionsOptions(); @Option( names = {"--revert-reason-enabled"}, @@ -1852,6 +1796,16 @@ private void configure() throws Exception { logger.info("Security Module: {}", securityModuleName); } + private Optional permissioningConfiguration() throws Exception { + return permissionsOptions.permissioningConfiguration( + jsonRpcHttpOptions, + rpcWebsocketOptions, + getEnodeDnsConfiguration(), + dataDir(), + logger, + commandLine); + } + private JsonRpcIpcConfiguration jsonRpcIpcConfiguration( final Boolean enabled, final Path ipcPath, final List rpcIpcApis) { final Path actualPath; @@ -2091,106 +2045,6 @@ public MetricsConfiguration metricsConfiguration() { .build(); } - private Optional permissioningConfiguration() throws Exception { - if (!(localPermissionsEnabled() || contractPermissionsEnabled())) { - if (jsonRpcHttpOptions.getRpcHttpApis().contains(RpcApis.PERM.name()) - || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.PERM.name())) { - logger.warn( - "Permissions are disabled. Cannot enable PERM APIs when not using Permissions."); - } - return Optional.empty(); - } - - final Optional localPermissioningConfigurationOptional; - if (localPermissionsEnabled()) { - final Optional nodePermissioningConfigFile = - Optional.ofNullable(permissionsOptionGroup.nodePermissionsConfigFile); - final Optional accountPermissioningConfigFile = - Optional.ofNullable(permissionsOptionGroup.accountPermissionsConfigFile); - - final LocalPermissioningConfiguration localPermissioningConfiguration = - PermissioningConfigurationBuilder.permissioningConfiguration( - permissionsOptionGroup.permissionsNodesEnabled, - getEnodeDnsConfiguration(), - nodePermissioningConfigFile.orElse(getDefaultPermissioningFilePath()), - permissionsOptionGroup.permissionsAccountsEnabled, - accountPermissioningConfigFile.orElse(getDefaultPermissioningFilePath())); - - localPermissioningConfigurationOptional = Optional.of(localPermissioningConfiguration); - } else { - if (permissionsOptionGroup.nodePermissionsConfigFile != null - && !permissionsOptionGroup.permissionsNodesEnabled) { - logger.warn( - "Node permissioning config file set {} but no permissions enabled", - permissionsOptionGroup.nodePermissionsConfigFile); - } - - if (permissionsOptionGroup.accountPermissionsConfigFile != null - && !permissionsOptionGroup.permissionsAccountsEnabled) { - logger.warn( - "Account permissioning config file set {} but no permissions enabled", - permissionsOptionGroup.accountPermissionsConfigFile); - } - localPermissioningConfigurationOptional = Optional.empty(); - } - - final SmartContractPermissioningConfiguration smartContractPermissioningConfiguration = - SmartContractPermissioningConfiguration.createDefault(); - - if (Boolean.TRUE.equals(permissionsOptionGroup.permissionsNodesContractEnabled)) { - if (permissionsOptionGroup.permissionsNodesContractAddress == null) { - throw new ParameterException( - this.commandLine, - "No node permissioning contract address specified. Cannot enable smart contract based node permissioning."); - } else { - smartContractPermissioningConfiguration.setSmartContractNodeAllowlistEnabled( - permissionsOptionGroup.permissionsNodesContractEnabled); - smartContractPermissioningConfiguration.setNodeSmartContractAddress( - permissionsOptionGroup.permissionsNodesContractAddress); - smartContractPermissioningConfiguration.setNodeSmartContractInterfaceVersion( - permissionsOptionGroup.permissionsNodesContractVersion); - } - } else if (permissionsOptionGroup.permissionsNodesContractAddress != null) { - logger.warn( - "Node permissioning smart contract address set {} but smart contract node permissioning is disabled.", - permissionsOptionGroup.permissionsNodesContractAddress); - } - - if (Boolean.TRUE.equals(permissionsOptionGroup.permissionsAccountsContractEnabled)) { - if (permissionsOptionGroup.permissionsAccountsContractAddress == null) { - throw new ParameterException( - this.commandLine, - "No account permissioning contract address specified. Cannot enable smart contract based account permissioning."); - } else { - smartContractPermissioningConfiguration.setSmartContractAccountAllowlistEnabled( - permissionsOptionGroup.permissionsAccountsContractEnabled); - smartContractPermissioningConfiguration.setAccountSmartContractAddress( - permissionsOptionGroup.permissionsAccountsContractAddress); - } - } else if (permissionsOptionGroup.permissionsAccountsContractAddress != null) { - logger.warn( - "Account permissioning smart contract address set {} but smart contract account permissioning is disabled.", - permissionsOptionGroup.permissionsAccountsContractAddress); - } - - final PermissioningConfiguration permissioningConfiguration = - new PermissioningConfiguration( - localPermissioningConfigurationOptional, - Optional.of(smartContractPermissioningConfiguration)); - - return Optional.of(permissioningConfiguration); - } - - private boolean localPermissionsEnabled() { - return permissionsOptionGroup.permissionsAccountsEnabled - || permissionsOptionGroup.permissionsNodesEnabled; - } - - private boolean contractPermissionsEnabled() { - return permissionsOptionGroup.permissionsNodesContractEnabled - || permissionsOptionGroup.permissionsAccountsContractEnabled; - } - private PrivacyParameters privacyParameters() { CommandLineUtils.checkOptionDependencies( @@ -2661,12 +2515,6 @@ private File resolveNodePrivateKeyFile(final File nodePrivateKeyFile) { .orElseGet(() -> KeyPairUtil.getDefaultKeyFile(dataDir())); } - private String getDefaultPermissioningFilePath() { - return dataDir() - + System.getProperty("file.separator") - + DefaultCommandValues.PERMISSIONING_CONFIG_LOCATION; - } - /** * Metrics System used by Besu * diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/PermissionsOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/PermissionsOptions.java new file mode 100644 index 00000000000..3f16432750e --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/PermissionsOptions.java @@ -0,0 +1,206 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.options.stable; + +import org.hyperledger.besu.cli.DefaultCommandValues; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis; +import org.hyperledger.besu.ethereum.p2p.peers.EnodeDnsConfiguration; +import org.hyperledger.besu.ethereum.permissioning.LocalPermissioningConfiguration; +import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; +import org.hyperledger.besu.ethereum.permissioning.PermissioningConfigurationBuilder; +import org.hyperledger.besu.ethereum.permissioning.SmartContractPermissioningConfiguration; + +import java.nio.file.Path; +import java.util.Optional; + +import org.slf4j.Logger; +import picocli.CommandLine; + +/** Handles configuration options for permissions in Besu. */ +public class PermissionsOptions { + @CommandLine.Option( + names = {"--permissions-nodes-config-file-enabled"}, + description = "Enable node level permissions (default: ${DEFAULT-VALUE})") + private final Boolean permissionsNodesEnabled = false; + + @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. + @CommandLine.Option( + names = {"--permissions-nodes-config-file"}, + description = + "Node permissioning config TOML file (default: a file named \"permissions_config.toml\" in the Besu data folder)") + private String nodePermissionsConfigFile = null; + + @CommandLine.Option( + names = {"--permissions-accounts-config-file-enabled"}, + description = "Enable account level permissions (default: ${DEFAULT-VALUE})") + private final Boolean permissionsAccountsEnabled = false; + + @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings. + @CommandLine.Option( + names = {"--permissions-accounts-config-file"}, + description = + "Account permissioning config TOML file (default: a file named \"permissions_config.toml\" in the Besu data folder)") + private String accountPermissionsConfigFile = null; + + @CommandLine.Option( + names = {"--permissions-nodes-contract-address"}, + description = "Address of the node permissioning smart contract", + arity = "1") + private final Address permissionsNodesContractAddress = null; + + @CommandLine.Option( + names = {"--permissions-nodes-contract-version"}, + description = "Version of the EEA Node Permissioning interface (default: ${DEFAULT-VALUE})") + private final Integer permissionsNodesContractVersion = 1; + + @CommandLine.Option( + names = {"--permissions-nodes-contract-enabled"}, + description = "Enable node level permissions via smart contract (default: ${DEFAULT-VALUE})") + private final Boolean permissionsNodesContractEnabled = false; + + @CommandLine.Option( + names = {"--permissions-accounts-contract-address"}, + description = "Address of the account permissioning smart contract", + arity = "1") + private final Address permissionsAccountsContractAddress = null; + + @CommandLine.Option( + names = {"--permissions-accounts-contract-enabled"}, + description = + "Enable account level permissions via smart contract (default: ${DEFAULT-VALUE})") + private final Boolean permissionsAccountsContractEnabled = false; + + /** + * Creates a PermissioningConfiguration based on the provided options. + * + * @param jsonRpcHttpOptions The JSON-RPC HTTP options + * @param rpcWebsocketOptions The RPC websocket options + * @param enodeDnsConfiguration The enode DNS configuration + * @param dataPath The data path + * @param logger The logger + * @param commandLine The command line + * @return An Optional PermissioningConfiguration instance + * @throws Exception If an error occurs while creating the configuration + */ + public Optional permissioningConfiguration( + final JsonRpcHttpOptions jsonRpcHttpOptions, + final RpcWebsocketOptions rpcWebsocketOptions, + final EnodeDnsConfiguration enodeDnsConfiguration, + final Path dataPath, + final Logger logger, + final CommandLine commandLine) + throws Exception { + if (!(localPermissionsEnabled() || contractPermissionsEnabled())) { + if (jsonRpcHttpOptions.getRpcHttpApis().contains(RpcApis.PERM.name()) + || rpcWebsocketOptions.getRpcWsApis().contains(RpcApis.PERM.name())) { + logger.warn( + "Permissions are disabled. Cannot enable PERM APIs when not using Permissions."); + } + return Optional.empty(); + } + + final Optional localPermissioningConfigurationOptional; + if (localPermissionsEnabled()) { + final Optional nodePermissioningConfigFile = + Optional.ofNullable(nodePermissionsConfigFile); + final Optional accountPermissioningConfigFile = + Optional.ofNullable(accountPermissionsConfigFile); + + final LocalPermissioningConfiguration localPermissioningConfiguration = + PermissioningConfigurationBuilder.permissioningConfiguration( + permissionsNodesEnabled, + enodeDnsConfiguration, + nodePermissioningConfigFile.orElse(getDefaultPermissioningFilePath(dataPath)), + permissionsAccountsEnabled, + accountPermissioningConfigFile.orElse(getDefaultPermissioningFilePath(dataPath))); + + localPermissioningConfigurationOptional = Optional.of(localPermissioningConfiguration); + } else { + if (nodePermissionsConfigFile != null && !permissionsNodesEnabled) { + logger.warn( + "Node permissioning config file set {} but no permissions enabled", + nodePermissionsConfigFile); + } + + if (accountPermissionsConfigFile != null && !permissionsAccountsEnabled) { + logger.warn( + "Account permissioning config file set {} but no permissions enabled", + accountPermissionsConfigFile); + } + localPermissioningConfigurationOptional = Optional.empty(); + } + + final SmartContractPermissioningConfiguration smartContractPermissioningConfiguration = + SmartContractPermissioningConfiguration.createDefault(); + + if (Boolean.TRUE.equals(permissionsNodesContractEnabled)) { + if (permissionsNodesContractAddress == null) { + throw new CommandLine.ParameterException( + commandLine, + "No node permissioning contract address specified. Cannot enable smart contract based node permissioning."); + } else { + smartContractPermissioningConfiguration.setSmartContractNodeAllowlistEnabled( + permissionsNodesContractEnabled); + smartContractPermissioningConfiguration.setNodeSmartContractAddress( + permissionsNodesContractAddress); + smartContractPermissioningConfiguration.setNodeSmartContractInterfaceVersion( + permissionsNodesContractVersion); + } + } else if (permissionsNodesContractAddress != null) { + logger.warn( + "Node permissioning smart contract address set {} but smart contract node permissioning is disabled.", + permissionsNodesContractAddress); + } + + if (Boolean.TRUE.equals(permissionsAccountsContractEnabled)) { + if (permissionsAccountsContractAddress == null) { + throw new CommandLine.ParameterException( + commandLine, + "No account permissioning contract address specified. Cannot enable smart contract based account permissioning."); + } else { + smartContractPermissioningConfiguration.setSmartContractAccountAllowlistEnabled( + permissionsAccountsContractEnabled); + smartContractPermissioningConfiguration.setAccountSmartContractAddress( + permissionsAccountsContractAddress); + } + } else if (permissionsAccountsContractAddress != null) { + logger.warn( + "Account permissioning smart contract address set {} but smart contract account permissioning is disabled.", + permissionsAccountsContractAddress); + } + + final PermissioningConfiguration permissioningConfiguration = + new PermissioningConfiguration( + localPermissioningConfigurationOptional, + Optional.of(smartContractPermissioningConfiguration)); + + return Optional.of(permissioningConfiguration); + } + + private boolean localPermissionsEnabled() { + return permissionsAccountsEnabled || permissionsNodesEnabled; + } + + private boolean contractPermissionsEnabled() { + return permissionsNodesContractEnabled || permissionsAccountsContractEnabled; + } + + private String getDefaultPermissioningFilePath(final Path dataPath) { + return dataPath + + System.getProperty("file.separator") + + DefaultCommandValues.PERMISSIONING_CONFIG_LOCATION; + } +} diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index c69d8861096..3850233d528 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -51,7 +51,6 @@ import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.config.MergeConfigOptions; import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; -import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.GasLimitCalculator; @@ -67,9 +66,6 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; -import org.hyperledger.besu.ethereum.permissioning.LocalPermissioningConfiguration; -import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; -import org.hyperledger.besu.ethereum.permissioning.SmartContractPermissioningConfiguration; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; @@ -96,7 +92,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -104,7 +99,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.common.collect.Lists; import com.google.common.io.Resources; import io.vertx.core.json.JsonObject; import org.apache.commons.io.FileUtils; @@ -129,7 +123,6 @@ public class BesuCommandTest extends CommandTestAbstract { private static final String ENCLAVE_PUBLIC_KEY = "A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGo="; private static final String VALID_NODE_ID = "6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0"; - private static final String PERMISSIONING_CONFIG_TOML = "/permissioning_config.toml"; private static final JsonRpcConfiguration DEFAULT_JSON_RPC_CONFIGURATION; private static final GraphQLConfiguration DEFAULT_GRAPH_QL_CONFIGURATION; private static final WebSocketConfiguration DEFAULT_WEB_SOCKET_CONFIGURATION; @@ -349,374 +342,6 @@ public void callingWithConfigOptionButInvalidValueTomlFileShouldDisplayHelp() th assertThat(commandOutput.toString(UTF_8)).isEmpty(); } - @Test - public void nodePermissionsSmartContractWithoutOptionMustError() { - parseCommand("--permissions-nodes-contract-address"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)) - .startsWith("Missing required parameter for option '--permissions-nodes-contract-address'"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissionsEnabledWithoutContractAddressMustError() { - parseCommand("--permissions-nodes-contract-enabled"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("No node permissioning contract address specified"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissionsEnabledWithInvalidContractAddressMustError() { - parseCommand( - "--permissions-nodes-contract-enabled", - "--permissions-nodes-contract-address", - "invalid-smart-contract-address"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)).contains("Invalid value"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissionsEnabledWithTooShortContractAddressMustError() { - parseCommand( - "--permissions-nodes-contract-enabled", "--permissions-nodes-contract-address", "0x1234"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)).contains("Invalid value"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissionsSmartContractMustUseOption() { - - final String smartContractAddress = "0x0000000000000000000000000000000000001234"; - - parseCommand( - "--permissions-nodes-contract-enabled", - "--permissions-nodes-contract-address", - smartContractAddress); - final SmartContractPermissioningConfiguration smartContractPermissioningConfiguration = - new SmartContractPermissioningConfiguration(); - smartContractPermissioningConfiguration.setNodeSmartContractAddress( - Address.fromHexString(smartContractAddress)); - smartContractPermissioningConfiguration.setSmartContractNodeAllowlistEnabled(true); - - verify(mockRunnerBuilder) - .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - final PermissioningConfiguration config = - permissioningConfigurationArgumentCaptor.getValue().get(); - assertThat(config.getSmartContractConfig().get()) - .usingRecursiveComparison() - .isEqualTo(smartContractPermissioningConfiguration); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissionsContractVersionDefaultValue() { - final SmartContractPermissioningConfiguration expectedConfig = - new SmartContractPermissioningConfiguration(); - expectedConfig.setNodeSmartContractAddress( - Address.fromHexString("0x0000000000000000000000000000000000001234")); - expectedConfig.setSmartContractNodeAllowlistEnabled(true); - expectedConfig.setNodeSmartContractInterfaceVersion(1); - - parseCommand( - "--permissions-nodes-contract-enabled", - "--permissions-nodes-contract-address", - "0x0000000000000000000000000000000000001234"); - - verify(mockRunnerBuilder) - .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - final PermissioningConfiguration config = - permissioningConfigurationArgumentCaptor.getValue().get(); - assertThat(config.getSmartContractConfig().get()) - .usingRecursiveComparison() - .isEqualTo(expectedConfig); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissionsContractVersionSetsValue() { - final SmartContractPermissioningConfiguration expectedConfig = - new SmartContractPermissioningConfiguration(); - expectedConfig.setNodeSmartContractAddress( - Address.fromHexString("0x0000000000000000000000000000000000001234")); - expectedConfig.setSmartContractNodeAllowlistEnabled(true); - expectedConfig.setNodeSmartContractInterfaceVersion(2); - - parseCommand( - "--permissions-nodes-contract-enabled", - "--permissions-nodes-contract-address", - "0x0000000000000000000000000000000000001234", - "--permissions-nodes-contract-version", - "2"); - - verify(mockRunnerBuilder) - .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - final PermissioningConfiguration config = - permissioningConfigurationArgumentCaptor.getValue().get(); - assertThat(config.getSmartContractConfig().get()) - .usingRecursiveComparison() - .isEqualTo(expectedConfig); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void accountPermissionsSmartContractWithoutOptionMustError() { - parseCommand("--permissions-accounts-contract-address"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)) - .startsWith( - "Missing required parameter for option '--permissions-accounts-contract-address'"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void accountPermissionsEnabledWithoutContractAddressMustError() { - parseCommand("--permissions-accounts-contract-enabled"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("No account permissioning contract address specified"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void accountPermissionsEnabledWithInvalidContractAddressMustError() { - parseCommand( - "--permissions-accounts-contract-enabled", - "--permissions-accounts-contract-address", - "invalid-smart-contract-address"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)).contains("Invalid value"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void accountPermissionsEnabledWithTooShortContractAddressMustError() { - parseCommand( - "--permissions-accounts-contract-enabled", - "--permissions-accounts-contract-address", - "0x1234"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)).contains("Invalid value"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void accountPermissionsSmartContractMustUseOption() { - final String smartContractAddress = "0x0000000000000000000000000000000000001234"; - - parseCommand( - "--permissions-accounts-contract-enabled", - "--permissions-accounts-contract-address", - smartContractAddress); - final SmartContractPermissioningConfiguration smartContractPermissioningConfiguration = - new SmartContractPermissioningConfiguration(); - smartContractPermissioningConfiguration.setAccountSmartContractAddress( - Address.fromHexString(smartContractAddress)); - smartContractPermissioningConfiguration.setSmartContractAccountAllowlistEnabled(true); - - verify(mockRunnerBuilder) - .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); - final PermissioningConfiguration permissioningConfiguration = - permissioningConfigurationArgumentCaptor.getValue().get(); - assertThat(permissioningConfiguration.getSmartContractConfig()).isPresent(); - - final SmartContractPermissioningConfiguration effectiveSmartContractConfig = - permissioningConfiguration.getSmartContractConfig().get(); - assertThat(effectiveSmartContractConfig.isSmartContractAccountAllowlistEnabled()).isTrue(); - assertThat(effectiveSmartContractConfig.getAccountSmartContractAddress()) - .isEqualTo(Address.fromHexString(smartContractAddress)); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissioningTomlPathWithoutOptionMustDisplayUsage() { - parseCommand("--permissions-nodes-config-file"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)) - .startsWith("Missing required parameter for option '--permissions-nodes-config-file'"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void accountPermissioningTomlPathWithoutOptionMustDisplayUsage() { - parseCommand("--permissions-accounts-config-file"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)) - .startsWith("Missing required parameter for option '--permissions-accounts-config-file'"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissioningEnabledWithNonexistentConfigFileMustError() { - parseCommand( - "--permissions-nodes-config-file-enabled", - "--permissions-nodes-config-file", - "file-does-not-exist"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)).contains("Configuration file does not exist"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void accountPermissioningEnabledWithNonexistentConfigFileMustError() { - parseCommand( - "--permissions-accounts-config-file-enabled", - "--permissions-accounts-config-file", - "file-does-not-exist"); - - Mockito.verifyNoInteractions(mockRunnerBuilder); - - assertThat(commandErrorOutput.toString(UTF_8)).contains("Configuration file does not exist"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissioningTomlFileWithNoPermissionsEnabledMustNotError() throws IOException { - - final URL configFile = this.getClass().getResource(PERMISSIONING_CONFIG_TOML); - final Path permToml = createTempFile("toml", Resources.toByteArray(configFile)); - parseCommand("--permissions-nodes-config-file", permToml.toString()); - - verify(mockRunnerBuilder).build(); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void accountPermissioningTomlFileWithNoPermissionsEnabledMustNotError() - throws IOException { - - final URL configFile = this.getClass().getResource(PERMISSIONING_CONFIG_TOML); - final Path permToml = createTempFile("toml", Resources.toByteArray(configFile)); - parseCommand("--permissions-accounts-config-file", permToml.toString()); - - verify(mockRunnerBuilder).build(); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void defaultPermissionsTomlFileWithNoPermissionsEnabledMustNotError() { - parseCommand("--p2p-enabled", "false"); - - verify(mockRunnerBuilder).build(); - - assertThat(commandErrorOutput.toString(UTF_8)).doesNotContain("no permissions enabled"); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void nodePermissioningTomlPathMustUseOption() throws IOException { - final List allowedNodes = - Lists.newArrayList( - EnodeURLImpl.fromString( - "enode://6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0@192.168.0.9:4567"), - EnodeURLImpl.fromString( - "enode://6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0@192.169.0.9:4568")); - - final URL configFile = this.getClass().getResource(PERMISSIONING_CONFIG_TOML); - final Path permToml = createTempFile("toml", Resources.toByteArray(configFile)); - - final String allowedNodesString = - allowedNodes.stream().map(Object::toString).collect(Collectors.joining(",")); - parseCommand( - "--permissions-nodes-config-file-enabled", - "--permissions-nodes-config-file", - permToml.toString(), - "--bootnodes", - allowedNodesString); - final LocalPermissioningConfiguration localPermissioningConfiguration = - LocalPermissioningConfiguration.createDefault(); - localPermissioningConfiguration.setNodePermissioningConfigFilePath(permToml.toString()); - localPermissioningConfiguration.setNodeAllowlist(allowedNodes); - - verify(mockRunnerBuilder) - .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); - verify(mockRunnerBuilder).build(); - - final PermissioningConfiguration config = - permissioningConfigurationArgumentCaptor.getValue().get(); - assertThat(config.getLocalConfig().get()) - .usingRecursiveComparison() - .isEqualTo(localPermissioningConfiguration); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void accountPermissioningTomlPathMustUseOption() throws IOException { - - final URL configFile = this.getClass().getResource(PERMISSIONING_CONFIG_TOML); - final Path permToml = createTempFile("toml", Resources.toByteArray(configFile)); - - parseCommand( - "--permissions-accounts-config-file-enabled", - "--permissions-accounts-config-file", - permToml.toString()); - final LocalPermissioningConfiguration localPermissioningConfiguration = - LocalPermissioningConfiguration.createDefault(); - localPermissioningConfiguration.setAccountPermissioningConfigFilePath(permToml.toString()); - localPermissioningConfiguration.setAccountAllowlist( - Collections.singletonList("0x0000000000000000000000000000000000000009")); - - verify(mockRunnerBuilder) - .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); - final PermissioningConfiguration permissioningConfiguration = - permissioningConfigurationArgumentCaptor.getValue().get(); - assertThat(permissioningConfiguration.getLocalConfig()).isPresent(); - - final LocalPermissioningConfiguration effectiveLocalPermissioningConfig = - permissioningConfiguration.getLocalConfig().get(); - assertThat(effectiveLocalPermissioningConfig.isAccountAllowlistEnabled()).isTrue(); - assertThat(effectiveLocalPermissioningConfig.getAccountPermissioningConfigFilePath()) - .isEqualTo(permToml.toString()); - - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - @Test public void tomlThatConfiguresEverythingExceptPermissioningToml() throws IOException { // Load a TOML that configures literally everything (except permissioning TOML config) @@ -3166,43 +2791,6 @@ public void privacyWithPruningMustError() { assertThat(commandOutput.toString(UTF_8)).isEmpty(); } - @Test - public void errorIsRaisedIfStaticNodesAreNotAllowed(final @TempDir Path testFolder) - throws IOException { - final Path staticNodesFile = testFolder.resolve("static-nodes.json"); - final Path permissioningConfig = testFolder.resolve("permissioning.json"); - - final EnodeURL staticNodeURI = - EnodeURLImpl.builder() - .nodeId( - "50203c6bfca6874370e71aecc8958529fd723feb05013dc1abca8fc1fff845c5259faba05852e9dfe5ce172a7d6e7c2a3a5eaa8b541c8af15ea5518bbff5f2fa") - .ipAddress("127.0.0.1") - .useDefaultPorts() - .build(); - - final EnodeURL allowedNode = - EnodeURLImpl.builder() - .nodeId( - "50203c6bfca6874370e71aecc8958529fd723feb05013dc1abca8fc1fff845c5259faba05852e9dfe5ce172a7d6e7c2a3a5eaa8b541c8af15ea5518bbff5f2fa") - .useDefaultPorts() - .ipAddress("127.0.0.1") - .listeningPort(30304) - .build(); - - Files.write(staticNodesFile, ("[\"" + staticNodeURI.toString() + "\"]").getBytes(UTF_8)); - Files.write( - permissioningConfig, - ("nodes-allowlist=[\"" + allowedNode.toString() + "\"]").getBytes(UTF_8)); - - parseCommand( - "--data-path=" + testFolder, - "--bootnodes", - "--permissions-nodes-config-file-enabled=true", - "--permissions-nodes-config-file=" + permissioningConfig); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains(staticNodeURI.toString(), "not in nodes-allowlist"); - } - @Test public void tomlThatHasInvalidOptions() throws IOException { final URL configFile = this.getClass().getResource("/complete_config.toml"); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/PermissionsOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/PermissionsOptionsTest.java new file mode 100644 index 00000000000..e93f712c6f2 --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/PermissionsOptionsTest.java @@ -0,0 +1,453 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.options; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; + +import org.hyperledger.besu.cli.CommandTestAbstract; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; +import org.hyperledger.besu.ethereum.permissioning.LocalPermissioningConfiguration; +import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; +import org.hyperledger.besu.ethereum.permissioning.SmartContractPermissioningConfiguration; +import org.hyperledger.besu.plugin.data.EnodeURL; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import com.google.common.collect.Lists; +import com.google.common.io.Resources; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class PermissionsOptionsTest extends CommandTestAbstract { + private static final String PERMISSIONING_CONFIG_TOML = "/permissioning_config.toml"; + + @Test + public void errorIsRaisedIfStaticNodesAreNotAllowed(final @TempDir Path testFolder) + throws IOException { + final Path staticNodesFile = testFolder.resolve("static-nodes.json"); + final Path permissioningConfig = testFolder.resolve("permissioning.json"); + + final EnodeURL staticNodeURI = + EnodeURLImpl.builder() + .nodeId( + "50203c6bfca6874370e71aecc8958529fd723feb05013dc1abca8fc1fff845c5259faba05852e9dfe5ce172a7d6e7c2a3a5eaa8b541c8af15ea5518bbff5f2fa") + .ipAddress("127.0.0.1") + .useDefaultPorts() + .build(); + + final EnodeURL allowedNode = + EnodeURLImpl.builder() + .nodeId( + "50203c6bfca6874370e71aecc8958529fd723feb05013dc1abca8fc1fff845c5259faba05852e9dfe5ce172a7d6e7c2a3a5eaa8b541c8af15ea5518bbff5f2fa") + .useDefaultPorts() + .ipAddress("127.0.0.1") + .listeningPort(30304) + .build(); + + Files.write(staticNodesFile, ("[\"" + staticNodeURI.toString() + "\"]").getBytes(UTF_8)); + Files.write( + permissioningConfig, + ("nodes-allowlist=[\"" + allowedNode.toString() + "\"]").getBytes(UTF_8)); + + parseCommand( + "--data-path=" + testFolder, + "--bootnodes", + "--permissions-nodes-config-file-enabled=true", + "--permissions-nodes-config-file=" + permissioningConfig); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains(staticNodeURI.toString(), "not in nodes-allowlist"); + } + + @Test + public void nodePermissionsSmartContractWithoutOptionMustError() { + parseCommand("--permissions-nodes-contract-address"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)) + .startsWith("Missing required parameter for option '--permissions-nodes-contract-address'"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissionsEnabledWithoutContractAddressMustError() { + parseCommand("--permissions-nodes-contract-enabled"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("No node permissioning contract address specified"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissionsEnabledWithInvalidContractAddressMustError() { + parseCommand( + "--permissions-nodes-contract-enabled", + "--permissions-nodes-contract-address", + "invalid-smart-contract-address"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)).contains("Invalid value"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissionsEnabledWithTooShortContractAddressMustError() { + parseCommand( + "--permissions-nodes-contract-enabled", "--permissions-nodes-contract-address", "0x1234"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)).contains("Invalid value"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissionsSmartContractMustUseOption() { + + final String smartContractAddress = "0x0000000000000000000000000000000000001234"; + + parseCommand( + "--permissions-nodes-contract-enabled", + "--permissions-nodes-contract-address", + smartContractAddress); + final SmartContractPermissioningConfiguration smartContractPermissioningConfiguration = + new SmartContractPermissioningConfiguration(); + smartContractPermissioningConfiguration.setNodeSmartContractAddress( + Address.fromHexString(smartContractAddress)); + smartContractPermissioningConfiguration.setSmartContractNodeAllowlistEnabled(true); + + verify(mockRunnerBuilder) + .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + final PermissioningConfiguration config = + permissioningConfigurationArgumentCaptor.getValue().get(); + assertThat(config.getSmartContractConfig().get()) + .usingRecursiveComparison() + .isEqualTo(smartContractPermissioningConfiguration); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissionsContractVersionDefaultValue() { + final SmartContractPermissioningConfiguration expectedConfig = + new SmartContractPermissioningConfiguration(); + expectedConfig.setNodeSmartContractAddress( + Address.fromHexString("0x0000000000000000000000000000000000001234")); + expectedConfig.setSmartContractNodeAllowlistEnabled(true); + expectedConfig.setNodeSmartContractInterfaceVersion(1); + + parseCommand( + "--permissions-nodes-contract-enabled", + "--permissions-nodes-contract-address", + "0x0000000000000000000000000000000000001234"); + + verify(mockRunnerBuilder) + .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + final PermissioningConfiguration config = + permissioningConfigurationArgumentCaptor.getValue().get(); + assertThat(config.getSmartContractConfig().get()) + .usingRecursiveComparison() + .isEqualTo(expectedConfig); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissionsContractVersionSetsValue() { + final SmartContractPermissioningConfiguration expectedConfig = + new SmartContractPermissioningConfiguration(); + expectedConfig.setNodeSmartContractAddress( + Address.fromHexString("0x0000000000000000000000000000000000001234")); + expectedConfig.setSmartContractNodeAllowlistEnabled(true); + expectedConfig.setNodeSmartContractInterfaceVersion(2); + + parseCommand( + "--permissions-nodes-contract-enabled", + "--permissions-nodes-contract-address", + "0x0000000000000000000000000000000000001234", + "--permissions-nodes-contract-version", + "2"); + + verify(mockRunnerBuilder) + .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + final PermissioningConfiguration config = + permissioningConfigurationArgumentCaptor.getValue().get(); + assertThat(config.getSmartContractConfig().get()) + .usingRecursiveComparison() + .isEqualTo(expectedConfig); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void accountPermissionsSmartContractWithoutOptionMustError() { + parseCommand("--permissions-accounts-contract-address"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)) + .startsWith( + "Missing required parameter for option '--permissions-accounts-contract-address'"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void accountPermissionsEnabledWithoutContractAddressMustError() { + parseCommand("--permissions-accounts-contract-enabled"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("No account permissioning contract address specified"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void accountPermissionsEnabledWithInvalidContractAddressMustError() { + parseCommand( + "--permissions-accounts-contract-enabled", + "--permissions-accounts-contract-address", + "invalid-smart-contract-address"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)).contains("Invalid value"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void accountPermissionsEnabledWithTooShortContractAddressMustError() { + parseCommand( + "--permissions-accounts-contract-enabled", + "--permissions-accounts-contract-address", + "0x1234"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)).contains("Invalid value"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void accountPermissionsSmartContractMustUseOption() { + final String smartContractAddress = "0x0000000000000000000000000000000000001234"; + + parseCommand( + "--permissions-accounts-contract-enabled", + "--permissions-accounts-contract-address", + smartContractAddress); + final SmartContractPermissioningConfiguration smartContractPermissioningConfiguration = + new SmartContractPermissioningConfiguration(); + smartContractPermissioningConfiguration.setAccountSmartContractAddress( + Address.fromHexString(smartContractAddress)); + smartContractPermissioningConfiguration.setSmartContractAccountAllowlistEnabled(true); + + verify(mockRunnerBuilder) + .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); + final PermissioningConfiguration permissioningConfiguration = + permissioningConfigurationArgumentCaptor.getValue().get(); + assertThat(permissioningConfiguration.getSmartContractConfig()).isPresent(); + + final SmartContractPermissioningConfiguration effectiveSmartContractConfig = + permissioningConfiguration.getSmartContractConfig().get(); + assertThat(effectiveSmartContractConfig.isSmartContractAccountAllowlistEnabled()).isTrue(); + assertThat(effectiveSmartContractConfig.getAccountSmartContractAddress()) + .isEqualTo(Address.fromHexString(smartContractAddress)); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissioningTomlPathWithoutOptionMustDisplayUsage() { + parseCommand("--permissions-nodes-config-file"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)) + .startsWith("Missing required parameter for option '--permissions-nodes-config-file'"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void accountPermissioningTomlPathWithoutOptionMustDisplayUsage() { + parseCommand("--permissions-accounts-config-file"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)) + .startsWith("Missing required parameter for option '--permissions-accounts-config-file'"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissioningEnabledWithNonexistentConfigFileMustError() { + parseCommand( + "--permissions-nodes-config-file-enabled", + "--permissions-nodes-config-file", + "file-does-not-exist"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)).contains("Configuration file does not exist"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void accountPermissioningEnabledWithNonexistentConfigFileMustError() { + parseCommand( + "--permissions-accounts-config-file-enabled", + "--permissions-accounts-config-file", + "file-does-not-exist"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + + assertThat(commandErrorOutput.toString(UTF_8)).contains("Configuration file does not exist"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissioningTomlFileWithNoPermissionsEnabledMustNotError() throws IOException { + + final URL configFile = this.getClass().getResource(PERMISSIONING_CONFIG_TOML); + final Path permToml = createTempFile("toml", Resources.toByteArray(configFile)); + parseCommand("--permissions-nodes-config-file", permToml.toString()); + + verify(mockRunnerBuilder).build(); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void accountPermissioningTomlFileWithNoPermissionsEnabledMustNotError() + throws IOException { + + final URL configFile = this.getClass().getResource(PERMISSIONING_CONFIG_TOML); + final Path permToml = createTempFile("toml", Resources.toByteArray(configFile)); + parseCommand("--permissions-accounts-config-file", permToml.toString()); + + verify(mockRunnerBuilder).build(); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void defaultPermissionsTomlFileWithNoPermissionsEnabledMustNotError() { + parseCommand("--p2p-enabled", "false"); + + verify(mockRunnerBuilder).build(); + + assertThat(commandErrorOutput.toString(UTF_8)).doesNotContain("no permissions enabled"); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void nodePermissioningTomlPathMustUseOption() throws IOException { + final List allowedNodes = + Lists.newArrayList( + EnodeURLImpl.fromString( + "enode://6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0@192.168.0.9:4567"), + EnodeURLImpl.fromString( + "enode://6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0@192.169.0.9:4568")); + + final URL configFile = this.getClass().getResource(PERMISSIONING_CONFIG_TOML); + final Path permToml = createTempFile("toml", Resources.toByteArray(configFile)); + + final String allowedNodesString = + allowedNodes.stream().map(Object::toString).collect(Collectors.joining(",")); + parseCommand( + "--permissions-nodes-config-file-enabled", + "--permissions-nodes-config-file", + permToml.toString(), + "--bootnodes", + allowedNodesString); + final LocalPermissioningConfiguration localPermissioningConfiguration = + LocalPermissioningConfiguration.createDefault(); + localPermissioningConfiguration.setNodePermissioningConfigFilePath(permToml.toString()); + localPermissioningConfiguration.setNodeAllowlist(allowedNodes); + + verify(mockRunnerBuilder) + .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + + final PermissioningConfiguration config = + permissioningConfigurationArgumentCaptor.getValue().get(); + assertThat(config.getLocalConfig().get()) + .usingRecursiveComparison() + .isEqualTo(localPermissioningConfiguration); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void accountPermissioningTomlPathMustUseOption() throws IOException { + + final URL configFile = this.getClass().getResource(PERMISSIONING_CONFIG_TOML); + final Path permToml = createTempFile("toml", Resources.toByteArray(configFile)); + + parseCommand( + "--permissions-accounts-config-file-enabled", + "--permissions-accounts-config-file", + permToml.toString()); + final LocalPermissioningConfiguration localPermissioningConfiguration = + LocalPermissioningConfiguration.createDefault(); + localPermissioningConfiguration.setAccountPermissioningConfigFilePath(permToml.toString()); + localPermissioningConfiguration.setAccountAllowlist( + Collections.singletonList("0x0000000000000000000000000000000000000009")); + + verify(mockRunnerBuilder) + .permissioningConfiguration(permissioningConfigurationArgumentCaptor.capture()); + final PermissioningConfiguration permissioningConfiguration = + permissioningConfigurationArgumentCaptor.getValue().get(); + assertThat(permissioningConfiguration.getLocalConfig()).isPresent(); + + final LocalPermissioningConfiguration effectiveLocalPermissioningConfig = + permissioningConfiguration.getLocalConfig().get(); + assertThat(effectiveLocalPermissioningConfig.isAccountAllowlistEnabled()).isTrue(); + assertThat(effectiveLocalPermissioningConfig.getAccountPermissioningConfigFilePath()) + .isEqualTo(permToml.toString()); + + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } +} From 7afe3a9ae30bd165c83d7234fce5367bdf943e46 Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Tue, 30 Jan 2024 12:50:50 +1100 Subject: [PATCH 54/56] Remove `--engine-jwt-enabled` deprecated option (#6491) Signed-off-by: Gabriel-Trintinalia --- CHANGELOG.md | 4 +++- .../main/java/org/hyperledger/besu/cli/BesuCommand.java | 7 ------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba9a4e7a2c2..33919806b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ ### Breaking Changes - The `trace-filter` method in JSON-RPC API now has a default block range limit of 1000, adjustable with `--rpc-max-trace-filter-range` (thanks @alyokaz) [#6446](https://github.com/hyperledger/besu/pull/6446) - Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): - - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. +- `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. +- `--engine-jwt-enabled` has been removed. Use `--engine-jwt-disabled` instead. [#6491](https://github.com/hyperledger/besu/pull/6491) + ### Deprecations diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 8e59b4c23dd..7cbb60f13cd 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -610,13 +610,6 @@ static class EngineRPCOptionGroup { description = "Path to file containing shared secret key for JWT signature verification") private final Path engineJwtKeyFile = null; - @Option( - names = {"--engine-jwt-enabled"}, - description = "deprecated option, engine jwt auth is enabled by default", - hidden = true) - @SuppressWarnings({"FieldCanBeFinal", "UnusedVariable"}) - private final Boolean deprecatedIsEngineAuthEnabled = true; - @Option( names = {"--engine-jwt-disabled"}, description = "Disable authentication for Engine APIs (default: ${DEFAULT-VALUE})") From 06406ce1088adab9b09bae622ccfcf8de85aa79f Mon Sep 17 00:00:00 2001 From: Stefan Pingel <16143240+pinges@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:45:28 +1000 Subject: [PATCH 55/56] only do 2 retries (#6482) Signed-off-by: stefan.pingel@consensys.net --- .../ethereum/eth/sync/backwardsync/BackwardSyncContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java index aee56cda22a..936b1eb2f8c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java @@ -44,7 +44,7 @@ public class BackwardSyncContext { private static final Logger LOG = LoggerFactory.getLogger(BackwardSyncContext.class); public static final int BATCH_SIZE = 200; - private static final int DEFAULT_MAX_RETRIES = 20; + private static final int DEFAULT_MAX_RETRIES = 2; private static final long MILLIS_DELAY_BETWEEN_PROGRESS_LOG = 10_000L; private static final long DEFAULT_MILLIS_BETWEEN_RETRIES = 5000; private static final int DEFAULT_MAX_CHAIN_EVENT_ENTRIES = BadBlockManager.MAX_BAD_BLOCKS_SIZE; From a563cf23c1a8e3fc48e0c40a484ab64d375b0462 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 30 Jan 2024 09:34:06 +0100 Subject: [PATCH 56/56] Upgrade Prometheus and Opentelemetry dependencies (#6422) Signed-off-by: Fabio Di Fabio Co-authored-by: Stefan Pingel <16143240+pinges@users.noreply.github.com> --- CHANGELOG.md | 5 +- gradle/verification-metadata.xml | 392 +++++++++--------- gradle/versions.gradle | 32 +- metrics/core/build.gradle | 2 +- .../opentelemetry/DebugMetricReader.java | 3 +- .../opentelemetry/OpenTelemetrySystem.java | 16 +- .../prometheus/PrometheusMetricsSystem.java | 51 ++- .../PrometheusMetricsSystemTest.java | 69 ++- 8 files changed, 344 insertions(+), 226 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33919806b73..39aec2f15fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 24.1.2-SNAPSHOT ### Breaking Changes +- Following the OpenMetrics convention, the updated Prometheus client adds the `_total` suffix to every metrics of type counter, with the effect that some existing metrics have been renamed to have this suffix. If you are using the official Besu Grafana dashboard [(available here)](https://grafana.com/grafana/dashboards/16455-besu-full/), just update it to the latest revision, that accepts the old and the new name of the affected metrics. If you have a custom dashboards or use the metrics in other ways, then you need to manually update it to support the new naming. - The `trace-filter` method in JSON-RPC API now has a default block range limit of 1000, adjustable with `--rpc-max-trace-filter-range` (thanks @alyokaz) [#6446](https://github.com/hyperledger/besu/pull/6446) - Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. @@ -12,6 +13,7 @@ ### Deprecations ### Additions and Improvements +- Upgrade Prometheus and Opentelemetry dependencies [#6422](https://github.com/hyperledger/besu/pull/6422) - Add `OperationTracer.tracePrepareTransaction`, where the sender account has not yet been altered[#6453](https://github.com/hyperledger/besu/pull/6453) - Improve the high spec flag by limiting it to a few column families [#6354](https://github.com/hyperledger/besu/pull/6354) - Log blob count when importing a block via Engine API [#6466](https://github.com/hyperledger/besu/pull/6466) @@ -30,7 +32,7 @@ - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 (thanks @h4l) [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) -- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. [#6423](https://github.com/hyperledger/besu/pull/6423) +- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis. This is to prevent possible DoS attacks in case a single transaction is taking too long to execute, and to have a stable block production rate. This could be a breaking change if an existing network needs to accept transactions that take more time to executed than the newly introduced limit. If it is mandatory for these networks to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. [#6423](https://github.com/hyperledger/besu/pull/6423) ### Deprecations @@ -63,6 +65,7 @@ Note, due to a CI race with the release job, the initial published version of 24 ~~https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.zip / sha256 b6b64f939e0bb4937ce90fc647e0a7073ce3e359c10352b502059955070a60c6 https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.tar.gz / sha256 cfcae04c30769bf338b0740ac65870f9346d3469931bb46cdba3b2f65d311e7a~~ + ## 24.1.0 ### Breaking Changes diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 673ac2df400..9a8866b5f1e 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1032,16 +1032,16 @@ - - - - - + + + + + @@ -1055,11 +1055,6 @@ - - - - - @@ -1068,6 +1063,11 @@ + + + + + @@ -1089,16 +1089,16 @@ - - - - - + + + + + @@ -2112,212 +2112,223 @@ - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + + + + - - + + - - + + + + + + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + + + + @@ -2423,25 +2434,20 @@ - - - - - - - - + + + - - + + - - - + + + - - + + @@ -2452,12 +2458,12 @@ - - - + + + - - + + @@ -2468,12 +2474,41 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + @@ -2816,19 +2851,6 @@ - - - - - - - - - - - - - diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 43fe774c463..23731c443d8 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -80,16 +80,16 @@ dependencyManagement { dependency group: 'io.netty', name: 'netty-transport-native-kqueue', version:'4.1.104.Final', classifier: 'osx-x86_64' dependency 'io.netty:netty-transport-native-unix-common:4.1.104.Final' - dependency 'io.opentelemetry:opentelemetry-api:1.24.0' - dependency 'io.opentelemetry:opentelemetry-exporter-otlp:1.24.0' - dependency 'io.opentelemetry:opentelemetry-extension-trace-propagators:1.24.0' - dependency 'io.opentelemetry.proto:opentelemetry-proto:0.19.0-alpha' - dependency 'io.opentelemetry:opentelemetry-sdk-metrics:1.24.0' - dependency 'io.opentelemetry:opentelemetry-sdk-trace:1.24.0' - dependency 'io.opentelemetry:opentelemetry-sdk:1.24.0' - dependency 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.24.0-alpha' - dependency 'io.opentelemetry:opentelemetry-semconv:1.24.0-alpha' - dependency 'io.opentelemetry.instrumentation:opentelemetry-okhttp-3.0:1.24.0-alpha' + dependency 'io.opentelemetry:opentelemetry-api:1.33.0' + dependency 'io.opentelemetry:opentelemetry-exporter-otlp:1.33.0' + dependency 'io.opentelemetry:opentelemetry-extension-trace-propagators:1.33.0' + dependency 'io.opentelemetry:opentelemetry-sdk-metrics:1.33.0' + dependency 'io.opentelemetry:opentelemetry-sdk-trace:1.33.0' + dependency 'io.opentelemetry:opentelemetry-sdk:1.33.0' + dependency 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.33.0' + dependency 'io.opentelemetry.instrumentation:opentelemetry-okhttp-3.0:1.32.0-alpha' + dependency 'io.opentelemetry.proto:opentelemetry-proto:1.0.0-alpha' + dependency 'io.opentelemetry.semconv:opentelemetry-semconv:1.23.1-alpha' dependency 'io.opentracing.contrib:opentracing-okhttp3:3.0.0' dependency 'io.opentracing:opentracing-api:0.33.0' @@ -97,11 +97,13 @@ dependencyManagement { dependency 'io.pkts:pkts-core:3.0.10' - dependency 'io.prometheus:simpleclient:0.9.0' - dependency 'io.prometheus:simpleclient_common:0.9.0' - dependency 'io.prometheus:simpleclient_hotspot:0.9.0' - dependency 'io.prometheus:simpleclient_pushgateway:0.9.0' - dependency 'io.prometheus:simpleclient_guava:0.16.0' + dependencySet(group: 'io.prometheus', version: '0.16.0') { + entry 'simpleclient' + entry 'simpleclient_common' + entry 'simpleclient_hotspot' + entry 'simpleclient_pushgateway' + entry 'simpleclient_guava' + } dependency 'io.reactivex.rxjava2:rxjava:2.2.21' diff --git a/metrics/core/build.gradle b/metrics/core/build.gradle index 295cdada976..bc92cb6b5fb 100644 --- a/metrics/core/build.gradle +++ b/metrics/core/build.gradle @@ -48,11 +48,11 @@ dependencies { implementation 'io.netty:netty-all' implementation 'io.opentelemetry:opentelemetry-api' implementation 'io.opentelemetry:opentelemetry-sdk' - implementation 'io.opentelemetry:opentelemetry-semconv' implementation 'io.opentelemetry:opentelemetry-sdk-trace' implementation 'io.opentelemetry:opentelemetry-sdk-metrics' implementation 'io.opentelemetry:opentelemetry-exporter-otlp' implementation 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure' + implementation 'io.opentelemetry.semconv:opentelemetry-semconv' implementation 'io.prometheus:simpleclient' implementation 'io.prometheus:simpleclient_common' diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/DebugMetricReader.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/DebugMetricReader.java index ea81e91aff9..a44ad1c1cee 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/DebugMetricReader.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/DebugMetricReader.java @@ -22,7 +22,6 @@ import io.opentelemetry.sdk.metrics.data.MetricData; import io.opentelemetry.sdk.metrics.export.CollectionRegistration; import io.opentelemetry.sdk.metrics.export.MetricReader; -import io.opentelemetry.sdk.metrics.internal.export.MetricProducer; import org.jetbrains.annotations.NotNull; class DebugMetricReader implements MetricReader { @@ -31,7 +30,7 @@ class DebugMetricReader implements MetricReader { public DebugMetricReader() {} public Collection getAllMetrics() { - return MetricProducer.asMetricProducer(this.registration).collectAllMetrics(); + return registration.collectAllMetrics(); } @Override diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java index 2c43eb8668e..c52dd037f76 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java @@ -50,6 +50,7 @@ import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.data.DoublePointData; @@ -61,7 +62,7 @@ import io.opentelemetry.sdk.metrics.data.SummaryPointData; import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; +import io.opentelemetry.semconv.ResourceAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,15 +111,18 @@ public OpenTelemetrySystem( .merge( Resource.create( Attributes.builder().put(ResourceAttributes.SERVICE_NAME, jobName).build())); - AutoConfiguredOpenTelemetrySdk autoSdk = + AutoConfiguredOpenTelemetrySdkBuilder autoSdkBuilder = AutoConfiguredOpenTelemetrySdk.builder() .addMeterProviderCustomizer( (provider, config) -> provider.setResource(resource).registerMetricReader(debugMetricReader)) - .addTracerProviderCustomizer((provider, config) -> provider.setResource(resource)) - .setResultAsGlobal(setAsGlobal) - .build(); - OpenTelemetrySdk sdk = autoSdk.getOpenTelemetrySdk(); + .addTracerProviderCustomizer((provider, config) -> provider.setResource(resource)); + + if (setAsGlobal) { + autoSdkBuilder.setResultAsGlobal(); + } + + OpenTelemetrySdk sdk = autoSdkBuilder.build().getOpenTelemetrySdk(); this.sdkMeterProvider = sdk.getSdkMeterProvider(); this.sdkTracerProvider = sdk.getSdkTracerProvider(); } diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java index fa07670521a..59211ee08a7 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java @@ -32,7 +32,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.DoubleSupplier; import java.util.function.Supplier; -import java.util.stream.Collectors; import java.util.stream.Stream; import com.google.common.collect.ImmutableSet; @@ -48,6 +47,7 @@ import io.prometheus.client.hotspot.MemoryPoolsExports; import io.prometheus.client.hotspot.StandardExports; import io.prometheus.client.hotspot.ThreadExports; +import io.vertx.core.impl.ConcurrentHashSet; /** The Prometheus metrics system. */ public class PrometheusMetricsSystem implements ObservableMetricsSystem { @@ -58,6 +58,7 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { cachedCounters = new ConcurrentHashMap<>(); private final Map> cachedTimers = new ConcurrentHashMap<>(); + private final Set totalSuffixedCounters = new ConcurrentHashSet<>(); private final Set enabledCategories; private final boolean timersEnabled; @@ -95,7 +96,7 @@ public LabelledMetric crea final String name, final String help, final String... labelNames) { - final String metricName = convertToPrometheusName(category, name); + final String metricName = convertToPrometheusCounterName(category, name); return cachedCounters.computeIfAbsent( metricName, (k) -> { @@ -185,9 +186,7 @@ private void addCollectorUnchecked(final MetricCategory category, final Collecto category, key -> Collections.newSetFromMap(new ConcurrentHashMap<>())); final List newSamples = - metric.collect().stream() - .map(metricFamilySamples -> metricFamilySamples.name) - .collect(Collectors.toList()); + metric.collect().stream().map(metricFamilySamples -> metricFamilySamples.name).toList(); metrics.stream() .filter( @@ -230,6 +229,9 @@ private Observation createObservationFromSample( if (familySamples.type == Collector.Type.SUMMARY) { return convertSummarySampleNamesToLabels(category, sample, familySamples); } + if (familySamples.type == Collector.Type.COUNTER) { + return convertCounterNamesToLabels(category, sample, familySamples); + } return new Observation( category, convertFromPrometheusName(category, sample.name), @@ -237,6 +239,20 @@ private Observation createObservationFromSample( sample.labelValues); } + private Observation convertCounterNamesToLabels( + final MetricCategory category, final Sample sample, final MetricFamilySamples familySamples) { + final List labelValues = new ArrayList<>(sample.labelValues); + if (sample.name.endsWith("_created")) { + labelValues.add("created"); + } + + return new Observation( + category, + convertFromPrometheusCounterName(category, familySamples.name), + sample.value, + labelValues); + } + private Observation convertHistogramSampleNamesToLabels( final MetricCategory category, final Sample sample, final MetricFamilySamples familySamples) { final List labelValues = new ArrayList<>(sample.labelValues); @@ -259,6 +275,8 @@ private Observation convertSummarySampleNamesToLabels( labelValues.add("sum"); } else if (sample.name.endsWith("_count")) { labelValues.add("count"); + } else if (sample.name.endsWith("_created")) { + labelValues.add("created"); } else { labelValues.add(labelValues.size() - 1, "quantile"); } @@ -280,11 +298,34 @@ public String convertToPrometheusName(final MetricCategory category, final Strin return prometheusPrefix(category) + name; } + /** + * Convert to prometheus counter name. Prometheus adds a _total suffix to the name if not present, + * so we remember if the original name already has it, to be able to covert back correctly + * + * @param category the category + * @param name the name + * @return the name as string + */ + public String convertToPrometheusCounterName(final MetricCategory category, final String name) { + if (name.endsWith("_total")) { + totalSuffixedCounters.add(name); + } + return convertToPrometheusName(category, name); + } + private String convertFromPrometheusName(final MetricCategory category, final String metricName) { final String prefix = prometheusPrefix(category); return metricName.startsWith(prefix) ? metricName.substring(prefix.length()) : metricName; } + private String convertFromPrometheusCounterName( + final MetricCategory category, final String metricName) { + final String unPrefixedName = convertFromPrometheusName(category, metricName); + return totalSuffixedCounters.contains(unPrefixedName + "_total") + ? unPrefixedName + "_total" + : unPrefixedName; + } + private String prometheusPrefix(final MetricCategory category) { return category.getApplicationPrefix().orElse("") + category.getName() + "_"; } diff --git a/metrics/core/src/test/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystemTest.java b/metrics/core/src/test/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystemTest.java index 7c62f1f0b6c..2ddf86d347b 100644 --- a/metrics/core/src/test/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystemTest.java +++ b/metrics/core/src/test/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystemTest.java @@ -17,6 +17,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static java.util.function.Predicate.not; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hyperledger.besu.metrics.BesuMetricCategory.DEFAULT_METRIC_CATEGORIES; @@ -51,6 +52,11 @@ public class PrometheusMetricsSystemTest { Comparator.comparing(observation -> observation.getCategory().getName()) .thenComparing(Observation::getMetricName) .thenComparing((o1, o2) -> o1.getLabels().equals(o2.getLabels()) ? 0 : 1); + private static final Comparator WITH_VALUES = + Comparator.comparing(observation -> observation.getCategory().getName()) + .thenComparing(Observation::getMetricName) + .thenComparing((o1, o2) -> o1.getLabels().equals(o2.getLabels()) ? 0 : 1) + .thenComparing((o1, o2) -> o1.getValue().equals(o2.getValue()) ? 0 : 1); @BeforeEach public void resetGlobalOpenTelemetry() { @@ -66,11 +72,17 @@ public void shouldCreateObservationFromCounter() { counter.inc(); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 1.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactlyInAnyOrder( + new Observation(PEERS, "connected", 1.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); counter.inc(); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 2.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 2.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); } @Test @@ -83,26 +95,36 @@ public void shouldHandleDuplicateCounterCreation() { counter1.labels().inc(); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 1.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 1.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); counter2.labels().inc(); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 2.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 2.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); } @Test public void shouldCreateSeparateObservationsForEachCounterLabelValue() { final LabelledMetric counter = - metricsSystem.createLabelledCounter(PEERS, "connected", "Some help string", "labelName"); + metricsSystem.createLabelledCounter( + PEERS, "connected_total", "Some help string", "labelName"); counter.labels("value1").inc(); counter.labels("value2").inc(); counter.labels("value1").inc(); assertThat(metricsSystem.streamObservations()) + .usingElementComparator(this::compareCounters) .containsExactlyInAnyOrder( - new Observation(PEERS, "connected", 2.0, singletonList("value1")), - new Observation(PEERS, "connected", 1.0, singletonList("value2"))); + new Observation(PEERS, "connected_total", 2.0, singletonList("value1")), + new Observation(PEERS, "connected_total", 1.0, singletonList("value2")), + new Observation(PEERS, "connected_total", null, List.of("value1", "created")), + new Observation(PEERS, "connected_total", null, List.of("value2", "created"))); } @Test @@ -138,11 +160,18 @@ public void shouldIncrementCounterBySpecifiedAmount() { counter.inc(5); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 5.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 5.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); counter.inc(6); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 11.0, emptyList())); + .usingDefaultElementComparator() + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 11.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); } @Test @@ -162,7 +191,8 @@ public void shouldCreateObservationsFromTimer() { new Observation(RPC, "request", null, asList("quantile", "0.99")), new Observation(RPC, "request", null, asList("quantile", "1.0")), new Observation(RPC, "request", null, singletonList("sum")), - new Observation(RPC, "request", null, singletonList("count"))); + new Observation(RPC, "request", null, singletonList("count")), + new Observation(RPC, "request", null, singletonList("created"))); } @Test @@ -192,7 +222,8 @@ public void shouldCreateObservationsFromTimerWithLabels() { new Observation(RPC, "request", null, asList("method", "quantile", "0.99")), new Observation(RPC, "request", null, asList("method", "quantile", "1.0")), new Observation(RPC, "request", null, asList("method", "sum")), - new Observation(RPC, "request", null, asList("method", "count"))); + new Observation(RPC, "request", null, asList("method", "count")), + new Observation(RPC, "request", null, asList("method", "created"))); } @Test @@ -251,6 +282,8 @@ public void shouldOnlyObserveEnabledMetrics() { counterR.labels("op").inc(); assertThat(localMetricSystem.streamObservations()) + .usingRecursiveFieldByFieldElementComparator() + .filteredOn(not(this::isCreatedSample)) .containsExactly(new Observation(RPC, "name", 1.0, singletonList("op"))); } @@ -280,4 +313,18 @@ public void returnsNoOpMetricsWhenPushEnabled() { assertThat(localMetricSystem).isInstanceOf(PrometheusMetricsSystem.class); } + + private boolean isCreatedSample(final Observation obs) { + // Simple client 0.10.0 add a _created sample to every counter, histogram and summary, that we + // may want to ignore + return obs.getLabels().contains("created"); + } + + private int compareCounters(final Observation obs1, final Observation obs2) { + // for created samples ignore values + if (obs1.getLabels().contains("created") && obs2.getLabels().contains("created")) { + return IGNORE_VALUES.compare(obs1, obs2); + } + return WITH_VALUES.compare(obs1, obs2); + } }