Merge main, fix conflicts, fix some tests

Signed-off-by: garyschulte <[email protected]>

.circleci/config.yml

@@ -4,14 +4,6 @@ orbs:
   win: circleci/[email protected]
 
 executors:
-  besu_executor_small:
-    docker:
-      - image: cimg/openjdk:17.0
-    resource_class: small
-    working_directory: ~/project
-    environment:
-      GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2
-
   besu_executor_med: # 2cpu, 4G ram
     docker:
       - image: cimg/openjdk:17.0
@@ -189,7 +181,7 @@ jobs:
       - capture_test_results
 
   integrationTests:
-    executor: xl_machine_executor
+    executor: xl_machine_executor # needs the machine executor since privacy test uses container tests (docker)
     steps:
       - prepare
       - attach_workspace:
@@ -219,8 +211,8 @@ jobs:
       - capture_test_results
 
   acceptanceTests:
-    parallelism: 4
-    executor: xl_machine_executor
+    parallelism: 2
+    executor: xl_machine_executor # needs the machine executor since privacy test uses container tests (docker)
     steps:
       - prepare
       - attach_workspace:
@@ -241,8 +233,7 @@ jobs:
       - capture_test_logs
 
   acceptanceTestsCliqueBft:
-    parallelism: 6
-    executor: xl_machine_executor
+    executor: besu_executor_xl
     steps:
       - prepare
       - attach_workspace:
@@ -251,20 +242,13 @@ jobs:
           name: AcceptanceTests (Non-Mainnet)
           no_output_timeout: 20m
           command: |
-            CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
-              | sed 's@.*/src/test/java/@@' \
-              | sed 's@/@.@g' \
-              | sed 's/.\{5\}$//' \
-              | circleci tests split --split-by=timings --timings-type=classname)
-            # Format the arguments to "./gradlew test"
-            GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
-            ./gradlew --no-daemon acceptanceTestCliqueBft $GRADLE_ARGS
+            ./gradlew --no-daemon acceptanceTestCliqueBft
       - capture_test_results
       - capture_test_logs
 
   acceptanceTestsPrivacy:
-    parallelism: 6
-    executor: xl_machine_executor
+    parallelism: 4
+    executor: xl_machine_executor # needs the machine executor since it uses container tests (docker)
     steps:
       - prepare
       - attach_workspace:
@@ -285,8 +269,7 @@ jobs:
       - capture_test_logs
 
   acceptanceTestsPermissioning:
-    parallelism: 6
-    executor: xl_machine_executor
+    executor: besu_executor_xl
     steps:
       - prepare
       - attach_workspace:
@@ -295,14 +278,7 @@ jobs:
           name: AcceptanceTests (Non-Mainnet)
           no_output_timeout: 20m
           command: |
-            CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
-              | sed 's@.*/src/test/java/@@' \
-              | sed 's@/@.@g' \
-              | sed 's/.\{5\}$//' \
-              | circleci tests split --split-by=timings --timings-type=classname)
-            # Format the arguments to "./gradlew test"
-            GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
-            ./gradlew --no-daemon acceptanceTestPermissioning $GRADLE_ARGS
+            ./gradlew --no-daemon acceptanceTestPermissioning
       - capture_test_results
       - capture_test_logs
 
@@ -448,12 +424,11 @@ workflows:
       - acceptanceTestsCliqueBft:
           requires:
             - assemble
-      - acceptanceTestsPrivacy:
-          requires:
-            - assemble
+            - acceptanceTests
       - acceptanceTestsPermissioning:
           requires:
             - assemble
+            - acceptanceTestsCliqueBft
       - buildDocker:
           requires:
             - assemble
@@ -525,3 +500,6 @@ workflows:
     jobs:
       - assemble
       - dockerScan
+      - acceptanceTestsPrivacy:
+          requires:
+            - assemble

.github/workflows/checks.yml

@@ -11,7 +11,7 @@ jobs:
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Java
         uses: actions/setup-java@v3
         with:
@@ -25,7 +25,7 @@ jobs:
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Java 17
         uses: actions/setup-java@v3
         with:

.github/workflows/codeql.yml

@@ -39,7 +39,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java
       uses: actions/setup-java@v3

.github/workflows/gradle-wrapper-validation.yml

@@ -7,5 +7,5 @@ jobs:
     name: "Gradle Wrapper Validation"
     runs-on: [besu-research-ubuntu-8]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: gradle/wrapper-validation-action@v1

.github/workflows/release.yml

@@ -6,7 +6,7 @@ jobs:
   dockerPromoteX64:
     runs-on: [besu-research-ubuntu-16]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-java@v3
         with:
           distribution: 'temurin' # See 'Supported distributions' for available options

.github/workflows/repolinter.yml

@@ -19,6 +19,6 @@ jobs:
     container: ghcr.io/todogroup/repolinter:v0.10.1
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Lint Repo
         run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown

.github/workflows/sonarcloud.yml

@@ -17,7 +17,7 @@ jobs:
     if: github.repository == 'hyperledger/besu'
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:

.gitignore

@@ -9,6 +9,7 @@
 .DS_Store
 .externalToolBuilders/
 .gradle/
+.vscode/
 .idea/
 .loadpath
 .metadata

CHANGELOG.md

@@ -1,30 +1,99 @@
 # Changelog
 
+## 23.10.4
+
 ### Breaking Changes
 
 ### Deprecations
+- Forest pruning (`pruning-enabled` options) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230)
 
 ### Additions and Improvements
+- Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212)
 
-### Bug Fixes
+### Bug fixes
 
-### Download Links
 
-## 23.10.1
-- Cache last n blocks by using a new Besu flag --cache-last-blocks=n [#6009](https://github.com/hyperledger/besu/pull/6009)
-- Optimize performances of RPC method Eth_feeHistory [#6011](https://github.com/hyperledger/besu/pull/6011)
+## 23.10.3
 
 ### Breaking Changes
 
 ### Deprecations
-- `--tx-pool-disable-locals` has been deprecated for removal in favor of `--tx-pool-no-local-priority`, no semantic change, only a renaming [#5959](https://github.com/hyperledger/besu/pull/5959)
+
+### Additions and Improvements
+- Implement debug_traceCall [#5885](https://github.com/hyperledger/besu/pull/5885)
+- Transactions that takes too long to evaluate, during block creation, are dropped from the txpool [#6163](https://github.com/hyperledger/besu/pull/6163)
+- New option `tx-pool-min-gas-price` to set a lower bound when accepting txs to the pool [#6098](https://github.com/hyperledger/besu/pull/6098)
+- Update OpenJDK latest Docker image to use Java 21 [#6189](https://github.com/hyperledger/besu/pull/6189)
+- Allow a transaction selection plugin to specify custom selection results [#6190](https://github.com/hyperledger/besu/pull/6190)
+- Add `rpc-gas-cap` to allow users to set gas limit to the RPC methods used to simulate transactions[#6156](https://github.com/hyperledger/besu/pull/6156)
+- Fix the unavailability of `address` field when returning an `Account` entity on GraphQL in case of unreachable world state [#6198](https://github.com/hyperledger/besu/pull/6198)
+- Update OpenJ9 Docker image to latest version [#6226](https://github.com/hyperledger/besu/pull/6226)
+- Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212)
+- Add `rocksdb usage` to the `storage` subcommand to allow users and dev to check columns families usage [#6185](https://github.com/hyperledger/besu/pull/6185)
+
+### Bug fixes
+- Fix Docker image name clash between Besu and evmtool [#6194](https://github.com/hyperledger/besu/pull/6194)
+- Fix `logIndex` in `eth_getTransactionReceipt` JSON RPC method [#6206](https://github.com/hyperledger/besu/pull/6206)
+
+## 23.10.2
+
+### Breaking Changes
+- TX pool eviction in the legacy TX pool now favours keeping oldest transactions (more likely to evict higher nonces, less likely to introduce nonce gaps) [#6106](https://github.com/hyperledger/besu/pull/6106) and [#6146](https://github.com/hyperledger/besu/pull/6146)
+
+### Deprecations
+
+### Additions and Improvements
+- Ethereum Classic Spiral network upgrade [#6078](https://github.com/hyperledger/besu/pull/6078)
+- Add a method to read from a `Memory` instance without altering its inner state [#6073](https://github.com/hyperledger/besu/pull/6073)
+- Accept `input` and `data` field for the payload of transaction-related RPC methods [#6094](https://github.com/hyperledger/besu/pull/6094)
+- Add APIs to set and get the min gas price a transaction must pay for being selected during block creation [#6097](https://github.com/hyperledger/besu/pull/6097)
+- TraceService: return results for transactions in block [#6086](https://github.com/hyperledger/besu/pull/6086)
+- New option `--min-priority-fee` that sets the minimum priority fee a transaction must meet to be selected for a block. [#6080](https://github.com/hyperledger/besu/pull/6080) [#6083](https://github.com/hyperledger/besu/pull/6083)
+- Implement new `miner_setMinPriorityFee` and `miner_getMinPriorityFee` RPC methods [#6080](https://github.com/hyperledger/besu/pull/6080)
+- Clique config option `createemptyblocks` to not create empty blocks [#6082](https://github.com/hyperledger/besu/pull/6082)
+- Upgrade EVM Reference Tests to v13 (Cancun) [#6114](https://github.com/hyperledger/besu/pull/6114)
+- Add `yParity` to GraphQL and JSON-RPC for relevant querise. [6119](https://github.com/hyperledger/besu/pull/6119)
+- Force tx replacement price bump to zero when zero base fee market is configured or `--min-gas-price` is set to 0. This allows for easier tx replacement in networks where there is not gas price. [#6079](https://github.com/hyperledger/besu/pull/6079)
+- Introduce the possibility to limit the time spent selecting pending transactions during block creation, using the new experimental option `Xblock-txs-selection-max-time` on PoS and PoW networks (by default set to 5000ms) or `Xpoa-block-txs-selection-max-time` on PoA networks (by default 75% of the min block time) [#6044](https://github.com/hyperledger/besu/pull/6044)
+- Remove LowestInvalidNonceCache from `legacy` transaction pool to make it more private networks friendly [#6148](https://github.com/hyperledger/besu/pull/6148)
+
+### Bug fixes
+- Upgrade netty to address CVE-2023-44487, CVE-2023-34462 [#6100](https://github.com/hyperledger/besu/pull/6100)
+- Upgrade grpc to address CVE-2023-32731, CVE-2023-33953, CVE-2023-44487, CVE-2023-4785 [#6100](https://github.com/hyperledger/besu/pull/6100)
+- Fix blob gas calculation in reference tests [#6107](https://github.com/hyperledger/besu/pull/6107)
+- Limit memory used in handling invalid blocks [#6138](https://github.com/hyperledger/besu/pull/6138)
+
+---
+
+### Download Links
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.2/besu-23.10.2.zip / sha256: 597ab71898d379180106baf24878239ed49acefea5772344fd359b0ff13fe19f
+
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.2/besu-23.10.2.tar.gz / sha256: 255818a5c6067a38aa8b565d8f32a49a172a7536a1d370673bbb75f548263c2c
+
+## 23.10.1
 
 ### Additions and Improvements
 - New option `--tx-pool-priority-senders` to specify a list of senders, that has the effect to prioritize any transactions sent by these senders from any source [#5959](https://github.com/hyperledger/besu/pull/5959)
+- Cache last n blocks by using a new Besu flag `--cache-last-blocks=n` [#6009](https://github.com/hyperledger/besu/pull/6009)
+- Optimize performances of RPC method `eth_feeHistory` [#6011](https://github.com/hyperledger/besu/pull/6011) [#6035](https://github.com/hyperledger/besu/pull/6035)
+- Logging summary of plugins at Info as part of the config overview [#5964](https://github.com/hyperledger/besu/pull/5964) [#6049](https://github.com/hyperledger/besu/pull/6049)
+- Layered tx pool memory improvements [#5985](https://github.com/hyperledger/besu/pull/5985) [#5974](https://github.com/hyperledger/besu/pull/5974)
+- Update Bouncy Castle to 1.76, and force the use of the `jdk18on` variant [#5748](https://github.com/hyperledger/besu/pull/5748)
+- Add GraphQL support for new fields in Cancun [#5923](https://github.com/hyperledger/besu/pull/5923) [#5975](https://github.com/hyperledger/besu/pull/5975)
+- Add new configuration options to the EVM Fluent APIs [#5930](https://github.com/hyperledger/besu/pull/5930)
+
+
+### Deprecations
+- `--tx-pool-disable-locals` has been deprecated for removal in favor of `--tx-pool-no-local-priority`, no semantic change, only a renaming [#5959](https://github.com/hyperledger/besu/pull/5959)
 
 ### Bug Fixes
+- Fix regression with t8n tool filling [#5979](https://github.com/hyperledger/besu/pull/5979)
+- Fix EOF and EIP-4788 regressions in reference tests  [#6060](https://github.com/hyperledger/besu/pull/6060)
 
 ### Download Links
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.1/besu-23.10.1.tar.gz / sha256: e27645f345583f3ee447e5418302382c6f8335d2da8707bdd20033aabd86ce4c
+
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.1/besu-23.10.1.zip / sha256: fb173acb93c72fbb74a6542051691ca2d3d5f54ea2f51026467a512f3a22106b
 
 ## 23.10.0
 ### Layered Transaction Pool: the new default transaction pool implementation

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java

@@ -22,6 +22,7 @@
 import org.hyperledger.besu.crypto.KeyPair;
 import org.hyperledger.besu.crypto.KeyPairUtil;
 import org.hyperledger.besu.datatypes.Address;
+import org.hyperledger.besu.ethereum.api.ApiConfiguration;
 import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration;
 import org.hyperledger.besu.ethereum.api.jsonrpc.ipc.JsonRpcIpcConfiguration;
 import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration;
@@ -107,6 +108,7 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable
   private final JsonRpcIpcConfiguration jsonRpcIpcConfiguration;
   private final MetricsConfiguration metricsConfiguration;
   private Optional<PermissioningConfiguration> permissioningConfiguration;
+  private final ApiConfiguration apiConfiguration;
   private final GenesisConfigurationProvider genesisConfigProvider;
   private final boolean devMode;
   private final NetworkName network;
@@ -139,6 +141,7 @@ public BesuNode(
       final JsonRpcIpcConfiguration jsonRpcIpcConfiguration,
       final MetricsConfiguration metricsConfiguration,
       final Optional<PermissioningConfiguration> permissioningConfiguration,
+      final ApiConfiguration apiConfiguration,
       final Optional<String> keyfilePath,
       final boolean devMode,
       final NetworkName network,
@@ -187,6 +190,7 @@ public BesuNode(
     this.jsonRpcIpcConfiguration = jsonRpcIpcConfiguration;
     this.metricsConfiguration = metricsConfiguration;
     this.permissioningConfiguration = permissioningConfiguration;
+    this.apiConfiguration = apiConfiguration;
     this.genesisConfigProvider = genesisConfigProvider;
     this.devMode = devMode;
     this.network = network;
@@ -806,4 +810,8 @@ public void setExitCode(final int exitValue) {
   public Map<String, String> getEnvironment() {
     return environment;
   }
+
+  public ApiConfiguration getApiConfiguration() {
+    return apiConfiguration;
+  }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java

@@ -110,9 +110,11 @@ public void startNode(final BesuNode node) {
       params.add(
           Integer.toString(node.getMiningParameters().getMinTransactionGasPrice().intValue()));
       params.add("--Xminer-remote-sealers-limit");
-      params.add(Integer.toString(node.getMiningParameters().getRemoteSealersLimit()));
+      params.add(
+          Integer.toString(node.getMiningParameters().getUnstable().getRemoteSealersLimit()));
       params.add("--Xminer-remote-sealers-hashrate-ttl");
-      params.add(Long.toString(node.getMiningParameters().getRemoteSealersTimeToLive()));
+      params.add(
+          Long.toString(node.getMiningParameters().getUnstable().getRemoteSealersTimeToLive()));
     }
     if (node.getMiningParameters().isStratumMiningEnabled()) {
       params.add("--miner-stratum-enabled");

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -44,6 +44,7 @@
 import org.hyperledger.besu.plugin.services.BesuEvents;
 import org.hyperledger.besu.plugin.services.PicoCLIOptions;
 import org.hyperledger.besu.plugin.services.PluginTransactionValidatorService;
+import org.hyperledger.besu.plugin.services.RpcEndpointService;
 import org.hyperledger.besu.plugin.services.SecurityModuleService;
 import org.hyperledger.besu.plugin.services.StorageService;
 import org.hyperledger.besu.plugin.services.TransactionSelectionService;
@@ -98,6 +99,7 @@ private BesuPluginContextImpl buildPluginContext(
     besuPluginContext.addService(StorageService.class, storageService);
     besuPluginContext.addService(SecurityModuleService.class, securityModuleService);
     besuPluginContext.addService(PicoCLIOptions.class, new PicoCLIOptionsImpl(commandLine));
+    besuPluginContext.addService(RpcEndpointService.class, new RpcEndpointServiceImpl());
     besuPluginContext.addService(
         TransactionSelectionService.class, new TransactionSelectionServiceImpl());
     besuPluginContext.addService(
@@ -223,6 +225,7 @@ public void startNode(final BesuNode node) {
 
     final RunnerBuilder runnerBuilder = new RunnerBuilder();
     runnerBuilder.permissioningConfiguration(node.getPermissioningConfiguration());
+    runnerBuilder.apiConfiguration(node.getApiConfiguration());
 
     runnerBuilder
         .vertx(Vertx.vertx())