Improve debug_traceBlock calls performance

CHANGELOG.md

@@ -20,6 +20,7 @@
 - Retrieve all transaction receipts for a block in one request [#6646](https://github.com/hyperledger/besu/pull/6646)
 - Implement EIP-7840: Add blob schedule to config files [#8042](https://github.com/hyperledger/besu/pull/8042)
 - Allow gasPrice (legacy) and 1559 gasPrice params to be specified simultaneously for `eth_call`, `eth_createAccessList`, and `eth_estimateGas` [#8059](https://github.com/hyperledger/besu/pull/8059)
+- Improve debug_traceBlock calls performance and reduce output size [#8076](https://github.com/hyperledger/besu/pull/8076)
 - Add support for EIP-7702 transaction in the txpool [#8018](https://github.com/hyperledger/besu/pull/8018) [#7984](https://github.com/hyperledger/besu/pull/7984)
 
 ### Bug fixes

besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright ConsenSys AG.
+ * Copyright contributors to Besu.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  * the License. You may obtain a copy of the License at
@@ -74,6 +74,7 @@
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.core.Synchronizer;
 import org.hyperledger.besu.ethereum.eth.manager.EthPeers;
+import org.hyperledger.besu.ethereum.eth.manager.EthScheduler;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.precompiles.privacy.FlexiblePrivacyPrecompiledContract;
@@ -836,7 +837,8 @@ public Runner build() {
               besuPluginContext.getNamedPlugins(),
               dataDir,
               rpcEndpointServiceImpl,
-              transactionSimulator);
+              transactionSimulator,
+              besuController.getProtocolManager().ethContext().getScheduler());
 
       jsonRpcHttpService =
           Optional.of(
@@ -882,7 +884,8 @@ public Runner build() {
               besuPluginContext.getNamedPlugins(),
               dataDir,
               rpcEndpointServiceImpl,
-              transactionSimulator);
+              transactionSimulator,
+              besuController.getProtocolManager().ethContext().getScheduler());
 
       final Optional<AuthenticationService> authToUse =
           engineJsonRpcConfiguration.get().isAuthenticationEnabled()
@@ -978,7 +981,8 @@ public Runner build() {
               besuPluginContext.getNamedPlugins(),
               dataDir,
               rpcEndpointServiceImpl,
-              transactionSimulator);
+              transactionSimulator,
+              besuController.getProtocolManager().ethContext().getScheduler());
 
       createLogsSubscriptionService(
           context.getBlockchain(), subscriptionManager, privacyParameters, blockchainQueries);
@@ -1059,7 +1063,8 @@ public Runner build() {
               besuPluginContext.getNamedPlugins(),
               dataDir,
               rpcEndpointServiceImpl,
-              transactionSimulator);
+              transactionSimulator,
+              besuController.getProtocolManager().ethContext().getScheduler());
 
       jsonRpcIpcService =
           Optional.of(
@@ -1099,7 +1104,8 @@ public Runner build() {
               besuPluginContext.getNamedPlugins(),
               dataDir,
               rpcEndpointServiceImpl,
-              transactionSimulator);
+              transactionSimulator,
+              besuController.getProtocolManager().ethContext().getScheduler());
     } else {
       inProcessRpcMethods = Map.of();
     }
@@ -1262,7 +1268,8 @@ private Map<String, JsonRpcMethod> jsonRpcMethods(
       final Map<String, BesuPlugin> namedPlugins,
       final Path dataDir,
       final RpcEndpointServiceImpl rpcEndpointServiceImpl,
-      final TransactionSimulator transactionSimulator) {
+      final TransactionSimulator transactionSimulator,
+      final EthScheduler ethScheduler) {
     // sync vertx for engine consensus API, to process requests in FIFO order;
     final Vertx consensusEngineServer = Vertx.vertx(new VertxOptions().setWorkerPoolSize(1));
 
@@ -1300,7 +1307,8 @@ private Map<String, JsonRpcMethod> jsonRpcMethods(
                 consensusEngineServer,
                 apiConfiguration,
                 enodeDnsConfiguration,
-                transactionSimulator);
+                transactionSimulator,
+                ethScheduler);
     methods.putAll(besuController.getAdditionalJsonRpcMethods(jsonRpcApis));
 
     final var pluginMethods =

ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright ConsenSys AG.
+ * Copyright contributors to Besu.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  * the License. You may obtain a copy of the License at
@@ -51,6 +51,7 @@
 import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
 import org.hyperledger.besu.nat.NatService;
+import org.hyperledger.besu.testutil.DeterministicEthScheduler;
 
 import java.math.BigInteger;
 import java.nio.file.Path;
@@ -228,6 +229,7 @@ public Map<String, JsonRpcMethod> methods() {
             Vertx.vertx(new VertxOptions().setWorkerPoolSize(1)),
             ImmutableApiConfiguration.builder().build(),
             Optional.empty(),
-            transactionSimulator);
+            transactionSimulator,
+            new DeterministicEthScheduler());
   }
 }

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractBlockParameterMethod.java

@@ -1,5 +1,5 @@
 /*
- * Copyright ConsenSys AG.
+ * Copyright contributors to Besu.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  * the License. You may obtain a copy of the License at

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractDebugTraceBlock.java

@@ -0,0 +1,151 @@
+/*
+ * Copyright contributors to Besu.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods;
+
+import static org.hyperledger.besu.services.pipeline.PipelineBuilder.createPipelineFrom;
+
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType;
+import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult;
+import org.hyperledger.besu.ethereum.api.query.BlockchainQueries;
+import org.hyperledger.besu.ethereum.core.Block;
+import org.hyperledger.besu.ethereum.debug.TraceOptions;
+import org.hyperledger.besu.ethereum.eth.manager.EthScheduler;
+import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor;
+import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
+import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
+import org.hyperledger.besu.ethereum.vm.DebugOperationTracer;
+import org.hyperledger.besu.metrics.BesuMetricCategory;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
+import org.hyperledger.besu.plugin.services.metrics.Counter;
+import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
+import org.hyperledger.besu.services.pipeline.Pipeline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Supplier;
+
+import com.google.common.base.Suppliers;
+
+public abstract class AbstractDebugTraceBlock implements JsonRpcMethod {
+
+  private final ProtocolSchedule protocolSchedule;
+  private final LabelledMetric<Counter> outputCounter;
+  private final Supplier<BlockchainQueries> blockchainQueriesSupplier;
+  private final EthScheduler ethScheduler;
+
+  public AbstractDebugTraceBlock(
+      final ProtocolSchedule protocolSchedule,
+      final BlockchainQueries blockchainQueries,
+      final ObservableMetricsSystem metricsSystem,
+      final EthScheduler ethScheduler) {
+    this.blockchainQueriesSupplier = Suppliers.ofInstance(blockchainQueries);
+    this.protocolSchedule = protocolSchedule;
+    this.outputCounter =
+        metricsSystem.createLabelledCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "transactions_debugTraceblock_pipeline_processed_total",
+            "Number of transactions processed for each block",
+            "step",
+            "action");
+    this.ethScheduler = ethScheduler;
+  }
+
+  protected BlockchainQueries getBlockchainQueries() {
+    return blockchainQueriesSupplier.get();
+  }
+
+  protected TraceOptions getTraceOptions(final JsonRpcRequestContext requestContext) {
+    final TraceOptions traceOptions;
+    try {
+      traceOptions =
+          requestContext
+              .getOptionalParameter(1, TransactionTraceParams.class)
+              .map(TransactionTraceParams::traceOptions)
+              .orElse(TraceOptions.DEFAULT);
+    } catch (JsonRpcParameter.JsonRpcParameterException e) {
+      throw new InvalidJsonRpcParameters(
+          "Invalid transaction trace parameter (index 1)",
+          RpcErrorType.INVALID_TRANSACTION_TRACE_PARAMS,
+          e);
+    }
+    return traceOptions;
+  }
+
+  protected Collection<DebugTraceTransactionResult> getTraces(
+      final JsonRpcRequestContext requestContext,
+      final TraceOptions traceOptions,
+      final Optional<Block> maybeBlock) {
+    return maybeBlock
+        .flatMap(
+            block ->
+                Tracer.processTracing(
+                    getBlockchainQueries(),
+                    block.getHash(),
+                    traceableState -> {
+                      List<DebugTraceTransactionResult> tracesList =
+                          Collections.synchronizedList(new ArrayList<>());
+                      final ProtocolSpec protocolSpec =
+                          protocolSchedule.getByBlockHeader(block.getHeader());
+                      final MainnetTransactionProcessor transactionProcessor =
+                          protocolSpec.getTransactionProcessor();
+                      final TraceBlock.ChainUpdater chainUpdater =
+                          new TraceBlock.ChainUpdater(traceableState);
+
+                      TransactionSource transactionSource = new TransactionSource(block);
+                      DebugOperationTracer debugOperationTracer =
+                          new DebugOperationTracer(traceOptions, true);
+                      ExecuteTransactionStep executeTransactionStep =
+                          new ExecuteTransactionStep(
+                              chainUpdater,
+                              transactionProcessor,
+                              getBlockchainQueries().getBlockchain(),
+                              debugOperationTracer,
+                              protocolSpec,
+                              block);
+                      DebugTraceTransactionStep debugTraceTransactionStep =
+                          new DebugTraceTransactionStep();
+                      Pipeline<TransactionTrace> traceBlockPipeline =
+                          createPipelineFrom(
+                                  "getTransactions",
+                                  transactionSource,
+                                  4,
+                                  outputCounter,
+                                  false,
+                                  "debug_trace_block")
+                              .thenProcess("executeTransaction", executeTransactionStep)
+                              .thenProcessAsyncOrdered(
+                                  "debugTraceTransactionStep", debugTraceTransactionStep, 4)
+                              .andFinishWith("collect_results", tracesList::add);
+
+                      try {
+                        ethScheduler.startPipeline(traceBlockPipeline).get();
+                      } catch (InterruptedException | ExecutionException e) {
+                        throw new RuntimeException(e);
+                      }
+                      return Optional.of(tracesList);
+                    }))
+        .orElse(null);
+  }
+}

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java

@@ -1,5 +1,5 @@
 /*
- * Copyright ConsenSys AG.
+ * Copyright contributors to Besu.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  * the License. You may obtain a copy of the License at
@@ -18,10 +18,6 @@
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException;
-import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams;
-import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTrace;
-import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer;
-import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
@@ -31,32 +27,32 @@
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.debug.TraceOptions;
+import org.hyperledger.besu.ethereum.eth.manager.EthScheduler;
+import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
+import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.rlp.RLP;
 import org.hyperledger.besu.ethereum.rlp.RLPException;
-import org.hyperledger.besu.ethereum.vm.DebugOperationTracer;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 
 import java.util.Collection;
 import java.util.Optional;
-import java.util.function.Supplier;
 
 import org.apache.tuweni.bytes.Bytes;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class DebugTraceBlock implements JsonRpcMethod {
+public class DebugTraceBlock extends AbstractDebugTraceBlock {
 
   private static final Logger LOG = LoggerFactory.getLogger(DebugTraceBlock.class);
-  private final Supplier<BlockTracer> blockTracerSupplier;
   private final BlockHeaderFunctions blockHeaderFunctions;
-  private final BlockchainQueries blockchainQueries;
 
   public DebugTraceBlock(
-      final Supplier<BlockTracer> blockTracerSupplier,
-      final BlockHeaderFunctions blockHeaderFunctions,
-      final BlockchainQueries blockchainQueries) {
-    this.blockTracerSupplier = blockTracerSupplier;
-    this.blockHeaderFunctions = blockHeaderFunctions;
-    this.blockchainQueries = blockchainQueries;
+      final ProtocolSchedule protocolSchedule,
+      final BlockchainQueries blockchainQueries,
+      final ObservableMetricsSystem metricsSystem,
+      final EthScheduler ethScheduler) {
+    super(protocolSchedule, blockchainQueries, metricsSystem, ethScheduler);
+    this.blockHeaderFunctions = ScheduleBasedBlockHeaderFunctions.create(protocolSchedule);
   }
 
   @Override
@@ -70,43 +66,22 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) {
     try {
       final String input = requestContext.getRequiredParameter(0, String.class);
       block = Block.readFrom(RLP.input(Bytes.fromHexString(input)), this.blockHeaderFunctions);
-    } catch (final RLPException e) {
+    } catch (final RLPException | IllegalArgumentException e) {
       LOG.debug("Failed to parse block RLP (index 0)", e);
       return new JsonRpcErrorResponse(
           requestContext.getRequest().getId(), RpcErrorType.INVALID_BLOCK_PARAMS);
     } catch (JsonRpcParameterException e) {
       throw new InvalidJsonRpcParameters(
           "Invalid block params (index 0)", RpcErrorType.INVALID_BLOCK_PARAMS, e);
     }
-    final TraceOptions traceOptions;
-    try {
-      traceOptions =
-          requestContext
-              .getOptionalParameter(1, TransactionTraceParams.class)
-              .map(TransactionTraceParams::traceOptions)
-              .orElse(TraceOptions.DEFAULT);
-    } catch (JsonRpcParameterException e) {
-      throw new InvalidJsonRpcParameters(
-          "Invalid transaction trace parameter (index 1)",
-          RpcErrorType.INVALID_TRANSACTION_TRACE_PARAMS,
-          e);
-    }
+    final TraceOptions traceOptions = getTraceOptions(requestContext);
 
-    if (this.blockchainQueries.blockByHash(block.getHeader().getParentHash()).isPresent()) {
+    if (getBlockchainQueries()
+        .getBlockchain()
+        .getBlockByHash(block.getHeader().getParentHash())
+        .isPresent()) {
       final Collection<DebugTraceTransactionResult> results =
-          Tracer.processTracing(
-                  blockchainQueries,
-                  Optional.of(block.getHeader()),
-                  mutableWorldState ->
-                      blockTracerSupplier
-                          .get()
-                          .trace(
-                              mutableWorldState,
-                              block,
-                              new DebugOperationTracer(traceOptions, true))
-                          .map(BlockTrace::getTransactionTraces)
-                          .map(DebugTraceTransactionResult::of))
-              .orElse(null);
+          getTraces(requestContext, traceOptions, Optional.ofNullable(block));
       return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), results);
     } else {
       return new JsonRpcErrorResponse(