Merge branch 'master' into move-committeeBits-at-the-end

CHANGELOG.md

@@ -5,6 +5,8 @@
 - Next release will introduce a breaking change to Teku's metrics. This is due to some metrics changing names after a library upgrade.
 We recommend all users of the `Teku - Detailed` dashboard to upgrade to version [Revision 12](https://grafana.com/api/dashboards/16737/revisions/12/download)  
 as soon as possible. Documentation with all metrics that have been renamed will be provided.
+- Next release will require Java 21. The current release is compatible, please consider upgrading before the next release.
+- From the next release, you will need to explicitly set `--data-storage-mode=(prune|archive)` unless you're using minimal data-storage-mode (which is the default behaviour).
 
 ## Current Releases
 
@@ -17,14 +19,4 @@ the [releases page](https://github.com/Consensys/teku/releases).
 
 ### Additions and Improvements
 
-- Added metadata fields to `/eth/v1/beacon/blob_sidecars/{block_id}` Beacon API response as per https://github.com/ethereum/beacon-APIs/pull/441
-- Added rest api endpoint `/teku/v1/beacon/state/finalized/slot/before/{slot}` to return most recent stored state at or before a specified slot.
-- The validator client will start using the `v2` variant of the beacon node block publishing
-  endpoints. In the cases where the block has been produced in the same beacon node, only equivocation validation will be done instead of the entire gossip validation.
-- Docker images are now based on ubuntu 24.04 LTS (noble)
-- The `teku vc` subcommand fails when no validator key source is provided. In order to run a validator client, one of the following options must be set:
-  `--validator-keys`, `--validators-external-signer-url` or `--validator-api-enabled`
-
 ### Bug Fixes
-- Fixed performance degradation introduced in 24.4.0 regarding archive state retrieval time.
-- Fixed file writer when storing database mode settings to file (related to https://github.com/Consensys/teku/issues/8357).

build.gradle

@@ -22,9 +22,9 @@ buildscript {
 plugins {
   id 'com.diffplug.spotless' version '6.25.0'
   id 'com.github.ben-manes.versions' version '0.51.0'
-  id 'com.github.jk1.dependency-license-report' version '2.7'
-  id 'io.spring.dependency-management' version '1.1.4'
-  id 'net.ltgt.errorprone' version '3.1.0' apply false
+  id 'com.github.jk1.dependency-license-report' version '2.8'
+  id 'io.spring.dependency-management' version '1.1.5'
+  id 'net.ltgt.errorprone' version '4.0.0' apply false
   id 'de.undercouch.download' version '5.6.0'
   id 'org.ajoberstar.grgit' version '5.2.2'
 }
@@ -312,7 +312,7 @@ allprojects {
   }
 }
 
-def refTestVersion = 'v1.5.0-alpha.2'
+def refTestVersion = 'v1.5.0-alpha.3'
 def blsRefTestVersion = 'v0.1.2'
 def slashingProtectionInterchangeRefTestVersion = 'v5.3.0'
 def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download'

eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/ReferenceTestFinder.java

@@ -31,12 +31,7 @@ public class ReferenceTestFinder {
       Path.of("src", "referenceTest", "resources", "consensus-spec-tests", "tests");
   private static final List<String> SUPPORTED_FORKS =
       List.of(
-          TestFork.PHASE0,
-          TestFork.ALTAIR,
-          TestFork.BELLATRIX,
-          TestFork.CAPELLA,
-          TestFork.DENEB,
-          TestFork.ELECTRA);
+          TestFork.PHASE0, TestFork.ALTAIR, TestFork.BELLATRIX, TestFork.CAPELLA, TestFork.DENEB);
 
   @MustBeClosed
   public static Stream<TestDefinition> findReferenceTests() throws IOException {

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java

@@ -17,6 +17,7 @@
 import static tech.pegasys.teku.infrastructure.logging.P2PLogger.P2P_LOG;
 import static tech.pegasys.teku.infrastructure.time.TimeUtilities.secondsToMillis;
 import static tech.pegasys.teku.spec.constants.NetworkConstants.INTERVALS_PER_SLOT;
+import static tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsValidationResult.INVALID;
 import static tech.pegasys.teku.statetransition.forkchoice.StateRootCollector.addParentStateRoots;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -540,21 +541,20 @@ private BlockImportResult importBlockAndState(
           payloadResult.getFailureCause().orElseThrow());
     }
 
+    LOG.debug("blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString);
+
     switch (blobSidecarsAndValidationResult.getValidationResult()) {
-      case VALID, NOT_REQUIRED -> LOG.debug(
-          "blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString);
       case NOT_AVAILABLE -> {
-        LOG.debug(
-            "blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString);
         return BlockImportResult.failedDataAvailabilityCheckNotAvailable(
             blobSidecarsAndValidationResult.getCause());
       }
       case INVALID -> {
-        LOG.error(
-            "blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString);
+        debugDataDumper.saveInvalidBlobSidecars(
+            blobSidecarsAndValidationResult.getBlobSidecars(), block);
         return BlockImportResult.failedDataAvailabilityCheckInvalid(
             blobSidecarsAndValidationResult.getCause());
       }
+      default -> {}
     }
 
     final ForkChoiceStrategy forkChoiceStrategy = getForkChoiceStrategy();

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataDumper.java

@@ -13,10 +13,12 @@
 
 package tech.pegasys.teku.statetransition.util;
 
+import java.util.List;
 import java.util.Optional;
 import java.util.function.Supplier;
 import org.apache.tuweni.bytes.Bytes;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 
 public interface DebugDataDumper {
@@ -42,6 +44,10 @@ public void saveInvalidBlock(
             final SignedBeaconBlock block,
             final String failureReason,
             final Optional<Throwable> failureCause) {}
+
+        @Override
+        public void saveInvalidBlobSidecars(
+            final List<BlobSidecar> blobSidecars, final SignedBeaconBlock block) {}
       };
 
   void saveGossipMessageDecodingError(
@@ -58,4 +64,6 @@ void saveGossipRejectedMessage(
 
   void saveInvalidBlock(
       SignedBeaconBlock block, String failureReason, Optional<Throwable> failureCause);
+
+  void saveInvalidBlobSidecars(List<BlobSidecar> blobSidecars, SignedBeaconBlock block);
 }

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumper.java

@@ -23,6 +23,7 @@
 import java.sql.Date;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.List;
 import java.util.Optional;
 import java.util.function.Supplier;
 import org.apache.logging.log4j.LogManager;
@@ -31,6 +32,7 @@
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.infrastructure.time.TimeProvider;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 
 public class DebugDataFileDumper implements DebugDataDumper {
@@ -41,6 +43,7 @@ public class DebugDataFileDumper implements DebugDataDumper {
   private static final String DECODING_ERROR_SUB_DIR = "decoding_error";
   private static final String REJECTED_SUB_DIR = "rejected";
   private static final String INVALID_BLOCK_DIR = "invalid_blocks";
+  private static final String INVALID_BLOB_SIDECARS_DIR = "invalid_blob_sidecars";
 
   private boolean enabled;
   private final Path directory;
@@ -125,14 +128,41 @@ public void saveInvalidBlock(
             "invalid block", Path.of(INVALID_BLOCK_DIR).resolve(fileName), block.sszSerialize());
     if (success) {
       LOG.warn(
-          "Rejecting invalid block at slot {} with root {} because {}",
+          "Rejecting invalid block at slot {} with root {}, reason: {}, cause: {}",
           slot,
           blockRoot,
           failureReason,
           failureCause.orElse(null));
     }
   }
 
+  @Override
+  public void saveInvalidBlobSidecars(
+      final List<BlobSidecar> blobSidecars, final SignedBeaconBlock block) {
+    if (!enabled) {
+      return;
+    }
+    final String kzgCommitmentsFileName =
+        String.format(
+            "%s_%s_kzg_commitments.ssz", block.getSlot(), block.getRoot().toUnprefixedHexString());
+    saveBytesToFile(
+        "kzg commitments",
+        Path.of(INVALID_BLOB_SIDECARS_DIR).resolve(kzgCommitmentsFileName),
+        block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow().sszSerialize());
+    blobSidecars.forEach(
+        blobSidecar -> {
+          final UInt64 slot = blobSidecar.getSlot();
+          final Bytes32 blockRoot = blobSidecar.getBlockRoot();
+          final UInt64 index = blobSidecar.getIndex();
+          final String fileName =
+              String.format("%s_%s_%s.ssz", slot, blockRoot.toUnprefixedHexString(), index);
+          saveBytesToFile(
+              "blob sidecar",
+              Path.of(INVALID_BLOB_SIDECARS_DIR).resolve(fileName),
+              blobSidecar.sszSerialize());
+        });
+  }
+
   @VisibleForTesting
   boolean saveBytesToFile(
       final String description, final Path relativeFilePath, final Bytes bytes) {

ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceTest.java

@@ -247,6 +247,31 @@ void onBlock_shouldFailIfBlobsAreNotAvailable() {
     verify(blobSidecarsAvailabilityChecker).getAvailabilityCheckResult();
   }
 
+  @Test
+  void onBlock_shouldFailIfBlobsAreInvalid() {
+    setupWithSpec(TestSpecFactory.createMinimalDeneb());
+    final SignedBlockAndState blockAndState = chainBuilder.generateBlockAtSlot(ONE);
+    storageSystem.chainUpdater().advanceCurrentSlotToAtLeast(blockAndState.getSlot());
+    final List<BlobSidecar> blobSidecars =
+        storageSystem
+            .chainStorage()
+            .getBlobSidecarsBySlotAndBlockRoot(blockAndState.getSlotAndBlockRoot())
+            .join();
+
+    when(blobSidecarsAvailabilityChecker.getAvailabilityCheckResult())
+        .thenReturn(
+            SafeFuture.completedFuture(
+                BlobSidecarsAndValidationResult.invalidResult(blobSidecars)));
+
+    importBlockAndAssertFailure(
+        blockAndState, FailureReason.FAILED_DATA_AVAILABILITY_CHECK_INVALID);
+
+    verify(blobSidecarManager).createAvailabilityChecker(blockAndState.getBlock());
+    verify(blobSidecarsAvailabilityChecker).initiateDataAvailabilityCheck();
+    verify(blobSidecarsAvailabilityChecker).getAvailabilityCheckResult();
+    verify(debugDataDumper).saveInvalidBlobSidecars(blobSidecars, blockAndState.getBlock());
+  }
+
   @Test
   void onBlock_consensusValidationShouldNotResolveWhenDataAvailabilityFails() {
     setupWithSpec(TestSpecFactory.createMinimalDeneb());

ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumperTest.java

@@ -24,6 +24,7 @@
 import java.sql.Date;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.List;
 import java.util.Optional;
 import org.apache.tuweni.bytes.Bytes;
 import org.junit.jupiter.api.Test;
@@ -33,12 +34,13 @@
 import tech.pegasys.teku.infrastructure.time.StubTimeProvider;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 import tech.pegasys.teku.spec.util.DataStructureUtil;
 
 class DebugDataFileDumperTest {
   final DataStructureUtil dataStructureUtil =
-      new DataStructureUtil(TestSpecFactory.createDefault());
+      new DataStructureUtil(TestSpecFactory.createMinimalDeneb());
   private final StubTimeProvider timeProvider = StubTimeProvider.withTimeInSeconds(10_000);
 
   @Test
@@ -91,6 +93,35 @@ void saveInvalidBlockToFile_shouldSaveToFile(@TempDir final Path tempDir) {
     checkBytesSavedToFile(expectedFile, block.sszSerialize());
   }
 
+  @Test
+  void saveInvalidBlobSidecars_shouldSaveToFiles(@TempDir final Path tempDir) {
+    final DebugDataFileDumper dumper = new DebugDataFileDumper(tempDir);
+    final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock();
+    final List<BlobSidecar> blobSidecars = dataStructureUtil.randomBlobSidecarsForBlock(block);
+    dumper.saveInvalidBlobSidecars(blobSidecars, block);
+
+    final String kzgCommitmentsFileName =
+        String.format(
+            "%s_%s_kzg_commitments.ssz", block.getSlot(), block.getRoot().toUnprefixedHexString());
+    final Path expectedKzgCommitmentsFileName =
+        tempDir.resolve("invalid_blob_sidecars").resolve(kzgCommitmentsFileName);
+    checkBytesSavedToFile(
+        expectedKzgCommitmentsFileName,
+        block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow().sszSerialize());
+
+    blobSidecars.forEach(
+        blobSidecar -> {
+          final String fileName =
+              String.format(
+                  "%s_%s_%s.ssz",
+                  blobSidecar.getSlot(),
+                  blobSidecar.getBlockRoot().toUnprefixedHexString(),
+                  blobSidecar.getIndex());
+          final Path expectedFile = tempDir.resolve("invalid_blob_sidecars").resolve(fileName);
+          checkBytesSavedToFile(expectedFile, blobSidecar.sszSerialize());
+        });
+  }
+
   @Test
   void saveBytesToFile_shouldNotThrowExceptionWhenNoDirectory(@TempDir final Path tempDir) {
     final DebugDataFileDumper dumper = new DebugDataFileDumper(tempDir);