diff --git a/CHANGELOG.md b/CHANGELOG.md index 00589266990..0aa7602676f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - Next release will introduce a breaking change to Teku's metrics. This is due to some metrics changing names after a library upgrade. We recommend all users of the `Teku - Detailed` dashboard to upgrade to version [Revision 12](https://grafana.com/api/dashboards/16737/revisions/12/download) as soon as possible. Documentation with all metrics that have been renamed will be provided. +- Next release will require Java 21. The current release is compatible, please consider upgrading before the next release. +- From the next release, you will need to explicitly set `--data-storage-mode=(prune|archive)` unless you're using minimal data-storage-mode (which is the default behaviour). ## Current Releases @@ -17,14 +19,4 @@ the [releases page](https://github.com/Consensys/teku/releases). ### Additions and Improvements -- Added metadata fields to `/eth/v1/beacon/blob_sidecars/{block_id}` Beacon API response as per https://github.com/ethereum/beacon-APIs/pull/441 -- Added rest api endpoint `/teku/v1/beacon/state/finalized/slot/before/{slot}` to return most recent stored state at or before a specified slot. -- The validator client will start using the `v2` variant of the beacon node block publishing - endpoints. In the cases where the block has been produced in the same beacon node, only equivocation validation will be done instead of the entire gossip validation. -- Docker images are now based on ubuntu 24.04 LTS (noble) -- The `teku vc` subcommand fails when no validator key source is provided. In order to run a validator client, one of the following options must be set: - `--validator-keys`, `--validators-external-signer-url` or `--validator-api-enabled` - ### Bug Fixes -- Fixed performance degradation introduced in 24.4.0 regarding archive state retrieval time. -- Fixed file writer when storing database mode settings to file (related to https://github.com/Consensys/teku/issues/8357). diff --git a/build.gradle b/build.gradle index dd1f5eab51b..54e9e617ee6 100644 --- a/build.gradle +++ b/build.gradle @@ -22,9 +22,9 @@ buildscript { plugins { id 'com.diffplug.spotless' version '6.25.0' id 'com.github.ben-manes.versions' version '0.51.0' - id 'com.github.jk1.dependency-license-report' version '2.7' - id 'io.spring.dependency-management' version '1.1.4' - id 'net.ltgt.errorprone' version '3.1.0' apply false + id 'com.github.jk1.dependency-license-report' version '2.8' + id 'io.spring.dependency-management' version '1.1.5' + id 'net.ltgt.errorprone' version '4.0.0' apply false id 'de.undercouch.download' version '5.6.0' id 'org.ajoberstar.grgit' version '5.2.2' } @@ -312,7 +312,7 @@ allprojects { } } -def refTestVersion = 'v1.5.0-alpha.2' +def refTestVersion = 'v1.5.0-alpha.3' def blsRefTestVersion = 'v0.1.2' def slashingProtectionInterchangeRefTestVersion = 'v5.3.0' def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download' diff --git a/eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/ReferenceTestFinder.java b/eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/ReferenceTestFinder.java index d58be7186fa..6d2ef5dc31a 100644 --- a/eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/ReferenceTestFinder.java +++ b/eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/ReferenceTestFinder.java @@ -31,12 +31,7 @@ public class ReferenceTestFinder { Path.of("src", "referenceTest", "resources", "consensus-spec-tests", "tests"); private static final List SUPPORTED_FORKS = List.of( - TestFork.PHASE0, - TestFork.ALTAIR, - TestFork.BELLATRIX, - TestFork.CAPELLA, - TestFork.DENEB, - TestFork.ELECTRA); + TestFork.PHASE0, TestFork.ALTAIR, TestFork.BELLATRIX, TestFork.CAPELLA, TestFork.DENEB); @MustBeClosed public static Stream findReferenceTests() throws IOException { diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java index ccf50d754dd..601d05a0f36 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java @@ -17,6 +17,7 @@ import static tech.pegasys.teku.infrastructure.logging.P2PLogger.P2P_LOG; import static tech.pegasys.teku.infrastructure.time.TimeUtilities.secondsToMillis; import static tech.pegasys.teku.spec.constants.NetworkConstants.INTERVALS_PER_SLOT; +import static tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsValidationResult.INVALID; import static tech.pegasys.teku.statetransition.forkchoice.StateRootCollector.addParentStateRoots; import com.google.common.annotations.VisibleForTesting; @@ -540,21 +541,20 @@ private BlockImportResult importBlockAndState( payloadResult.getFailureCause().orElseThrow()); } + LOG.debug("blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString); + switch (blobSidecarsAndValidationResult.getValidationResult()) { - case VALID, NOT_REQUIRED -> LOG.debug( - "blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString); case NOT_AVAILABLE -> { - LOG.debug( - "blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString); return BlockImportResult.failedDataAvailabilityCheckNotAvailable( blobSidecarsAndValidationResult.getCause()); } case INVALID -> { - LOG.error( - "blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString); + debugDataDumper.saveInvalidBlobSidecars( + blobSidecarsAndValidationResult.getBlobSidecars(), block); return BlockImportResult.failedDataAvailabilityCheckInvalid( blobSidecarsAndValidationResult.getCause()); } + default -> {} } final ForkChoiceStrategy forkChoiceStrategy = getForkChoiceStrategy(); diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataDumper.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataDumper.java index aebd1fc32e2..e8af45dba06 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataDumper.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataDumper.java @@ -13,10 +13,12 @@ package tech.pegasys.teku.statetransition.util; +import java.util.List; import java.util.Optional; import java.util.function.Supplier; import org.apache.tuweni.bytes.Bytes; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; public interface DebugDataDumper { @@ -42,6 +44,10 @@ public void saveInvalidBlock( final SignedBeaconBlock block, final String failureReason, final Optional failureCause) {} + + @Override + public void saveInvalidBlobSidecars( + final List blobSidecars, final SignedBeaconBlock block) {} }; void saveGossipMessageDecodingError( @@ -58,4 +64,6 @@ void saveGossipRejectedMessage( void saveInvalidBlock( SignedBeaconBlock block, String failureReason, Optional failureCause); + + void saveInvalidBlobSidecars(List blobSidecars, SignedBeaconBlock block); } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumper.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumper.java index a5b93ca645c..ccdd1da1710 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumper.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumper.java @@ -23,6 +23,7 @@ import java.sql.Date; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.List; import java.util.Optional; import java.util.function.Supplier; import org.apache.logging.log4j.LogManager; @@ -31,6 +32,7 @@ import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.time.TimeProvider; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; public class DebugDataFileDumper implements DebugDataDumper { @@ -41,6 +43,7 @@ public class DebugDataFileDumper implements DebugDataDumper { private static final String DECODING_ERROR_SUB_DIR = "decoding_error"; private static final String REJECTED_SUB_DIR = "rejected"; private static final String INVALID_BLOCK_DIR = "invalid_blocks"; + private static final String INVALID_BLOB_SIDECARS_DIR = "invalid_blob_sidecars"; private boolean enabled; private final Path directory; @@ -125,7 +128,7 @@ public void saveInvalidBlock( "invalid block", Path.of(INVALID_BLOCK_DIR).resolve(fileName), block.sszSerialize()); if (success) { LOG.warn( - "Rejecting invalid block at slot {} with root {} because {}", + "Rejecting invalid block at slot {} with root {}, reason: {}, cause: {}", slot, blockRoot, failureReason, @@ -133,6 +136,33 @@ public void saveInvalidBlock( } } + @Override + public void saveInvalidBlobSidecars( + final List blobSidecars, final SignedBeaconBlock block) { + if (!enabled) { + return; + } + final String kzgCommitmentsFileName = + String.format( + "%s_%s_kzg_commitments.ssz", block.getSlot(), block.getRoot().toUnprefixedHexString()); + saveBytesToFile( + "kzg commitments", + Path.of(INVALID_BLOB_SIDECARS_DIR).resolve(kzgCommitmentsFileName), + block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow().sszSerialize()); + blobSidecars.forEach( + blobSidecar -> { + final UInt64 slot = blobSidecar.getSlot(); + final Bytes32 blockRoot = blobSidecar.getBlockRoot(); + final UInt64 index = blobSidecar.getIndex(); + final String fileName = + String.format("%s_%s_%s.ssz", slot, blockRoot.toUnprefixedHexString(), index); + saveBytesToFile( + "blob sidecar", + Path.of(INVALID_BLOB_SIDECARS_DIR).resolve(fileName), + blobSidecar.sszSerialize()); + }); + } + @VisibleForTesting boolean saveBytesToFile( final String description, final Path relativeFilePath, final Bytes bytes) { diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceTest.java index 1098ad72f81..70d9ca7e103 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceTest.java @@ -247,6 +247,31 @@ void onBlock_shouldFailIfBlobsAreNotAvailable() { verify(blobSidecarsAvailabilityChecker).getAvailabilityCheckResult(); } + @Test + void onBlock_shouldFailIfBlobsAreInvalid() { + setupWithSpec(TestSpecFactory.createMinimalDeneb()); + final SignedBlockAndState blockAndState = chainBuilder.generateBlockAtSlot(ONE); + storageSystem.chainUpdater().advanceCurrentSlotToAtLeast(blockAndState.getSlot()); + final List blobSidecars = + storageSystem + .chainStorage() + .getBlobSidecarsBySlotAndBlockRoot(blockAndState.getSlotAndBlockRoot()) + .join(); + + when(blobSidecarsAvailabilityChecker.getAvailabilityCheckResult()) + .thenReturn( + SafeFuture.completedFuture( + BlobSidecarsAndValidationResult.invalidResult(blobSidecars))); + + importBlockAndAssertFailure( + blockAndState, FailureReason.FAILED_DATA_AVAILABILITY_CHECK_INVALID); + + verify(blobSidecarManager).createAvailabilityChecker(blockAndState.getBlock()); + verify(blobSidecarsAvailabilityChecker).initiateDataAvailabilityCheck(); + verify(blobSidecarsAvailabilityChecker).getAvailabilityCheckResult(); + verify(debugDataDumper).saveInvalidBlobSidecars(blobSidecars, blockAndState.getBlock()); + } + @Test void onBlock_consensusValidationShouldNotResolveWhenDataAvailabilityFails() { setupWithSpec(TestSpecFactory.createMinimalDeneb()); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumperTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumperTest.java index 1cd40a66f8d..04b5510af69 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumperTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/DebugDataFileDumperTest.java @@ -24,6 +24,7 @@ import java.sql.Date; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.List; import java.util.Optional; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; @@ -33,12 +34,13 @@ import tech.pegasys.teku.infrastructure.time.StubTimeProvider; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.util.DataStructureUtil; class DebugDataFileDumperTest { final DataStructureUtil dataStructureUtil = - new DataStructureUtil(TestSpecFactory.createDefault()); + new DataStructureUtil(TestSpecFactory.createMinimalDeneb()); private final StubTimeProvider timeProvider = StubTimeProvider.withTimeInSeconds(10_000); @Test @@ -91,6 +93,35 @@ void saveInvalidBlockToFile_shouldSaveToFile(@TempDir final Path tempDir) { checkBytesSavedToFile(expectedFile, block.sszSerialize()); } + @Test + void saveInvalidBlobSidecars_shouldSaveToFiles(@TempDir final Path tempDir) { + final DebugDataFileDumper dumper = new DebugDataFileDumper(tempDir); + final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(); + final List blobSidecars = dataStructureUtil.randomBlobSidecarsForBlock(block); + dumper.saveInvalidBlobSidecars(blobSidecars, block); + + final String kzgCommitmentsFileName = + String.format( + "%s_%s_kzg_commitments.ssz", block.getSlot(), block.getRoot().toUnprefixedHexString()); + final Path expectedKzgCommitmentsFileName = + tempDir.resolve("invalid_blob_sidecars").resolve(kzgCommitmentsFileName); + checkBytesSavedToFile( + expectedKzgCommitmentsFileName, + block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow().sszSerialize()); + + blobSidecars.forEach( + blobSidecar -> { + final String fileName = + String.format( + "%s_%s_%s.ssz", + blobSidecar.getSlot(), + blobSidecar.getBlockRoot().toUnprefixedHexString(), + blobSidecar.getIndex()); + final Path expectedFile = tempDir.resolve("invalid_blob_sidecars").resolve(fileName); + checkBytesSavedToFile(expectedFile, blobSidecar.sszSerialize()); + }); + } + @Test void saveBytesToFile_shouldNotThrowExceptionWhenNoDirectory(@TempDir final Path tempDir) { final DebugDataFileDumper dumper = new DebugDataFileDumper(tempDir); diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 13ae3ca6763..ddfdd64f331 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -1,9 +1,9 @@ dependencyManagement { dependencies { - dependency 'com.fasterxml.jackson.core:jackson-databind:2.17.0' - dependency 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.0' - dependency 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.17.0' - dependency 'com.fasterxml.jackson.module:jackson-module-kotlin:2.17.0' + dependency 'com.fasterxml.jackson.core:jackson-databind:2.17.1' + dependency 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.1' + dependency 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.17.1' + dependency 'com.fasterxml.jackson.module:jackson-module-kotlin:2.17.1' dependencySet(group: 'com.google.errorprone', version: '2.26.1') { entry 'error_prone_annotation' @@ -25,24 +25,24 @@ dependencyManagement { entry 'mockwebserver' } - dependency 'info.picocli:picocli:4.7.5' + dependency 'info.picocli:picocli:4.7.6' - dependencySet(group: 'io.javalin', version: '6.1.3') { + dependencySet(group: 'io.javalin', version: '6.1.6') { entry 'javalin' entry 'javalin-rendering' } - dependency 'io.libp2p:jvm-libp2p:1.1.0-RELEASE' - dependency 'tech.pegasys:jblst:0.3.11' + dependency 'io.libp2p:jvm-libp2p:1.1.1-RELEASE' + dependency 'tech.pegasys:jblst:0.3.12' dependency 'tech.pegasys:jc-kzg-4844:1.0.0' - dependency 'org.hdrhistogram:HdrHistogram:2.1.12' + dependency 'org.hdrhistogram:HdrHistogram:2.2.2' - dependency 'org.jetbrains.kotlin:kotlin-stdlib:1.9.23' + dependency 'org.jetbrains.kotlin:kotlin-stdlib:2.0.0' dependency 'org.mock-server:mockserver-junit-jupiter:5.15.0' - dependencySet(group: 'io.swagger.core.v3', version: '2.2.21') { + dependencySet(group: 'io.swagger.core.v3', version: '2.2.22') { entry 'swagger-parser' entry 'swagger-core' entry 'swagger-models' @@ -50,27 +50,27 @@ dependencyManagement { } // On update don't forget to change version in tech.pegasys.teku.infrastructure.restapi.SwaggerUIBuilder - dependency 'org.webjars:swagger-ui:5.17.0' + dependency 'org.webjars:swagger-ui:5.17.14' dependency 'org.thymeleaf:thymeleaf:3.1.2.RELEASE' - dependency 'io.github.classgraph:classgraph:4.8.172' - dependencySet(group: 'com.github.oshi', version: '6.6.0') { + dependency 'io.github.classgraph:classgraph:4.8.173' + dependencySet(group: 'com.github.oshi', version: '6.6.1') { entry 'oshi-core' entry 'oshi-core-java11' } - dependencySet(group: 'io.netty', version: '4.1.109.Final') { + dependencySet(group: 'io.netty', version: '4.1.111.Final') { entry 'netty-handler' entry 'netty-codec-http' } - dependencySet(group: 'io.vertx', version: '4.5.7') { + dependencySet(group: 'io.vertx', version: '4.5.8') { entry 'vertx-codegen' entry 'vertx-core' entry 'vertx-unit' entry 'vertx-web' } - dependency 'io.projectreactor:reactor-core:3.6.5' + dependency 'io.projectreactor:reactor-core:3.6.7' dependency 'it.unimi.dsi:fastutil:8.5.12' @@ -98,7 +98,7 @@ dependencyManagement { dependency 'org.apiguardian:apiguardian-api:1.1.2' - dependency 'org.assertj:assertj-core:3.25.3' + dependency 'org.assertj:assertj-core:3.26.0' dependency 'org.awaitility:awaitility:4.2.1' @@ -113,7 +113,7 @@ dependencyManagement { entry 'junit-jupiter-params' } - dependencySet(group: 'org.mockito', version: '5.11.0') { + dependencySet(group: 'org.mockito', version: '5.12.0') { entry 'mockito-core' entry 'mockito-junit-jupiter' } @@ -128,7 +128,7 @@ dependencyManagement { dependency 'org.fusesource.leveldbjni:leveldbjni-win32:1.8' dependency 'tech.pegasys:leveldb-native:0.3.1' - dependencySet(group: "org.web3j", version: "4.11.2") { + dependencySet(group: "org.web3j", version: "4.12.0") { entry 'core' entry 'abi' entry 'crypto' @@ -150,7 +150,7 @@ dependencyManagement { entry('plugin-api') } - dependencySet(group: 'org.testcontainers', version: '1.19.7') { + dependencySet(group: 'org.testcontainers', version: '1.19.8') { entry "testcontainers" entry "junit-jupiter" } @@ -163,7 +163,7 @@ dependencyManagement { exclude 'org.apache.tuweni:units' } - dependencySet(group: 'org.jupnp', version: '3.0.1') { + dependencySet(group: 'org.jupnp', version: '3.0.2') { entry "org.jupnp" entry "org.jupnp.support" } @@ -174,6 +174,6 @@ dependencyManagement { entry 'jjwt-jackson' } - dependency 'net.jqwik:jqwik:1.8.4' + dependency 'net.jqwik:jqwik:1.8.5' } } diff --git a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java index 1083cfa1f0d..863e7d4af1e 100644 --- a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java +++ b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java @@ -532,6 +532,15 @@ public void warnIgnoringWeakSubjectivityPeriod() { Color.YELLOW)); } + public void warnUsageOfImplicitPruneDataStorageMode() { + log.warn( + print( + "Prune mode being used as default without a explicit --data-storage-mode option. This will NOT be " + + "supported in future Teku versions. Please add --data-storage-mode=prune to your CLI arguments" + + " or config file if you want to keep using PRUNE.", + Color.YELLOW)); + } + private void logWithColorIfLevelGreaterThanInfo( final Level level, final String msg, final ColorConsolePrinter.Color color) { final boolean useColor = level.compareTo(Level.INFO) < 0; diff --git a/infrastructure/restapi/src/main/java/tech/pegasys/teku/infrastructure/restapi/SwaggerUIBuilder.java b/infrastructure/restapi/src/main/java/tech/pegasys/teku/infrastructure/restapi/SwaggerUIBuilder.java index 46ef88aac98..a1194120dc6 100644 --- a/infrastructure/restapi/src/main/java/tech/pegasys/teku/infrastructure/restapi/SwaggerUIBuilder.java +++ b/infrastructure/restapi/src/main/java/tech/pegasys/teku/infrastructure/restapi/SwaggerUIBuilder.java @@ -30,7 +30,7 @@ public class SwaggerUIBuilder { // Version here MUST match `swagger-ui` library version - private static final String SWAGGER_UI_VERSION = "5.17.0"; + private static final String SWAGGER_UI_VERSION = "5.17.14"; private static final String SWAGGER_UI_PATH = "/swagger-ui"; private static final String SWAGGER_HOSTED_PATH = "/webjars/swagger-ui/" + SWAGGER_UI_VERSION; diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/DatabaseStorageModeFileHelper.java b/storage/src/main/java/tech/pegasys/teku/storage/server/DatabaseStorageModeFileHelper.java new file mode 100644 index 00000000000..56147650ec8 --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/DatabaseStorageModeFileHelper.java @@ -0,0 +1,48 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class DatabaseStorageModeFileHelper { + + private static final Logger LOG = LogManager.getLogger(); + + public static Optional readStateStorageMode(final Path path) { + if (!Files.exists(path)) { + return Optional.empty(); + } + + try { + final StateStorageMode dbStorageMode = + StateStorageMode.valueOf(Files.readString(path).trim()); + LOG.debug("Read previous storage mode as {}", dbStorageMode); + return Optional.of(dbStorageMode); + } catch (final IllegalArgumentException ex) { + throw DatabaseStorageException.unrecoverable( + "Invalid database storage mode file (" + + path + + "). Run your node using '--data-storage-mode' option to configure the correct storage mode.", + ex); + } catch (final IOException ex) { + throw new UncheckedIOException("Failed to read storage mode from file", ex); + } + } +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/StorageConfiguration.java b/storage/src/main/java/tech/pegasys/teku/storage/server/StorageConfiguration.java index a573f6d522d..b19d00db4cb 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/StorageConfiguration.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/StorageConfiguration.java @@ -13,14 +13,12 @@ package tech.pegasys.teku.storage.server; +import static tech.pegasys.teku.infrastructure.logging.StatusLogger.STATUS_LOG; import static tech.pegasys.teku.storage.server.StateStorageMode.MINIMAL; import static tech.pegasys.teku.storage.server.StateStorageMode.NOT_SET; import static tech.pegasys.teku.storage.server.StateStorageMode.PRUNE; import static tech.pegasys.teku.storage.server.VersionedDatabaseFactory.STORAGE_MODE_PATH; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.util.Optional; @@ -33,7 +31,6 @@ import tech.pegasys.teku.spec.Spec; public class StorageConfiguration { - public static final boolean DEFAULT_STORE_NON_CANONICAL_BLOCKS_ENABLED = false; public static final int DEFAULT_STATE_REBUILD_TIMEOUT_SECONDS = 120; public static final long DEFAULT_STORAGE_FREQUENCY = 2048L; @@ -267,11 +264,24 @@ public StorageConfiguration build() { private void determineDataStorageMode() { if (dataConfig != null) { final DataDirLayout dataDirLayout = DataDirLayout.createFrom(dataConfig); + final Path beaconDataDirectory = dataDirLayout.getBeaconDataDirectory(); + + Optional storageModeFromStoredFile; + try { + storageModeFromStoredFile = + DatabaseStorageModeFileHelper.readStateStorageMode( + beaconDataDirectory.resolve(STORAGE_MODE_PATH)); + } catch (final DatabaseStorageException e) { + if (dataStorageMode == NOT_SET) { + throw e; + } else { + storageModeFromStoredFile = Optional.empty(); + } + } + this.dataStorageMode = determineStorageDefault( - dataDirLayout.getBeaconDataDirectory().toFile().exists(), - getStorageModeFromPersistedDatabase(dataDirLayout), - dataStorageMode); + beaconDataDirectory.toFile().exists(), storageModeFromStoredFile, dataStorageMode); } else { if (dataStorageMode.equals(NOT_SET)) { dataStorageMode = PRUNE; @@ -279,27 +289,11 @@ private void determineDataStorageMode() { } } - private Optional getStorageModeFromPersistedDatabase( - final DataDirLayout dataDirLayout) { - final Path dbStorageModeFile = - dataDirLayout.getBeaconDataDirectory().resolve(STORAGE_MODE_PATH); - if (!Files.exists(dbStorageModeFile)) { - return Optional.empty(); - } - try { - final StateStorageMode dbStorageMode = - StateStorageMode.valueOf(Files.readString(dbStorageModeFile).trim()); - LOG.debug("Read previous storage mode as {}", dbStorageMode); - return Optional.of(dbStorageMode); - } catch (final IOException ex) { - throw new UncheckedIOException("Failed to read storage mode from file", ex); - } - } - public Builder stateRebuildTimeoutSeconds(final int stateRebuildTimeoutSeconds) { if (stateRebuildTimeoutSeconds < 10 || stateRebuildTimeoutSeconds > 300) { LOG.warn( - "State rebuild timeout is set outside of sensible defaults of 10 -> 300, {} was defined. Cannot be below 1, will allow the value to exceed 300.", + "State rebuild timeout is set outside of sensible defaults of 10 -> 300, {} was defined. Cannot be below " + + "1, will allow the value to exceed 300.", stateRebuildTimeoutSeconds); } this.stateRebuildTimeoutSeconds = Math.max(stateRebuildTimeoutSeconds, 1); @@ -315,6 +309,20 @@ static StateStorageMode determineStorageDefault( if (modeRequested != NOT_SET) { return modeRequested; } - return maybeHistoricStorageMode.orElse(isExistingStore ? PRUNE : MINIMAL); + + if (maybeHistoricStorageMode.isPresent()) { + final StateStorageMode stateStorageMode = maybeHistoricStorageMode.get(); + if (stateStorageMode == PRUNE) { + STATUS_LOG.warnUsageOfImplicitPruneDataStorageMode(); + } + return stateStorageMode; + } + + if (isExistingStore) { + STATUS_LOG.warnUsageOfImplicitPruneDataStorageMode(); + return PRUNE; + } else { + return MINIMAL; + } } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/VersionedDatabaseFactory.java b/storage/src/main/java/tech/pegasys/teku/storage/server/VersionedDatabaseFactory.java index 429be0d0053..16cac554b7a 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/VersionedDatabaseFactory.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/VersionedDatabaseFactory.java @@ -20,7 +20,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes; @@ -82,29 +81,7 @@ public VersionedDatabaseFactory( this.dbStorageModeFile = this.dataDirectory.toPath().resolve(STORAGE_MODE_PATH).toFile(); dbSettingFileSyncDataAccessor = SyncDataAccessor.create(dataDirectory.toPath()); - this.stateStorageMode = - getStateStorageModeFromConfigOrDisk(Optional.of(config.getDataStorageMode())); - } - - private StateStorageMode getStateStorageModeFromConfigOrDisk( - final Optional maybeConfiguredStorageMode) { - try { - final File storageModeFile = this.dataDirectory.toPath().resolve(STORAGE_MODE_PATH).toFile(); - if (storageModeFile.exists() && maybeConfiguredStorageMode.isPresent()) { - final StateStorageMode storageModeFromFile = - StateStorageMode.valueOf(Files.readString(storageModeFile.toPath()).trim()); - if (!storageModeFromFile.equals(maybeConfiguredStorageMode.get())) { - LOG.info( - "Storage mode that was persisted differs from the command specified option; file: {}, CLI: {}", - () -> storageModeFromFile, - maybeConfiguredStorageMode::get); - } - } - } catch (IOException e) { - LOG.error("Failed to read storage mode file", e); - } - - return maybeConfiguredStorageMode.orElse(StateStorageMode.DEFAULT_MODE); + this.stateStorageMode = config.getDataStorageMode(); } @Override diff --git a/storage/src/test/java/tech/pegasys/teku/storage/server/DatabaseStorageModeFileHelperTest.java b/storage/src/test/java/tech/pegasys/teku/storage/server/DatabaseStorageModeFileHelperTest.java new file mode 100644 index 00000000000..d0bd3c2c88d --- /dev/null +++ b/storage/src/test/java/tech/pegasys/teku/storage/server/DatabaseStorageModeFileHelperTest.java @@ -0,0 +1,65 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.fail; +import static tech.pegasys.teku.storage.server.VersionedDatabaseFactory.STORAGE_MODE_PATH; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DatabaseStorageModeFileHelperTest { + + @Test + public void shouldReadValidStorageModeFile(@TempDir final Path tempDir) { + final Path dbStorageModeFile = + createDatabaseStorageModeFile(tempDir, StateStorageMode.MINIMAL.toString()); + assertThat(DatabaseStorageModeFileHelper.readStateStorageMode(dbStorageModeFile)) + .hasValue(StateStorageMode.MINIMAL); + } + + @Test + public void shouldReadEmptyWhenFileDoesNotExist(@TempDir final Path tempDir) { + final Path dbStorageModeFile = tempDir.resolve("foo"); + assertThat(DatabaseStorageModeFileHelper.readStateStorageMode(dbStorageModeFile)).isEmpty(); + } + + @Test + public void shouldThrowErrorIfFileHasInvalidValue(@TempDir final Path tempDir) { + final Path dbStorageModeFile = createDatabaseStorageModeFile(tempDir, "hello"); + assertThatThrownBy(() -> DatabaseStorageModeFileHelper.readStateStorageMode(dbStorageModeFile)) + .isInstanceOf(DatabaseStorageException.class); + } + + @Test + public void shouldThrowErrorIfFileIsEmpty(@TempDir final Path tempDir) { + final Path dbStorageModeFile = createDatabaseStorageModeFile(tempDir, ""); + assertThatThrownBy(() -> DatabaseStorageModeFileHelper.readStateStorageMode(dbStorageModeFile)) + .isInstanceOf(DatabaseStorageException.class); + } + + private Path createDatabaseStorageModeFile(final Path path, final String value) { + try { + return Files.writeString(path.resolve(STORAGE_MODE_PATH), value); + } catch (IOException e) { + fail(e); + return null; + } + } +} diff --git a/storage/src/test/java/tech/pegasys/teku/storage/server/StorageConfigurationTest.java b/storage/src/test/java/tech/pegasys/teku/storage/server/StorageConfigurationTest.java index 63de8a8ca57..b766dfa3995 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/server/StorageConfigurationTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/server/StorageConfigurationTest.java @@ -13,21 +13,36 @@ package tech.pegasys.teku.storage.server; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; import static tech.pegasys.teku.storage.server.StateStorageMode.ARCHIVE; import static tech.pegasys.teku.storage.server.StateStorageMode.MINIMAL; import static tech.pegasys.teku.storage.server.StateStorageMode.NOT_SET; import static tech.pegasys.teku.storage.server.StateStorageMode.PRUNE; +import static tech.pegasys.teku.storage.server.VersionedDatabaseFactory.STORAGE_MODE_PATH; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Optional; import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import tech.pegasys.teku.ethereum.execution.types.Eth1Address; +import tech.pegasys.teku.service.serviceutils.layout.DataConfig; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.TestSpecFactory; public class StorageConfigurationTest { + final Spec spec = TestSpecFactory.createMinimalPhase0(); + final Eth1Address eth1Address = + Eth1Address.fromHexString("0x77f7bED277449F51505a4C54550B074030d989bC"); + public static Stream getStateStorageDefaultScenarios() { ArrayList args = new ArrayList<>(); args.add(Arguments.of(false, Optional.empty(), NOT_SET, MINIMAL)); @@ -58,4 +73,41 @@ void shouldDetermineCorrectStorageModeGivenInputs( isExistingStore, maybePreviousStorageMode, requestedMode)) .isEqualTo(expectedResult); } + + @Test + public void shouldFailIfDatabaseStorageModeFileIsInvalidAndNoExplicitOptionIsSet( + @TempDir final Path dir) throws IOException { + createInvalidStorageModeFile(dir); + final DataConfig dataConfig = DataConfig.builder().beaconDataPath(dir).build(); + + final StorageConfiguration.Builder storageConfigBuilder = + StorageConfiguration.builder() + .specProvider(spec) + .dataConfig(dataConfig) + .eth1DepositContract(eth1Address); + + assertThatThrownBy(storageConfigBuilder::build).isInstanceOf(DatabaseStorageException.class); + } + + @Test + public void shouldSucceedIfDatabaseStorageModeFileIsInvalidAndExplicitOptionIsSet( + @TempDir final Path dir) throws IOException { + createInvalidStorageModeFile(dir); + final DataConfig dataConfig = DataConfig.builder().beaconDataPath(dir).build(); + + final StorageConfiguration storageConfig = + StorageConfiguration.builder() + .specProvider(spec) + .dataConfig(dataConfig) + .eth1DepositContract(eth1Address) + .dataStorageMode(ARCHIVE) + .build(); + + assertThat(storageConfig.getDataStorageMode()).isEqualTo(ARCHIVE); + } + + private static void createInvalidStorageModeFile(final Path dir) throws IOException { + // An empty storage mode path is invalid + Files.createFile(dir.resolve(STORAGE_MODE_PATH)); + } }