From 81c214785653ad99225441fdfa634e0db9bfe66f Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Tue, 15 Oct 2024 21:02:31 -0600 Subject: [PATCH 01/15] temporary commit to use protobuf from temporal branch block-repeated-fix-tss-message, hopefully by the time this ready to merge to main, we can use a specific commit hash Signed-off-by: Alfredo Gutierrez --- stream/build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stream/build.gradle.kts b/stream/build.gradle.kts index d2d8d391a..eaf519835 100644 --- a/stream/build.gradle.kts +++ b/stream/build.gradle.kts @@ -33,10 +33,10 @@ tasks.withType().configureEach { tasks.cloneHederaProtobufs { // uncomment below to use a specific tag // tag = "v0.53.0" or a specific commit like "0047255" - tag = "d5e6988" + // tag = "d5e6988" // uncomment below to use a specific branch - // branch = "main" + branch = "block-repeated-fix-tss-message" } sourceSets { From 3b2be5034d395d47f0b949460c2733c57f3325f2 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Tue, 15 Oct 2024 21:05:06 -0600 Subject: [PATCH 02/15] simple changes to allow the simulator to stream List instead of individual block items. Pending to add a config with the desired batch size. Pending to add more tests with different batch sizes. Signed-off-by: Alfredo Gutierrez --- .../simulator/BlockStreamSimulatorApp.java | 3 ++- .../grpc/PublishStreamGrpcClient.java | 5 +++-- .../grpc/PublishStreamGrpcClientImpl.java | 21 ++++++++++++------- .../grpc/PublishStreamGrpcClientImplTest.java | 3 ++- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java b/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java index 96bec9f94..f0e9ce29a 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java +++ b/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java @@ -26,6 +26,7 @@ import com.swirlds.config.api.Configuration; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import javax.inject.Inject; @@ -130,7 +131,7 @@ private void constantRateStreaming() break; } - publishStreamGrpcClient.streamBlockItem(blockItem); + publishStreamGrpcClient.streamBlockItem(List.of(blockItem)); blockItemsStreamed++; Thread.sleep(delayMSBetweenBlockItems, delayNSBetweenBlockItems); diff --git a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClient.java b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClient.java index 51cead506..4795acc7f 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClient.java +++ b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClient.java @@ -18,6 +18,7 @@ import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; +import java.util.List; /** * The PublishStreamGrpcClient interface provides the methods to stream the block and block item. @@ -26,10 +27,10 @@ public interface PublishStreamGrpcClient { /** * Streams the block item. * - * @param blockItem the block item to be streamed + * @param blockItems list of the block item to be streamed * @return true if the block item is streamed successfully, false otherwise */ - boolean streamBlockItem(BlockItem blockItem); + boolean streamBlockItem(List blockItems); /** * Streams the block. diff --git a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java index d8d85803d..41d8e27d8 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java +++ b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java @@ -26,6 +26,8 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.stub.StreamObserver; +import java.util.ArrayList; +import java.util.List; import javax.inject.Inject; /** @@ -57,11 +59,15 @@ public PublishStreamGrpcClientImpl(@NonNull GrpcConfig grpcConfig) { * stream. */ @Override - public boolean streamBlockItem(BlockItem blockItem) { + public boolean streamBlockItem(List blockItems) { + + List blockItemsProtoc = new ArrayList<>(); + for (BlockItem blockItem : blockItems) { + blockItemsProtoc.add(Translator.fromPbj(blockItem)); + } + requestStreamObserver.onNext( - PublishStreamRequest.newBuilder() - .setBlockItem(Translator.fromPbj(blockItem)) - .build()); + PublishStreamRequest.newBuilder().addAllBlockItems(blockItemsProtoc).build()); return true; } @@ -72,12 +78,13 @@ public boolean streamBlockItem(BlockItem blockItem) { */ @Override public boolean streamBlock(Block block) { + List blockItemsProtoc = new ArrayList<>(); for (BlockItem blockItem : block.items()) { - streamBlockItem(blockItem); + blockItemsProtoc.add(Translator.fromPbj(blockItem)); } - // wait for ack on the block - // if and when the ack is received return true + requestStreamObserver.onNext( + PublishStreamRequest.newBuilder().addAllBlockItems(blockItemsProtoc).build()); return true; } diff --git a/simulator/src/test/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImplTest.java b/simulator/src/test/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImplTest.java index 526aaafce..69a2b0ea9 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImplTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImplTest.java @@ -23,6 +23,7 @@ import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; import java.io.IOException; +import java.util.List; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -45,7 +46,7 @@ void streamBlockItem() { BlockItem blockItem = BlockItem.newBuilder().build(); PublishStreamGrpcClientImpl publishStreamGrpcClient = new PublishStreamGrpcClientImpl(grpcConfig); - boolean result = publishStreamGrpcClient.streamBlockItem(blockItem); + boolean result = publishStreamGrpcClient.streamBlockItem(List.of(blockItem)); assertTrue(result); } From da2cc0666059358714023dc7e78e9ed3839b07ab Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Wed, 16 Oct 2024 16:45:24 -0600 Subject: [PATCH 03/15] Changes to Helidon and Services Signed-off-by: Alfredo Gutierrez --- .../ConsumerStreamResponseObserver.java | 44 ++++++----- .../server/mediator/LiveStreamMediator.java | 3 +- .../mediator/LiveStreamMediatorImpl.java | 21 ++++-- .../block/server/notifier/Notifier.java | 4 +- .../block/server/notifier/NotifierImpl.java | 16 ++-- .../PersistenceInjectionModule.java | 3 +- .../StreamPersistenceHandlerImpl.java | 18 +++-- .../storage/write/BlockAsDirWriter.java | 20 +++-- .../write/BlockAsDirWriterBuilder.java | 3 +- .../producer/ProducerBlockItemObserver.java | 74 ++++++++++--------- .../hedera/block/server/producer/Util.java | 7 +- 11 files changed, 125 insertions(+), 88 deletions(-) diff --git a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java index 07f9822c7..44c68dc6b 100644 --- a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java +++ b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java @@ -17,7 +17,6 @@ package com.hedera.block.server.consumer; import static com.hedera.block.server.Translator.fromPbj; -import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItemsConsumed; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; @@ -27,14 +26,17 @@ import com.hedera.block.server.events.LivenessCalculator; import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.metrics.BlockNodeMetricTypes; import com.hedera.block.server.metrics.MetricsService; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.OneOf; +import com.swirlds.metrics.api.Counter; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; import java.time.InstantSource; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -55,7 +57,7 @@ public class ConsumerStreamResponseObserver private final AtomicBoolean isResponsePermitted = new AtomicBoolean(true); private final ResponseSender statusResponseSender = new StatusResponseSender(); - private final ResponseSender blockItemResponseSender = new BlockItemResponseSender(); + private final ResponseSender blockItemsResponseSender = new BlockItemsResponseSender(); private static final String PROTOCOL_VIOLATION_MESSAGE = "Protocol Violation. %s is OneOf type %s but %s is null.\n%s"; @@ -186,7 +188,7 @@ private ResponseSender getResponseSender( subscribeStreamResponse.response(); return switch (oneOfTypeOneOf.kind()) { case STATUS -> statusResponseSender; - case BLOCK_ITEM -> blockItemResponseSender; + case BLOCK_ITEMS -> blockItemsResponseSender; default -> throw new IllegalArgumentException( "Unknown response type: " + oneOfTypeOneOf.kind()); }; @@ -196,35 +198,39 @@ private interface ResponseSender { void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse); } - private final class BlockItemResponseSender implements ResponseSender { + private final class BlockItemsResponseSender implements ResponseSender { private boolean streamStarted = false; public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) { - // Only start sending BlockItems after we've reached - // the beginning of a block. - final BlockItem blockItem = subscribeStreamResponse.blockItem(); - if (blockItem == null) { + if (subscribeStreamResponse.blockItems() == null) { final String message = PROTOCOL_VIOLATION_MESSAGE.formatted( "SubscribeStreamResponse", - "BLOCK_ITEM", - "block_item", + "BLOCK_ITEMS", + "block_items", subscribeStreamResponse); LOGGER.log(ERROR, message); throw new IllegalArgumentException(message); - } else { - if (!streamStarted && blockItem.hasBlockHeader()) { - streamStarted = true; - } + } + + final List blockItems = subscribeStreamResponse.blockItems().blockItems(); + // Only start sending BlockItems after we've reached + // the beginning of a block. + if (!streamStarted && blockItems.getFirst().hasBlockHeader()) { + streamStarted = true; + } - if (streamStarted) { - LOGGER.log(DEBUG, "Sending BlockItem downstream: " + blockItem); + if (streamStarted) { + LOGGER.log(DEBUG, "Sending BlockItem Batch downstream"); - // Increment counter - metricsService.get(LiveBlockItemsConsumed).increment(); - subscribeStreamResponseObserver.onNext(fromPbj(subscribeStreamResponse)); + final Counter liveBlockItemsConsumed = + metricsService.get(BlockNodeMetricTypes.Counter.LiveBlockItemsConsumed); + // Increment counter manually + for (int i = 0; i < blockItems.size(); i++) { + liveBlockItemsConsumed.increment(); } + subscribeStreamResponseObserver.onNext(fromPbj(subscribeStreamResponse)); } } } diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java index 716afab32..a738a58a7 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java @@ -19,10 +19,11 @@ import com.hedera.block.server.notifier.Notifiable; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.BlockItem; +import java.util.List; /** * Use this interface to combine the contract for mediating the live stream of blocks from the * Hedera network with the contract to be notified of critical system events. */ public interface LiveStreamMediator - extends StreamMediator, Notifiable {} + extends StreamMediator, SubscribeStreamResponse>, Notifiable {} diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java index 0ba032c94..78ca051df 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java @@ -30,9 +30,12 @@ import com.hedera.block.server.service.ServiceStatus; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.SubscribeStreamResponseCode; +import com.hedera.hapi.block.SubscribeStreamResponseSet; import com.hedera.hapi.block.stream.BlockItem; import com.lmax.disruptor.BatchEventProcessor; +import com.swirlds.metrics.api.Counter; import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.List; import java.util.Map; /** @@ -87,24 +90,30 @@ class LiveStreamMediatorImpl extends SubscriptionHandlerBase blockItems) { if (serviceStatus.isRunning()) { // Publish the block for all subscribers to receive - LOGGER.log(DEBUG, "Publishing BlockItem: " + blockItem); + LOGGER.log(DEBUG, "Publishing BlockItem"); + final SubscribeStreamResponseSet blockItemsSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItems).build(); final var subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); + SubscribeStreamResponse.newBuilder().blockItems(blockItemsSet).build(); ringBuffer.publishEvent((event, sequence) -> event.set(subscribeStreamResponse)); // Increment the block item counter - metricsService.get(LiveBlockItems).increment(); + Counter liveBlockItems = metricsService.get(LiveBlockItems); + for (int i = 0; i < blockItems.size(); i++) { + liveBlockItems.increment(); + } } else { LOGGER.log(ERROR, "StreamMediator is not accepting BlockItems"); diff --git a/server/src/main/java/com/hedera/block/server/notifier/Notifier.java b/server/src/main/java/com/hedera/block/server/notifier/Notifier.java index 8d4cc5086..137ae3679 100644 --- a/server/src/main/java/com/hedera/block/server/notifier/Notifier.java +++ b/server/src/main/java/com/hedera/block/server/notifier/Notifier.java @@ -19,9 +19,11 @@ import com.hedera.block.server.mediator.StreamMediator; import com.hedera.hapi.block.PublishStreamResponse; import com.hedera.hapi.block.stream.BlockItem; +import java.util.List; /** * Use this interface to combine the contract for streaming block items with the contract to be * notified of critical system events. */ -public interface Notifier extends StreamMediator, Notifiable {} +public interface Notifier + extends StreamMediator, PublishStreamResponse>, Notifiable {} diff --git a/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java b/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java index a5338088b..4023cf9e4 100644 --- a/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java +++ b/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java @@ -34,6 +34,7 @@ import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import java.security.NoSuchAlgorithmException; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; import javax.inject.Inject; import javax.inject.Singleton; @@ -104,18 +105,18 @@ public void notifyUnrecoverableError() { /** * Publishes the given block item to all subscribed producers. * - * @param blockItem the block item from the persistence layer to publish a response to upstream - * producers + * @param blockItems the block items from the persistence layer to publish a response to + * upstream producers */ @Override - public void publish(@NonNull BlockItem blockItem) { + public void publish(@NonNull List blockItems) { try { if (serviceStatus.isRunning()) { // Publish the block item to the subscribers final var publishStreamResponse = PublishStreamResponse.newBuilder() - .acknowledgement(buildAck(blockItem)) + .acknowledgement(buildAck(blockItems)) .build(); ringBuffer.publishEvent((event, sequence) -> event.set(publishStreamResponse)); @@ -150,16 +151,17 @@ static PublishStreamResponse buildErrorStreamResponse() { /** * Protected method meant for testing. Builds an Acknowledgement for the block item. * - * @param blockItem the block item to build the Acknowledgement for + * @param blockItems the block items to build the Acknowledgement for * @return the Acknowledgement for the block item * @throws NoSuchAlgorithmException if the hash algorithm is not supported */ @NonNull - Acknowledgement buildAck(@NonNull final BlockItem blockItem) throws NoSuchAlgorithmException { + Acknowledgement buildAck(@NonNull final List blockItems) + throws NoSuchAlgorithmException { final ItemAcknowledgement itemAck = ItemAcknowledgement.newBuilder() // TODO: Replace this with a real hash generator - .itemHash(Bytes.wrap(getFakeHash(blockItem))) + .itemsHash(Bytes.wrap(getFakeHash(blockItems))) .build(); return Acknowledgement.newBuilder().itemAck(itemAck).build(); diff --git a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java index 1efa810f6..08ebc22b2 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java @@ -31,6 +31,7 @@ import dagger.Module; import dagger.Provides; import java.io.IOException; +import java.util.List; import javax.inject.Singleton; /** A Dagger module for providing dependencies for Persistence Module. */ @@ -45,7 +46,7 @@ public interface PersistenceInjectionModule { */ @Provides @Singleton - static BlockWriter providesBlockWriter(BlockNodeContext blockNodeContext) { + static BlockWriter> providesBlockWriter(BlockNodeContext blockNodeContext) { try { return BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); } catch (IOException e) { diff --git a/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java index a34638b9b..90eee841f 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java +++ b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java @@ -34,6 +34,7 @@ import com.hedera.pbj.runtime.OneOf; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; +import java.util.List; import java.util.Optional; import javax.inject.Inject; import javax.inject.Singleton; @@ -53,7 +54,7 @@ public class StreamPersistenceHandlerImpl private final System.Logger LOGGER = System.getLogger(getClass().getName()); private final SubscriptionHandler subscriptionHandler; - private final BlockWriter blockWriter; + private final BlockWriter> blockWriter; private final Notifier notifier; private final MetricsService metricsService; private final ServiceStatus serviceStatus; @@ -78,7 +79,7 @@ public class StreamPersistenceHandlerImpl public StreamPersistenceHandlerImpl( @NonNull final SubscriptionHandler subscriptionHandler, @NonNull final Notifier notifier, - @NonNull final BlockWriter blockWriter, + @NonNull final BlockWriter> blockWriter, @NonNull final BlockNodeContext blockNodeContext, @NonNull final ServiceStatus serviceStatus) { this.subscriptionHandler = subscriptionHandler; @@ -105,9 +106,8 @@ public void onEvent(ObjectEvent event, long l, boolean final OneOf oneOfTypeOneOf = subscribeStreamResponse.response(); switch (oneOfTypeOneOf.kind()) { - case BLOCK_ITEM -> { - final BlockItem blockItem = subscribeStreamResponse.blockItem(); - if (blockItem == null) { + case BLOCK_ITEMS -> { + if (subscribeStreamResponse.blockItems() == null) { final String message = PROTOCOL_VIOLATION_MESSAGE.formatted( "SubscribeStreamResponse", @@ -118,12 +118,14 @@ public void onEvent(ObjectEvent event, long l, boolean metricsService.get(StreamPersistenceHandlerError).increment(); throw new BlockStreamProtocolException(message); } else { - // Persist the BlockItem - Optional result = blockWriter.write(blockItem); + // Persist the BlockItems + List blockItems = + subscribeStreamResponse.blockItems().blockItems(); + Optional> result = blockWriter.write(blockItems); if (result.isPresent()) { // Publish the block item back upstream to the notifier // to send responses to producers. - notifier.publish(blockItem); + notifier.publish(blockItems); } } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java index 7bee82409..afed2cd6e 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java @@ -36,6 +36,7 @@ import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -48,7 +49,7 @@ * to remove the current, incomplete block (directory) before re-throwing the exception to the * caller. */ -class BlockAsDirWriter implements BlockWriter { +class BlockAsDirWriter implements BlockWriter> { private final Logger LOGGER = System.getLogger(getClass().getName()); @@ -94,20 +95,23 @@ class BlockAsDirWriter implements BlockWriter { /** * Writes the given block item to the filesystem. * - * @param blockItem the block item to write + * @param blockItems the block item to write * @throws IOException if an error occurs while writing the block item */ @Override - public Optional write(@NonNull final BlockItem blockItem) throws IOException { + public Optional> write(@NonNull final List blockItems) + throws IOException { - if (blockItem.hasBlockHeader()) { - resetState(blockItem); + if (blockItems.getFirst().hasBlockHeader()) { + resetState(blockItems.getFirst()); } final Path blockItemFilePath = calculateBlockItemPath(); for (int retries = 0; ; retries++) { try { - write(blockItemFilePath, blockItem); + for (BlockItem blockItem : blockItems) { + write(blockItemFilePath, blockItem); + } break; } catch (IOException e) { @@ -128,9 +132,9 @@ public Optional write(@NonNull final BlockItem blockItem) throws IOEx } } - if (blockItem.hasBlockProof()) { + if (blockItems.getLast().hasBlockProof()) { metricsService.get(BlocksPersisted).increment(); - return Optional.of(blockItem); + return Optional.of(blockItems); } return Optional.empty(); diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java index f509f2a9e..ee6fee813 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java @@ -27,6 +27,7 @@ import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; +import java.util.List; import java.util.Set; /** @@ -103,7 +104,7 @@ public BlockAsDirWriterBuilder blockRemover(@NonNull BlockRemover blockRemover) * @throws IOException when an error occurs while persisting block items to storage. */ @NonNull - public BlockWriter build() throws IOException { + public BlockWriter> build() throws IOException { return new BlockAsDirWriter(blockRemover, filePerms, blockNodeContext); } } diff --git a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java index db65ce6cf..92c9e8228 100644 --- a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java +++ b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java @@ -38,10 +38,13 @@ import com.hedera.hapi.block.PublishStreamResponseCode; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.ParseException; +import com.swirlds.metrics.api.Counter; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; import java.time.InstantSource; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -59,7 +62,7 @@ public class ProducerBlockItemObserver private final StreamObserver publishStreamResponseObserver; private final SubscriptionHandler subscriptionHandler; - private final Publisher publisher; + private final Publisher> publisher; private final ServiceStatus serviceStatus; private final MetricsService metricsService; @@ -85,8 +88,8 @@ public class ProducerBlockItemObserver * to the upstream producer via the responseStreamObserver. * * @param producerLivenessClock the clock used to calculate the producer liveness. - * @param publisher the block item publisher to used to pass block items to consumers as they - * arrive from the upstream producer. + * @param publisher the block item list publisher to used to pass block item lists to consumers + * as they arrive from the upstream producer. * @param subscriptionHandler the subscription handler used to * @param publishStreamResponseObserver the response stream observer to send responses back to * the upstream producer for each block item processed. @@ -97,7 +100,7 @@ public class ProducerBlockItemObserver */ public ProducerBlockItemObserver( @NonNull final InstantSource producerLivenessClock, - @NonNull final Publisher publisher, + @NonNull final Publisher> publisher, @NonNull final SubscriptionHandler subscriptionHandler, @NonNull final StreamObserver @@ -157,42 +160,47 @@ public ProducerBlockItemObserver( public void onNext( @NonNull final com.hedera.hapi.block.protoc.PublishStreamRequest publishStreamRequest) { - try { - LOGGER.log(DEBUG, "Received PublishStreamRequest from producer"); - final BlockItem blockItem = - toPbj(BlockItem.PROTOBUF, publishStreamRequest.getBlockItem().toByteArray()); - LOGGER.log(DEBUG, "Received block item: " + blockItem); + LOGGER.log(DEBUG, "Received PublishStreamRequest from producer"); + final List blockItemsPbj = new ArrayList<>(); + final Counter liveBlockItemsReceived = metricsService.get(LiveBlockItemsReceived); + for (final var blockItemProtoc : publishStreamRequest.getBlockItemsList()) { + try { + final BlockItem blockItem = + toPbj(BlockItem.PROTOBUF, blockItemProtoc.toByteArray()); + blockItemsPbj.add(blockItem); + } catch (ParseException e) { + final var errorResponse = buildErrorStreamResponse(); + publishStreamResponseObserver.onNext(errorResponse); + LOGGER.log( + ERROR, + "Error parsing inbound block item from a producer: " + blockItemProtoc, + e); + + // Stop the server + serviceStatus.stopWebServer(getClass().getName()); + } + liveBlockItemsReceived.increment(); + } - metricsService.get(LiveBlockItemsReceived).increment(); + // is this log just too much? see below log for proposed... + LOGGER.log(DEBUG, "Received block item batch with {} items.", blockItemsPbj.size()); - // Publish the block to all the subscribers unless - // there's an issue with the StreamMediator. - if (serviceStatus.isRunning()) { - // Refresh the producer liveness - livenessCalculator.refresh(); + // Publish the block to all the subscribers unless + // there's an issue with the StreamMediator. + if (serviceStatus.isRunning()) { + // Refresh the producer liveness + livenessCalculator.refresh(); - // Publish the block to the mediator - publisher.publish(blockItem); + // Publish the block to the mediator + publisher.publish(blockItemsPbj); - } else { - LOGGER.log(ERROR, getClass().getName() + " is not accepting BlockItems"); + } else { + LOGGER.log(ERROR, getClass().getName() + " is not accepting BlockItems"); - // Close the upstream connection to the producer(s) - final var errorResponse = buildErrorStreamResponse(); - publishStreamResponseObserver.onNext(errorResponse); - LOGGER.log(ERROR, "Error PublishStreamResponse sent to upstream producer"); - } - } catch (ParseException e) { + // Close the upstream connection to the producer(s) final var errorResponse = buildErrorStreamResponse(); publishStreamResponseObserver.onNext(errorResponse); - LOGGER.log( - ERROR, - "Error parsing inbound block item from a producer: " - + publishStreamRequest.getBlockItem(), - e); - - // Stop the server - serviceStatus.stopWebServer(getClass().getName()); + LOGGER.log(ERROR, "Error PublishStreamResponse sent to upstream producer"); } } diff --git a/server/src/main/java/com/hedera/block/server/producer/Util.java b/server/src/main/java/com/hedera/block/server/producer/Util.java index 144ffa1ec..d6ad74488 100644 --- a/server/src/main/java/com/hedera/block/server/producer/Util.java +++ b/server/src/main/java/com/hedera/block/server/producer/Util.java @@ -20,6 +20,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.List; /** Utility class for the BlockNode service. */ public final class Util { @@ -29,14 +30,14 @@ private Util() {} * Gets a fake hash for the given block item. This is a placeholder and should be replaced with * real hash functionality once the hedera-protobufs types are integrated. * - * @param blockItem the block item to get the fake hash for + * @param blockItems the block item to get the fake hash for * @return the fake hash for the given block item * @throws NoSuchAlgorithmException thrown if the SHA-384 algorithm is not available */ - public static byte[] getFakeHash(@NonNull final BlockItem blockItem) + public static byte[] getFakeHash(@NonNull final List blockItems) throws NoSuchAlgorithmException { // Calculate the SHA-384 hash MessageDigest digest = MessageDigest.getInstance("SHA-384"); - return digest.digest(BlockItem.PROTOBUF.toBytes(blockItem).toByteArray()); + return digest.digest(BlockItem.PROTOBUF.toBytes(blockItems.getLast()).toByteArray()); } } From 27a259a58a1d57b9ee11d1112eaeed515dc3d318 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Fri, 18 Oct 2024 15:33:54 -0600 Subject: [PATCH 04/15] Fixing Unit Tests Signed-off-by: Alfredo Gutierrez --- .../block/server/BlockStreamService.java | 4 +- .../com/hedera/block/server/Constants.java | 4 +- .../storage/write/BlockAsDirWriter.java | 42 +++++++------- .../BlockStreamServiceIntegrationTest.java | 55 ++++++++++++------- .../block/server/BlockStreamServiceTest.java | 46 ++++++++++++---- .../ConsumerStreamResponseObserverTest.java | 38 ++++++++++--- .../mediator/LiveStreamMediatorImplTest.java | 48 +++++++++------- .../mediator/MediatorInjectionModuleTest.java | 3 +- .../server/notifier/NotifierImplTest.java | 20 +++---- .../PersistenceInjectionModuleTest.java | 5 +- .../StreamPersistenceHandlerImplTest.java | 25 +++++++-- .../storage/read/BlockAsDirReaderTest.java | 40 ++++++++------ .../storage/remove/BlockAsDirRemoverTest.java | 11 ++-- .../storage/write/BlockAsDirWriterTest.java | 40 +++++++------- .../ProducerBlockItemObserverTest.java | 16 +++--- 15 files changed, 241 insertions(+), 156 deletions(-) diff --git a/server/src/main/java/com/hedera/block/server/BlockStreamService.java b/server/src/main/java/com/hedera/block/server/BlockStreamService.java index 4639157be..dbd8462aa 100644 --- a/server/src/main/java/com/hedera/block/server/BlockStreamService.java +++ b/server/src/main/java/com/hedera/block/server/BlockStreamService.java @@ -18,7 +18,7 @@ import static com.hedera.block.server.Constants.CLIENT_STREAMING_METHOD_NAME; import static com.hedera.block.server.Constants.SERVER_STREAMING_METHOD_NAME; -import static com.hedera.block.server.Constants.SERVICE_NAME; +import static com.hedera.block.server.Constants.SERVICE_NAME_BLOCK_STREAM; import static com.hedera.block.server.Constants.SINGLE_BLOCK_METHOD_NAME; import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.Translator.toPbj; @@ -126,7 +126,7 @@ public Descriptors.FileDescriptor proto() { @NonNull @Override public String serviceName() { - return SERVICE_NAME; + return SERVICE_NAME_BLOCK_STREAM; } /** diff --git a/server/src/main/java/com/hedera/block/server/Constants.java b/server/src/main/java/com/hedera/block/server/Constants.java index 9f12b5948..4b7a894d2 100644 --- a/server/src/main/java/com/hedera/block/server/Constants.java +++ b/server/src/main/java/com/hedera/block/server/Constants.java @@ -32,7 +32,9 @@ private Constants() {} @NonNull public static final String LOGGING_PROPERTIES = "logging.properties"; /** Constant mapped to the name of the service in the .proto file */ - @NonNull public static final String SERVICE_NAME = "BlockStreamService"; + @NonNull public static final String SERVICE_NAME_BLOCK_STREAM = "BlockStreamService"; + + @NonNull public static final String SERVICE_NAME_BLOCK_ACCESS = "BlockAccessService"; /** Constant mapped to the publishBlockStream service method name in the .proto file */ @NonNull public static final String CLIENT_STREAMING_METHOD_NAME = "publishBlockStream"; diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java index afed2cd6e..74fe72a1d 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java @@ -106,28 +106,28 @@ public Optional> write(@NonNull final List blockItems resetState(blockItems.getFirst()); } - final Path blockItemFilePath = calculateBlockItemPath(); - for (int retries = 0; ; retries++) { - try { - for (BlockItem blockItem : blockItems) { + for (BlockItem blockItem : blockItems) { + final Path blockItemFilePath = calculateBlockItemPath(); + for (int retries = 0; ; retries++) { + try { write(blockItemFilePath, blockItem); - } - break; - } catch (IOException e) { - - LOGGER.log(ERROR, "Error writing the BlockItem protobuf to a file: ", e); - - // Remove the block if repairing the permissions fails - if (retries > 0) { - // Attempt to remove the block - blockRemover.remove(Long.parseLong(currentBlockDir.toString())); - throw e; - } else { - // Attempt to repair the permissions on the block path - // and the blockItem path - repairPermissions(blockNodeRootPath); - repairPermissions(calculateBlockPath()); - LOGGER.log(INFO, "Retrying to write the BlockItem protobuf to a file"); + break; + } catch (IOException e) { + + LOGGER.log(ERROR, "Error writing the BlockItem protobuf to a file: ", e); + + // Remove the block if repairing the permissions fails + if (retries > 0) { + // Attempt to remove the block + blockRemover.remove(Long.parseLong(currentBlockDir.toString())); + throw e; + } else { + // Attempt to repair the permissions on the block path + // and the blockItem path + repairPermissions(blockNodeRootPath); + repairPermissions(calculateBlockPath()); + LOGGER.log(INFO, "Retrying to write the BlockItem protobuf to a file"); + } } } } diff --git a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java b/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java index e0f3dc992..3665c4ea6 100644 --- a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java @@ -57,6 +57,7 @@ import com.hedera.hapi.block.SubscribeStreamRequest; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.SubscribeStreamResponseCode; +import com.hedera.hapi.block.SubscribeStreamResponseSet; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.io.buffer.Bytes; @@ -133,7 +134,7 @@ public class BlockStreamServiceIntegrationTest { @Mock private WebServer webServer; @Mock private BlockReader blockReader; - @Mock private BlockWriter blockWriter; + @Mock private BlockWriter> blockWriter; private static final String TEMP_DIR = "block-node-unit-test-dir"; @@ -196,16 +197,16 @@ public void testPublishBlockStreamRegistrationAndExecution() List blockItems = generateBlockItems(1); for (int i = 0; i < blockItems.size(); i++) { if (i == 9) { - when(blockWriter.write(blockItems.get(i))) - .thenReturn(Optional.of(blockItems.get(i))); + when(blockWriter.write(List.of(blockItems.get(i)))) + .thenReturn(Optional.of(List.of(blockItems.get(i)))); } else { - when(blockWriter.write(blockItems.get(i))).thenReturn(Optional.empty()); + when(blockWriter.write(List.of(blockItems.get(i)))).thenReturn(Optional.empty()); } } for (BlockItem blockItem : blockItems) { final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockItem).build(); + PublishStreamRequest.newBuilder().blockItems(blockItem).build(); // Calling onNext() as Helidon does with each block item for // the first producer. @@ -235,7 +236,7 @@ public void testPublishBlockStreamRegistrationAndExecution() .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.get(9)))); // Only 1 response is expected per block sent - final Acknowledgement itemAck = buildAck(blockItems.get(9)); + final Acknowledgement itemAck = buildAck(List.of(blockItems.get(9))); final PublishStreamResponse publishStreamResponse = PublishStreamResponse.newBuilder().acknowledgement(itemAck).build(); @@ -295,7 +296,9 @@ public void testSubscribeBlockStream() throws IOException { // Build the BlockItem final List blockItems = generateBlockItems(1); final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockItems.getFirst()).build(); + PublishStreamRequest.newBuilder() + .blockItems(List.of(blockItems.getFirst())) + .build(); // Calling onNext() with a BlockItem streamObserver.onNext(fromPbj(publishStreamRequest)); @@ -303,10 +306,14 @@ public void testSubscribeBlockStream() throws IOException { // Verify the counter was incremented assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); - verify(blockWriter, timeout(testTimeout).times(1)).write(blockItems.getFirst()); + verify(blockWriter, timeout(testTimeout).times(1)).write(List.of(blockItems.getFirst())); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder() + .blockItems(List.of(blockItems.getFirst())) + .build(); final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build(); + SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); verify(subscribeStreamObserver1, timeout(testTimeout).times(1)) .onNext(fromPbj(subscribeStreamResponse)); @@ -320,7 +327,7 @@ public void testSubscribeBlockStream() throws IOException { public void testFullHappyPath() throws IOException { int numberOfBlocks = 100; - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); final BlockStreamService blockStreamService = buildBlockStreamService(blockWriter); @@ -338,7 +345,7 @@ public void testFullHappyPath() throws IOException { final List blockItems = generateBlockItems(numberOfBlocks); for (BlockItem blockItem : blockItems) { final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockItem).build(); + PublishStreamRequest.newBuilder().blockItems(blockItem).build(); streamObserver.onNext(fromPbj(publishStreamRequest)); } @@ -377,7 +384,7 @@ public void testFullWithSubscribersAddedDynamically() { for (int i = 0; i < blockItems.size(); i++) { final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockItems.get(i)).build(); + PublishStreamRequest.newBuilder().blockItems(blockItems.get(i)).build(); // Add a new subscriber if (i == 51) { @@ -467,7 +474,7 @@ public void testSubAndUnsubWhileStreaming() throws InterruptedException { streamObserver.onNext( fromPbj( PublishStreamRequest.newBuilder() - .blockItem(blockItems.get(i)) + .blockItems(blockItems.get(i)) .build())); // Remove 1st subscriber @@ -553,9 +560,9 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep final List blockItems = generateBlockItems(1); // Use a spy to make sure the write() method throws an IOException - final BlockWriter blockWriter = + final BlockWriter> blockWriter = spy(BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build()); - doThrow(IOException.class).when(blockWriter).write(blockItems.getFirst()); + doThrow(IOException.class).when(blockWriter).write(blockItems); final var streamMediator = buildStreamMediator(consumers, serviceStatus); final var notifier = new NotifierImpl(streamMediator, blockNodeContext, serviceStatus); @@ -585,7 +592,7 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep // Transmit a BlockItem final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockItems.getFirst()).build(); + PublishStreamRequest.newBuilder().blockItems(blockItems).build(); streamObserver.onNext(fromPbj(publishStreamRequest)); // Use verify to make sure the serviceStatus.stopRunning() method is called @@ -620,8 +627,11 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep // The BlockItem expected to pass through since it was published // before the IOException was thrown. + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItems).build(); + final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build(); + SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); verify(subscribeStreamObserver1, timeout(testTimeout).times(1)) .onNext(fromPbj(subscribeStreamResponse)); verify(subscribeStreamObserver2, timeout(testTimeout).times(1)) @@ -709,7 +719,9 @@ private static void verifySubscribeStreamResponse( } private static SubscribeStreamResponse buildSubscribeStreamResponse(BlockItem blockItem) { - return SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItem).build(); + return SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); } private static PublishStreamResponse buildEndOfStreamResponse() { @@ -720,7 +732,8 @@ private static PublishStreamResponse buildEndOfStreamResponse() { return PublishStreamResponse.newBuilder().status(endOfStream).build(); } - private BlockStreamService buildBlockStreamService(final BlockWriter blockWriter) { + private BlockStreamService buildBlockStreamService( + final BlockWriter> blockWriter) { final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = buildStreamMediator(new ConcurrentHashMap<>(32), serviceStatus); @@ -752,11 +765,11 @@ private LiveStreamMediator buildStreamMediator( .build(); } - public static Acknowledgement buildAck(@NonNull final BlockItem blockItem) + public static Acknowledgement buildAck(@NonNull final List blockItems) throws NoSuchAlgorithmException { ItemAcknowledgement itemAck = ItemAcknowledgement.newBuilder() - .itemHash(Bytes.wrap(getFakeHash(blockItem))) + .itemsHash(Bytes.wrap(getFakeHash(blockItems))) .build(); return Acknowledgement.newBuilder().itemAck(itemAck).build(); diff --git a/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java b/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java index 0e624ad39..458b3d0d6 100644 --- a/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java @@ -66,10 +66,12 @@ import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -83,7 +85,7 @@ public class BlockStreamServiceTest { @Mock private BlockReader blockReader; - @Mock private BlockWriter blockWriter; + @Mock private BlockWriter> blockWriter; @Mock private ServiceStatus serviceStatus; @@ -129,10 +131,10 @@ public void testServiceName() { blockNodeContext); // Verify the service name - assertEquals(Constants.SERVICE_NAME, blockStreamService.serviceName()); + assertEquals(Constants.SERVICE_NAME_BLOCK_STREAM, blockStreamService.serviceName()); // Verify other methods not invoked - verify(streamMediator, timeout(testTimeout).times(0)).publish(any(BlockItem.class)); + verify(streamMediator, timeout(testTimeout).times(0)).publish(any()); } @Test @@ -151,11 +153,37 @@ public void testProto() { blockNodeContext); Descriptors.FileDescriptor fileDescriptor = blockStreamService.proto(); - // Verify the current rpc methods - assertEquals(5, fileDescriptor.getServices().getFirst().getMethods().size()); + // Verify the current rpc methods on + Descriptors.ServiceDescriptor blockStreamServiceDescriptor = + fileDescriptor.getServices().stream() + .filter( + service -> + service.getName() + .equals(Constants.SERVICE_NAME_BLOCK_STREAM)) + .findFirst() + .orElse(null); + + Assertions.assertNotNull( + blockStreamServiceDescriptor, + "Service descriptor not found for: " + Constants.SERVICE_NAME_BLOCK_STREAM); + assertEquals(2, blockStreamServiceDescriptor.getMethods().size()); + + Descriptors.ServiceDescriptor blockAccessServiceDescriptor = + fileDescriptor.getServices().stream() + .filter( + service -> + service.getName() + .equals(Constants.SERVICE_NAME_BLOCK_ACCESS)) + .findFirst() + .orElse(null); + Assertions.assertNotNull( + blockAccessServiceDescriptor, + "Service descriptor not found for: " + Constants.SERVICE_NAME_BLOCK_ACCESS); + assertEquals(1, blockAccessServiceDescriptor.getMethods().size()); // Verify other methods not invoked - verify(streamMediator, timeout(testTimeout).times(0)).publish(any(BlockItem.class)); + verify(streamMediator, timeout(testTimeout).times(0)) + .publish(Mockito.>any()); } @Test @@ -178,12 +206,10 @@ void testSingleBlockHappyPath() throws IOException, ParseException { when(serviceStatus.isRunning()).thenReturn(true); // Generate and persist a block - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); final List blockItems = generateBlockItems(1); - for (BlockItem blockItem : blockItems) { - blockWriter.write(blockItem); - } + blockWriter.write(blockItems); // Get the block so we can verify the response payload final Optional blockOpt = blockReader.read(1); diff --git a/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java b/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java index 6eb7053f0..3ef939b07 100644 --- a/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java @@ -30,6 +30,7 @@ import com.hedera.block.server.mediator.StreamMediator; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.SubscribeStreamResponseSet; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.input.EventHeader; @@ -90,8 +91,10 @@ public void testProducerTimeoutWithinWindow() { final BlockHeader blockHeader = BlockHeader.newBuilder().number(1).build(); final BlockItem blockItem = BlockItem.newBuilder().blockHeader(blockHeader).build(); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItem).build(); final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); + SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); when(objectEvent.get()).thenReturn(subscribeStreamResponse); @@ -142,8 +145,10 @@ public void testResponseNotPermittedAfterCancel() { testClock, streamMediator, serverCallStreamObserver, testContext); final List blockItems = generateBlockItems(1); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItems).build(); final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build(); + SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); when(objectEvent.get()).thenReturn(subscribeStreamResponse); // Confirm that the observer is called with the first BlockItem @@ -168,8 +173,10 @@ public void testResponseNotPermittedAfterClose() { testClock, streamMediator, serverCallStreamObserver, testContext); final List blockItems = generateBlockItems(1); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItems).build(); final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build(); + SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); when(objectEvent.get()).thenReturn(subscribeStreamResponse); // Confirm that the observer is called with the first BlockItem @@ -204,14 +211,22 @@ public void testConsumerNotToSendBeforeBlockHeader() { final EventHeader eventHeader = EventHeader.newBuilder().eventCore(EventCore.newBuilder().build()).build(); final BlockItem blockItem = BlockItem.newBuilder().eventHeader(eventHeader).build(); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItem).build(); final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); + SubscribeStreamResponse.newBuilder() + .blockItems(subscribeStreamResponseSet) + .build(); when(objectEvent.get()).thenReturn(subscribeStreamResponse); } else { final BlockProof blockProof = BlockProof.newBuilder().block(i).build(); final BlockItem blockItem = BlockItem.newBuilder().blockProof(blockProof).build(); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItem).build(); final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); + SubscribeStreamResponse.newBuilder() + .blockItems(subscribeStreamResponseSet) + .build(); when(objectEvent.get()).thenReturn(subscribeStreamResponse); } @@ -219,8 +234,10 @@ public void testConsumerNotToSendBeforeBlockHeader() { } final BlockItem blockItem = BlockItem.newBuilder().build(); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItem).build(); final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); + SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); // Confirm that the observer was called with the next BlockItem // since we never send a BlockItem with a Header to start the stream. @@ -235,10 +252,15 @@ public void testSubscriberStreamResponseIsBlockItemWhenBlockItemIsNull() { // being created with a null BlockItem. Here, I have to used a spy() to even // manufacture this scenario. This should not happen in production. final BlockItem blockItem = BlockItem.newBuilder().build(); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItem).build(); final SubscribeStreamResponse subscribeStreamResponse = - spy(SubscribeStreamResponse.newBuilder().blockItem(blockItem).build()); + spy( + SubscribeStreamResponse.newBuilder() + .blockItems(subscribeStreamResponseSet) + .build()); - when(subscribeStreamResponse.blockItem()).thenReturn(null); + when(subscribeStreamResponse.blockItems()).thenReturn(null); when(objectEvent.get()).thenReturn(subscribeStreamResponse); final var consumerBlockItemObserver = diff --git a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java index 2b97dc0b0..1b06334bf 100644 --- a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java +++ b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java @@ -43,6 +43,7 @@ import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.SubscribeStreamResponseCode; +import com.hedera.hapi.block.SubscribeStreamResponseSet; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.output.BlockHeader; import com.swirlds.metrics.api.LongGauge; @@ -67,7 +68,7 @@ public class LiveStreamMediatorImplTest { @Mock private BlockNodeEventHandler> observer2; @Mock private BlockNodeEventHandler> observer3; - @Mock private BlockWriter blockWriter; + @Mock private BlockWriter> blockWriter; @Mock private Notifier notifier; @Mock @@ -157,21 +158,21 @@ public void testMediatorPersistenceWithoutSubscribers() throws IOException { final BlockItem blockItem = BlockItem.newBuilder().build(); // register the stream validator - when(blockWriter.write(blockItem)).thenReturn(Optional.empty()); + when(blockWriter.write(List.of(blockItem))).thenReturn(Optional.empty()); final var streamValidator = new StreamPersistenceHandlerImpl( streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); streamMediator.subscribe(streamValidator); // Acting as a producer, notify the mediator of a new block - streamMediator.publish(blockItem); + streamMediator.publish(List.of(blockItem)); // Verify the counter was incremented assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); // Confirm the BlockStorage write method was // called despite the absence of subscribers - verify(blockWriter, timeout(testTimeout).times(1)).write(blockItem); + verify(blockWriter, timeout(testTimeout).times(1)).write(List.of(blockItem)); } @Test @@ -213,18 +214,20 @@ public void testMediatorPublishEventToSubscribers() throws IOException { final BlockHeader blockHeader = BlockHeader.newBuilder().number(1).build(); final BlockItem blockItem = BlockItem.newBuilder().blockHeader(blockHeader).build(); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItem).build(); final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); + SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); // register the stream validator - when(blockWriter.write(blockItem)).thenReturn(Optional.empty()); + when(blockWriter.write(List.of(blockItem))).thenReturn(Optional.empty()); final var streamValidator = new StreamPersistenceHandlerImpl( streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); streamMediator.subscribe(streamValidator); // Acting as a producer, notify the mediator of a new block - streamMediator.publish(blockItem); + streamMediator.publish(List.of(blockItem)); assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); @@ -237,7 +240,7 @@ public void testMediatorPublishEventToSubscribers() throws IOException { .onNext(fromPbj(subscribeStreamResponse)); // Confirm the BlockStorage write method was called - verify(blockWriter, timeout(testTimeout).times(1)).write(blockItem); + verify(blockWriter, timeout(testTimeout).times(1)).write(List.of(blockItem)); } @Test @@ -318,7 +321,7 @@ public void testOnCancelSubscriptionHandling() throws IOException { final List blockItems = generateBlockItems(1); // register the stream validator - when(blockWriter.write(blockItems.getFirst())).thenReturn(Optional.empty()); + when(blockWriter.write(List.of(blockItems.getFirst()))).thenReturn(Optional.empty()); final var streamValidator = new StreamPersistenceHandlerImpl( streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); @@ -333,7 +336,7 @@ public void testOnCancelSubscriptionHandling() throws IOException { assertTrue(streamMediator.isSubscribed(testConsumerBlockItemObserver)); // Simulate the producer notifying the mediator of a new block - streamMediator.publish(blockItems.getFirst()); + streamMediator.publish(List.of(blockItems.getFirst())); // Simulate the consumer cancelling the stream testConsumerBlockItemObserver.getOnCancel().run(); @@ -348,7 +351,7 @@ public void testOnCancelSubscriptionHandling() throws IOException { assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); // Confirm the BlockStorage write method was called - verify(blockWriter, timeout(testTimeout).times(1)).write(blockItems.getFirst()); + verify(blockWriter, timeout(testTimeout).times(1)).write(List.of(blockItems.getFirst())); // Confirm the stream validator is still subscribed assertTrue(streamMediator.isSubscribed(streamValidator)); @@ -368,7 +371,7 @@ public void testOnCloseSubscriptionHandling() throws IOException { final List blockItems = generateBlockItems(1); // register the stream validator - when(blockWriter.write(blockItems.getFirst())).thenReturn(Optional.empty()); + when(blockWriter.write(List.of(blockItems.getFirst()))).thenReturn(Optional.empty()); final var streamValidator = new StreamPersistenceHandlerImpl( streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); @@ -382,7 +385,7 @@ public void testOnCloseSubscriptionHandling() throws IOException { assertTrue(streamMediator.isSubscribed(testConsumerBlockItemObserver)); // Simulate the producer notifying the mediator of a new block - streamMediator.publish(blockItems.getFirst()); + streamMediator.publish(List.of(blockItems.getFirst())); // Simulate the consumer completing the stream testConsumerBlockItemObserver.getOnClose().run(); @@ -397,7 +400,7 @@ public void testOnCloseSubscriptionHandling() throws IOException { assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); // Confirm the BlockStorage write method was called - verify(blockWriter, timeout(testTimeout).times(1)).write(blockItems.getFirst()); + verify(blockWriter, timeout(testTimeout).times(1)).write(List.of(blockItems.getFirst())); // Confirm the stream validator is still subscribed assertTrue(streamMediator.isSubscribed(streamValidator)); @@ -444,9 +447,9 @@ public void testMediatorBlocksPublishAfterException() throws IOException, Interr // However, we will need to support multiple producers in the // future. In that case, we need to make sure a second producer // is not able to publish a block after the first producer fails. - doThrow(new IOException()).when(blockWriter).write(firstBlockItem); + doThrow(new IOException()).when(blockWriter).write(List.of(firstBlockItem)); - streamMediator.publish(firstBlockItem); + streamMediator.publish(List.of(firstBlockItem)); Thread.sleep(testTimeout); @@ -457,10 +460,11 @@ public void testMediatorBlocksPublishAfterException() throws IOException, Interr assertEquals(1, blockNodeContext.metricsService().get(LiveBlockStreamMediatorError).get()); // Send another block item after the exception - streamMediator.publish(blockItems.get(1)); - + streamMediator.publish(List.of(blockItems.get(1))); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(firstBlockItem).build(); final var subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(firstBlockItem).build(); + SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); verify(streamObserver1, timeout(testTimeout).times(1)) .onNext(fromPbj(subscribeStreamResponse)); verify(streamObserver2, timeout(testTimeout).times(1)) @@ -478,7 +482,7 @@ public void testMediatorBlocksPublishAfterException() throws IOException, Interr verify(streamObserver3, timeout(testTimeout).times(1)).onNext(fromPbj(endOfStreamResponse)); // verify write method only called once despite the second block being published. - verify(blockWriter, timeout(testTimeout).times(1)).write(firstBlockItem); + verify(blockWriter, timeout(testTimeout).times(1)).write(List.of(firstBlockItem)); } @Test @@ -515,7 +519,9 @@ public void testUnsubscribeWhenNotSubscribed() throws IOException { private static class TestConsumerStreamResponseObserver extends ConsumerStreamResponseObserver { public TestConsumerStreamResponseObserver( @NonNull final InstantSource producerLivenessClock, - @NonNull final StreamMediator streamMediator, + @NonNull + final StreamMediator, SubscribeStreamResponse> + streamMediator, @NonNull final StreamObserver responseStreamObserver, diff --git a/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java index 5ba33db46..65baf1b8a 100644 --- a/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java @@ -24,6 +24,7 @@ import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.BlockItem; import java.io.IOException; +import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -46,7 +47,7 @@ void testProvidesStreamMediator() throws IOException { BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); // Call the method under test - StreamMediator streamMediator = + StreamMediator, SubscribeStreamResponse> streamMediator = MediatorInjectionModule.providesLiveStreamMediator(blockNodeContext, serviceStatus); // Verify that the streamMediator is correctly instantiated diff --git a/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java b/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java index 4124f9b0e..d56df4c91 100644 --- a/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java +++ b/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java @@ -54,7 +54,7 @@ public class NotifierImplTest { @Mock private Notifiable mediator; - @Mock private Publisher publisher; + @Mock private Publisher> publisher; @Mock private ServiceStatus serviceStatus; @Mock private SubscriptionHandler subscriptionHandler; @@ -132,13 +132,11 @@ public void testRegistration() throws NoSuchAlgorithmException { "Expected the notifier to have observer3 subscribed"); List blockItems = generateBlockItems(1); - notifier.publish(blockItems.getFirst()); + notifier.publish(blockItems); // Verify the response was received by all observers final var publishStreamResponse = - PublishStreamResponse.newBuilder() - .acknowledgement(buildAck(blockItems.getFirst())) - .build(); + PublishStreamResponse.newBuilder().acknowledgement(buildAck(blockItems)).build(); verify(streamObserver1, timeout(testTimeout).times(1)) .onNext(fromPbj(publishStreamResponse)); verify(streamObserver2, timeout(testTimeout).times(1)) @@ -222,7 +220,7 @@ public void testTimeoutExpiredHandling() throws InterruptedException { "Expected the notifier to have observer3 subscribed"); List blockItems = generateBlockItems(1); - notifier.publish(blockItems.getFirst()); + notifier.publish(blockItems); Thread.sleep(testTimeout); @@ -284,7 +282,7 @@ public void testPublishThrowsNoSuchAlgorithmException() { "Expected the notifier to have observer3 subscribed"); List blockItems = generateBlockItems(1); - notifier.publish(blockItems.getFirst()); + notifier.publish(blockItems); final PublishStreamResponse errorResponse = buildErrorStreamResponse(); verify(streamObserver1, timeout(testTimeout).times(1)).onNext(fromPbj(errorResponse)); @@ -340,13 +338,11 @@ public void testServiceStatusNotRunning() throws NoSuchAlgorithmException { "Expected the notifier to have observer3 subscribed"); final List blockItems = generateBlockItems(1); - notifier.publish(blockItems.getFirst()); + notifier.publish(blockItems); // Verify once the serviceStatus is not running that we do not publish the responses final var publishStreamResponse = - PublishStreamResponse.newBuilder() - .acknowledgement(buildAck(blockItems.getFirst())) - .build(); + PublishStreamResponse.newBuilder().acknowledgement(buildAck(blockItems)).build(); verify(streamObserver1, timeout(testTimeout).times(0)) .onNext(fromPbj(publishStreamResponse)); verify(streamObserver2, timeout(testTimeout).times(0)) @@ -365,7 +361,7 @@ public TestNotifier( @Override @NonNull - Acknowledgement buildAck(@NonNull final BlockItem blockItem) + Acknowledgement buildAck(@NonNull final List blockItems) throws NoSuchAlgorithmException { throw new NoSuchAlgorithmException("Test exception"); } diff --git a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java index 61763d498..ee3a0e89f 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java @@ -35,6 +35,7 @@ import com.hedera.hapi.block.stream.BlockItem; import com.swirlds.config.api.Configuration; import java.io.IOException; +import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -48,7 +49,7 @@ class PersistenceInjectionModuleTest { @Mock private PersistenceStorageConfig persistenceStorageConfig; @Mock private SubscriptionHandler subscriptionHandler; @Mock private Notifier notifier; - @Mock private BlockWriter blockWriter; + @Mock private BlockWriter> blockWriter; @Mock private ServiceStatus serviceStatus; @BeforeEach @@ -62,7 +63,7 @@ void setup() throws IOException { @Test void testProvidesBlockWriter() { - BlockWriter blockWriter = + BlockWriter> blockWriter = PersistenceInjectionModule.providesBlockWriter(blockNodeContext); assertNotNull(blockWriter); diff --git a/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java b/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java index 87c23d5f0..45be5407c 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java @@ -35,6 +35,7 @@ import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.SubscribeStreamResponseSet; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.OneOf; import java.io.IOException; @@ -49,7 +50,7 @@ public class StreamPersistenceHandlerImplTest { @Mock private SubscriptionHandler subscriptionHandler; - @Mock private BlockWriter blockWriter; + @Mock private BlockWriter> blockWriter; @Mock private Notifier notifier; @@ -76,8 +77,10 @@ public void testOnEventWhenServiceIsNotRunning() { serviceStatus); final List blockItems = generateBlockItems(1); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItems).build(); final var subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build(); + SubscribeStreamResponse.newBuilder().blockItems(subscribeStreamResponseSet).build(); final ObjectEvent event = new ObjectEvent<>(); event.set(subscribeStreamResponse); @@ -85,7 +88,7 @@ public void testOnEventWhenServiceIsNotRunning() { // Indirectly confirm the branch we're in by verifying // these methods were not called. - verify(notifier, never()).publish(blockItems.getFirst()); + verify(notifier, never()).publish(blockItems); verify(metricsService, never()).get(StreamPersistenceHandlerError); } @@ -104,11 +107,16 @@ public void testBlockItemIsNull() throws IOException { serviceStatus); final List blockItems = generateBlockItems(1); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItems).build(); final var subscribeStreamResponse = - spy(SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build()); + spy( + SubscribeStreamResponse.newBuilder() + .blockItems(subscribeStreamResponseSet) + .build()); // Force the block item to be null - when(subscribeStreamResponse.blockItem()).thenReturn(null); + when(subscribeStreamResponse.blockItems()).thenReturn(null); final ObjectEvent event = new ObjectEvent<>(); event.set(subscribeStreamResponse); @@ -133,8 +141,13 @@ public void testSubscribeStreamResponseTypeUnknown() throws IOException { serviceStatus); final List blockItems = generateBlockItems(1); + final SubscribeStreamResponseSet subscribeStreamResponseSet = + SubscribeStreamResponseSet.newBuilder().blockItems(blockItems).build(); final var subscribeStreamResponse = - spy(SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build()); + spy( + SubscribeStreamResponse.newBuilder() + .blockItems(subscribeStreamResponseSet) + .build()); // Force the block item to be UNSET final OneOf illegalOneOf = diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java index e3b7cac84..375631314 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java @@ -93,10 +93,10 @@ public void testReadBlockDoesNotExist() throws IOException, ParseException { public void testReadPermsRepairSucceeded() throws IOException, ParseException { final List blockItems = generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); for (BlockItem blockItem : blockItems) { - blockWriter.write(blockItem); + blockWriter.write(List.of(blockItem)); } // Make the block unreadable @@ -113,11 +113,12 @@ public void testReadPermsRepairSucceeded() throws IOException, ParseException { public void testRemoveBlockReadPermsRepairFailed() throws IOException, ParseException { final List blockItems = generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (BlockItem blockItem : blockItems) { - blockWriter.write(blockItem); - } + // for (BlockItem blockItem : blockItems) { + // blockWriter.write(blockItem); + // } + blockWriter.write(blockItems); // Make the block unreadable removeBlockReadPerms(1, config); @@ -136,11 +137,12 @@ public void testRemoveBlockReadPermsRepairFailed() throws IOException, ParseExce public void testRemoveBlockItemReadPerms() throws IOException { final List blockItems = generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (BlockItem blockItem : blockItems) { - blockWriter.write(blockItem); - } + // for (BlockItem blockItem : blockItems) { + // blockWriter.write(blockItem); + // } + blockWriter.write(blockItems); removeBlockItemReadPerms(1, 1, config); @@ -168,11 +170,12 @@ public void testRepairReadPermsFails() throws IOException, ParseException { final List blockItems = generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (final BlockItem blockItem : blockItems) { - blockWriter.write(blockItem); - } + // for (final BlockItem blockItem : blockItems) { + // blockWriter.write(blockItem); + // } + blockWriter.write(blockItems); removeBlockReadPerms(1, config); @@ -206,11 +209,12 @@ public void testBlockNodePathReadFails() throws IOException, ParseException { public void testParseExceptionHandling() throws IOException, ParseException { final List blockItems = generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (final BlockItem blockItem : blockItems) { - blockWriter.write(blockItem); - } + // for (final BlockItem blockItem : blockItems) { + // blockWriter.write(blockItem); + // } + blockWriter.write(blockItems); // Read the block back and confirm it's read successfully final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java index bbe824d93..9e1542733 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java @@ -69,10 +69,10 @@ public void testRemoveNonExistentBlock() throws IOException, ParseException { // Write a block final var blockItems = PersistTestUtils.generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); for (final BlockItem blockItem : blockItems) { - blockWriter.write(blockItem); + blockWriter.write(List.of(blockItem)); } // Remove a block that does not exist @@ -102,11 +102,10 @@ public void testRemoveBlockWithPermException() throws IOException, ParseExceptio // Write a block final List blockItems = PersistTestUtils.generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (final BlockItem blockItem : blockItems) { - blockWriter.write(blockItem); - } + + blockWriter.write(blockItems); // Set up the BlockRemover with permissions that will prevent the block from being removed BlockRemover blockRemover = new BlockAsDirRemover(testPath, TestUtils.getNoPerms()); diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java index ace12e14b..d9520ce03 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java @@ -93,20 +93,20 @@ public void testWriterAndReaderHappyPath() throws IOException, ParseException { // Write a block final List blockItems = generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext) .filePerms(FileUtils.defaultPerms) .build(); for (int i = 0; i < 10; i++) { if (i == 9) { - Optional result = blockWriter.write(blockItems.get(i)); + Optional> result = blockWriter.write(List.of(blockItems.get(i))); if (result.isPresent()) { - assertEquals(blockItems.get(i), result.get()); + assertEquals(blockItems.get(i), result.get().get(0)); } else { fail("The optional should contain the last block proof block item"); } } else { - Optional result = blockWriter.write(blockItems.get(i)); + Optional> result = blockWriter.write(List.of(blockItems.get(i))); assertTrue(result.isEmpty()); } } @@ -143,7 +143,7 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { final List blockItems = generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); // Change the permissions on the block node root directory @@ -151,7 +151,7 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { // The first BlockItem contains a header which will create a new block directory. // The BlockWriter will attempt to repair the permissions and should succeed. - Optional result = blockWriter.write(blockItems.getFirst()); + Optional> result = blockWriter.write(List.of(blockItems.getFirst())); assertFalse(result.isPresent()); // Confirm we're able to read 1 block item @@ -164,7 +164,7 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { // Remove all permissions on the block directory and // attempt to write the next block item removeBlockAllPerms(1, testConfig); - result = blockWriter.write(blockItems.get(1)); + result = blockWriter.write(List.of(blockItems.get(1))); assertFalse(result.isPresent()); // There should now be 2 blockItems in the block @@ -175,7 +175,7 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { // Remove read permission on the block directory removeBlockReadPerms(1, testConfig); - result = blockWriter.write(blockItems.get(2)); + result = blockWriter.write(List.of(blockItems.get(2))); assertFalse(result.isPresent()); // There should now be 3 blockItems in the block @@ -193,25 +193,25 @@ public void testUnrecoverableIOExceptionOnWrite() throws IOException { new BlockAsDirRemover(Path.of(testConfig.rootPath()), FileUtils.defaultPerms); // Use a spy to simulate an IOException when the first block item is written - final BlockWriter blockWriter = + final BlockWriter> blockWriter = spy( BlockAsDirWriterBuilder.newBuilder(blockNodeContext) .blockRemover(blockRemover) .build()); - doThrow(IOException.class).when(blockWriter).write(blockItems.getFirst()); - assertThrows(IOException.class, () -> blockWriter.write(blockItems.getFirst())); + doThrow(IOException.class).when(blockWriter).write(blockItems); + assertThrows(IOException.class, () -> blockWriter.write(blockItems)); } @Test public void testRemoveRootDirReadPerm() throws IOException, ParseException { final List blockItems = generateBlockItems(1); - final BlockWriter blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); // Write the first block item to create the block // directory - Optional result = blockWriter.write(blockItems.getFirst()); + Optional> result = blockWriter.write(List.of(blockItems.getFirst())); assertFalse(result.isPresent()); // Remove root dir read permissions and @@ -223,14 +223,14 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { // items for (int i = 1; i < 10; i++) { if (i == 9) { - result = blockWriter.write(blockItems.get(i)); + result = blockWriter.write(List.of(blockItems.get(i))); if (result.isPresent()) { - assertEquals(blockItems.get(i), result.get()); + assertEquals(blockItems.get(i), result.get().get(0)); } else { fail("The optional should contain the last block proof block item"); } } else { - result = blockWriter.write(blockItems.get(i)); + result = blockWriter.write(List.of(blockItems.get(i))); assertTrue(result.isEmpty()); } } @@ -259,7 +259,7 @@ public void testPartialBlockRemoval() throws IOException, ParseException { for (int i = 0; i < 23; i++) { // Prepare the block writer to call the actual write method // for 23 block items - doCallRealMethod().when(blockWriter).write(same(blockItems.get(i))); + doCallRealMethod().when(blockWriter).write(same(List.of(blockItems.get(i)))); } // Simulate an IOException when writing the 24th block item @@ -268,12 +268,12 @@ public void testPartialBlockRemoval() throws IOException, ParseException { // Now make the calls for (int i = 0; i < 23; i++) { - Optional result = blockWriter.write(blockItems.get(i)); + Optional> result = blockWriter.write(List.of(blockItems.get(i))); if (i == 9 || i == 19) { // The last block item in each block is the block proof // and should be returned by the write method assertTrue(result.isPresent()); - assertEquals(blockItems.get(i), result.get()); + assertEquals(blockItems.get(i), result.get().get(0)); } else { // The write method should return an empty optional assertTrue(result.isEmpty()); @@ -281,7 +281,7 @@ public void testPartialBlockRemoval() throws IOException, ParseException { } // Verify the IOException was thrown on the 23rd block item - assertThrows(IOException.class, () -> blockWriter.write(blockItems.get(23))); + assertThrows(IOException.class, () -> blockWriter.write(List.of(blockItems.get(23)))); // Verify the partially written block was removed final BlockReader blockReader = diff --git a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java index 571e1e2c1..66de5f4c9 100644 --- a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java @@ -59,7 +59,7 @@ public class ProducerBlockItemObserverTest { @Mock private InstantSource testClock; - @Mock private Publisher publisher; + @Mock private Publisher> publisher; @Mock private SubscriptionHandler subscriptionHandler; @Mock @@ -135,7 +135,7 @@ public void testBlockItemThrowsParseException() throws IOException { // create the PublishStreamRequest with the spy block item final com.hedera.hapi.block.protoc.PublishStreamRequest protocPublishStreamRequest = com.hedera.hapi.block.protoc.PublishStreamRequest.newBuilder() - .setBlockItem(protocBlockItem) + .addBlockItems(protocBlockItem) .build(); // call the producerBlockItemObserver @@ -149,7 +149,9 @@ public void testBlockItemThrowsParseException() throws IOException { fromPbj(PublishStreamResponse.newBuilder().status(endOfStream).build()); // verify the ProducerBlockItemObserver has sent an error response - verify(publishStreamResponseObserver, timeout(testTimeout).times(1)) + verify( + publishStreamResponseObserver, + timeout(testTimeout).atLeast(1)) // It fixes if set it to 2, but why??? .onNext(fromPbj(PublishStreamResponse.newBuilder().status(endOfStream).build())); verify(serviceStatus, timeout(testTimeout).times(1)).stopWebServer(any()); @@ -170,7 +172,7 @@ public void testResponseNotPermittedAfterCancel() throws NoSuchAlgorithmExceptio final List blockItems = generateBlockItems(1); final ItemAcknowledgement itemAck = ItemAcknowledgement.newBuilder() - .itemHash(Bytes.wrap(getFakeHash(blockItems.getLast()))) + .itemsHash(Bytes.wrap(getFakeHash(blockItems))) .build(); final PublishStreamResponse publishStreamResponse = PublishStreamResponse.newBuilder() @@ -207,7 +209,7 @@ public void testResponseNotPermittedAfterClose() throws NoSuchAlgorithmException final List blockItems = generateBlockItems(1); final ItemAcknowledgement itemAck = ItemAcknowledgement.newBuilder() - .itemHash(Bytes.wrap(getFakeHash(blockItems.getLast()))) + .itemsHash(Bytes.wrap(getFakeHash(blockItems))) .build(); final PublishStreamResponse publishStreamResponse = PublishStreamResponse.newBuilder() @@ -245,7 +247,7 @@ public void testOnlyErrorStreamResponseAllowedAfterStatusChange() { final List blockItems = generateBlockItems(1); final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockItems.getFirst()).build(); + PublishStreamRequest.newBuilder().blockItems(blockItems).build(); // Confirm that the observer is called with the first BlockItem producerBlockItemObserver.onNext(fromPbj(publishStreamRequest)); @@ -263,7 +265,7 @@ public void testOnlyErrorStreamResponseAllowedAfterStatusChange() { private static class TestProducerBlockItemObserver extends ProducerBlockItemObserver { public TestProducerBlockItemObserver( @NonNull final InstantSource clock, - @NonNull final Publisher publisher, + @NonNull final Publisher> publisher, @NonNull final SubscriptionHandler subscriptionHandler, @NonNull final StreamObserver From 544dd6bc551ec6c5b14bc28f8f3c44f4527d2fe4 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Fri, 18 Oct 2024 17:19:05 -0600 Subject: [PATCH 05/15] Refactor BlockStreamService to extract BlockAccessService Signed-off-by: Alfredo Gutierrez --- .../server/BlockNodeAppInjectionModule.java | 1 + .../com/hedera/block/server/Constants.java | 3 +- .../block/server/grpc/BlockAccessService.java | 176 ++++++++++++ .../server/{ => grpc}/BlockStreamService.java | 106 +------- server/src/main/java/module-info.java | 1 + .../hedera/block/server/BlockNodeAppTest.java | 1 + .../server/grpc/BlockAccessServiceTest.java | 250 ++++++++++++++++++ .../BlockStreamServiceIntegrationTest.java | 8 +- .../{ => grpc}/BlockStreamServiceTest.java | 206 +-------------- .../server/notifier/NotifierImplTest.java | 2 +- 10 files changed, 448 insertions(+), 306 deletions(-) create mode 100644 server/src/main/java/com/hedera/block/server/grpc/BlockAccessService.java rename server/src/main/java/com/hedera/block/server/{ => grpc}/BlockStreamService.java (62%) create mode 100644 server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java rename server/src/test/java/com/hedera/block/server/{ => grpc}/BlockStreamServiceIntegrationTest.java (99%) rename server/src/test/java/com/hedera/block/server/{ => grpc}/BlockStreamServiceTest.java (50%) diff --git a/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java b/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java index f23cab5a7..3aafbc4b9 100644 --- a/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java @@ -17,6 +17,7 @@ package com.hedera.block.server; import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.grpc.BlockStreamService; import com.hedera.block.server.metrics.MetricsService; import com.swirlds.config.api.Configuration; import dagger.Binds; diff --git a/server/src/main/java/com/hedera/block/server/Constants.java b/server/src/main/java/com/hedera/block/server/Constants.java index 4b7a894d2..e471d213d 100644 --- a/server/src/main/java/com/hedera/block/server/Constants.java +++ b/server/src/main/java/com/hedera/block/server/Constants.java @@ -31,9 +31,10 @@ private Constants() {} */ @NonNull public static final String LOGGING_PROPERTIES = "logging.properties"; - /** Constant mapped to the name of the service in the .proto file */ + /** Constant mapped to the name of the BlockStream service in the .proto file */ @NonNull public static final String SERVICE_NAME_BLOCK_STREAM = "BlockStreamService"; + /** Constant mapped to the name of the BlockAccess service in the .proto file */ @NonNull public static final String SERVICE_NAME_BLOCK_ACCESS = "BlockAccessService"; /** Constant mapped to the publishBlockStream service method name in the .proto file */ diff --git a/server/src/main/java/com/hedera/block/server/grpc/BlockAccessService.java b/server/src/main/java/com/hedera/block/server/grpc/BlockAccessService.java new file mode 100644 index 000000000..aae176ef2 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/grpc/BlockAccessService.java @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.grpc; + +import static com.hedera.block.server.Constants.SINGLE_BLOCK_METHOD_NAME; +import static com.hedera.block.server.Translator.fromPbj; +import static com.hedera.block.server.Translator.toPbj; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SingleBlocksNotFound; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SingleBlocksRetrieved; +import static java.lang.System.Logger.Level.DEBUG; +import static java.lang.System.Logger.Level.ERROR; + +import com.google.protobuf.Descriptors; +import com.google.protobuf.InvalidProtocolBufferException; +import com.hedera.block.server.Constants; +import com.hedera.block.server.metrics.MetricsService; +import com.hedera.block.server.persistence.storage.read.BlockReader; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.hapi.block.SingleBlockRequest; +import com.hedera.hapi.block.SingleBlockResponseCode; +import com.hedera.hapi.block.protoc.BlockService; +import com.hedera.hapi.block.protoc.SingleBlockResponse; +import com.hedera.hapi.block.stream.Block; +import com.hedera.pbj.runtime.ParseException; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.grpc.stub.StreamObserver; +import io.helidon.webserver.grpc.GrpcService; +import java.io.IOException; +import java.util.Optional; +import javax.inject.Inject; + +/** + * The BlockAccessService class provides a gRPC service to access blocks. + * + *

This service provides a unary gRPC method to retrieve a single block by block number. + */ +public class BlockAccessService implements GrpcService { + private final System.Logger LOGGER = System.getLogger(getClass().getName()); + + private final ServiceStatus serviceStatus; + private final BlockReader blockReader; + private final MetricsService metricsService; + + /** + * Constructs a new BlockAccessService instance with the given dependencies. + * + * @param serviceStatus used to query the service status + * @param blockReader used to retrieve blocks + * @param metricsService used to observe metrics + */ + @Inject + public BlockAccessService( + @NonNull ServiceStatus serviceStatus, + @NonNull BlockReader blockReader, + @NonNull MetricsService metricsService) { + this.serviceStatus = serviceStatus; + this.blockReader = blockReader; + this.metricsService = metricsService; + } + + @Override + public Descriptors.FileDescriptor proto() { + return BlockService.getDescriptor(); + } + + @Override + public String serviceName() { + return Constants.SERVICE_NAME_BLOCK_ACCESS; + } + + @Override + public void update(Routing routing) { + routing.unary(SINGLE_BLOCK_METHOD_NAME, this::protocSingleBlock); + } + + void protocSingleBlock( + @NonNull final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest, + @NonNull final StreamObserver singleBlockResponseStreamObserver) { + LOGGER.log(DEBUG, "Executing Unary singleBlock gRPC method"); + + try { + final SingleBlockRequest pbjSingleBlockRequest = + toPbj(SingleBlockRequest.PROTOBUF, singleBlockRequest.toByteArray()); + singleBlock(pbjSingleBlockRequest, singleBlockResponseStreamObserver); + } catch (ParseException e) { + LOGGER.log(ERROR, "Error parsing protoc SingleBlockRequest: {0}", singleBlockRequest); + singleBlockResponseStreamObserver.onNext(buildSingleBlockNotAvailableResponse()); + } + } + + private void singleBlock( + @NonNull final SingleBlockRequest singleBlockRequest, + @NonNull + final StreamObserver + singleBlockResponseStreamObserver) { + + LOGGER.log(DEBUG, "Executing Unary singleBlock gRPC method"); + + if (serviceStatus.isRunning()) { + final long blockNumber = singleBlockRequest.blockNumber(); + try { + final Optional blockOpt = blockReader.read(blockNumber); + if (blockOpt.isPresent()) { + LOGGER.log(DEBUG, "Successfully returning block number: {0}", blockNumber); + singleBlockResponseStreamObserver.onNext( + fromPbjSingleBlockSuccessResponse(blockOpt.get())); + + metricsService.get(SingleBlocksRetrieved).increment(); + } else { + LOGGER.log(DEBUG, "Block number {0} not found", blockNumber); + singleBlockResponseStreamObserver.onNext(buildSingleBlockNotFoundResponse()); + metricsService.get(SingleBlocksNotFound).increment(); + } + } catch (IOException e) { + LOGGER.log(ERROR, "Error reading block number: {0}", blockNumber); + singleBlockResponseStreamObserver.onNext(buildSingleBlockNotAvailableResponse()); + } catch (ParseException e) { + LOGGER.log(ERROR, "Error parsing block number: {0}", blockNumber); + singleBlockResponseStreamObserver.onNext(buildSingleBlockNotAvailableResponse()); + } + } else { + LOGGER.log(ERROR, "Unary singleBlock gRPC method is not currently running"); + singleBlockResponseStreamObserver.onNext(buildSingleBlockNotAvailableResponse()); + } + + // Send the response + singleBlockResponseStreamObserver.onCompleted(); + } + + @NonNull + static com.hedera.hapi.block.protoc.SingleBlockResponse buildSingleBlockNotAvailableResponse() { + final com.hedera.hapi.block.SingleBlockResponse response = + com.hedera.hapi.block.SingleBlockResponse.newBuilder() + .status(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE) + .build(); + + return fromPbj(response); + } + + @NonNull + static com.hedera.hapi.block.protoc.SingleBlockResponse buildSingleBlockNotFoundResponse() + throws InvalidProtocolBufferException { + final com.hedera.hapi.block.SingleBlockResponse response = + com.hedera.hapi.block.SingleBlockResponse.newBuilder() + .status(SingleBlockResponseCode.READ_BLOCK_NOT_FOUND) + .build(); + + return fromPbj(response); + } + + @NonNull + static com.hedera.hapi.block.protoc.SingleBlockResponse fromPbjSingleBlockSuccessResponse( + @NonNull final Block block) { + final com.hedera.hapi.block.SingleBlockResponse singleBlockResponse = + com.hedera.hapi.block.SingleBlockResponse.newBuilder() + .status(SingleBlockResponseCode.READ_BLOCK_SUCCESS) + .block(block) + .build(); + + return fromPbj(singleBlockResponse); + } +} diff --git a/server/src/main/java/com/hedera/block/server/BlockStreamService.java b/server/src/main/java/com/hedera/block/server/grpc/BlockStreamService.java similarity index 62% rename from server/src/main/java/com/hedera/block/server/BlockStreamService.java rename to server/src/main/java/com/hedera/block/server/grpc/BlockStreamService.java index dbd8462aa..a7295c6e0 100644 --- a/server/src/main/java/com/hedera/block/server/BlockStreamService.java +++ b/server/src/main/java/com/hedera/block/server/grpc/BlockStreamService.java @@ -14,22 +14,17 @@ * limitations under the License. */ -package com.hedera.block.server; +package com.hedera.block.server.grpc; import static com.hedera.block.server.Constants.CLIENT_STREAMING_METHOD_NAME; import static com.hedera.block.server.Constants.SERVER_STREAMING_METHOD_NAME; import static com.hedera.block.server.Constants.SERVICE_NAME_BLOCK_STREAM; -import static com.hedera.block.server.Constants.SINGLE_BLOCK_METHOD_NAME; import static com.hedera.block.server.Translator.fromPbj; -import static com.hedera.block.server.Translator.toPbj; -import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SingleBlocksNotFound; -import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SingleBlocksRetrieved; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; import com.google.protobuf.Descriptors; -import com.google.protobuf.InvalidProtocolBufferException; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.consumer.ConsumerStreamResponseObserver; import com.hedera.block.server.events.BlockNodeEventHandler; @@ -40,26 +35,19 @@ import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.producer.ProducerBlockItemObserver; import com.hedera.block.server.service.ServiceStatus; -import com.hedera.hapi.block.SingleBlockRequest; -import com.hedera.hapi.block.SingleBlockResponse; -import com.hedera.hapi.block.SingleBlockResponseCode; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.SubscribeStreamResponseCode; import com.hedera.hapi.block.protoc.BlockService; import com.hedera.hapi.block.stream.Block; -import com.hedera.pbj.runtime.ParseException; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.StreamObserver; import io.helidon.webserver.grpc.GrpcService; -import java.io.IOException; import java.time.Clock; -import java.util.Optional; import javax.inject.Inject; /** * The BlockStreamService class defines the gRPC service for the block stream service. It provides - * the implementation for the bidirectional streaming, server streaming, and unary methods defined - * in the proto file. + * the implementation for the bidirectional streaming, server streaming as defined in the proto file. */ public class BlockStreamService implements GrpcService { @@ -140,7 +128,6 @@ public String serviceName() { public void update(@NonNull final Routing routing) { routing.bidi(CLIENT_STREAMING_METHOD_NAME, this::protocPublishBlockStream); routing.serverStream(SERVER_STREAMING_METHOD_NAME, this::protocSubscribeBlockStream); - routing.unary(SINGLE_BLOCK_METHOD_NAME, this::protocSingleBlock); } StreamObserver protocPublishBlockStream( @@ -198,62 +185,6 @@ void protocSubscribeBlockStream( } } - void protocSingleBlock( - @NonNull final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest, - @NonNull - final StreamObserver - singleBlockResponseStreamObserver) { - LOGGER.log(DEBUG, "Executing Unary singleBlock gRPC method"); - - try { - final SingleBlockRequest pbjSingleBlockRequest = - toPbj(SingleBlockRequest.PROTOBUF, singleBlockRequest.toByteArray()); - singleBlock(pbjSingleBlockRequest, singleBlockResponseStreamObserver); - } catch (ParseException e) { - LOGGER.log(ERROR, "Error parsing protoc SingleBlockRequest: {0}", singleBlockRequest); - singleBlockResponseStreamObserver.onNext(buildSingleBlockNotAvailableResponse()); - } - } - - private void singleBlock( - @NonNull final SingleBlockRequest singleBlockRequest, - @NonNull - final StreamObserver - singleBlockResponseStreamObserver) { - - LOGGER.log(DEBUG, "Executing Unary singleBlock gRPC method"); - - if (serviceStatus.isRunning()) { - final long blockNumber = singleBlockRequest.blockNumber(); - try { - final Optional blockOpt = blockReader.read(blockNumber); - if (blockOpt.isPresent()) { - LOGGER.log(DEBUG, "Successfully returning block number: {0}", blockNumber); - singleBlockResponseStreamObserver.onNext( - fromPbjSingleBlockSuccessResponse(blockOpt.get())); - - metricsService.get(SingleBlocksRetrieved).increment(); - } else { - LOGGER.log(DEBUG, "Block number {0} not found", blockNumber); - singleBlockResponseStreamObserver.onNext(buildSingleBlockNotFoundResponse()); - metricsService.get(SingleBlocksNotFound).increment(); - } - } catch (IOException e) { - LOGGER.log(ERROR, "Error reading block number: {0}", blockNumber); - singleBlockResponseStreamObserver.onNext(buildSingleBlockNotAvailableResponse()); - } catch (ParseException e) { - LOGGER.log(ERROR, "Error parsing block number: {0}", blockNumber); - singleBlockResponseStreamObserver.onNext(buildSingleBlockNotAvailableResponse()); - } - } else { - LOGGER.log(ERROR, "Unary singleBlock gRPC method is not currently running"); - singleBlockResponseStreamObserver.onNext(buildSingleBlockNotAvailableResponse()); - } - - // Send the response - singleBlockResponseStreamObserver.onCompleted(); - } - // TODO: Fix this error type once it's been standardized in `hedera-protobufs` // this should not be success @NonNull @@ -266,37 +197,4 @@ private void singleBlock( return fromPbj(response); } - - @NonNull - static com.hedera.hapi.block.protoc.SingleBlockResponse buildSingleBlockNotAvailableResponse() { - final SingleBlockResponse response = - SingleBlockResponse.newBuilder() - .status(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE) - .build(); - - return fromPbj(response); - } - - @NonNull - static com.hedera.hapi.block.protoc.SingleBlockResponse buildSingleBlockNotFoundResponse() - throws InvalidProtocolBufferException { - final SingleBlockResponse response = - SingleBlockResponse.newBuilder() - .status(SingleBlockResponseCode.READ_BLOCK_NOT_FOUND) - .build(); - - return fromPbj(response); - } - - @NonNull - static com.hedera.hapi.block.protoc.SingleBlockResponse fromPbjSingleBlockSuccessResponse( - @NonNull final Block block) { - final SingleBlockResponse singleBlockResponse = - SingleBlockResponse.newBuilder() - .status(SingleBlockResponseCode.READ_BLOCK_SUCCESS) - .block(block) - .build(); - - return fromPbj(singleBlockResponse); - } } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index ccd73557f..579ce946b 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -17,6 +17,7 @@ exports com.hedera.block.server.persistence; exports com.hedera.block.server.notifier; exports com.hedera.block.server.service; + exports com.hedera.block.server.grpc; requires com.hedera.block.stream; requires com.google.protobuf; diff --git a/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java b/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java index f12a1bc27..41e0a3a85 100644 --- a/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.hedera.block.server.grpc.BlockStreamService; import com.hedera.block.server.health.HealthService; import com.hedera.block.server.service.ServiceStatus; import io.helidon.webserver.WebServer; diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java new file mode 100644 index 000000000..2ff8ace12 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java @@ -0,0 +1,250 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.grpc; + +import static com.hedera.block.server.Constants.SINGLE_BLOCK_METHOD_NAME; +import static com.hedera.block.server.grpc.BlockAccessService.buildSingleBlockNotAvailableResponse; +import static com.hedera.block.server.grpc.BlockAccessService.buildSingleBlockNotFoundResponse; +import static com.hedera.block.server.grpc.BlockAccessService.fromPbjSingleBlockSuccessResponse; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static java.lang.System.Logger.Level.INFO; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.mediator.LiveStreamMediator; +import com.hedera.block.server.notifier.Notifier; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; +import com.hedera.block.server.persistence.storage.read.BlockReader; +import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.block.server.util.TestConfigUtil; +import com.hedera.block.server.util.TestUtils; +import com.hedera.hapi.block.protoc.SingleBlockResponse; +import com.hedera.hapi.block.stream.Block; +import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.pbj.runtime.ParseException; +import io.grpc.stub.ServerCalls; +import io.grpc.stub.StreamObserver; +import io.helidon.webserver.grpc.GrpcService; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class BlockAccessServiceTest { + + @Mock private Notifier notifier; + + @Mock private StreamObserver responseObserver; + + @Mock private LiveStreamMediator streamMediator; + + @Mock private BlockReader blockReader; + + @Mock private BlockWriter> blockWriter; + + @Mock private ServiceStatus serviceStatus; + + private final System.Logger LOGGER = System.getLogger(getClass().getName()); + + private static final String TEMP_DIR = "block-node-unit-test-dir"; + + private static final int testTimeout = 1000; + + private Path testPath; + private BlockNodeContext blockNodeContext; + private PersistenceStorageConfig config; + + @BeforeEach + public void setUp() throws IOException { + testPath = Files.createTempDirectory(TEMP_DIR); + LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); + + blockNodeContext = + TestConfigUtil.getTestBlockNodeContext( + Map.of("persistence.storage.rootPath", testPath.toString())); + config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + } + + @AfterEach + public void tearDown() { + TestUtils.deleteDirectory(testPath.toFile()); + } + + @Test + void testSingleBlockHappyPath() throws IOException, ParseException { + + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); + + final BlockAccessService blockAccessService = + new BlockAccessService( + serviceStatus, blockReader, blockNodeContext.metricsService()); + + // Enable the serviceStatus + when(serviceStatus.isRunning()).thenReturn(true); + + // Generate and persist a block + final BlockWriter> blockWriter = + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + final List blockItems = generateBlockItems(1); + blockWriter.write(blockItems); + + // Get the block so we can verify the response payload + final Optional blockOpt = blockReader.read(1); + if (blockOpt.isEmpty()) { + fail("Block 1 should be present"); + return; + } + + // Build a response to verify what's passed to the response observer + final com.hedera.hapi.block.protoc.SingleBlockResponse expectedSingleBlockResponse = + fromPbjSingleBlockSuccessResponse(blockOpt.get()); + + // Build a request to invoke the service + final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = + com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() + .setBlockNumber(1) + .build(); + + // Call the service + blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); + verify(responseObserver, times(1)).onNext(expectedSingleBlockResponse); + } + + @Test + void testSingleBlockNotFoundPath() throws IOException, ParseException { + + // Get the block so we can verify the response payload + when(blockReader.read(1)).thenReturn(Optional.empty()); + + // Build a response to verify what's passed to the response observer + final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotFound = + buildSingleBlockNotFoundResponse(); + + // Build a request to invoke the service + final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = + com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() + .setBlockNumber(1) + .build(); + + final BlockAccessService blockAccessService = + new BlockAccessService( + serviceStatus, blockReader, blockNodeContext.metricsService()); + + // Enable the serviceStatus + when(serviceStatus.isRunning()).thenReturn(true); + + blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); + verify(responseObserver, times(1)).onNext(expectedNotFound); + } + + @Test + void testSingleBlockServiceNotAvailable() { + + final BlockAccessService blockAccessService = + new BlockAccessService( + serviceStatus, blockReader, blockNodeContext.metricsService()); + + // Set the service status to not running + when(serviceStatus.isRunning()).thenReturn(false); + + final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotAvailable = + buildSingleBlockNotAvailableResponse(); + + // Build a request to invoke the service + final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = + com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() + .setBlockNumber(1) + .build(); + blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); + verify(responseObserver, times(1)).onNext(expectedNotAvailable); + } + + @Test + public void testSingleBlockIOExceptionPath() throws IOException, ParseException { + final BlockAccessService blockAccessService = + new BlockAccessService( + serviceStatus, blockReader, blockNodeContext.metricsService()); + + when(serviceStatus.isRunning()).thenReturn(true); + when(blockReader.read(1)).thenThrow(new IOException("Test exception")); + + final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotAvailable = + buildSingleBlockNotAvailableResponse(); + + // Build a request to invoke the service + final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = + com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() + .setBlockNumber(1) + .build(); + blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); + verify(responseObserver, times(1)).onNext(expectedNotAvailable); + } + + @Test + public void testSingleBlockParseExceptionPath() throws IOException, ParseException { + final BlockAccessService blockAccessService = + new BlockAccessService( + serviceStatus, blockReader, blockNodeContext.metricsService()); + + when(serviceStatus.isRunning()).thenReturn(true); + when(blockReader.read(1)).thenThrow(new ParseException("Test exception")); + + final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotAvailable = + buildSingleBlockNotAvailableResponse(); + + // Build a request to invoke the service + final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = + com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() + .setBlockNumber(1) + .build(); + blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); + verify(responseObserver, times(1)).onNext(expectedNotAvailable); + } + + @Test + public void testUpdateInvokesRoutingWithLambdas() { + + final BlockAccessService blockAccessService = + new BlockAccessService( + serviceStatus, blockReader, blockNodeContext.metricsService()); + + GrpcService.Routing routing = mock(GrpcService.Routing.class); + blockAccessService.update(routing); + + verify(routing, timeout(testTimeout).times(1)) + .unary(eq(SINGLE_BLOCK_METHOD_NAME), any(ServerCalls.UnaryMethod.class)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceIntegrationTest.java similarity index 99% rename from server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java rename to server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceIntegrationTest.java index 3665c4ea6..0ebd46c9c 100644 --- a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceIntegrationTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.hedera.block.server; +package com.hedera.block.server.grpc; import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; @@ -578,6 +578,10 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep notifier, blockNodeContext); + final BlockAccessService blockAccessService = + new BlockAccessService( + serviceStatus, blockReader, blockNodeContext.metricsService()); + // Subscribe the consumers blockStreamService.protocSubscribeBlockStream( subscribeStreamRequest, subscribeStreamObserver1); @@ -616,7 +620,7 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep .build(); // Simulate a consumer attempting to connect to the Block Node after the exception. - blockStreamService.protocSingleBlock(singleBlockRequest, singleBlockResponseStreamObserver); + blockAccessService.protocSingleBlock(singleBlockRequest, singleBlockResponseStreamObserver); // Build a request to invoke the subscribeBlockStream service final SubscribeStreamRequest subscribeStreamRequest = diff --git a/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java similarity index 50% rename from server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java rename to server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java index 458b3d0d6..f3becede7 100644 --- a/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java @@ -14,21 +14,15 @@ * limitations under the License. */ -package com.hedera.block.server; +package com.hedera.block.server.grpc; -import static com.hedera.block.server.BlockStreamService.buildSingleBlockNotAvailableResponse; -import static com.hedera.block.server.BlockStreamService.buildSingleBlockNotFoundResponse; -import static com.hedera.block.server.BlockStreamService.fromPbjSingleBlockSuccessResponse; import static com.hedera.block.server.Constants.CLIENT_STREAMING_METHOD_NAME; import static com.hedera.block.server.Constants.SERVER_STREAMING_METHOD_NAME; -import static com.hedera.block.server.Constants.SINGLE_BLOCK_METHOD_NAME; import static com.hedera.block.server.Translator.fromPbj; -import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static com.hedera.block.server.util.PersistTestUtils.reverseByteArray; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.INFO; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; @@ -39,14 +33,13 @@ import static org.mockito.Mockito.when; import com.google.protobuf.Descriptors; +import com.hedera.block.server.Constants; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.mediator.LiveStreamMediator; import com.hedera.block.server.notifier.Notifier; import com.hedera.block.server.persistence.StreamPersistenceHandlerImpl; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; @@ -55,7 +48,6 @@ import com.hedera.hapi.block.SingleBlockResponseCode; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; -import com.hedera.pbj.runtime.ParseException; import io.grpc.stub.ServerCalls; import io.grpc.stub.StreamObserver; import io.helidon.webserver.grpc.GrpcService; @@ -64,7 +56,6 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.Optional; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -131,7 +122,8 @@ public void testServiceName() { blockNodeContext); // Verify the service name - assertEquals(Constants.SERVICE_NAME_BLOCK_STREAM, blockStreamService.serviceName()); + Assertions.assertEquals( + Constants.SERVICE_NAME_BLOCK_STREAM, blockStreamService.serviceName()); // Verify other methods not invoked verify(streamMediator, timeout(testTimeout).times(0)).publish(any()); @@ -186,177 +178,6 @@ public void testProto() { .publish(Mockito.>any()); } - @Test - void testSingleBlockHappyPath() throws IOException, ParseException { - - final var blockNodeEventHandler = - new StreamPersistenceHandlerImpl( - streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); - final BlockStreamService blockStreamService = - new BlockStreamService( - streamMediator, - blockReader, - serviceStatus, - blockNodeEventHandler, - notifier, - blockNodeContext); - - // Enable the serviceStatus - when(serviceStatus.isRunning()).thenReturn(true); - - // Generate and persist a block - final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - final List blockItems = generateBlockItems(1); - blockWriter.write(blockItems); - - // Get the block so we can verify the response payload - final Optional blockOpt = blockReader.read(1); - if (blockOpt.isEmpty()) { - fail("Block 1 should be present"); - return; - } - - // Build a response to verify what's passed to the response observer - final com.hedera.hapi.block.protoc.SingleBlockResponse expectedSingleBlockResponse = - fromPbjSingleBlockSuccessResponse(blockOpt.get()); - - // Build a request to invoke the service - final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - .setBlockNumber(1) - .build(); - - // Call the service - blockStreamService.protocSingleBlock(singleBlockRequest, responseObserver); - verify(responseObserver, times(1)).onNext(expectedSingleBlockResponse); - } - - @Test - void testSingleBlockNotFoundPath() throws IOException, ParseException { - - // Get the block so we can verify the response payload - when(blockReader.read(1)).thenReturn(Optional.empty()); - - // Build a response to verify what's passed to the response observer - final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotFound = - buildSingleBlockNotFoundResponse(); - - // Build a request to invoke the service - final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - .setBlockNumber(1) - .build(); - - // Call the service - final var blockNodeEventHandler = - new StreamPersistenceHandlerImpl( - streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - final BlockStreamService blockStreamService = - new BlockStreamService( - streamMediator, - blockReader, - serviceStatus, - blockNodeEventHandler, - notifier, - blockNodeContext); - - // Enable the serviceStatus - when(serviceStatus.isRunning()).thenReturn(true); - - blockStreamService.protocSingleBlock(singleBlockRequest, responseObserver); - verify(responseObserver, times(1)).onNext(expectedNotFound); - } - - @Test - void testSingleBlockServiceNotAvailable() { - - final var blockNodeEventHandler = - new StreamPersistenceHandlerImpl( - streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - final BlockStreamService blockStreamService = - new BlockStreamService( - streamMediator, - blockReader, - serviceStatus, - blockNodeEventHandler, - notifier, - blockNodeContext); - - // Set the service status to not running - when(serviceStatus.isRunning()).thenReturn(false); - - final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotAvailable = - buildSingleBlockNotAvailableResponse(); - - // Build a request to invoke the service - final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - .setBlockNumber(1) - .build(); - blockStreamService.protocSingleBlock(singleBlockRequest, responseObserver); - verify(responseObserver, times(1)).onNext(expectedNotAvailable); - } - - @Test - public void testSingleBlockIOExceptionPath() throws IOException, ParseException { - final var blockNodeEventHandler = - new StreamPersistenceHandlerImpl( - streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - final BlockStreamService blockStreamService = - new BlockStreamService( - streamMediator, - blockReader, - serviceStatus, - blockNodeEventHandler, - notifier, - blockNodeContext); - - when(serviceStatus.isRunning()).thenReturn(true); - when(blockReader.read(1)).thenThrow(new IOException("Test exception")); - - final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotAvailable = - buildSingleBlockNotAvailableResponse(); - - // Build a request to invoke the service - final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - .setBlockNumber(1) - .build(); - blockStreamService.protocSingleBlock(singleBlockRequest, responseObserver); - verify(responseObserver, times(1)).onNext(expectedNotAvailable); - } - - @Test - public void testSingleBlockParseExceptionPath() throws IOException, ParseException { - final var blockNodeEventHandler = - new StreamPersistenceHandlerImpl( - streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - final BlockStreamService blockStreamService = - new BlockStreamService( - streamMediator, - blockReader, - serviceStatus, - blockNodeEventHandler, - notifier, - blockNodeContext); - - when(serviceStatus.isRunning()).thenReturn(true); - when(blockReader.read(1)).thenThrow(new ParseException("Test exception")); - - final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotAvailable = - buildSingleBlockNotAvailableResponse(); - - // Build a request to invoke the service - final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - .setBlockNumber(1) - .build(); - blockStreamService.protocSingleBlock(singleBlockRequest, responseObserver); - verify(responseObserver, times(1)).onNext(expectedNotAvailable); - } - @Test public void testUpdateInvokesRoutingWithLambdas() { @@ -371,8 +192,6 @@ public void testUpdateInvokesRoutingWithLambdas() { .serverStream( eq(SERVER_STREAMING_METHOD_NAME), any(ServerCalls.ServerStreamingMethod.class)); - verify(routing, timeout(testTimeout).times(1)) - .unary(eq(SINGLE_BLOCK_METHOD_NAME), any(ServerCalls.UnaryMethod.class)); } private BlockStreamService getBlockStreamService() { @@ -393,18 +212,9 @@ private BlockStreamService getBlockStreamService() { @Test public void testProtocParseExceptionHandling() { // TODO: We might be able to remove this test once we can remove the Translator class - - final var blockNodeEventHandler = - new StreamPersistenceHandlerImpl( - streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - final BlockStreamService blockStreamService = - new BlockStreamService( - streamMediator, - blockReader, - serviceStatus, - blockNodeEventHandler, - notifier, - blockNodeContext); + final BlockAccessService blockAccessService = + new BlockAccessService( + serviceStatus, blockReader, blockNodeContext.metricsService()); // Build a request to invoke the service final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = @@ -422,7 +232,7 @@ public void testProtocParseExceptionHandling() { .status(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE) .build(); // Call the service - blockStreamService.protocSingleBlock(singleBlockRequest, responseObserver); + blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); verify(responseObserver, times(1)).onNext(fromPbj(expectedSingleBlockErrorResponse)); } } diff --git a/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java b/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java index d56df4c91..a4c926180 100644 --- a/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java +++ b/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java @@ -16,8 +16,8 @@ package com.hedera.block.server.notifier; -import static com.hedera.block.server.BlockStreamServiceIntegrationTest.buildAck; import static com.hedera.block.server.Translator.fromPbj; +import static com.hedera.block.server.grpc.BlockStreamServiceIntegrationTest.buildAck; import static com.hedera.block.server.notifier.NotifierImpl.buildErrorStreamResponse; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static org.junit.jupiter.api.Assertions.assertFalse; From d6a80bf32e3cb60ea6c9fc9fc33513542908a40a Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Sat, 19 Oct 2024 13:11:19 -0600 Subject: [PATCH 06/15] Enable BlockAccessService as part of the GRPC Routes in the BlockNodeApp, fixed scripts for smoke-test and created an DI Injection Module for GRPC Services. Signed-off-by: Alfredo Gutierrez --- .../com/hedera/block/server/BlockNodeApp.java | 18 ++++--- .../server/BlockNodeAppInjectionModule.java | 13 ----- .../grpc/GrpcServiceInjectionModule.java | 48 +++++++++++++++++++ .../hedera/block/server/BlockNodeAppTest.java | 3 ++ server/src/test/resources/block_service.proto | 5 +- server/src/test/resources/get-block.sh | 2 +- server/src/test/resources/producer.sh | 4 +- 7 files changed, 71 insertions(+), 22 deletions(-) create mode 100644 server/src/main/java/com/hedera/block/server/grpc/GrpcServiceInjectionModule.java diff --git a/server/src/main/java/com/hedera/block/server/BlockNodeApp.java b/server/src/main/java/com/hedera/block/server/BlockNodeApp.java index 77d49a363..f8d9393ab 100644 --- a/server/src/main/java/com/hedera/block/server/BlockNodeApp.java +++ b/server/src/main/java/com/hedera/block/server/BlockNodeApp.java @@ -19,13 +19,14 @@ import static java.lang.System.Logger; import static java.lang.System.Logger.Level.INFO; +import com.hedera.block.server.grpc.BlockAccessService; +import com.hedera.block.server.grpc.BlockStreamService; import com.hedera.block.server.health.HealthService; import com.hedera.block.server.service.ServiceStatus; import edu.umd.cs.findbugs.annotations.NonNull; import io.helidon.webserver.WebServer; import io.helidon.webserver.WebServerConfig; import io.helidon.webserver.grpc.GrpcRouting; -import io.helidon.webserver.grpc.GrpcService; import io.helidon.webserver.http.HttpRouting; import java.io.IOException; import javax.inject.Inject; @@ -42,7 +43,8 @@ public class BlockNodeApp { private final ServiceStatus serviceStatus; private final HealthService healthService; - private final GrpcService blockStreamService; + private final BlockStreamService blockStreamService; + private final BlockAccessService blockAccessService; private final WebServerConfig.Builder webServerBuilder; /** @@ -50,19 +52,22 @@ public class BlockNodeApp { * * @param serviceStatus has the status of the service * @param healthService handles the health API requests - * @param blockStreamService handles the GRPC API requests + * @param blockStreamService handles the block stream requests * @param webServerBuilder used to build the web server and start it + * @param blockAccessService grpc service for block access */ @Inject public BlockNodeApp( @NonNull ServiceStatus serviceStatus, @NonNull HealthService healthService, - @NonNull GrpcService blockStreamService, - @NonNull WebServerConfig.Builder webServerBuilder) { + @NonNull BlockStreamService blockStreamService, + @NonNull WebServerConfig.Builder webServerBuilder, + @NonNull BlockAccessService blockAccessService) { this.serviceStatus = serviceStatus; this.healthService = healthService; this.blockStreamService = blockStreamService; this.webServerBuilder = webServerBuilder; + this.blockAccessService = blockAccessService; } /** @@ -72,7 +77,8 @@ public BlockNodeApp( */ public void start() throws IOException { - final GrpcRouting.Builder grpcRouting = GrpcRouting.builder().service(blockStreamService); + final GrpcRouting.Builder grpcRouting = + GrpcRouting.builder().service(blockStreamService).service(blockAccessService); final HttpRouting.Builder httpRouting = HttpRouting.builder().register(healthService.getHealthRootPath(), healthService); diff --git a/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java b/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java index 3aafbc4b9..719a90b73 100644 --- a/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java @@ -17,14 +17,11 @@ package com.hedera.block.server; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.grpc.BlockStreamService; import com.hedera.block.server.metrics.MetricsService; import com.swirlds.config.api.Configuration; -import dagger.Binds; import dagger.Module; import dagger.Provides; import io.helidon.webserver.WebServerConfig; -import io.helidon.webserver.grpc.GrpcService; import javax.inject.Singleton; /** @@ -48,16 +45,6 @@ static BlockNodeContext provideBlockNodeContext( return new BlockNodeContext(metricsService, config); } - /** - * Provides a block stream service singleton using DI. - * - * @param blockStreamService should come from DI - * @return a block stream service singleton - */ - @Singleton - @Binds - GrpcService bindBlockStreamService(BlockStreamService blockStreamService); - /** * Provides a web server config builder singleton using DI. * diff --git a/server/src/main/java/com/hedera/block/server/grpc/GrpcServiceInjectionModule.java b/server/src/main/java/com/hedera/block/server/grpc/GrpcServiceInjectionModule.java new file mode 100644 index 000000000..f2463cf24 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/grpc/GrpcServiceInjectionModule.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.grpc; + +import dagger.Binds; +import dagger.Module; +import javax.inject.Singleton; + +/** + * A Dagger Module for GRPC services that are at the BlockNode Services. + */ +@Module +public interface GrpcServiceInjectionModule { + + /** + * Provides a block stream service singleton using DI. + * + * @param blockStreamService should come from DI + * @return a block stream service singleton + */ + @Singleton + @Binds + BlockStreamService bindBlockStreamService(BlockStreamService blockStreamService); + + /** + * Provides a block access service singleton using DI. + * + * @param blockAccessService should come from DI + * @return a block access service singleton + */ + @Singleton + @Binds + BlockAccessService bindBlockAccessService(BlockAccessService blockAccessService); +} diff --git a/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java b/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java index 41e0a3a85..9a78ada39 100644 --- a/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.hedera.block.server.grpc.BlockAccessService; import com.hedera.block.server.grpc.BlockStreamService; import com.hedera.block.server.health.HealthService; import com.hedera.block.server.service.ServiceStatus; @@ -44,6 +45,8 @@ class BlockNodeAppTest { @Mock private BlockStreamService blockStreamService; + @Mock private BlockAccessService blockAccessService; + @Mock private WebServerConfig.Builder webServerBuilder; @Mock private WebServer webServer; diff --git a/server/src/test/resources/block_service.proto b/server/src/test/resources/block_service.proto index 6bd87c170..dca29e47c 100644 --- a/server/src/test/resources/block_service.proto +++ b/server/src/test/resources/block_service.proto @@ -126,12 +126,15 @@ enum SubscribeStreamResponseCode { service BlockStreamService { // rpc serverStatus(ServerStatusRequest) returns (ServerStatusResponse); - rpc singleBlock(SingleBlockRequest) returns (SingleBlockResponse); // rpc stateSnapshot(StateSnapshotRequest) returns (StateSnapshotResponse); rpc publishBlockStream (stream PublishStreamRequest) returns (stream PublishStreamResponse); rpc subscribeBlockStream(SubscribeStreamRequest) returns (stream SubscribeStreamResponse); } +service BlockAccessService { + rpc singleBlock(SingleBlockRequest) returns (SingleBlockResponse); +} + // block.proto message Block { diff --git a/server/src/test/resources/get-block.sh b/server/src/test/resources/get-block.sh index 8837d43e3..407106b07 100755 --- a/server/src/test/resources/get-block.sh +++ b/server/src/test/resources/get-block.sh @@ -15,7 +15,7 @@ echo "Param is: $1" # Use environment variables or default values GRPC_SERVER=${GRPC_SERVER:-"localhost:8080"} -GRPC_METHOD=${GRPC_METHOD:-"com.hedera.hapi.block.BlockStreamService/singleBlock"} +GRPC_METHOD=${GRPC_METHOD:-"com.hedera.hapi.block.BlockAccessService/singleBlock"} PATH_TO_PROTO="./block_service.proto" echo "Requesting block $1..." diff --git a/server/src/test/resources/producer.sh b/server/src/test/resources/producer.sh index e35a9a38e..abc0ae47b 100755 --- a/server/src/test/resources/producer.sh +++ b/server/src/test/resources/producer.sh @@ -1,4 +1,6 @@ #!/bin/bash +# set -x + usage_error() { echo "Usage: $0 [positive-integer]" @@ -100,7 +102,7 @@ event_template=$(cat "templates/event_template.json") sleep 0.01 done - if [ $iter -eq $2 ]; then + if [ "$iter" -eq "$2" ]; then exit 0 fi ((iter++)) From 2d777995c9d52d36b06f053cd94b2fa8c15271ca Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Sat, 19 Oct 2024 14:15:27 -0600 Subject: [PATCH 07/15] Added missing UT to complete coverage Signed-off-by: Alfredo Gutierrez --- .../server/grpc/BlockAccessServiceTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java index 2ff8ace12..e57d7bb3d 100644 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java @@ -22,6 +22,8 @@ import static com.hedera.block.server.grpc.BlockAccessService.fromPbjSingleBlockSuccessResponse; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static java.lang.System.Logger.Level.INFO; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -31,6 +33,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.protobuf.Descriptors; +import com.hedera.block.server.Constants; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.mediator.LiveStreamMediator; import com.hedera.block.server.notifier.Notifier; @@ -56,6 +60,7 @@ import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -103,6 +108,39 @@ public void tearDown() { TestUtils.deleteDirectory(testPath.toFile()); } + @Test + void testProto() { + BlockAccessService blockAccessService = + new BlockAccessService( + serviceStatus, blockReader, blockNodeContext.metricsService()); + Descriptors.FileDescriptor fileDescriptor = blockAccessService.proto(); + // Verify the current rpc methods on + Descriptors.ServiceDescriptor blockAccessServiceDescriptor = + fileDescriptor.getServices().stream() + .filter( + service -> + service.getName() + .equals(Constants.SERVICE_NAME_BLOCK_ACCESS)) + .findFirst() + .orElse(null); + + Assertions.assertNotNull( + blockAccessServiceDescriptor, + "Service descriptor not found for: " + Constants.SERVICE_NAME_BLOCK_ACCESS); + assertEquals(1, blockAccessServiceDescriptor.getMethods().size()); + + assertNotNull(blockAccessServiceDescriptor.getName(), blockAccessService.serviceName()); + + // Verify the current rpc methods on the service + Descriptors.MethodDescriptor singleBlockMethod = + blockAccessServiceDescriptor.getMethods().stream() + .filter(method -> method.getName().equals(SINGLE_BLOCK_METHOD_NAME)) + .findFirst() + .orElse(null); + + assertEquals(SINGLE_BLOCK_METHOD_NAME, singleBlockMethod.getName()); + } + @Test void testSingleBlockHappyPath() throws IOException, ParseException { From d59bcafae70ecc0b53fd98b2310992dba794b1cd Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Sun, 20 Oct 2024 00:21:20 -0600 Subject: [PATCH 08/15] - Refactored BlockStreamConfig and extracted BlockGeneratorConfig Properties to its own record class. - Added generator.blockItemsBatchSize that defines the size of the batch to send. Signed-off-by: Alfredo Gutierrez --- .../simulator/BlockStreamSimulatorApp.java | 28 ++- .../config/ConfigInjectionModule.java | 13 ++ .../config/SimulatorConfigExtension.java | 3 +- .../config/data/BlockGeneratorConfig.java | 164 ++++++++++++++++++ .../config/data/BlockStreamConfig.java | 134 +++++++++----- .../BlockAsDirBlockStreamManager.java | 8 +- .../BlockAsFileBlockStreamManager.java | 4 +- .../generator/BlockAsFileLargeDataSets.java | 4 +- .../generator/GeneratorInjectionModule.java | 4 +- .../grpc/PublishStreamGrpcClientImpl.java | 26 ++- .../simulator/BlockStreamSimulatorTest.java | 34 ++-- .../config/ConfigInjectionModuleTest.java | 3 +- .../config/data/BlockStreamConfigTest.java | 122 +++++++------ .../BlockAsDirBlockStreamManagerTest.java | 20 +-- .../BlockAsFileBlockStreamManagerTest.java | 26 ++- .../BlockAsFileLargeDataSetsTest.java | 48 +++-- .../GeneratorInjectionModuleTest.java | 36 ++-- .../grpc/PublishStreamGrpcClientImplTest.java | 16 +- 18 files changed, 486 insertions(+), 207 deletions(-) create mode 100644 simulator/src/main/java/com/hedera/block/simulator/config/data/BlockGeneratorConfig.java diff --git a/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java b/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java index f0e9ce29a..3d25aa1da 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java +++ b/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java @@ -121,18 +121,18 @@ private void constantRateStreaming() int blockItemsStreamed = 0; while (streamBlockItem) { - // get block item - BlockItem blockItem = blockStreamManager.getNextBlockItem(); + // get block + Block block = blockStreamManager.getNextBlock(); - if (blockItem == null) { + if (block == null) { LOGGER.log( System.Logger.Level.INFO, "Block Stream Simulator has reached the end of the block items"); break; } - publishStreamGrpcClient.streamBlockItem(List.of(blockItem)); - blockItemsStreamed++; + streamInBatches(block); + blockItemsStreamed += block.items().size(); Thread.sleep(delayMSBetweenBlockItems, delayNSBetweenBlockItems); @@ -146,6 +146,24 @@ private void constantRateStreaming() } } + private void streamInBatches(Block block) { + final int blockItemsNumberOfBatches = + block.items().size() % blockStreamConfig.blockItemsBatchSize(); + for (int i = 0; i < blockItemsNumberOfBatches; i++) { + + int blockItemsBatchSize = blockStreamConfig.blockItemsBatchSize(); + int startIndexOfBlockItems = i * blockItemsBatchSize; + int endIndexOfBlockItems = (i + 1) * blockItemsBatchSize; + if (endIndexOfBlockItems > block.items().size()) { + endIndexOfBlockItems = block.items().size(); + } + + List blockItems = + block.items().subList(startIndexOfBlockItems, endIndexOfBlockItems); + publishStreamGrpcClient.streamBlockItem(blockItems); + } + } + /** * Returns whether the block stream simulator is running. * diff --git a/simulator/src/main/java/com/hedera/block/simulator/config/ConfigInjectionModule.java b/simulator/src/main/java/com/hedera/block/simulator/config/ConfigInjectionModule.java index ff2d4efb3..b752fa595 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/config/ConfigInjectionModule.java +++ b/simulator/src/main/java/com/hedera/block/simulator/config/ConfigInjectionModule.java @@ -16,6 +16,7 @@ package com.hedera.block.simulator.config; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.data.BlockStreamConfig; import com.hedera.block.simulator.config.data.GrpcConfig; import com.swirlds.config.api.Configuration; @@ -50,4 +51,16 @@ static BlockStreamConfig provideBlockStreamConfig(Configuration configuration) { static GrpcConfig provideGrpcConfig(Configuration configuration) { return configuration.getConfigData(GrpcConfig.class); } + + /** + * Provides the block generator configuration. + * + * @param configuration the configuration to be used by the block generator + * @return the block generator configuration + */ + @Singleton + @Provides + static BlockGeneratorConfig provideBlockGeneratorConfig(Configuration configuration) { + return configuration.getConfigData(BlockGeneratorConfig.class); + } } diff --git a/simulator/src/main/java/com/hedera/block/simulator/config/SimulatorConfigExtension.java b/simulator/src/main/java/com/hedera/block/simulator/config/SimulatorConfigExtension.java index 61f526ca8..b53f581b4 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/config/SimulatorConfigExtension.java +++ b/simulator/src/main/java/com/hedera/block/simulator/config/SimulatorConfigExtension.java @@ -17,6 +17,7 @@ package com.hedera.block.simulator.config; import com.google.auto.service.AutoService; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.data.BlockStreamConfig; import com.hedera.block.simulator.config.data.GrpcConfig; import com.swirlds.config.api.ConfigurationExtension; @@ -35,6 +36,6 @@ public SimulatorConfigExtension() { @NonNull @Override public Set> getConfigDataTypes() { - return Set.of(BlockStreamConfig.class, GrpcConfig.class); + return Set.of(BlockStreamConfig.class, GrpcConfig.class, BlockGeneratorConfig.class); } } diff --git a/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockGeneratorConfig.java b/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockGeneratorConfig.java new file mode 100644 index 000000000..23378eeed --- /dev/null +++ b/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockGeneratorConfig.java @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.simulator.config.data; + +import com.hedera.block.simulator.config.types.GenerationMode; +import com.swirlds.config.api.ConfigData; +import com.swirlds.config.api.ConfigProperty; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Defines the configuration for the BlockStreamManager (Generator) of blocks in the Hedera Block Simulator. + * + * @param generationMode the mode of block generation (e.g., directory-based) + * @param folderRootPath the root path of the folder containing block files + * @param managerImplementation the implementation class name of the block stream manager + * @param paddedLength the length to which block identifiers are padded + * @param fileExtension the file extension used for block files + */ +@ConfigData("generator") +public record BlockGeneratorConfig( + @ConfigProperty(defaultValue = "DIR") GenerationMode generationMode, + @ConfigProperty(defaultValue = "") String folderRootPath, + @ConfigProperty(defaultValue = "BlockAsFileBlockStreamManager") + String managerImplementation, + @ConfigProperty(defaultValue = "36") int paddedLength, + @ConfigProperty(defaultValue = ".blk.gz") String fileExtension) { + + /** + * Constructs a new {@code BlockGeneratorConfig} instance with validation. + * + * @throws IllegalArgumentException if the folder root path is not absolute or the folder does not exist + */ + public BlockGeneratorConfig { + // Verify folderRootPath property + Path path = Path.of(folderRootPath); + + // If folderRootPath is empty, set it to the default data directory + if (folderRootPath.isEmpty()) { + path = Paths.get("").toAbsolutePath().resolve("src/main/resources/block-0.0.3"); + } + // Check if absolute + if (!path.isAbsolute()) { + throw new IllegalArgumentException(folderRootPath + " Root path must be absolute"); + } + // Check if the folder exists + if (Files.notExists(path) && generationMode == GenerationMode.DIR) { + throw new IllegalArgumentException("Folder does not exist: " + path); + } + + folderRootPath = path.toString(); + } + + /** + * Creates a new {@link Builder} for constructing a {@code BlockGeneratorConfig}. + * + * @return a new {@code Builder} instance + */ + public static Builder builder() { + return new Builder(); + } + + /** + * A builder class for creating instances of {@link BlockGeneratorConfig}. + */ + public static class Builder { + private GenerationMode generationMode = GenerationMode.DIR; + private String folderRootPath = ""; + private String managerImplementation = "BlockAsFileBlockStreamManager"; + private int paddedLength = 36; + private String fileExtension = ".blk.gz"; + + /** + * Creates a new instance of the {@code Builder} class with default configuration values. + */ + public Builder() { + // Default constructor + } + + /** + * Sets the generation mode for block generation. + * + * @param generationMode the {@link GenerationMode} to use + * @return this {@code Builder} instance + */ + public Builder generationMode(GenerationMode generationMode) { + this.generationMode = generationMode; + return this; + } + + /** + * Sets the root path of the folder containing block files. + * + * @param folderRootPath the absolute path to the folder + * @return this {@code Builder} instance + */ + public Builder folderRootPath(String folderRootPath) { + this.folderRootPath = folderRootPath; + return this; + } + + /** + * Sets the implementation class name of the block stream manager. + * + * @param managerImplementation the class name of the manager implementation + * @return this {@code Builder} instance + */ + public Builder managerImplementation(String managerImplementation) { + this.managerImplementation = managerImplementation; + return this; + } + + /** + * Sets the length to which block identifiers are padded. + * + * @param paddedLength the padded length + * @return this {@code Builder} instance + */ + public Builder paddedLength(int paddedLength) { + this.paddedLength = paddedLength; + return this; + } + + /** + * Sets the file extension used for block files. + * + * @param fileExtension the file extension (e.g., ".blk.gz") + * @return this {@code Builder} instance + */ + public Builder fileExtension(String fileExtension) { + this.fileExtension = fileExtension; + return this; + } + + /** + * Builds a new {@link BlockGeneratorConfig} instance with the configured values. + * + * @return a new {@code BlockGeneratorConfig} + */ + public BlockGeneratorConfig build() { + return new BlockGeneratorConfig( + generationMode, + folderRootPath, + managerImplementation, + paddedLength, + fileExtension); + } + } +} diff --git a/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockStreamConfig.java b/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockStreamConfig.java index ecc6d50aa..a5a6a657c 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockStreamConfig.java +++ b/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockStreamConfig.java @@ -16,64 +16,120 @@ package com.hedera.block.simulator.config.data; -import com.hedera.block.simulator.config.types.GenerationMode; import com.hedera.block.simulator.config.types.StreamingMode; import com.swirlds.config.api.ConfigData; import com.swirlds.config.api.ConfigProperty; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; /** - * The BlockStreamConfig class defines the configuration data for the block stream. + * Defines the configuration data for the block stream in the Hedera Block Simulator. * - * @param generationMode the mode of generation for the block stream - * @param folderRootPath the root path of the folder containing the block stream - * @param delayBetweenBlockItems the delay between block items - * @param managerImplementation the implementation of the block stream manager - * @param maxBlockItemsToStream the maximum number of block items to stream - * @param paddedLength the padded length of 0 the block file format - * @param fileExtension the file extension of the block file format - * @param streamingMode the mode of streaming for the block stream - * @param millisecondsPerBlock the milliseconds per block + * @param delayBetweenBlockItems the delay in microseconds between streaming each block item + * @param maxBlockItemsToStream the maximum number of block items to stream before stopping + * @param streamingMode the mode of streaming for the block stream (e.g., time-based, count-based) + * @param millisecondsPerBlock the duration in milliseconds for each block when using time-based streaming + * @param blockItemsBatchSize the number of block items to stream in each batch */ @ConfigData("blockStream") public record BlockStreamConfig( - @ConfigProperty(defaultValue = "DIR") GenerationMode generationMode, - @ConfigProperty(defaultValue = "") String folderRootPath, @ConfigProperty(defaultValue = "1_500_000") int delayBetweenBlockItems, - @ConfigProperty(defaultValue = "BlockAsFileBlockStreamManager") - String managerImplementation, @ConfigProperty(defaultValue = "10_000") int maxBlockItemsToStream, - @ConfigProperty(defaultValue = "36") int paddedLength, - @ConfigProperty(defaultValue = ".blk.gz") String fileExtension, @ConfigProperty(defaultValue = "MILLIS_PER_BLOCK") StreamingMode streamingMode, - @ConfigProperty(defaultValue = "1000") int millisecondsPerBlock) { + @ConfigProperty(defaultValue = "1000") int millisecondsPerBlock, + @ConfigProperty(defaultValue = "1000") int blockItemsBatchSize) { /** - * Constructor to set the default root path if not provided, it will be set to the data - * directory in the current working directory + * Creates a new {@link Builder} instance for constructing a {@code BlockStreamConfig}. + * + * @return a new {@code Builder} */ - public BlockStreamConfig { - // verify rootPath prop - Path path = Path.of(folderRootPath); + public static Builder builder() { + return new Builder(); + } + + /** + * A builder for creating instances of {@link BlockStreamConfig}. + */ + public static class Builder { + private int delayBetweenBlockItems = 1_500_000; + private int maxBlockItemsToStream = 10_000; + private StreamingMode streamingMode = StreamingMode.MILLIS_PER_BLOCK; + private int millisecondsPerBlock = 1000; + private int blockItemsBatchSize = 1000; - // if rootPath is empty, set it to the default data directory - if (folderRootPath.isEmpty()) { - path = - Paths.get(folderRootPath) - .toAbsolutePath() - .resolve("src/main/resources/block-0.0.3"); + /** + * Creates a new instance of the {@code Builder} class with default configuration values. + */ + public Builder() { + // Default constructor } - // Check if absolute - if (!path.isAbsolute()) { - throw new IllegalArgumentException(folderRootPath + " Root path must be absolute"); + + /** + * Sets the delay between streaming each block item. + * + * @param delayBetweenBlockItems the delay in microseconds + * @return this {@code Builder} instance + */ + public Builder delayBetweenBlockItems(int delayBetweenBlockItems) { + this.delayBetweenBlockItems = delayBetweenBlockItems; + return this; } - // Check if the folder exists - if (Files.notExists(path) && generationMode == GenerationMode.DIR) { - throw new IllegalArgumentException("Folder does not exist: " + path); + + /** + * Sets the maximum number of block items to stream. + * + * @param maxBlockItemsToStream the maximum number of items + * @return this {@code Builder} instance + */ + public Builder maxBlockItemsToStream(int maxBlockItemsToStream) { + this.maxBlockItemsToStream = maxBlockItemsToStream; + return this; } - folderRootPath = path.toString(); + /** + * Sets the streaming mode for the block stream. + * + * @param streamingMode the {@link StreamingMode} to use + * @return this {@code Builder} instance + */ + public Builder streamingMode(StreamingMode streamingMode) { + this.streamingMode = streamingMode; + return this; + } + + /** + * Sets the duration for each block when using time-based streaming. + * + * @param millisecondsPerBlock the duration in milliseconds + * @return this {@code Builder} instance + */ + public Builder millisecondsPerBlock(int millisecondsPerBlock) { + this.millisecondsPerBlock = millisecondsPerBlock; + return this; + } + + /** + * Sets the number of block items to stream in each batch. + * + * @param blockItemsBatchSize the batch size + * @return this {@code Builder} instance + */ + public Builder blockItemsBatchSize(int blockItemsBatchSize) { + this.blockItemsBatchSize = blockItemsBatchSize; + return this; + } + + /** + * Builds a new {@link BlockStreamConfig} instance with the configured values. + * + * @return a new {@code BlockStreamConfig} + */ + public BlockStreamConfig build() { + return new BlockStreamConfig( + delayBetweenBlockItems, + maxBlockItemsToStream, + streamingMode, + millisecondsPerBlock, + blockItemsBatchSize); + } } } diff --git a/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsDirBlockStreamManager.java b/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsDirBlockStreamManager.java index ec1d9e9c5..3b3d7892e 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsDirBlockStreamManager.java +++ b/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsDirBlockStreamManager.java @@ -21,7 +21,7 @@ import static java.lang.System.Logger.Level.ERROR; import static java.lang.System.Logger.Level.INFO; -import com.hedera.block.simulator.config.data.BlockStreamConfig; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.types.GenerationMode; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; @@ -57,11 +57,11 @@ public class BlockAsDirBlockStreamManager implements BlockStreamManager { * Constructor to initialize the BlockAsDirBlockStreamManager with the block stream * configuration. * - * @param blockStreamConfig the block stream configuration + * @param blockGeneratorConfig the block stream configuration */ @Inject - public BlockAsDirBlockStreamManager(@NonNull BlockStreamConfig blockStreamConfig) { - this.rootFolder = blockStreamConfig.folderRootPath(); + public BlockAsDirBlockStreamManager(@NonNull BlockGeneratorConfig blockGeneratorConfig) { + this.rootFolder = blockGeneratorConfig.folderRootPath(); try { this.loadBlocks(); } catch (IOException | ParseException | IllegalArgumentException e) { diff --git a/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManager.java b/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManager.java index 56dba5765..c6a0e0524 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManager.java +++ b/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManager.java @@ -21,7 +21,7 @@ import static java.lang.System.Logger.Level.ERROR; import static java.lang.System.Logger.Level.INFO; -import com.hedera.block.simulator.config.data.BlockStreamConfig; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.types.GenerationMode; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; @@ -56,7 +56,7 @@ public class BlockAsFileBlockStreamManager implements BlockStreamManager { * @param blockStreamConfig the block stream config */ @Inject - public BlockAsFileBlockStreamManager(@NonNull BlockStreamConfig blockStreamConfig) { + public BlockAsFileBlockStreamManager(@NonNull BlockGeneratorConfig blockStreamConfig) { this.rootFolder = blockStreamConfig.folderRootPath(); try { this.loadBlocks(); diff --git a/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileLargeDataSets.java b/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileLargeDataSets.java index 0e7c493c8..8fecfb772 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileLargeDataSets.java +++ b/simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileLargeDataSets.java @@ -19,7 +19,7 @@ import static com.hedera.block.simulator.generator.Utils.readFileBytes; import static java.lang.System.Logger.Level.INFO; -import com.hedera.block.simulator.config.data.BlockStreamConfig; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.types.GenerationMode; import com.hedera.block.simulator.exception.BlockSimulatorParsingException; import com.hedera.hapi.block.stream.Block; @@ -49,7 +49,7 @@ public class BlockAsFileLargeDataSets implements BlockStreamManager { * @param config the block stream configuration */ @Inject - public BlockAsFileLargeDataSets(@NonNull BlockStreamConfig config) { + public BlockAsFileLargeDataSets(@NonNull BlockGeneratorConfig config) { this.blockstreamPath = config.folderRootPath(); this.formatString = "%0" + config.paddedLength() + "d" + config.fileExtension(); } diff --git a/simulator/src/main/java/com/hedera/block/simulator/generator/GeneratorInjectionModule.java b/simulator/src/main/java/com/hedera/block/simulator/generator/GeneratorInjectionModule.java index f77f816ae..1b23a1ccb 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/generator/GeneratorInjectionModule.java +++ b/simulator/src/main/java/com/hedera/block/simulator/generator/GeneratorInjectionModule.java @@ -16,7 +16,7 @@ package com.hedera.block.simulator.generator; -import com.hedera.block.simulator.config.data.BlockStreamConfig; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import dagger.Module; import dagger.Provides; import javax.inject.Singleton; @@ -35,7 +35,7 @@ public interface GeneratorInjectionModule { */ @Singleton @Provides - static BlockStreamManager providesBlockStreamManager(BlockStreamConfig config) { + static BlockStreamManager providesBlockStreamManager(BlockGeneratorConfig config) { if ("BlockAsDirBlockStreamManager".equalsIgnoreCase(config.managerImplementation())) { return new BlockAsDirBlockStreamManager(config); diff --git a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java index 41d8e27d8..0514865e9 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java +++ b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java @@ -17,6 +17,7 @@ package com.hedera.block.simulator.grpc; import com.hedera.block.simulator.Translator; +import com.hedera.block.simulator.config.data.BlockStreamConfig; import com.hedera.block.simulator.config.data.GrpcConfig; import com.hedera.hapi.block.protoc.BlockStreamServiceGrpc; import com.hedera.hapi.block.protoc.PublishStreamRequest; @@ -37,14 +38,17 @@ public class PublishStreamGrpcClientImpl implements PublishStreamGrpcClient { private final BlockStreamServiceGrpc.BlockStreamServiceStub stub; private final StreamObserver requestStreamObserver; + private final BlockStreamConfig blockStreamConfig; /** * Creates a new PublishStreamGrpcClientImpl instance. * * @param grpcConfig the gRPC configuration + * @param blockStreamConfig the block stream configuration */ @Inject - public PublishStreamGrpcClientImpl(@NonNull GrpcConfig grpcConfig) { + public PublishStreamGrpcClientImpl( + @NonNull GrpcConfig grpcConfig, @NonNull BlockStreamConfig blockStreamConfig) { ManagedChannel channel = ManagedChannelBuilder.forAddress(grpcConfig.serverAddress(), grpcConfig.port()) .usePlaintext() @@ -52,6 +56,7 @@ public PublishStreamGrpcClientImpl(@NonNull GrpcConfig grpcConfig) { stub = BlockStreamServiceGrpc.newStub(channel); PublishStreamObserver publishStreamObserver = new PublishStreamObserver(); requestStreamObserver = stub.publishBlockStream(publishStreamObserver); + this.blockStreamConfig = blockStreamConfig; } /** @@ -83,8 +88,23 @@ public boolean streamBlock(Block block) { blockItemsProtoc.add(Translator.fromPbj(blockItem)); } - requestStreamObserver.onNext( - PublishStreamRequest.newBuilder().addAllBlockItems(blockItemsProtoc).build()); + final int blockItemsNumberOfBatches = + block.items().size() % blockStreamConfig.blockItemsBatchSize(); + for (int i = 0; i < blockItemsNumberOfBatches; i++) { + + int blockItemsBatchSize = blockStreamConfig.blockItemsBatchSize(); + int startIndexOfBlockItems = i * blockItemsBatchSize; + int endIndexOfBlockItems = (i + 1) * blockItemsBatchSize; + if (endIndexOfBlockItems > block.items().size()) { + endIndexOfBlockItems = block.items().size(); + } + + List streamingBatch = + blockItemsProtoc.subList(startIndexOfBlockItems, endIndexOfBlockItems); + + requestStreamObserver.onNext( + PublishStreamRequest.newBuilder().addAllBlockItems(streamingBatch).build()); + } return true; } diff --git a/simulator/src/test/java/com/hedera/block/simulator/BlockStreamSimulatorTest.java b/simulator/src/test/java/com/hedera/block/simulator/BlockStreamSimulatorTest.java index f08338226..57e997cab 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/BlockStreamSimulatorTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/BlockStreamSimulatorTest.java @@ -80,23 +80,33 @@ void start_logsStartedMessage() } @Test - void start_exitByBlockNull() + void start_constantRateStreaming() throws InterruptedException, BlockSimulatorParsingException, IOException { + BlockItem blockItem = + BlockItem.newBuilder() + .blockHeader(BlockHeader.newBuilder().number(1L).build()) + .build(); + + Block block1 = Block.newBuilder().items(blockItem).build(); + Block block2 = Block.newBuilder().items(blockItem, blockItem, blockItem).build(); + BlockStreamManager blockStreamManager = Mockito.mock(BlockStreamManager.class); - when(blockStreamManager.getNextBlockItem()).thenReturn(BlockItem.newBuilder().build()); + when(blockStreamManager.getNextBlock()).thenReturn(block1, block2, null); Configuration configuration = TestUtils.getTestConfiguration( Map.of( "blockStream.maxBlockItemsToStream", "2", - "blockStream.BlockAsFileBlockStreamManager", + "generator.managerImplementation", "BlockAsFileLargeDataSets", - "blockStream.rootPath", + "generator.rootPath", getAbsoluteFolder("src/test/resources/block-0.0.3-blk/"), "blockStream.streamingMode", - "CONSTANT_RATE")); + "CONSTANT_RATE", + "blockStream.blockItemsBatchSize", + "2")); BlockStreamSimulatorApp blockStreamSimulator = new BlockStreamSimulatorApp( @@ -116,7 +126,7 @@ void stop_doesNotThrowException() { } @Test - void start_millisPerSecond() + void start_millisPerBlockStreaming() throws InterruptedException, IOException, BlockSimulatorParsingException { BlockStreamManager blockStreamManager = Mockito.mock(BlockStreamManager.class); BlockItem blockItem = @@ -131,9 +141,9 @@ void start_millisPerSecond() Map.of( "blockStream.maxBlockItemsToStream", "2", - "blockStream.BlockAsFileBlockStreamManager", + "generator.managerImplementation", "BlockAsFileLargeDataSets", - "blockStream.rootPath", + "generator.rootPath", getAbsoluteFolder("src/test/resources/block-0.0.3-blk/"), "blockStream.streamingMode", "MILLIS_PER_BLOCK")); @@ -176,14 +186,16 @@ void start_millisPerSecond_streamingLagVerifyWarnLog() Map.of( "blockStream.maxBlockItemsToStream", "2", - "blockStream.BlockAsFileBlockStreamManager", + "generator.managerImplementation", "BlockAsFileLargeDataSets", - "blockStream.rootPath", + "generator.rootPath", getAbsoluteFolder("src/test/resources/block-0.0.3-blk/"), "blockStream.streamingMode", "MILLIS_PER_BLOCK", "blockStream.millisecondsPerBlock", - "10")); + "10", + "blockStream.blockItemsBatchSize", + "1")); BlockStreamSimulatorApp blockStreamSimulator = new BlockStreamSimulatorApp( diff --git a/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java b/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java index 4b8f2dbae..2fc28118c 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java @@ -18,7 +18,6 @@ import com.hedera.block.simulator.config.data.BlockStreamConfig; import com.hedera.block.simulator.config.data.GrpcConfig; -import com.hedera.block.simulator.config.types.GenerationMode; import com.swirlds.config.api.Configuration; import com.swirlds.config.api.ConfigurationBuilder; import com.swirlds.config.extensions.sources.ClasspathFileConfigSource; @@ -52,7 +51,7 @@ void provideBlockStreamConfig() { ConfigInjectionModule.provideBlockStreamConfig(configuration); Assertions.assertNotNull(blockStreamConfig); - Assertions.assertEquals(GenerationMode.DIR, blockStreamConfig.generationMode()); + Assertions.assertEquals(1000, blockStreamConfig.blockItemsBatchSize()); } @Test diff --git a/simulator/src/test/java/com/hedera/block/simulator/config/data/BlockStreamConfigTest.java b/simulator/src/test/java/com/hedera/block/simulator/config/data/BlockStreamConfigTest.java index 7f7bf49fc..821735e04 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/config/data/BlockStreamConfigTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/config/data/BlockStreamConfigTest.java @@ -27,18 +27,47 @@ class BlockStreamConfigTest { - private final int delayBetweenBlockItems = 1_500_000; - private final String blockStreamManagerImplementation = "BlockAsFileBlockStreamManager"; - private final int maxBlockItemsToStream = 10_000; - private final int paddedLength = 36; - private final String fileExtension = ".blk"; - private final StreamingMode streamingMode = StreamingMode.CONSTANT_RATE; - private final int millisPerBlock = 1000; - private String getAbsoluteFolder(String relativePath) { return Paths.get(relativePath).toAbsolutePath().toString(); } + private BlockStreamConfig.Builder getBlockStreamConfigBuilder() { + final StreamingMode streamingMode = StreamingMode.CONSTANT_RATE; + final int delayBetweenBlockItems = 1_500_000; + final int maxBlockItemsToStream = 10_000; + final int millisPerBlock = 1000; + final int blockItemsBatchSize = 1000; + + return BlockStreamConfig.builder() + .delayBetweenBlockItems(delayBetweenBlockItems) + .maxBlockItemsToStream(maxBlockItemsToStream) + .streamingMode(streamingMode) + .millisecondsPerBlock(millisPerBlock) + .blockItemsBatchSize(blockItemsBatchSize); + } + + private BlockGeneratorConfig.Builder getBlockGeneratorConfigBuilder() { + String folderRootPath = "src/main/resources/block-0.0.3/"; + GenerationMode generationMode = GenerationMode.DIR; + + String blockStreamManagerImplementation = "BlockAsFileBlockStreamManager"; + int paddedLength = 36; + String fileExtension = ".blk"; + return BlockGeneratorConfig.builder() + .generationMode(generationMode) + .folderRootPath(folderRootPath) + .managerImplementation(blockStreamManagerImplementation) + .paddedLength(paddedLength) + .fileExtension(fileExtension); + } + + @Test + void testStreamConfigBuilder() { + BlockStreamConfig config = getBlockStreamConfigBuilder().build(); + // assert + assertEquals(StreamingMode.CONSTANT_RATE, config.streamingMode()); + } + @Test void testValidAbsolutePath() { // Setup valid folder path and generation mode @@ -51,17 +80,11 @@ void testValidAbsolutePath() { assertTrue(Files.exists(path), "The folder must exist for this test."); // No exception should be thrown - BlockStreamConfig config = - new BlockStreamConfig( - generationMode, - folderRootPath, - delayBetweenBlockItems, - blockStreamManagerImplementation, - maxBlockItemsToStream, - paddedLength, - fileExtension, - streamingMode, - millisPerBlock); + BlockGeneratorConfig config = + getBlockGeneratorConfigBuilder() + .folderRootPath(folderRootPath) + .generationMode(generationMode) + .build(); assertEquals(folderRootPath, config.folderRootPath()); assertEquals(GenerationMode.DIR, config.generationMode()); @@ -72,19 +95,12 @@ void testEmptyFolderRootPath() { // Setup empty folder root path and generation mode String folderRootPath = ""; GenerationMode generationMode = GenerationMode.DIR; + BlockGeneratorConfig.Builder builder = + getBlockGeneratorConfigBuilder() + .folderRootPath(folderRootPath) + .generationMode(generationMode); - // No exception should be thrown, and the default folder should be used - BlockStreamConfig config = - new BlockStreamConfig( - generationMode, - folderRootPath, - delayBetweenBlockItems, - blockStreamManagerImplementation, - maxBlockItemsToStream, - paddedLength, - fileExtension, - streamingMode, - millisPerBlock); + BlockGeneratorConfig config = builder.build(); // Verify that the path is set to the default Path expectedPath = Paths.get("src/main/resources/block-0.0.3/").toAbsolutePath(); @@ -103,16 +119,10 @@ void testRelativeFolderPathThrowsException() { assertThrows( IllegalArgumentException.class, () -> - new BlockStreamConfig( - generationMode, - relativeFolderPath, - delayBetweenBlockItems, - blockStreamManagerImplementation, - maxBlockItemsToStream, - paddedLength, - fileExtension, - streamingMode, - millisPerBlock)); + getBlockGeneratorConfigBuilder() + .folderRootPath(relativeFolderPath) + .generationMode(generationMode) + .build()); // Verify the exception message assertEquals(relativeFolderPath + " Root path must be absolute", exception.getMessage()); @@ -133,16 +143,10 @@ void testNonExistentFolderThrowsException() { assertThrows( IllegalArgumentException.class, () -> - new BlockStreamConfig( - generationMode, - folderRootPath, - delayBetweenBlockItems, - blockStreamManagerImplementation, - maxBlockItemsToStream, - paddedLength, - fileExtension, - streamingMode, - millisPerBlock)); + getBlockGeneratorConfigBuilder() + .folderRootPath(folderRootPath) + .generationMode(generationMode) + .build()); // Verify the exception message assertEquals("Folder does not exist: " + path, exception.getMessage()); @@ -155,17 +159,11 @@ void testGenerationModeNonDirDoesNotCheckFolderExistence() { GenerationMode generationMode = GenerationMode.ADHOC; // No exception should be thrown because generation mode is not DIR - BlockStreamConfig config = - new BlockStreamConfig( - generationMode, - folderRootPath, - delayBetweenBlockItems, - blockStreamManagerImplementation, - maxBlockItemsToStream, - paddedLength, - fileExtension, - streamingMode, - millisPerBlock); + BlockGeneratorConfig config = + getBlockGeneratorConfigBuilder() + .folderRootPath(folderRootPath) + .generationMode(generationMode) + .build(); // Verify that the configuration was created successfully assertEquals(folderRootPath, config.folderRootPath()); diff --git a/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsDirBlockStreamManagerTest.java b/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsDirBlockStreamManagerTest.java index be19eb0ad..09f418c4f 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsDirBlockStreamManagerTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsDirBlockStreamManagerTest.java @@ -20,9 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.hedera.block.simulator.config.data.BlockStreamConfig; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.types.GenerationMode; -import com.hedera.block.simulator.config.types.StreamingMode; import com.hedera.block.simulator.exception.BlockSimulatorParsingException; import java.io.IOException; import java.nio.file.Paths; @@ -75,17 +74,10 @@ void BlockAsFileBlockStreamManagerInvalidRootPath() { } private BlockStreamManager getBlockAsDirBlockStreamManager(String rootFolder) { - BlockStreamConfig blockStreamConfig = - new BlockStreamConfig( - GenerationMode.DIR, - rootFolder, - 1_500_000, - "BlockAsDirBlockStreamManager", - 10_000, - 36, - ".blk", - StreamingMode.CONSTANT_RATE, - 1000); - return new BlockAsDirBlockStreamManager(blockStreamConfig); + final BlockGeneratorConfig blockGeneratorConfig = + new BlockGeneratorConfig( + GenerationMode.DIR, rootFolder, "BlockAsDirBlockStreamManager", 36, ".blk"); + + return new BlockAsDirBlockStreamManager(blockGeneratorConfig); } } diff --git a/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManagerTest.java b/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManagerTest.java index b5a950239..882db4c0d 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManagerTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManagerTest.java @@ -18,9 +18,8 @@ import static org.junit.jupiter.api.Assertions.*; -import com.hedera.block.simulator.config.data.BlockStreamConfig; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.types.GenerationMode; -import com.hedera.block.simulator.config.types.StreamingMode; import com.hedera.block.simulator.exception.BlockSimulatorParsingException; import java.io.IOException; import java.nio.file.Paths; @@ -77,17 +76,16 @@ void BlockAsFileBlockStreamManagerInvalidRootPath() { } private BlockAsFileBlockStreamManager getBlockAsFileBlockStreamManager(String rootFolder) { - BlockStreamConfig blockStreamConfig = - new BlockStreamConfig( - GenerationMode.DIR, - rootFolder, - 1_500_000, - "BlockAsFileBlockStreamManager", - 10_000, - 36, - ".blk", - StreamingMode.CONSTANT_RATE, - 1000); - return new BlockAsFileBlockStreamManager(blockStreamConfig); + + BlockGeneratorConfig blockGeneratorConfig = + BlockGeneratorConfig.builder() + .generationMode(GenerationMode.DIR) + .folderRootPath(rootFolder) + .managerImplementation("BlockAsFileBlockStreamManager") + .paddedLength(36) + .fileExtension(".blk") + .build(); + + return new BlockAsFileBlockStreamManager(blockGeneratorConfig); } } diff --git a/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsFileLargeDataSetsTest.java b/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsFileLargeDataSetsTest.java index 4aeb049e8..e03ad357f 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsFileLargeDataSetsTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/generator/BlockAsFileLargeDataSetsTest.java @@ -18,9 +18,8 @@ import static org.junit.jupiter.api.Assertions.*; -import com.hedera.block.simulator.config.data.BlockStreamConfig; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.types.GenerationMode; -import com.hedera.block.simulator.config.types.StreamingMode; import com.hedera.block.simulator.exception.BlockSimulatorParsingException; import com.hedera.hapi.block.stream.BlockItem; import java.io.File; @@ -96,19 +95,17 @@ void gettingNextBlockItemThrowsParsingException(@TempDir Path tempDir) throws IO byte[] invalidData = "invalid block data".getBytes(); Files.write(currentBlockFilePath, invalidData); - BlockStreamConfig blockStreamConfig = - new BlockStreamConfig( - GenerationMode.DIR, - blockDirPath.toString(), - 1_500_000, - "BlockAsFileBlockStreamManager", - 10_000, - 36, - ".blk", - StreamingMode.CONSTANT_RATE, - 1000); + final BlockGeneratorConfig blockGeneratorConfig = + BlockGeneratorConfig.builder() + .generationMode(GenerationMode.DIR) + .folderRootPath(blockDirPath.toString()) + .managerImplementation("BlockAsFileBlockStreamManager") + .paddedLength(36) + .fileExtension(".blk") + .build(); + BlockAsFileLargeDataSets blockStreamManager = - new BlockAsFileLargeDataSets(blockStreamConfig); + new BlockAsFileLargeDataSets(blockGeneratorConfig); assertThrows( BlockSimulatorParsingException.class, @@ -118,18 +115,17 @@ void gettingNextBlockItemThrowsParsingException(@TempDir Path tempDir) throws IO private BlockAsFileLargeDataSets getBlockAsFileLargeDatasetsBlockStreamManager( String rootFolder) { - BlockStreamConfig blockStreamConfig = - new BlockStreamConfig( - GenerationMode.DIR, - rootFolder, - 1_500_000, - "BlockAsFileBlockStreamManager", - 10_000, - 36, - ".blk", - StreamingMode.CONSTANT_RATE, - 1000); - return new BlockAsFileLargeDataSets(blockStreamConfig); + + final BlockGeneratorConfig blockGeneratorConfig = + BlockGeneratorConfig.builder() + .generationMode(GenerationMode.DIR) + .folderRootPath(rootFolder) + .managerImplementation("BlockAsFileBlockStreamManager") + .paddedLength(36) + .fileExtension(".blk") + .build(); + + return new BlockAsFileLargeDataSets(blockGeneratorConfig); } private static String getAbsoluteFolder(String relativePath) { diff --git a/simulator/src/test/java/com/hedera/block/simulator/generator/GeneratorInjectionModuleTest.java b/simulator/src/test/java/com/hedera/block/simulator/generator/GeneratorInjectionModuleTest.java index cf6a069f5..ad5654d86 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/generator/GeneratorInjectionModuleTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/generator/GeneratorInjectionModuleTest.java @@ -19,7 +19,7 @@ import static org.junit.jupiter.api.Assertions.*; import com.hedera.block.simulator.TestUtils; -import com.hedera.block.simulator.config.data.BlockStreamConfig; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import java.io.IOException; import java.util.Map; import org.junit.jupiter.api.Test; @@ -29,15 +29,15 @@ class GeneratorInjectionModuleTest { @Test void providesBlockStreamManager_AsFileLargeDataSets() throws IOException { - BlockStreamConfig blockStreamConfig = + BlockGeneratorConfig blockGeneratorConfig = TestUtils.getTestConfiguration( Map.of( - "blockStream.managerImplementation", + "generator.managerImplementation", "BlockAsFileLargeDataSets")) - .getConfigData(BlockStreamConfig.class); + .getConfigData(BlockGeneratorConfig.class); BlockStreamManager blockStreamManager = - GeneratorInjectionModule.providesBlockStreamManager(blockStreamConfig); + GeneratorInjectionModule.providesBlockStreamManager(blockGeneratorConfig); assertEquals( blockStreamManager.getClass().getName(), BlockAsFileLargeDataSets.class.getName()); @@ -45,11 +45,11 @@ void providesBlockStreamManager_AsFileLargeDataSets() throws IOException { @Test void providesBlockStreamManager_AsFile() throws IOException { - BlockStreamConfig blockStreamConfig = - TestUtils.getTestConfiguration().getConfigData(BlockStreamConfig.class); + BlockGeneratorConfig blockGeneratorConfig = + TestUtils.getTestConfiguration().getConfigData(BlockGeneratorConfig.class); BlockStreamManager blockStreamManager = - GeneratorInjectionModule.providesBlockStreamManager(blockStreamConfig); + GeneratorInjectionModule.providesBlockStreamManager(blockGeneratorConfig); assertEquals( blockStreamManager.getClass().getName(), @@ -58,29 +58,29 @@ void providesBlockStreamManager_AsFile() throws IOException { @Test void providesBlockStreamManager_AsDir() throws IOException { - BlockStreamConfig blockStreamConfig = + BlockGeneratorConfig blockGeneratorConfig = TestUtils.getTestConfiguration( Map.of( - "blockStream.managerImplementation", + "generator.managerImplementation", "BlockAsDirBlockStreamManager")) - .getConfigData(BlockStreamConfig.class); + .getConfigData(BlockGeneratorConfig.class); BlockStreamManager blockStreamManager = - GeneratorInjectionModule.providesBlockStreamManager(blockStreamConfig); + GeneratorInjectionModule.providesBlockStreamManager(blockGeneratorConfig); assertEquals( - blockStreamManager.getClass().getName(), - BlockAsDirBlockStreamManager.class.getName()); + BlockAsDirBlockStreamManager.class.getName(), + blockStreamManager.getClass().getName()); } @Test void providesBlockStreamManager_default() throws IOException { - BlockStreamConfig blockStreamConfig = - TestUtils.getTestConfiguration(Map.of("blockStream.managerImplementation", "")) - .getConfigData(BlockStreamConfig.class); + BlockGeneratorConfig blockGeneratorConfig = + TestUtils.getTestConfiguration(Map.of("generator.managerImplementation", "")) + .getConfigData(BlockGeneratorConfig.class); BlockStreamManager blockStreamManager = - GeneratorInjectionModule.providesBlockStreamManager(blockStreamConfig); + GeneratorInjectionModule.providesBlockStreamManager(blockGeneratorConfig); assertEquals( blockStreamManager.getClass().getName(), diff --git a/simulator/src/test/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImplTest.java b/simulator/src/test/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImplTest.java index 69a2b0ea9..a60aed355 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImplTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImplTest.java @@ -19,11 +19,13 @@ import static org.junit.jupiter.api.Assertions.*; import com.hedera.block.simulator.TestUtils; +import com.hedera.block.simulator.config.data.BlockStreamConfig; import com.hedera.block.simulator.config.data.GrpcConfig; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; import java.io.IOException; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -31,11 +33,15 @@ class PublishStreamGrpcClientImplTest { GrpcConfig grpcConfig; + BlockStreamConfig blockStreamConfig; @BeforeEach void setUp() throws IOException { grpcConfig = TestUtils.getTestConfiguration().getConfigData(GrpcConfig.class); + blockStreamConfig = + TestUtils.getTestConfiguration(Map.of("blockStream.blockItemsBatchSize", "2")) + .getConfigData(BlockStreamConfig.class); } @AfterEach @@ -45,7 +51,7 @@ void tearDown() {} void streamBlockItem() { BlockItem blockItem = BlockItem.newBuilder().build(); PublishStreamGrpcClientImpl publishStreamGrpcClient = - new PublishStreamGrpcClientImpl(grpcConfig); + new PublishStreamGrpcClientImpl(grpcConfig, blockStreamConfig); boolean result = publishStreamGrpcClient.streamBlockItem(List.of(blockItem)); assertTrue(result); } @@ -55,9 +61,15 @@ void streamBlock() { BlockItem blockItem = BlockItem.newBuilder().build(); Block block = Block.newBuilder().items(blockItem).build(); + Block block1 = Block.newBuilder().items(blockItem, blockItem, blockItem).build(); + PublishStreamGrpcClientImpl publishStreamGrpcClient = - new PublishStreamGrpcClientImpl(grpcConfig); + new PublishStreamGrpcClientImpl(grpcConfig, blockStreamConfig); + boolean result = publishStreamGrpcClient.streamBlock(block); assertTrue(result); + + boolean result1 = publishStreamGrpcClient.streamBlock(block1); + assertTrue(result1); } } From bf75e9c63926d0b997938192fcd64634fcfde65c Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Sun, 20 Oct 2024 22:31:23 -0600 Subject: [PATCH 09/15] - More fixes to the simulator and it's tests. - Updated the README.md and app.properties example file. - Simplify some logic Signed-off-by: Alfredo Gutierrez --- simulator/README.md | 16 +++++++++--- .../simulator/BlockStreamSimulatorApp.java | 26 +++---------------- .../config/data/BlockStreamConfig.java | 2 +- .../grpc/PublishStreamGrpcClientImpl.java | 4 +-- .../simulator/grpc/PublishStreamObserver.java | 1 + simulator/src/main/resources/app.properties | 9 ++++--- .../simulator/BlockStreamSimulatorTest.java | 17 ++++++------ .../com/hedera/block/simulator/TestUtils.java | 10 +++---- .../GeneratorInjectionModuleTest.java | 15 +++++++++-- 9 files changed, 51 insertions(+), 49 deletions(-) diff --git a/simulator/README.md b/simulator/README.md index 99084be72..837871a15 100644 --- a/simulator/README.md +++ b/simulator/README.md @@ -38,17 +38,25 @@ There are 2 configuration sets: ### BlockStreamConfig Uses the prefix `blockStream` so all properties should start with `blockStream.` +| Key | Description | Default Value | +|---------------------------|-------------------------------------------------------------------------------------------------------------------------------------|-----------------| +| `delayBetweenBlockItems` | The delay between each block item in nanoseconds, only applicable when streamingMode=CONSTANT_RATE | `1_500_000` | +| `maxBlockItemsToStream` | exit condition for the simulator and the circular implementations such as `BlockAsDir` or `BlockAsFile` implementations | `10_000` | +| `streamingMode` | can either be `CONSTANT_RATE` or `MILLIS_PER_BLOCK` | `CONSTANT_RATE` | +| `millisecondsPerBlock` | if streamingMode is `MILLIS_PER_BLOCK` this will be the time to wait between blocks in milliseconds | `1_000` | +| `blockItemsBatchSize` | the number of block items to send in a single batch, however if a block has less block items, it will send all the items in a block | `1_000` | + +### GeneratorConfig +Uses the prefix `generator` so all properties should start with `generator.` | Key | Description | Default Value | |--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------| | `generationMode` | The desired generation Mode to use, it can only be `DIR` or `AD_HOC` | `DIR` | | `folderRootPath` | If the generationMode is DIR this will be used as the source of the recording to stream to the Block-Node | `` | -| `delayBetweenBlockItems` | The delay between each block item in nanoseconds, only applicable when streamingMode=CONSTANT_RATE | `1_500_000` | | `managerImplementation` | The desired implementation of the BlockStreamManager to use, it can only be `BlockAsDirBlockStreamManager`, `BlockAsFileBlockStreamManager` or `BlockAsFileLargeDataSets` | `BlockAsFileBlockStreamManager` | -| `maxBlockItemsToStream` | exit condition for the simulator and the circular implementations such as `BlockAsDir` or `BlockAsFile` implementations | `10_000` | | `paddedLength` | on the `BlockAsFileLargeDataSets` implementation, the length of the padded left zeroes `000001.blk.gz` | 36 | | `fileExtension` | on the `BlockAsFileLargeDataSets` implementation, the extension of the files to be streamed | `.blk.gz` | -| `streamingMode` | can either be `CONSTANT_RATE` or `MILLIS_PER_BLOCK`, if `CONSTANT_RATE` | `CONSTANT_RATE` | -| `millisecondsPerBlock` | if streamingMode is `MILLIS_PER_BLOCK` this will be the time to wait between blocks in milliseconds | `1_000` | + + ### GrpcConfig Uses the prefix `grpc` so all properties should start with `grpc.` diff --git a/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java b/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java index 3d25aa1da..e7b04ef0b 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java +++ b/simulator/src/main/java/com/hedera/block/simulator/BlockStreamSimulatorApp.java @@ -22,11 +22,9 @@ import com.hedera.block.simulator.generator.BlockStreamManager; import com.hedera.block.simulator.grpc.PublishStreamGrpcClient; import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; import com.swirlds.config.api.Configuration; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; -import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import javax.inject.Inject; @@ -104,7 +102,9 @@ private void millisPerBlockStreaming() } else { LOGGER.log( System.Logger.Level.WARNING, - "Block Server is running behind, Streaming took longer than max expected: " + "Block Server is running behind. Streaming took: " + + (elapsedTime / 1_000_000) + + "ms - Longer than max expected of: " + millisecondsPerBlock + " milliseconds"); } @@ -131,7 +131,7 @@ private void constantRateStreaming() break; } - streamInBatches(block); + publishStreamGrpcClient.streamBlock(block); blockItemsStreamed += block.items().size(); Thread.sleep(delayMSBetweenBlockItems, delayNSBetweenBlockItems); @@ -146,24 +146,6 @@ private void constantRateStreaming() } } - private void streamInBatches(Block block) { - final int blockItemsNumberOfBatches = - block.items().size() % blockStreamConfig.blockItemsBatchSize(); - for (int i = 0; i < blockItemsNumberOfBatches; i++) { - - int blockItemsBatchSize = blockStreamConfig.blockItemsBatchSize(); - int startIndexOfBlockItems = i * blockItemsBatchSize; - int endIndexOfBlockItems = (i + 1) * blockItemsBatchSize; - if (endIndexOfBlockItems > block.items().size()) { - endIndexOfBlockItems = block.items().size(); - } - - List blockItems = - block.items().subList(startIndexOfBlockItems, endIndexOfBlockItems); - publishStreamGrpcClient.streamBlockItem(blockItems); - } - } - /** * Returns whether the block stream simulator is running. * diff --git a/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockStreamConfig.java b/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockStreamConfig.java index a5a6a657c..d2e6a9860 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockStreamConfig.java +++ b/simulator/src/main/java/com/hedera/block/simulator/config/data/BlockStreamConfig.java @@ -32,7 +32,7 @@ @ConfigData("blockStream") public record BlockStreamConfig( @ConfigProperty(defaultValue = "1_500_000") int delayBetweenBlockItems, - @ConfigProperty(defaultValue = "10_000") int maxBlockItemsToStream, + @ConfigProperty(defaultValue = "100_000") int maxBlockItemsToStream, @ConfigProperty(defaultValue = "MILLIS_PER_BLOCK") StreamingMode streamingMode, @ConfigProperty(defaultValue = "1000") int millisecondsPerBlock, @ConfigProperty(defaultValue = "1000") int blockItemsBatchSize) { diff --git a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java index 0514865e9..484304c2c 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java +++ b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java @@ -89,8 +89,8 @@ public boolean streamBlock(Block block) { } final int blockItemsNumberOfBatches = - block.items().size() % blockStreamConfig.blockItemsBatchSize(); - for (int i = 0; i < blockItemsNumberOfBatches; i++) { + block.items().size() / blockStreamConfig.blockItemsBatchSize(); + for (int i = 0; i <= blockItemsNumberOfBatches; i++) { int blockItemsBatchSize = blockStreamConfig.blockItemsBatchSize(); int startIndexOfBlockItems = i * blockItemsBatchSize; diff --git a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamObserver.java b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamObserver.java index 0a48f0846..6c7a3136f 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamObserver.java +++ b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamObserver.java @@ -41,6 +41,7 @@ public void onNext(PublishStreamResponse publishStreamResponse) { @Override public void onError(Throwable throwable) { logger.log(Logger.Level.ERROR, "Error: " + throwable.toString()); + // TODO: Stop the stream, retry, or stop simulation } /** what will the stream observer do when the stream is completed */ diff --git a/simulator/src/main/resources/app.properties b/simulator/src/main/resources/app.properties index b030a898a..2999dc112 100644 --- a/simulator/src/main/resources/app.properties +++ b/simulator/src/main/resources/app.properties @@ -1,4 +1,7 @@ -#blockStream.delayBetweenBlockItems=3_000_000 -#blockStream.folderRootPath=/Users/user/Downloads/block-0.0.3-perf1 -#blockStream.managerImplementation=BlockAsFileLargeDataSets +#generator.folderRootPath=/Users/user/Downloads/block-0.0.3-perf +#generator.managerImplementation=BlockAsFileLargeDataSets #blockStream.maxBlockItemsToStream=100_000_000 +#blockStream.streamingMode=MILLIS_PER_BLOCK +#blockStream.millisecondsPerBlock=500 +#blockStream.blockItemsBatchSize=1_000 +#blockStream.delayBetweenBlockItems=3_000_000 diff --git a/simulator/src/test/java/com/hedera/block/simulator/BlockStreamSimulatorTest.java b/simulator/src/test/java/com/hedera/block/simulator/BlockStreamSimulatorTest.java index 57e997cab..0c0c54436 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/BlockStreamSimulatorTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/BlockStreamSimulatorTest.java @@ -16,7 +16,9 @@ package com.hedera.block.simulator; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import com.hedera.block.simulator.exception.BlockSimulatorParsingException; @@ -173,7 +175,7 @@ void start_millisPerSecond_streamingLagVerifyWarnLog() // simulate that the first block takes 15ms to stream, when the limit is 10, to force to go // over WARN Path. - when(publishStreamGrpcClient.streamBlock(block)) + when(publishStreamGrpcClient.streamBlock(any())) .thenAnswer( invocation -> { Thread.sleep(15); @@ -184,12 +186,12 @@ void start_millisPerSecond_streamingLagVerifyWarnLog() Configuration configuration = TestUtils.getTestConfiguration( Map.of( - "blockStream.maxBlockItemsToStream", - "2", "generator.managerImplementation", - "BlockAsFileLargeDataSets", + "BlockAsFileBlockStreamManager", "generator.rootPath", getAbsoluteFolder("src/test/resources/block-0.0.3-blk/"), + "blockStream.maxBlockItemsToStream", + "2", "blockStream.streamingMode", "MILLIS_PER_BLOCK", "blockStream.millisecondsPerBlock", @@ -211,10 +213,7 @@ void start_millisPerSecond_streamingLagVerifyWarnLog() logRecord -> logRecord .getMessage() - .contains( - "Block Server is running behind, Streaming" - + " took longer than max expected: 10" - + " milliseconds")); + .contains("Block Server is running behind")); assertTrue(found_log); } diff --git a/simulator/src/test/java/com/hedera/block/simulator/TestUtils.java b/simulator/src/test/java/com/hedera/block/simulator/TestUtils.java index 952c0be49..dc093abda 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/TestUtils.java +++ b/simulator/src/test/java/com/hedera/block/simulator/TestUtils.java @@ -17,10 +17,9 @@ package com.hedera.block.simulator; import com.hedera.block.simulator.config.TestConfigBuilder; -import com.hedera.block.simulator.config.data.BlockStreamConfig; -import com.hedera.block.simulator.config.data.GrpcConfig; import com.swirlds.config.api.Configuration; import com.swirlds.config.extensions.sources.ClasspathFileConfigSource; +import com.swirlds.config.extensions.sources.SimpleConfigSource; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Path; @@ -41,12 +40,11 @@ public static Configuration getTestConfiguration(@NonNull Map cu for (Map.Entry entry : customProperties.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); - testConfigBuilder = testConfigBuilder.withValue(key, value); + testConfigBuilder = + testConfigBuilder.withSource( + new SimpleConfigSource(key, value).withOrdinal(500)); } - testConfigBuilder = testConfigBuilder.withConfigDataType(BlockStreamConfig.class); - testConfigBuilder = testConfigBuilder.withConfigDataType(GrpcConfig.class); - return testConfigBuilder.getOrCreateConfig(); } diff --git a/simulator/src/test/java/com/hedera/block/simulator/generator/GeneratorInjectionModuleTest.java b/simulator/src/test/java/com/hedera/block/simulator/generator/GeneratorInjectionModuleTest.java index ad5654d86..65022ed75 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/generator/GeneratorInjectionModuleTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/generator/GeneratorInjectionModuleTest.java @@ -46,7 +46,13 @@ void providesBlockStreamManager_AsFileLargeDataSets() throws IOException { @Test void providesBlockStreamManager_AsFile() throws IOException { BlockGeneratorConfig blockGeneratorConfig = - TestUtils.getTestConfiguration().getConfigData(BlockGeneratorConfig.class); + TestUtils.getTestConfiguration( + Map.of( + "generator.managerImplementation", + "BlockAsFileBlockStreamManager", + "generator.folderRootPath", + "")) + .getConfigData(BlockGeneratorConfig.class); BlockStreamManager blockStreamManager = GeneratorInjectionModule.providesBlockStreamManager(blockGeneratorConfig); @@ -76,7 +82,12 @@ void providesBlockStreamManager_AsDir() throws IOException { @Test void providesBlockStreamManager_default() throws IOException { BlockGeneratorConfig blockGeneratorConfig = - TestUtils.getTestConfiguration(Map.of("generator.managerImplementation", "")) + TestUtils.getTestConfiguration( + Map.of( + "generator.managerImplementation", + "", + "generator.folderRootPath", + "")) .getConfigData(BlockGeneratorConfig.class); BlockStreamManager blockStreamManager = From 7086c45dda27c5f733a158a2fa21a20d16ea56a3 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Sun, 20 Oct 2024 22:49:38 -0600 Subject: [PATCH 10/15] added missing UT Signed-off-by: Alfredo Gutierrez --- .../simulator/config/ConfigInjectionModuleTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java b/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java index 2fc28118c..2b4ed8c4c 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java @@ -16,6 +16,7 @@ package com.hedera.block.simulator.config; +import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.data.BlockStreamConfig; import com.hedera.block.simulator.config.data.GrpcConfig; import com.swirlds.config.api.Configuration; @@ -62,4 +63,14 @@ void provideGrpcConfig() { Assertions.assertEquals("localhost", grpcConfig.serverAddress()); Assertions.assertEquals(8080, grpcConfig.port()); } + + @Test + void provideBlockGeneratorConfig() { + BlockGeneratorConfig blockGeneratorConfig = + ConfigInjectionModule.provideBlockGeneratorConfig(configuration); + + Assertions.assertNotNull(blockGeneratorConfig); + Assertions.assertEquals( + "BlockAsFileBlockStreamManager", blockGeneratorConfig.managerImplementation()); + } } From a0dd2c4205e37cca29520a74f0e960b4becad429 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Mon, 21 Oct 2024 16:04:11 -0600 Subject: [PATCH 11/15] Addressing PR Review comments, suggestions and improvements. Signed-off-by: Alfredo Gutierrez --- .../hedera/block/common/utils/ChunkUtils.java | 46 +++++++++++++++++++ .../ConsumerStreamResponseObserver.java | 20 ++++---- .../block/server/grpc/BlockAccessService.java | 4 +- .../block/server/grpc/BlockStreamService.java | 11 ----- .../mediator/LiveStreamMediatorImpl.java | 8 +--- .../producer/ProducerBlockItemObserver.java | 5 +- .../server/grpc/BlockAccessServiceTest.java | 17 +------ .../BlockStreamServiceIntegrationTest.java | 11 +---- .../server/grpc/BlockStreamServiceTest.java | 13 +----- .../storage/read/BlockAsDirReaderTest.java | 21 +-------- .../ProducerBlockItemObserverTest.java | 3 +- simulator/build.gradle.kts | 2 + .../grpc/PublishStreamGrpcClientImpl.java | 19 ++------ simulator/src/main/java/module-info.java | 1 + simulator/src/main/resources/app.properties | 14 +++--- .../config/ConfigInjectionModuleTest.java | 9 ++-- 16 files changed, 92 insertions(+), 112 deletions(-) create mode 100644 common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java diff --git a/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java b/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java new file mode 100644 index 000000000..30d8e4e31 --- /dev/null +++ b/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.common.utils; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public final class ChunkUtils { + public static List> chunkify(final Collection collection, final int chunkSize) { + Objects.requireNonNull(collection); + if (chunkSize <= 0) { + throw new IllegalArgumentException("Chunk size must be greater than 0"); + } + if (collection.isEmpty()) { + return Collections.emptyList(); // or throw, depends on how we want to handle + } + final List localCollection = List.copyOf(collection); + final int localCollectionSize = localCollection.size(); + return IntStream.iterate(0, i -> i < localCollectionSize, i -> i + chunkSize) + .mapToObj( + i -> + localCollection.subList( + i, Math.min(i + chunkSize, localCollectionSize))) + .collect(Collectors.toList()); + } + + private ChunkUtils() {} +} diff --git a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java index 44c68dc6b..20bf465cf 100644 --- a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java +++ b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java @@ -31,12 +31,12 @@ import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.OneOf; -import com.swirlds.metrics.api.Counter; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; import java.time.InstantSource; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -214,22 +214,22 @@ public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) throw new IllegalArgumentException(message); } - final List blockItems = subscribeStreamResponse.blockItems().blockItems(); + final List blockItems = + Objects.requireNonNull(subscribeStreamResponse.blockItems()).blockItems(); // Only start sending BlockItems after we've reached // the beginning of a block. if (!streamStarted && blockItems.getFirst().hasBlockHeader()) { + LOGGER.log( + DEBUG, + "Sending BlockItem Batch downstream for block: " + + blockItems.getFirst().blockHeader().number()); streamStarted = true; } if (streamStarted) { - LOGGER.log(DEBUG, "Sending BlockItem Batch downstream"); - - final Counter liveBlockItemsConsumed = - metricsService.get(BlockNodeMetricTypes.Counter.LiveBlockItemsConsumed); - // Increment counter manually - for (int i = 0; i < blockItems.size(); i++) { - liveBlockItemsConsumed.increment(); - } + metricsService + .get(BlockNodeMetricTypes.Counter.LiveBlockItemsReceived) + .add(blockItems.size()); subscribeStreamResponseObserver.onNext(fromPbj(subscribeStreamResponse)); } } diff --git a/server/src/main/java/com/hedera/block/server/grpc/BlockAccessService.java b/server/src/main/java/com/hedera/block/server/grpc/BlockAccessService.java index aae176ef2..23652b9b4 100644 --- a/server/src/main/java/com/hedera/block/server/grpc/BlockAccessService.java +++ b/server/src/main/java/com/hedera/block/server/grpc/BlockAccessService.java @@ -16,6 +16,7 @@ package com.hedera.block.server.grpc; +import static com.hedera.block.server.Constants.SERVICE_NAME_BLOCK_ACCESS; import static com.hedera.block.server.Constants.SINGLE_BLOCK_METHOD_NAME; import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.Translator.toPbj; @@ -26,7 +27,6 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.InvalidProtocolBufferException; -import com.hedera.block.server.Constants; import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.service.ServiceStatus; @@ -79,7 +79,7 @@ public Descriptors.FileDescriptor proto() { @Override public String serviceName() { - return Constants.SERVICE_NAME_BLOCK_ACCESS; + return SERVICE_NAME_BLOCK_ACCESS; } @Override diff --git a/server/src/main/java/com/hedera/block/server/grpc/BlockStreamService.java b/server/src/main/java/com/hedera/block/server/grpc/BlockStreamService.java index a7295c6e0..929fd3e60 100644 --- a/server/src/main/java/com/hedera/block/server/grpc/BlockStreamService.java +++ b/server/src/main/java/com/hedera/block/server/grpc/BlockStreamService.java @@ -30,15 +30,12 @@ import com.hedera.block.server.events.BlockNodeEventHandler; import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.mediator.LiveStreamMediator; -import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.notifier.Notifier; -import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.producer.ProducerBlockItemObserver; import com.hedera.block.server.service.ServiceStatus; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.SubscribeStreamResponseCode; import com.hedera.hapi.block.protoc.BlockService; -import com.hedera.hapi.block.stream.Block; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.StreamObserver; import io.helidon.webserver.grpc.GrpcService; @@ -55,10 +52,7 @@ public class BlockStreamService implements GrpcService { private final LiveStreamMediator streamMediator; private final ServiceStatus serviceStatus; - private final BlockReader blockReader; - private final BlockNodeContext blockNodeContext; - private final MetricsService metricsService; private final Notifier notifier; @@ -68,26 +62,21 @@ public class BlockStreamService implements GrpcService { * * @param streamMediator the stream mediator to proxy block items from the producer to the * subscribers and manage the subscription lifecycle for subscribers - * @param blockReader the block reader to fetch blocks from storage for unary singleBlock - * service calls * @param serviceStatus the service status provides methods to check service availability and to * stop the service and web server in the event of an unrecoverable exception */ @Inject BlockStreamService( @NonNull final LiveStreamMediator streamMediator, - @NonNull final BlockReader blockReader, @NonNull final ServiceStatus serviceStatus, @NonNull final BlockNodeEventHandler> streamPersistenceHandler, @NonNull final Notifier notifier, @NonNull final BlockNodeContext blockNodeContext) { - this.blockReader = blockReader; this.serviceStatus = serviceStatus; this.notifier = notifier; this.blockNodeContext = blockNodeContext; - this.metricsService = blockNodeContext.metricsService(); streamMediator.subscribe(streamPersistenceHandler); this.streamMediator = streamMediator; diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java index 78ca051df..4f00126b6 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java @@ -33,7 +33,6 @@ import com.hedera.hapi.block.SubscribeStreamResponseSet; import com.hedera.hapi.block.stream.BlockItem; import com.lmax.disruptor.BatchEventProcessor; -import com.swirlds.metrics.api.Counter; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.List; import java.util.Map; @@ -109,11 +108,8 @@ public void publish(@NonNull final List blockItems) { SubscribeStreamResponse.newBuilder().blockItems(blockItemsSet).build(); ringBuffer.publishEvent((event, sequence) -> event.set(subscribeStreamResponse)); - // Increment the block item counter - Counter liveBlockItems = metricsService.get(LiveBlockItems); - for (int i = 0; i < blockItems.size(); i++) { - liveBlockItems.increment(); - } + // Increment the block item counter by all block items published + metricsService.get(LiveBlockItems).add(blockItems.size()); } else { LOGGER.log(ERROR, "StreamMediator is not accepting BlockItems"); diff --git a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java index 92c9e8228..d54616ff4 100644 --- a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java +++ b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java @@ -162,8 +162,10 @@ public void onNext( LOGGER.log(DEBUG, "Received PublishStreamRequest from producer"); final List blockItemsPbj = new ArrayList<>(); + final Counter liveBlockItemsReceived = metricsService.get(LiveBlockItemsReceived); - for (final var blockItemProtoc : publishStreamRequest.getBlockItemsList()) { + for (final com.hedera.hapi.block.stream.protoc.BlockItem blockItemProtoc : + publishStreamRequest.getBlockItemsList()) { try { final BlockItem blockItem = toPbj(BlockItem.PROTOBUF, blockItemProtoc.toByteArray()); @@ -182,7 +184,6 @@ public void onNext( liveBlockItemsReceived.increment(); } - // is this log just too much? see below log for proposed... LOGGER.log(DEBUG, "Received block item batch with {} items.", blockItemsPbj.size()); // Publish the block to all the subscribers unless diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java index e57d7bb3d..4ac263b10 100644 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java @@ -21,7 +21,6 @@ import static com.hedera.block.server.grpc.BlockAccessService.buildSingleBlockNotFoundResponse; import static com.hedera.block.server.grpc.BlockAccessService.fromPbjSingleBlockSuccessResponse; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; -import static java.lang.System.Logger.Level.INFO; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; @@ -36,8 +35,6 @@ import com.google.protobuf.Descriptors; import com.hedera.block.server.Constants; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.mediator.LiveStreamMediator; -import com.hedera.block.server.notifier.Notifier; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; @@ -54,7 +51,6 @@ import io.grpc.stub.StreamObserver; import io.helidon.webserver.grpc.GrpcService; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -64,38 +60,29 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) class BlockAccessServiceTest { - @Mock private Notifier notifier; - @Mock private StreamObserver responseObserver; - @Mock private LiveStreamMediator streamMediator; - @Mock private BlockReader blockReader; @Mock private BlockWriter> blockWriter; @Mock private ServiceStatus serviceStatus; - private final System.Logger LOGGER = System.getLogger(getClass().getName()); - - private static final String TEMP_DIR = "block-node-unit-test-dir"; - private static final int testTimeout = 1000; - private Path testPath; + @TempDir private Path testPath; private BlockNodeContext blockNodeContext; private PersistenceStorageConfig config; @BeforeEach public void setUp() throws IOException { - testPath = Files.createTempDirectory(TEMP_DIR); - LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); blockNodeContext = TestConfigUtil.getTestBlockNodeContext( diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceIntegrationTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceIntegrationTest.java index 0ebd46c9c..a5a6c07be 100644 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceIntegrationTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceIntegrationTest.java @@ -173,7 +173,6 @@ public void testPublishBlockStreamRegistrationAndExecution() final BlockStreamService blockStreamService = new BlockStreamService( streamMediator, - blockReader, serviceStatus, blockNodeEventHandler, notifier, @@ -275,7 +274,6 @@ public void testSubscribeBlockStream() throws IOException { final BlockStreamService blockStreamService = new BlockStreamService( streamMediator, - blockReader, serviceStatus, blockNodeEventHandler, notifier, @@ -449,7 +447,6 @@ public void testSubAndUnsubWhileStreaming() throws InterruptedException { final var blockStreamService = new BlockStreamService( streamMediator, - blockReader, serviceStatus, blockNodeEventHandler, notifier, @@ -572,7 +569,6 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep final var blockStreamService = new BlockStreamService( streamMediator, - blockReader, serviceStatus, blockNodeEventHandler, notifier, @@ -747,12 +743,7 @@ private BlockStreamService buildBlockStreamService( streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); return new BlockStreamService( - streamMediator, - blockReader, - serviceStatus, - blockNodeEventHandler, - notifier, - blockNodeContext); + streamMediator, serviceStatus, blockNodeEventHandler, notifier, blockNodeContext); } private LiveStreamMediator buildStreamMediator( diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java index f3becede7..d7ed0d9e2 100644 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java @@ -21,7 +21,6 @@ import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.util.PersistTestUtils.reverseByteArray; import static java.lang.System.Logger; -import static java.lang.System.Logger.Level.INFO; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.eq; @@ -52,7 +51,6 @@ import io.grpc.stub.StreamObserver; import io.helidon.webserver.grpc.GrpcService; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -61,6 +59,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @@ -82,19 +81,14 @@ public class BlockStreamServiceTest { private final Logger LOGGER = System.getLogger(getClass().getName()); - private static final String TEMP_DIR = "block-node-unit-test-dir"; - private static final int testTimeout = 1000; - private Path testPath; + @TempDir private Path testPath; private BlockNodeContext blockNodeContext; private PersistenceStorageConfig config; @BeforeEach public void setUp() throws IOException { - testPath = Files.createTempDirectory(TEMP_DIR); - LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - blockNodeContext = TestConfigUtil.getTestBlockNodeContext( Map.of("persistence.storage.rootPath", testPath.toString())); @@ -115,7 +109,6 @@ public void testServiceName() { final BlockStreamService blockStreamService = new BlockStreamService( streamMediator, - blockReader, serviceStatus, blockNodeEventHandler, notifier, @@ -138,7 +131,6 @@ public void testProto() { final BlockStreamService blockStreamService = new BlockStreamService( streamMediator, - blockReader, serviceStatus, blockNodeEventHandler, notifier, @@ -201,7 +193,6 @@ private BlockStreamService getBlockStreamService() { final BlockStreamService blockStreamService = new BlockStreamService( streamMediator, - blockReader, serviceStatus, blockNodeEventHandler, notifier, diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java index 375631314..9e87265d2 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java @@ -21,7 +21,6 @@ import static com.hedera.block.server.util.PersistTestUtils.reverseByteArray; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.ERROR; -import static java.lang.System.Logger.Level.INFO; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; @@ -52,23 +51,19 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; public class BlockAsDirReaderTest { private final Logger LOGGER = System.getLogger(getClass().getName()); - private static final String TEMP_DIR = "block-node-unit-test-dir"; - - private Path testPath; + @TempDir private Path testPath; private BlockNodeContext blockNodeContext; private PersistenceStorageConfig config; @BeforeEach public void setUp() throws IOException { - testPath = Files.createTempDirectory(TEMP_DIR); - LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - blockNodeContext = TestConfigUtil.getTestBlockNodeContext( Map.of("persistence.storage.rootPath", testPath.toString())); @@ -115,9 +110,6 @@ public void testRemoveBlockReadPermsRepairFailed() throws IOException, ParseExce final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - // for (BlockItem blockItem : blockItems) { - // blockWriter.write(blockItem); - // } blockWriter.write(blockItems); // Make the block unreadable @@ -139,9 +131,6 @@ public void testRemoveBlockItemReadPerms() throws IOException { final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - // for (BlockItem blockItem : blockItems) { - // blockWriter.write(blockItem); - // } blockWriter.write(blockItems); removeBlockItemReadPerms(1, 1, config); @@ -172,9 +161,6 @@ public void testRepairReadPermsFails() throws IOException, ParseException { final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - // for (final BlockItem blockItem : blockItems) { - // blockWriter.write(blockItem); - // } blockWriter.write(blockItems); removeBlockReadPerms(1, config); @@ -211,9 +197,6 @@ public void testParseExceptionHandling() throws IOException, ParseException { final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - // for (final BlockItem blockItem : blockItems) { - // blockWriter.write(blockItem); - // } blockWriter.write(blockItems); // Read the block back and confirm it's read successfully diff --git a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java index 66de5f4c9..5f0e9bf0c 100644 --- a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java @@ -151,7 +151,8 @@ public void testBlockItemThrowsParseException() throws IOException { // verify the ProducerBlockItemObserver has sent an error response verify( publishStreamResponseObserver, - timeout(testTimeout).atLeast(1)) // It fixes if set it to 2, but why??? + timeout(testTimeout) + .atLeast(1)) // TODO: it calls more than 1 usually 2, but why? .onNext(fromPbj(PublishStreamResponse.newBuilder().status(endOfStream).build())); verify(serviceStatus, timeout(testTimeout).times(1)).stopWebServer(any()); diff --git a/simulator/build.gradle.kts b/simulator/build.gradle.kts index ea112413a..27925b2c5 100644 --- a/simulator/build.gradle.kts +++ b/simulator/build.gradle.kts @@ -19,6 +19,8 @@ plugins { id("com.hedera.block.simulator") } +dependencies { implementation(project(":common")) } + description = "Hedera Block Stream Simulator" application { diff --git a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java index 484304c2c..1d192076e 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java +++ b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamGrpcClientImpl.java @@ -16,6 +16,7 @@ package com.hedera.block.simulator.grpc; +import com.hedera.block.common.utils.ChunkUtils; import com.hedera.block.simulator.Translator; import com.hedera.block.simulator.config.data.BlockStreamConfig; import com.hedera.block.simulator.config.data.GrpcConfig; @@ -88,20 +89,10 @@ public boolean streamBlock(Block block) { blockItemsProtoc.add(Translator.fromPbj(blockItem)); } - final int blockItemsNumberOfBatches = - block.items().size() / blockStreamConfig.blockItemsBatchSize(); - for (int i = 0; i <= blockItemsNumberOfBatches; i++) { - - int blockItemsBatchSize = blockStreamConfig.blockItemsBatchSize(); - int startIndexOfBlockItems = i * blockItemsBatchSize; - int endIndexOfBlockItems = (i + 1) * blockItemsBatchSize; - if (endIndexOfBlockItems > block.items().size()) { - endIndexOfBlockItems = block.items().size(); - } - - List streamingBatch = - blockItemsProtoc.subList(startIndexOfBlockItems, endIndexOfBlockItems); - + List> streamingBatches = + ChunkUtils.chunkify(blockItemsProtoc, blockStreamConfig.blockItemsBatchSize()); + for (List streamingBatch : + streamingBatches) { requestStreamObserver.onNext( PublishStreamRequest.newBuilder().addAllBlockItems(streamingBatch).build()); } diff --git a/simulator/src/main/java/module-info.java b/simulator/src/main/java/module-info.java index ddca45813..0b694464b 100644 --- a/simulator/src/main/java/module-info.java +++ b/simulator/src/main/java/module-info.java @@ -8,6 +8,7 @@ requires static com.github.spotbugs.annotations; requires static com.google.auto.service; + requires com.hedera.block.common; requires com.hedera.block.stream; requires com.google.protobuf; requires com.hedera.pbj.runtime; diff --git a/simulator/src/main/resources/app.properties b/simulator/src/main/resources/app.properties index 2999dc112..e5f8d8892 100644 --- a/simulator/src/main/resources/app.properties +++ b/simulator/src/main/resources/app.properties @@ -1,7 +1,7 @@ -#generator.folderRootPath=/Users/user/Downloads/block-0.0.3-perf -#generator.managerImplementation=BlockAsFileLargeDataSets -#blockStream.maxBlockItemsToStream=100_000_000 -#blockStream.streamingMode=MILLIS_PER_BLOCK -#blockStream.millisecondsPerBlock=500 -#blockStream.blockItemsBatchSize=1_000 -#blockStream.delayBetweenBlockItems=3_000_000 +generator.folderRootPath=/Users/user/Downloads/block-0.0.3-perf +generator.managerImplementation=BlockAsFileLargeDataSets +blockStream.maxBlockItemsToStream=100_000_000 +blockStream.streamingMode=MILLIS_PER_BLOCK +blockStream.millisecondsPerBlock=500 +blockStream.blockItemsBatchSize=1_000 +blockStream.delayBetweenBlockItems=3_000_000 diff --git a/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java b/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java index 2b4ed8c4c..c499c5076 100644 --- a/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java +++ b/simulator/src/test/java/com/hedera/block/simulator/config/ConfigInjectionModuleTest.java @@ -16,16 +16,16 @@ package com.hedera.block.simulator.config; +import com.hedera.block.simulator.TestUtils; import com.hedera.block.simulator.config.data.BlockGeneratorConfig; import com.hedera.block.simulator.config.data.BlockStreamConfig; import com.hedera.block.simulator.config.data.GrpcConfig; import com.swirlds.config.api.Configuration; import com.swirlds.config.api.ConfigurationBuilder; import com.swirlds.config.extensions.sources.ClasspathFileConfigSource; -import com.swirlds.config.extensions.sources.SystemEnvironmentConfigSource; -import com.swirlds.config.extensions.sources.SystemPropertiesConfigSource; import java.io.IOException; import java.nio.file.Path; +import java.util.Map; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -38,11 +38,12 @@ class ConfigInjectionModuleTest { static void setUpAll() throws IOException { configuration = ConfigurationBuilder.create() - .withSource(SystemEnvironmentConfigSource.getInstance()) - .withSource(SystemPropertiesConfigSource.getInstance()) .withSource(new ClasspathFileConfigSource(Path.of("app.properties"))) .autoDiscoverExtensions() .build(); + configuration = + TestUtils.getTestConfiguration( + Map.of("generator.managerImplementation", "BlockAsFileBlockStreamManager")); } @Test From ea98ce35794c4a6c57a2766060ea2cf33019c696 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Mon, 21 Oct 2024 16:24:47 -0600 Subject: [PATCH 12/15] restoring app.properties to defaults Signed-off-by: Alfredo Gutierrez --- simulator/src/main/resources/app.properties | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/simulator/src/main/resources/app.properties b/simulator/src/main/resources/app.properties index e5f8d8892..2999dc112 100644 --- a/simulator/src/main/resources/app.properties +++ b/simulator/src/main/resources/app.properties @@ -1,7 +1,7 @@ -generator.folderRootPath=/Users/user/Downloads/block-0.0.3-perf -generator.managerImplementation=BlockAsFileLargeDataSets -blockStream.maxBlockItemsToStream=100_000_000 -blockStream.streamingMode=MILLIS_PER_BLOCK -blockStream.millisecondsPerBlock=500 -blockStream.blockItemsBatchSize=1_000 -blockStream.delayBetweenBlockItems=3_000_000 +#generator.folderRootPath=/Users/user/Downloads/block-0.0.3-perf +#generator.managerImplementation=BlockAsFileLargeDataSets +#blockStream.maxBlockItemsToStream=100_000_000 +#blockStream.streamingMode=MILLIS_PER_BLOCK +#blockStream.millisecondsPerBlock=500 +#blockStream.blockItemsBatchSize=1_000 +#blockStream.delayBetweenBlockItems=3_000_000 From 0092362ab2427935e0c8f04b5d2b3997a793dfd8 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Tue, 22 Oct 2024 11:02:11 -0600 Subject: [PATCH 13/15] Addressing more feedback from PR Reviews Signed-off-by: Alfredo Gutierrez --- .../java/com/hedera/block/common/utils/ChunkUtils.java | 9 ++++++++- .../hedera/block/server/grpc/BlockAccessServiceTest.java | 7 ------- .../hedera/block/server/grpc/BlockStreamServiceTest.java | 7 ------- .../persistence/storage/read/BlockAsDirReaderTest.java | 9 --------- simulator/build.gradle.kts | 2 +- .../block/simulator/grpc/PublishStreamObserver.java | 2 +- 6 files changed, 10 insertions(+), 26 deletions(-) diff --git a/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java b/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java index 30d8e4e31..fa75c372e 100644 --- a/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java +++ b/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java @@ -16,6 +16,7 @@ package com.hedera.block.common.utils; +import edu.umd.cs.findbugs.annotations.NonNull; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -23,8 +24,14 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +/** Utility class for chunking collections. */ public final class ChunkUtils { - public static List> chunkify(final Collection collection, final int chunkSize) { + /** Chunks a collection into a list of lists of the specified size. + * @param collection the collection to chunk, if the collection is empty, an empty list is returned. + * @param chunkSize the size of each chunk + * */ + public static List> chunkify( + @NonNull final Collection collection, final int chunkSize) { Objects.requireNonNull(collection); if (chunkSize <= 0) { throw new IllegalArgumentException("Chunk size must be greater than 0"); diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java index 4ac263b10..12c824963 100644 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java @@ -42,7 +42,6 @@ import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.protoc.SingleBlockResponse; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; @@ -55,7 +54,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -90,11 +88,6 @@ public void setUp() throws IOException { config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); } - @AfterEach - public void tearDown() { - TestUtils.deleteDirectory(testPath.toFile()); - } - @Test void testProto() { BlockAccessService blockAccessService = diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java index d7ed0d9e2..56a35c8af 100644 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java @@ -42,7 +42,6 @@ import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.SingleBlockResponse; import com.hedera.hapi.block.SingleBlockResponseCode; import com.hedera.hapi.block.stream.Block; @@ -54,7 +53,6 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -95,11 +93,6 @@ public void setUp() throws IOException { config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); } - @AfterEach - public void tearDown() { - TestUtils.deleteDirectory(testPath.toFile()); - } - @Test public void testServiceName() { diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java index 9e87265d2..4e2fc2141 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java @@ -20,7 +20,6 @@ import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static com.hedera.block.server.util.PersistTestUtils.reverseByteArray; import static java.lang.System.Logger; -import static java.lang.System.Logger.Level.ERROR; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; @@ -48,7 +47,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -70,13 +68,6 @@ public void setUp() throws IOException { config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); } - @AfterEach - public void tearDown() { - if (!TestUtils.deleteDirectory(testPath.toFile())) { - LOGGER.log(ERROR, "Failed to delete temp directory: " + testPath.toString()); - } - } - @Test public void testReadBlockDoesNotExist() throws IOException, ParseException { final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); diff --git a/simulator/build.gradle.kts b/simulator/build.gradle.kts index 27925b2c5..e8012d4b1 100644 --- a/simulator/build.gradle.kts +++ b/simulator/build.gradle.kts @@ -19,7 +19,7 @@ plugins { id("com.hedera.block.simulator") } -dependencies { implementation(project(":common")) } +// dependencies { implementation(project(":common")) } description = "Hedera Block Stream Simulator" diff --git a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamObserver.java b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamObserver.java index 6c7a3136f..8880c65b1 100644 --- a/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamObserver.java +++ b/simulator/src/main/java/com/hedera/block/simulator/grpc/PublishStreamObserver.java @@ -41,7 +41,7 @@ public void onNext(PublishStreamResponse publishStreamResponse) { @Override public void onError(Throwable throwable) { logger.log(Logger.Level.ERROR, "Error: " + throwable.toString()); - // TODO: Stop the stream, retry, or stop simulation + // @todo(286) - handle the error } /** what will the stream observer do when the stream is completed */ From 5369431cf9f3bdd8cfabd383dbaded5b067f5ba1 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Tue, 22 Oct 2024 11:06:47 -0600 Subject: [PATCH 14/15] addressing warnings and adding unit test for Chunkify Util Signed-off-by: Alfredo Gutierrez --- .../hedera/block/common/utils/ChunkUtils.java | 2 + common/src/main/java/module-info.java | 3 + .../block/common/utils/ChunkUtilsTest.java | 82 +++++++++++++++++++ .../com/hedera/block/suites/BaseSuite.java | 2 + .../PositiveDataPersistenceTests.java | 1 + 5 files changed, 90 insertions(+) create mode 100644 common/src/test/java/com/hedera/block/common/utils/ChunkUtilsTest.java diff --git a/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java b/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java index fa75c372e..3c2481ef6 100644 --- a/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java +++ b/common/src/main/java/com/hedera/block/common/utils/ChunkUtils.java @@ -29,6 +29,8 @@ public final class ChunkUtils { /** Chunks a collection into a list of lists of the specified size. * @param collection the collection to chunk, if the collection is empty, an empty list is returned. * @param chunkSize the size of each chunk + * @param the type of the collection + * @return a list of lists of the specified size * */ public static List> chunkify( @NonNull final Collection collection, final int chunkSize) { diff --git a/common/src/main/java/module-info.java b/common/src/main/java/module-info.java index a6f56aab4..9f80aea65 100644 --- a/common/src/main/java/module-info.java +++ b/common/src/main/java/module-info.java @@ -1,3 +1,6 @@ +/** + * Module info for common module. + */ module com.hedera.block.common { exports com.hedera.block.common.constants; exports com.hedera.block.common.utils; diff --git a/common/src/test/java/com/hedera/block/common/utils/ChunkUtilsTest.java b/common/src/test/java/com/hedera/block/common/utils/ChunkUtilsTest.java new file mode 100644 index 000000000..4c20f6e9f --- /dev/null +++ b/common/src/test/java/com/hedera/block/common/utils/ChunkUtilsTest.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.common.utils; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class ChunkUtilsTest { + + @Test + public void testEmptyCollection() { + List emptyList = Collections.emptyList(); + List> chunks = ChunkUtils.chunkify(emptyList, 3); + assertTrue(chunks.isEmpty(), "Chunks of empty collection should be empty"); + } + + @Test + public void testChunkSizeZero() { + List list = Arrays.asList(1, 2, 3); + Exception exception = + assertThrows(IllegalArgumentException.class, () -> ChunkUtils.chunkify(list, 0)); + assertEquals("Chunk size must be greater than 0", exception.getMessage()); + } + + @Test + public void testChunkSizeNegative() { + List list = Arrays.asList(1, 2, 3); + Exception exception = + assertThrows(IllegalArgumentException.class, () -> ChunkUtils.chunkify(list, -1)); + assertEquals("Chunk size must be greater than 0", exception.getMessage()); + } + + @Test + public void testChunkifyCollectionSmallerThanChunkSize() { + List list = Arrays.asList(1, 2); + List> chunks = ChunkUtils.chunkify(list, 5); + assertEquals(1, chunks.size(), "Should return one chunk"); + assertEquals(list, chunks.get(0), "Chunk should contain the entire collection"); + } + + @Test + public void testChunkifyCollectionExactlyDivisible() { + List list = Arrays.asList(1, 2, 3, 4); + List> chunks = ChunkUtils.chunkify(list, 2); + assertEquals(2, chunks.size(), "Should return two chunks"); + assertEquals(Arrays.asList(1, 2), chunks.get(0), "First chunk mismatch"); + assertEquals(Arrays.asList(3, 4), chunks.get(1), "Second chunk mismatch"); + } + + @Test + public void testChunkifyCollectionNotExactlyDivisible() { + List list = Arrays.asList(1, 2, 3, 4, 5); + List> chunks = ChunkUtils.chunkify(list, 2); + assertEquals(3, chunks.size(), "Should return three chunks"); + assertEquals(Arrays.asList(1, 2), chunks.get(0), "First chunk mismatch"); + assertEquals(Arrays.asList(3, 4), chunks.get(1), "Second chunk mismatch"); + assertEquals(Arrays.asList(5), chunks.get(2), "Third chunk mismatch"); + } + + @Test + public void testNullCollection() { + assertThrows(NullPointerException.class, () -> ChunkUtils.chunkify(null, 3)); + } +} diff --git a/suites/src/main/java/com/hedera/block/suites/BaseSuite.java b/suites/src/main/java/com/hedera/block/suites/BaseSuite.java index 5c226bc0f..cbcaed78e 100644 --- a/suites/src/main/java/com/hedera/block/suites/BaseSuite.java +++ b/suites/src/main/java/com/hedera/block/suites/BaseSuite.java @@ -71,6 +71,8 @@ public BaseSuite() { * Setup method to be executed before all tests. * *

This method initializes the Block Node server container using Testcontainers. + * + * @throws IOException if an I/O error occurs */ @BeforeAll public static void setup() throws IOException { diff --git a/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java b/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java index 1e4ffba16..0e9670e0a 100644 --- a/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java +++ b/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java @@ -19,6 +19,7 @@ import com.hedera.block.suites.BaseSuite; import org.junit.jupiter.api.DisplayName; +/** Positive Data Persistence Tests */ @DisplayName("Positive Data Persistence Tests") public class PositiveDataPersistenceTests extends BaseSuite { /** Default constructor for the {@link PositiveDataPersistenceTests} class. */ From 925b9f7d77af52ccd62b2a5c166a5beb65010220 Mon Sep 17 00:00:00 2001 From: Alfredo Gutierrez Date: Tue, 22 Oct 2024 11:14:26 -0600 Subject: [PATCH 15/15] replaced with single import Signed-off-by: Alfredo Gutierrez --- .../java/com/hedera/block/common/utils/ChunkUtilsTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/src/test/java/com/hedera/block/common/utils/ChunkUtilsTest.java b/common/src/test/java/com/hedera/block/common/utils/ChunkUtilsTest.java index 4c20f6e9f..5f206110b 100644 --- a/common/src/test/java/com/hedera/block/common/utils/ChunkUtilsTest.java +++ b/common/src/test/java/com/hedera/block/common/utils/ChunkUtilsTest.java @@ -16,7 +16,9 @@ package com.hedera.block.common.utils; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; import java.util.Collections;