From 31594da366f407f96d62282f0998a8b071b0ede8 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Mon, 10 Feb 2025 21:32:58 +0530 Subject: [PATCH 01/15] first draft of getBlobs --- beacon_chain/el/el_manager.nim | 65 +++++++++++++++++++++++++++++ beacon_chain/nimbus_beacon_node.nim | 63 ++++++++++++++++++++++++---- vendor/nim-web3 | 2 +- 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/beacon_chain/el/el_manager.nim b/beacon_chain/el/el_manager.nim index c5c2e3c93c..91c5901671 100644 --- a/beacon_chain/el/el_manager.nim +++ b/beacon_chain/el/el_manager.nim @@ -57,6 +57,7 @@ type GetPayloadV3Response | GetPayloadV4Response + contract(DepositContract): proc deposit(pubkey: PubKeyBytes, withdrawalCredentials: WithdrawalCredentialsBytes, @@ -108,6 +109,8 @@ const # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.3/src/engine/shanghai.md#request-2 GETPAYLOAD_TIMEOUT = 1.seconds + # https://github.com/ethereum/execution-apis/blob/ad9102b11212d51b736a0413c8655a8da93e55fc/src/engine/cancun.md#request-3 + GETBLOBS_TIMEOUT = 1.seconds connectionStateChangeHysteresisThreshold = 15 ## How many unsuccesful/successful requests we must see ## before declaring the connection as degraded/restored @@ -862,6 +865,13 @@ proc sendNewPayloadToSingleEL( payload, versioned_hashes, Hash32 parent_beacon_block_root, executionRequests) +proc sendGetBlobsToSingleEL( + connection: ELConnection, + versioned_hashes: seq[engine_api.VersionedHash] +): Future[GetBlobsV1Response] {.async: (raises: [CatchableError]).} = + let rpcClient = await connection.connectedRpcClient() + await rpcClient.engine_getBlobsV1(versioned_hashes) + type StatusRelation = enum newStatusIsPreferable @@ -990,6 +1000,61 @@ proc lazyWait(futures: seq[FutureBase]) {.async: (raises: []).} = if len(pending) > 0: await noCancel allFutures(pending) +proc sendGetBlobs*( + m: ELManager, + blck: electra.SignedBeaconBlock | fulu.SignedBeaconBlock +): Future[Opt[seq[BlobAndProofV1]]] {.async: (raises: [CancelledError]).} = + if m.elConnections.len == 0: + return err() + let + timeout = GETBLOBS_TIMEOUT + deadline = sleepAsync(timeout) + + var bestResponse = Opt.none(int) + + while true: + let + requests = m.elConnections.mapIt( + sendGetBlobsToSingleEL(it, mapIt( + blck.message.body.blob_kzg_commitments, + engine_api.VersionedHash(kzg_commitment_to_versioned_hash(it))))) + timeoutExceeded = + try: + await allFutures(requests).wait(deadline) + false + except AsyncTimeoutError: + true + except CancelledError as exc: + let pending = + requests.filterIt(not(it.finished())).mapIt(it.cancelAndWait()) + await noCancel allFutures(pending) + raise exc + + for idx, req in requests: + if not(req.finished()): + warn "Timeout while getting blob and proof", + url = m.elConnections[idx].engineUrl.url, + reason = req.error.msg + else: + if bestResponse.isNone: + bestResponse = Opt.some(idx) + + let pending = + requests.filterIt(not(it.finished())).mapIt(it.cancelAndWait()) + await noCancel allFutures(pending) + + if bestResponse.isSome(): + return ok(requests[bestResponse.get()].value().blobsAndProofs) + + else: + # should not reach this case + discard + + if timeoutExceeded: + break + + err() + proc sendNewPayload*( m: ELManager, blck: SomeForkyBeaconBlock, diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index e4d8152ac0..e1c7b40ac0 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -8,9 +8,10 @@ {.push raises: [].} import - std/[os, random, terminal, times, exitprocs], + std/[os, random, terminal, times, exitprocs, sequtils], chronos, chronicles, metrics, metrics/chronos_httpserver, + ssz_serialization/types, stew/[byteutils, io2], eth/p2p/discoveryv5/[enr, random2], ./consensus_object_pools/[ @@ -460,21 +461,67 @@ proc initFullNode( maybeFinalized: bool): Future[Result[void, VerifierError]] {.async: (raises: [CancelledError]).} = withBlck(signedBlock): - when consensusFork >= ConsensusFork.Deneb: + when consensusFork >= ConsensusFork.Electra: + # Pull blobs and proofs from the EL blob pool + let blobsFromElOpt = await node.elManager.sendGetBlobs(forkyBlck) + if blobsFromElOpt.isSome(): + let blobsEl = blobsFromElOpt.get() + # check lengths of array[BlobAndProofV1] with blob + # kzg commitments of the signed block + if blobsEl.len == forkyBlck.message.body.blob_kzg_commitments.len: + # create blob sidecars from EL instead + var + kzgBlbs: deneb.Blobs + kzgPrfs: deneb.KzgProofs + + for idx in 0..= ConsensusFork.Deneb and + consensusFork < ConsensusFork.Electra: if not blobQuarantine[].hasBlobs(forkyBlck): # We don't have all the blobs for this block, so we have # to put it in blobless quarantine. if not quarantine[].addBlobless(dag.finalizedHead.slot, forkyBlck): - err(VerifierError.UnviableFork) + return err(VerifierError.UnviableFork) else: - err(VerifierError.MissingParent) + return err(VerifierError.MissingParent) else: let blobs = blobQuarantine[].popBlobs(forkyBlck.root, forkyBlck) - await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, - Opt.some(blobs), - maybeFinalized = maybeFinalized) + return await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, + Opt.some(blobs), + maybeFinalized = maybeFinalized) + else: - await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, + return await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, Opt.none(BlobSidecars), maybeFinalized = maybeFinalized) rmanBlockLoader = proc( diff --git a/vendor/nim-web3 b/vendor/nim-web3 index a3bc5ad48e..f3c86a6674 160000 --- a/vendor/nim-web3 +++ b/vendor/nim-web3 @@ -1 +1 @@ -Subproject commit a3bc5ad48e2b05fa253ba68bbd5b84e4ea234f50 +Subproject commit f3c86a6674070d1f68d7fec6e0572aba94bae096 From a1d10c76473641aa0d2fde427dc38414e45669dc Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Tue, 11 Feb 2025 13:35:27 +0530 Subject: [PATCH 02/15] bump nim-web3 to 5443261a0246614e0eb2f749a4f161e5e1832520 --- vendor/nim-web3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-web3 b/vendor/nim-web3 index f3c86a6674..5443261a02 160000 --- a/vendor/nim-web3 +++ b/vendor/nim-web3 @@ -1 +1 @@ -Subproject commit f3c86a6674070d1f68d7fec6e0572aba94bae096 +Subproject commit 5443261a0246614e0eb2f749a4f161e5e1832520 From e6edb6b9e140f9d887a03462f9fe12336a00ab18 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Wed, 12 Feb 2025 01:38:36 +0530 Subject: [PATCH 03/15] don't delay in calling addBlock --- beacon_chain/nimbus_beacon_node.nim | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index e1c7b40ac0..831fe9b0e0 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -464,6 +464,8 @@ proc initFullNode( when consensusFork >= ConsensusFork.Electra: # Pull blobs and proofs from the EL blob pool let blobsFromElOpt = await node.elManager.sendGetBlobs(forkyBlck) + debugEcho "pulled blobs from el" + debugEcho blobsFromElOpt.get.len if blobsFromElOpt.isSome(): let blobsEl = blobsFromElOpt.get() # check lengths of array[BlobAndProofV1] with blob @@ -480,14 +482,14 @@ proc initFullNode( let blob_sidecars_el = create_blob_sidecars(forkyBlck, kzgPrfs, kzgBlbs) - # populate blob quarantine to tackle blob loop - for blb_el in blob_sidecars_el: - blobQuarantine[].put(newClone blb_el) + # # populate blob quarantine to tackle blob loop + # for blb_el in blob_sidecars_el: + # blobQuarantine[].put(newClone blb_el) - # now pop blobQuarantine and make block available for attestation - let blobs = blobQuarantine[].popBlobs(forkyBlck.root, forkyBlck) + # # now pop blobQuarantine and make block available for attestation + # let blobs = blobQuarantine[].popBlobs(forkyBlck.root, forkyBlck) return await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, - Opt.some(blobs), + Opt.some(blob_sidecars_el), maybeFinalized = maybeFinalized) # in case EL does not support `engine_getBlobsV1` From 93f3e3cb455a2223bc19c26ae663b0726e756655 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Wed, 12 Feb 2025 01:45:20 +0530 Subject: [PATCH 04/15] just add debug logs --- beacon_chain/nimbus_beacon_node.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 831fe9b0e0..f7c95741b7 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -482,12 +482,12 @@ proc initFullNode( let blob_sidecars_el = create_blob_sidecars(forkyBlck, kzgPrfs, kzgBlbs) - # # populate blob quarantine to tackle blob loop - # for blb_el in blob_sidecars_el: - # blobQuarantine[].put(newClone blb_el) + # populate blob quarantine to tackle blob loop + for blb_el in blob_sidecars_el: + blobQuarantine[].put(newClone blb_el) - # # now pop blobQuarantine and make block available for attestation - # let blobs = blobQuarantine[].popBlobs(forkyBlck.root, forkyBlck) + # now pop blobQuarantine and make block available for attestation + let blobs = blobQuarantine[].popBlobs(forkyBlck.root, forkyBlck) return await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, Opt.some(blob_sidecars_el), maybeFinalized = maybeFinalized) From f2c1b0a02a1b07fbda9371d149d417da083d00a7 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Wed, 12 Feb 2025 02:01:54 +0530 Subject: [PATCH 05/15] fix --- beacon_chain/nimbus_beacon_node.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index f7c95741b7..8aa814e58b 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -489,7 +489,7 @@ proc initFullNode( # now pop blobQuarantine and make block available for attestation let blobs = blobQuarantine[].popBlobs(forkyBlck.root, forkyBlck) return await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, - Opt.some(blob_sidecars_el), + Opt.some(blobs), maybeFinalized = maybeFinalized) # in case EL does not support `engine_getBlobsV1` From 31ce92f12343132ceb27d05cda3819db1e7c1856 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Thu, 13 Feb 2025 04:42:44 +0530 Subject: [PATCH 06/15] address review comments, and rework strategy --- beacon_chain/el/el_manager.nim | 7 +- .../gossip_processing/eth2_processor.nim | 45 +++++++++++-- beacon_chain/nimbus_beacon_node.nim | 67 +++---------------- 3 files changed, 53 insertions(+), 66 deletions(-) diff --git a/beacon_chain/el/el_manager.nim b/beacon_chain/el/el_manager.nim index 91c5901671..6b36ebeb6b 100644 --- a/beacon_chain/el/el_manager.nim +++ b/beacon_chain/el/el_manager.nim @@ -1007,8 +1007,7 @@ proc sendGetBlobs*( if m.elConnections.len == 0: return err() let - timeout = GETBLOBS_TIMEOUT - deadline = sleepAsync(timeout) + deadline = sleepAsync(GETBLOBS_TIMEOUT) var bestResponse = Opt.none(int) @@ -1046,10 +1045,6 @@ proc sendGetBlobs*( if bestResponse.isSome(): return ok(requests[bestResponse.get()].value().blobsAndProofs) - else: - # should not reach this case - discard - if timeoutExceeded: break diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index 276d7e7c3c..beadcc697d 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -12,6 +12,7 @@ import chronicles, chronos, metrics, taskpools, ../spec/[helpers, forks], + ../el/el_manager, ../consensus_object_pools/[ blob_quarantine, block_clearance, block_quarantine, blockchain_dag, attestation_pool, light_client_pool, sync_committee_msg_pool, @@ -138,6 +139,10 @@ type # ---------------------------------------------------------------- batchCrypto*: ref BatchCrypto + # EL integration + # ---------------------------------------------------------------- + elManager*: ELManager + # Missing information # ---------------------------------------------------------------- quarantine*: ref Quarantine @@ -169,6 +174,7 @@ proc new*(T: type Eth2Processor, blobQuarantine: ref BlobQuarantine, rng: ref HmacDrbgContext, getBeaconTime: GetBeaconTimeFn, + elManager: ELManager, taskpool: Taskpool ): ref Eth2Processor = (ref Eth2Processor)( @@ -186,6 +192,7 @@ proc new*(T: type Eth2Processor, quarantine: quarantine, blobQuarantine: blobQuarantine, getCurrentBeaconTime: getBeaconTime, + elManager: elManager, batchCrypto: BatchCrypto.new( rng = rng, # Only run eager attestation signature verification if we're not @@ -267,9 +274,40 @@ proc processSignedBeaconBlock*( v proc processBlobSidecar*( - self: var Eth2Processor, src: MsgSource, - blobSidecar: deneb.BlobSidecar, subnet_id: BlobId): ValidationRes = + self: ref Eth2Processor, src: MsgSource, + blobSidecar: deneb.BlobSidecar, subnet_id: BlobId): + Future[ValidationRes] {.async: (raises: [CancelledError]).} = template block_header: untyped = blobSidecar.signed_block_header.message + let block_root = hash_tree_root(block_header) + + if (let o = self.quarantine[].popBlobless(block_root); o.isSome): + let blobless = o.get() + withBlck(blobless): + when consensusFork >= ConsensusFork.Electra: + let blobsFromElOpt = await self.elManager.sendGetBlobs(forkyBlck) + debugEcho "pulled blobs from el" + debugEcho blobsFromElOpt.get.len + if blobsFromElOpt.get.len > 0 and blobsFromElOpt.isSome(): + let blobsEl = blobsFromElOpt.get() + # check lengths of array[BlobAndProofV1] with blobs + # kzg commitments of the signed block + if blobsEl.len == forkyBlck.message.body.blob_kzg_commitments.len: + var + kzgblbs: deneb.Blobs + kzgprfs: deneb.KzgProofs + for idx in 0..= ConsensusFork.Deneb: if self.blobQuarantine[].hasBlobs(forkyBlck): diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 8aa814e58b..0a867b2520 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -461,69 +461,23 @@ proc initFullNode( maybeFinalized: bool): Future[Result[void, VerifierError]] {.async: (raises: [CancelledError]).} = withBlck(signedBlock): - when consensusFork >= ConsensusFork.Electra: - # Pull blobs and proofs from the EL blob pool - let blobsFromElOpt = await node.elManager.sendGetBlobs(forkyBlck) - debugEcho "pulled blobs from el" - debugEcho blobsFromElOpt.get.len - if blobsFromElOpt.isSome(): - let blobsEl = blobsFromElOpt.get() - # check lengths of array[BlobAndProofV1] with blob - # kzg commitments of the signed block - if blobsEl.len == forkyBlck.message.body.blob_kzg_commitments.len: - # create blob sidecars from EL instead - var - kzgBlbs: deneb.Blobs - kzgPrfs: deneb.KzgProofs - - for idx in 0..= ConsensusFork.Deneb and + when consensusFork >= ConsensusFork.Deneb and consensusFork < ConsensusFork.Electra: if not blobQuarantine[].hasBlobs(forkyBlck): # We don't have all the blobs for this block, so we have # to put it in blobless quarantine. if not quarantine[].addBlobless(dag.finalizedHead.slot, forkyBlck): - return err(VerifierError.UnviableFork) + err(VerifierError.UnviableFork) else: - return err(VerifierError.MissingParent) + err(VerifierError.MissingParent) else: let blobs = blobQuarantine[].popBlobs(forkyBlck.root, forkyBlck) - return await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, + await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, Opt.some(blobs), maybeFinalized = maybeFinalized) else: - return await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, + await blockProcessor[].addBlock(MsgSource.gossip, signedBlock, Opt.none(BlobSidecars), maybeFinalized = maybeFinalized) rmanBlockLoader = proc( @@ -548,7 +502,8 @@ proc initFullNode( config.doppelgangerDetection, blockProcessor, node.validatorMonitor, dag, attestationPool, validatorChangePool, node.attachedValidators, syncCommitteeMsgPool, - lightClientPool, quarantine, blobQuarantine, rng, getBeaconTime, taskpool) + lightClientPool, quarantine, blobQuarantine, rng, getBeaconTime, + node.elManager, taskpool) syncManagerFlags = if node.config.longRangeSync != LongRangeSyncMode.Lenient: {SyncManagerFlag.NoGenesisSync} @@ -2160,12 +2115,12 @@ proc installMessageValidators(node: BeaconNode) = for it in 0.BlobId ..< subnetCount.BlobId: closureScope: # Needed for inner `proc`; don't lift it out of loop. let subnet_id = it - node.network.addValidator( + node.network.addAsyncValidator( getBlobSidecarTopic(digest, subnet_id), proc ( blobSidecar: deneb.BlobSidecar - ): ValidationResult = - toValidationResult( - node.processor[].processBlobSidecar( + ): Future[ValidationResult] {.async: (raises: [CancelledError]).} = + return toValidationResult( + await node.processor.processBlobSidecar( MsgSource.gossip, blobSidecar, subnet_id))) node.installLightClientMessageValidators() From fe9d83a17c272b5f737c4df6986f96f28a047ebe Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Thu, 13 Feb 2025 04:58:08 +0530 Subject: [PATCH 07/15] update copyright years --- beacon_chain/gossip_processing/eth2_processor.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index beadcc697d..3a357f2c56 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2018-2024 Status Research & Development GmbH +# Copyright (c) 2018-2025 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). From 2463df546af43fe92b50213969da29027dfb7cd6 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Thu, 13 Feb 2025 04:59:27 +0530 Subject: [PATCH 08/15] cleanup whitespace --- beacon_chain/el/el_manager.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/beacon_chain/el/el_manager.nim b/beacon_chain/el/el_manager.nim index 6b36ebeb6b..58a89247f7 100644 --- a/beacon_chain/el/el_manager.nim +++ b/beacon_chain/el/el_manager.nim @@ -57,7 +57,6 @@ type GetPayloadV3Response | GetPayloadV4Response - contract(DepositContract): proc deposit(pubkey: PubKeyBytes, withdrawalCredentials: WithdrawalCredentialsBytes, From 015850dd260bce50af43ed287b997950344be590 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Thu, 13 Feb 2025 05:01:46 +0530 Subject: [PATCH 09/15] cleanup condition --- beacon_chain/nimbus_beacon_node.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 0a867b2520..c71de445a4 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -461,8 +461,7 @@ proc initFullNode( maybeFinalized: bool): Future[Result[void, VerifierError]] {.async: (raises: [CancelledError]).} = withBlck(signedBlock): - when consensusFork >= ConsensusFork.Deneb and - consensusFork < ConsensusFork.Electra: + when consensusFork >= ConsensusFork.Deneb: if not blobQuarantine[].hasBlobs(forkyBlck): # We don't have all the blobs for this block, so we have # to put it in blobless quarantine. From dec6ea112c1226b1d062d3757a96c146d7ee5fe1 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Fri, 14 Feb 2025 01:10:37 +0530 Subject: [PATCH 10/15] fixes + bump nim-web3 to e9640d65eca5618291438bf6e98f6ea21f4c1d03 --- beacon_chain/el/el_manager.nim | 4 ++-- vendor/nim-web3 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_chain/el/el_manager.nim b/beacon_chain/el/el_manager.nim index 58a89247f7..20c6864148 100644 --- a/beacon_chain/el/el_manager.nim +++ b/beacon_chain/el/el_manager.nim @@ -1001,7 +1001,7 @@ proc lazyWait(futures: seq[FutureBase]) {.async: (raises: []).} = proc sendGetBlobs*( m: ELManager, - blck: electra.SignedBeaconBlock | fulu.SignedBeaconBlock + blck: electra.SignedBeaconBlock | fulu.SignedBeaconBlock, ): Future[Opt[seq[BlobAndProofV1]]] {.async: (raises: [CancelledError]).} = if m.elConnections.len == 0: return err() @@ -1042,7 +1042,7 @@ proc sendGetBlobs*( await noCancel allFutures(pending) if bestResponse.isSome(): - return ok(requests[bestResponse.get()].value().blobsAndProofs) + return ok(requests[bestResponse.get()].value()) if timeoutExceeded: break diff --git a/vendor/nim-web3 b/vendor/nim-web3 index 5443261a02..e9640d65ec 160000 --- a/vendor/nim-web3 +++ b/vendor/nim-web3 @@ -1 +1 @@ -Subproject commit 5443261a0246614e0eb2f749a4f161e5e1832520 +Subproject commit e9640d65eca5618291438bf6e98f6ea21f4c1d03 From 62030250016a266015ebff2c9e7f3320f1a85e9f Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Fri, 14 Feb 2025 11:29:17 +0530 Subject: [PATCH 11/15] fix list --- beacon_chain/gossip_processing/eth2_processor.nim | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index 3a357f2c56..bc5fa957d7 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -8,9 +8,10 @@ {.push raises: [].} import - std/tables, + std/[tables, sequtils], chronicles, chronos, metrics, taskpools, + ssz_serialization/types, ../spec/[helpers, forks], ../el/el_manager, ../consensus_object_pools/[ @@ -292,14 +293,11 @@ proc processBlobSidecar*( # check lengths of array[BlobAndProofV1] with blobs # kzg commitments of the signed block if blobsEl.len == forkyBlck.message.body.blob_kzg_commitments.len: - var - kzgblbs: deneb.Blobs - kzgprfs: deneb.KzgProofs - for idx in 0.. Date: Sat, 15 Feb 2025 18:10:22 +0530 Subject: [PATCH 12/15] rework and cleanup --- .../gossip_processing/eth2_processor.nim | 33 ++++++++++++++----- beacon_chain/nimbus_beacon_node.nim | 2 +- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index bc5fa957d7..a52cdf1315 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -274,20 +274,17 @@ proc processSignedBeaconBlock*( v -proc processBlobSidecar*( - self: ref Eth2Processor, src: MsgSource, - blobSidecar: deneb.BlobSidecar, subnet_id: BlobId): - Future[ValidationRes] {.async: (raises: [CancelledError]).} = - template block_header: untyped = blobSidecar.signed_block_header.message - let block_root = hash_tree_root(block_header) +proc validateBlobSidecarFromEL( + self: ref Eth2Processor, + block_root: Eth2Digest): + Future[Result[void, ValidationError]] {.async: (raises: [CancelledError]).} = if (let o = self.quarantine[].popBlobless(block_root); o.isSome): let blobless = o.get() withBlck(blobless): when consensusFork >= ConsensusFork.Electra: - let blobsFromElOpt = await self.elManager.sendGetBlobs(forkyBlck) - debugEcho "pulled blobs from el" - debugEcho blobsFromElOpt.get.len + let blobsFromElOpt = + await self.elManager.sendGetBlobs(forkyBlck) if blobsFromElOpt.get.len > 0 and blobsFromElOpt.isSome(): let blobsEl = blobsFromElOpt.get() # check lengths of array[BlobAndProofV1] with blobs @@ -307,6 +304,24 @@ proc processBlobSidecar*( MsgSource.gossip, blobless, Opt.some(self.blobQuarantine[].popBlobs(block_root, forkyBlck))) + return ok() + + else: + return errIgnore("EL did not respond with blobs and proofs") + +proc processBlobSidecar*( + self: ref Eth2Processor, src: MsgSource, + blobSidecar: deneb.BlobSidecar, subnet_id: BlobId): + Future[ValidationRes] {.async: (raises: [CancelledError]).} = + template block_header: untyped = blobSidecar.signed_block_header.message + let block_root = hash_tree_root(block_header) + + let vEl = + await self.validateBlobSidecarFromEL(block_root) + + if vEl.isOk(): + return vEl + let wallTime = self.getCurrentBeaconTime() (_, wallSlot) = wallTime.toSlot() diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index c71de445a4..924e526939 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -8,7 +8,7 @@ {.push raises: [].} import - std/[os, random, terminal, times, exitprocs, sequtils], + std/[os, random, terminal, times, exitprocs], chronos, chronicles, metrics, metrics/chronos_httpserver, ssz_serialization/types, From e16a4d8e285363acd441dc5dc3b718af01ec0fc3 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Mon, 17 Feb 2025 02:25:40 +0530 Subject: [PATCH 13/15] add some debug logs on successfully bypassing blob gossip validation --- beacon_chain/gossip_processing/eth2_processor.nim | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index a52cdf1315..7c3e799df3 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -277,7 +277,8 @@ proc processSignedBeaconBlock*( proc validateBlobSidecarFromEL( self: ref Eth2Processor, block_root: Eth2Digest): - Future[Result[void, ValidationError]] {.async: (raises: [CancelledError]).} = + Future[Result[void, ValidationError]] + {.async: (raises: [CancelledError]).} = if (let o = self.quarantine[].popBlobless(block_root); o.isSome): let blobless = o.get() @@ -287,13 +288,15 @@ proc validateBlobSidecarFromEL( await self.elManager.sendGetBlobs(forkyBlck) if blobsFromElOpt.get.len > 0 and blobsFromElOpt.isSome(): let blobsEl = blobsFromElOpt.get() + # check lengths of array[BlobAndProofV1] with blobs # kzg commitments of the signed block if blobsEl.len == forkyBlck.message.body.blob_kzg_commitments.len: let blob_sidecars_el = create_blob_sidecars( forkyBlck, - deneb.KzgProofs.init(blobsEl.mapIt(kzg.KzgProof(bytes: it.proof.data))), + deneb.KzgProofs.init(blobsEl.mapIt(kzg.KzgProof( + bytes: it.proof.data))), deneb.Blobs.init(blobsEl.mapIt(it.blob.data))) for blb_el in blob_sidecars_el: @@ -304,6 +307,8 @@ proc validateBlobSidecarFromEL( MsgSource.gossip, blobless, Opt.some(self.blobQuarantine[].popBlobs(block_root, forkyBlck))) + debug "Pulled blobs from EL, bypassing blob gossip validation", + blobs_from_el = blobsEl.len return ok() else: From 999fb04d25b5aea1274f993c5177b657dc9ab439 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Mon, 3 Mar 2025 12:53:59 +0530 Subject: [PATCH 14/15] only bypass gossip when EL can sends blobs against ALL kzg commitments --- .../consensus_object_pools/block_quarantine.nim | 10 ++++++++++ beacon_chain/gossip_processing/eth2_processor.nim | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/beacon_chain/consensus_object_pools/block_quarantine.nim b/beacon_chain/consensus_object_pools/block_quarantine.nim index da33499be1..0a5c603b10 100644 --- a/beacon_chain/consensus_object_pools/block_quarantine.nim +++ b/beacon_chain/consensus_object_pools/block_quarantine.nim @@ -381,6 +381,16 @@ func popBlobless*( else: Opt.none(ForkedSignedBeaconBlock) +func getBlobless*( + quarantine: var Quarantine, + root: Eth2Digest): Opt[ForkedSignedBeaconBlock] = + if quarantine.blobless.hasKey(root): + Opt.some((quarantine.blobless.getOrDefault( + root, + default(ForkedSignedBeaconBlock)))) + else: + Opt.none(ForkedSignedBeaconBlock) + func popColumnless*( quarantine: var Quarantine, root: Eth2Digest): Opt[ForkedSignedBeaconBlock] = diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index 7c3e799df3..45a65f7fd1 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -280,7 +280,7 @@ proc validateBlobSidecarFromEL( Future[Result[void, ValidationError]] {.async: (raises: [CancelledError]).} = - if (let o = self.quarantine[].popBlobless(block_root); o.isSome): + if (let o = self.quarantine[].getBlobless(block_root); o.isSome): let blobless = o.get() withBlck(blobless): when consensusFork >= ConsensusFork.Electra: @@ -292,6 +292,11 @@ proc validateBlobSidecarFromEL( # check lengths of array[BlobAndProofV1] with blobs # kzg commitments of the signed block if blobsEl.len == forkyBlck.message.body.blob_kzg_commitments.len: + + # we have got all blobs from EL, now we can + # conveniently the blobless block from quarantine + discard self.quarantine[].popBlobless(block_root) + let blob_sidecars_el = create_blob_sidecars( forkyBlck, From 086066a63ec99b4743fab7beedbc3f2cd2d7d6f1 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh Date: Mon, 3 Mar 2025 13:01:33 +0530 Subject: [PATCH 15/15] fix copyright year --- beacon_chain/consensus_object_pools/block_quarantine.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_chain/consensus_object_pools/block_quarantine.nim b/beacon_chain/consensus_object_pools/block_quarantine.nim index 0a5c603b10..be8b300fcd 100644 --- a/beacon_chain/consensus_object_pools/block_quarantine.nim +++ b/beacon_chain/consensus_object_pools/block_quarantine.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2018-2024 Status Research & Development GmbH +# Copyright (c) 2018-2025 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).