From 1026ea2d9443204fbb2656e98c4cebaf101402e5 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 31 Jan 2025 11:05:21 -0500 Subject: [PATCH 01/15] Draft distributed payload retrieval Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 26 ++-- .../v2/distributed_payload_retriever.go | 116 ++++++++++++++++++ api/clients/v2/mock/retrieval_client.go | 54 ++++++-- api/clients/v2/relay_payload_retriever.go | 105 +++------------- api/clients/v2/retrieval_client.go | 42 ++++--- .../v2/test/relay_payload_retriever_test.go | 4 +- api/clients/v2/verification/blob_verifier.go | 55 +++++++++ .../v2/verification/conversion_utils.go | 16 +++ api/clients/v2/verification/eigenda_cert.go | 4 +- core/v2/core_test.go | 2 +- core/v2/types.go | 4 +- core/v2/validator.go | 5 +- disperser/controller/encoding_manager.go | 2 +- inabox/tests/integration_v2_test.go | 38 +++++- retriever/v2/server.go | 9 +- test/v2/test_client.go | 14 ++- 16 files changed, 350 insertions(+), 146 deletions(-) create mode 100644 api/clients/v2/distributed_payload_retriever.go create mode 100644 api/clients/v2/verification/blob_verifier.go diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index 9dae41ed08..b0f77f2d97 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -45,13 +45,13 @@ type PayloadClientConfig struct { BlobVersion v2.BlobVersion } -// RelayPayloadRetrieverConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed -// by a RelayPayloadRetriever -type RelayPayloadRetrieverConfig struct { +// PayloadRetrieverConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed +// by a PayloadRetriever +type PayloadRetrieverConfig struct { PayloadClientConfig - // The timeout duration for relay calls to retrieve blobs. - RelayTimeout time.Duration + // The timeout duration for calls to retrieve blobs. + FetchTimeout time.Duration } // PayloadDisperserConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed @@ -125,13 +125,13 @@ func (cc *PayloadClientConfig) checkAndSetDefaults() error { return nil } -// GetDefaultRelayPayloadRetrieverConfig creates a RelayPayloadRetrieverConfig with default values +// GetDefaultPayloadRetrieverConfig creates a GetDefaultPayloadRetrieverConfig with default values // // NOTE: EthRpcUrl and EigenDACertVerifierAddr do not have defined defaults. These must always be specifically configured. -func GetDefaultRelayPayloadRetrieverConfig() *RelayPayloadRetrieverConfig { - return &RelayPayloadRetrieverConfig{ +func GetDefaultPayloadRetrieverConfig() *PayloadRetrieverConfig { + return &PayloadRetrieverConfig{ PayloadClientConfig: *getDefaultPayloadClientConfig(), - RelayTimeout: 5 * time.Second, + FetchTimeout: 5 * time.Second, } } @@ -140,15 +140,15 @@ func GetDefaultRelayPayloadRetrieverConfig() *RelayPayloadRetrieverConfig { // 1. If a config value is 0, and a 0 value makes sense, do nothing. // 2. If a config value is 0, but a 0 value doesn't make sense and a default value is defined, then set it to the default. // 3. If a config value is 0, but a 0 value doesn't make sense and a default value isn't defined, return an error. -func (rc *RelayPayloadRetrieverConfig) checkAndSetDefaults() error { +func (rc *PayloadRetrieverConfig) checkAndSetDefaults() error { err := rc.PayloadClientConfig.checkAndSetDefaults() if err != nil { return err } - defaultConfig := GetDefaultRelayPayloadRetrieverConfig() - if rc.RelayTimeout == 0 { - rc.RelayTimeout = defaultConfig.RelayTimeout + defaultConfig := GetDefaultPayloadRetrieverConfig() + if rc.FetchTimeout == 0 { + rc.FetchTimeout = defaultConfig.FetchTimeout } return nil diff --git a/api/clients/v2/distributed_payload_retriever.go b/api/clients/v2/distributed_payload_retriever.go new file mode 100644 index 0000000000..528435fd2a --- /dev/null +++ b/api/clients/v2/distributed_payload_retriever.go @@ -0,0 +1,116 @@ +package clients + +import ( + "context" + "fmt" + + "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/api/clients/v2/verification" + "github.com/Layr-Labs/eigenda/core" + corev2 "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigensdk-go/logging" + "github.com/consensys/gnark-crypto/ecc/bn254" +) + +// DistributedPayloadRetriever provides the ability to get payloads from the EigenDA nodes directly +// +// This struct is goroutine safe. +type DistributedPayloadRetriever struct { + logger logging.Logger + config PayloadRetrieverConfig + codec codecs.BlobCodec + retrievalClient RetrievalClient + g1Srs []bn254.G1Affine +} + +var _ PayloadRetriever = &DistributedPayloadRetriever{} + +// GetPayload iteratively attempts to retrieve a given blob from the quorums listed in the EigenDACert. +// +// If the blob is successfully retrieved, then the blob is verified. If the verification succeeds, the blob is decoded +// to yield the payload (the original user data, with no padding or any modification), and the payload is returned. +func (pr *DistributedPayloadRetriever) GetPayload( + ctx context.Context, + eigenDACert *verification.EigenDACert, +) ([]byte, error) { + + blobKey, err := eigenDACert.ComputeBlobKey() + if err != nil { + return nil, fmt.Errorf("compute blob key: %w", err) + } + + blobHeader := eigenDACert.BlobInclusionInfo.BlobCertificate.BlobHeader + convertedCommitment, err := verification.BlobCommitmentsBindingToInternal(&blobHeader.Commitment) + if err != nil { + return nil, fmt.Errorf("convert commitments binding to internal: %w", err) + } + + // TODO (litt3): Add a feature which keeps chunks from previous quorums, and just fills in gaps + for _, quorumID := range blobHeader.QuorumNumbers { + blobBytes, err := pr.getBlobWithTimeout( + ctx, + *blobKey, + blobHeader.Version, + *convertedCommitment, + eigenDACert.BatchHeader.ReferenceBlockNumber, + quorumID) + + if err != nil { + pr.logger.Warn( + "blob couldn't be retrieved from quorum", + "blobKey", blobKey.Hex(), + "quorumId", quorumID, + "error", err) + continue + } + + err = verification.CheckBlobLength(blobBytes, convertedCommitment.Length) + if err != nil { + pr.logger.Warn("check blob length", "blobKey", blobKey.Hex(), "quorumID", quorumID, "error", err) + continue + } + err = verification.VerifyBlobAgainstCert(blobKey, blobBytes, convertedCommitment.Commitment, pr.g1Srs) + if err != nil { + pr.logger.Warn("verify blob against cert", "blobKey", blobKey.Hex(), "quorumID", quorumID, "error", err) + continue + } + + payload, err := pr.codec.DecodeBlob(blobBytes) + if err != nil { + pr.logger.Error( + `Cert verification was successful, but decode blob failed! + This is likely a problem with the local blob codec configuration, + but could potentially indicate a maliciously generated blob certificate. + It should not be possible for an honestly generated certificate to verify + for an invalid blob!`, + "blobKey", blobKey.Hex(), "quorumID", quorumID, "eigenDACert", eigenDACert, "error", err) + return nil, fmt.Errorf("decode blob: %w", err) + } + + return payload, nil + } + + return nil, fmt.Errorf("unable to retrieve payload from quorums %v", blobHeader.QuorumNumbers) +} + +// getBlobWithTimeout attempts to get a blob from a given quorum, and times out based on config.FetchTimeout +func (pr *DistributedPayloadRetriever) getBlobWithTimeout( + ctx context.Context, + blobKey corev2.BlobKey, + blobVersion corev2.BlobVersion, + blobCommitments encoding.BlobCommitments, + referenceBlockNumber uint32, + quorumID core.QuorumID) ([]byte, error) { + + timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.FetchTimeout) + defer cancel() + + return pr.retrievalClient.GetBlob( + timeoutCtx, + blobKey, + blobVersion, + blobCommitments, + uint64(referenceBlockNumber), + quorumID) +} diff --git a/api/clients/v2/mock/retrieval_client.go b/api/clients/v2/mock/retrieval_client.go index 6f0e23b723..66c8653383 100644 --- a/api/clients/v2/mock/retrieval_client.go +++ b/api/clients/v2/mock/retrieval_client.go @@ -1,27 +1,61 @@ +// Code generated by mockery v2.50.0. DO NOT EDIT. + package mock import ( "context" - "github.com/Layr-Labs/eigenda/api/clients/v2" - "github.com/Layr-Labs/eigenda/core" - corev2 "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigenda/encoding" "github.com/stretchr/testify/mock" + + v2 "github.com/Layr-Labs/eigenda/core/v2" ) +// MockRetrievalClient is an autogenerated mock type for the MockRetrievalClient type type MockRetrievalClient struct { mock.Mock } -var _ clients.RetrievalClient = (*MockRetrievalClient)(nil) +// GetBlob provides a mock function with given fields: ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID +func (_m *MockRetrievalClient) GetBlob(ctx context.Context, blobKey v2.BlobKey, blobVersion uint16, blobCommitments encoding.BlobCommitments, referenceBlockNumber uint64, quorumID uint8) ([]byte, error) { + ret := _m.Called(ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID) + + if len(ret) == 0 { + panic("no return value specified for GetBlob") + } -func NewRetrievalClient() *MockRetrievalClient { - return &MockRetrievalClient{} + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, v2.BlobKey, uint16, encoding.BlobCommitments, uint64, uint8) ([]byte, error)); ok { + return rf(ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID) + } + if rf, ok := ret.Get(0).(func(context.Context, v2.BlobKey, uint16, encoding.BlobCommitments, uint64, uint8) []byte); ok { + r0 = rf(ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, v2.BlobKey, uint16, encoding.BlobCommitments, uint64, uint8) error); ok { + r1 = rf(ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -func (c *MockRetrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHeader, referenceBlockNumber uint64, quorumID core.QuorumID) ([]byte, error) { - args := c.Called() +// NewMockRetrievalClient creates a new instance of MockRetrievalClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockRetrievalClient(t interface { + mock.TestingT + Cleanup(func()) +}) *MockRetrievalClient { + mock := &MockRetrievalClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) - result := args.Get(0) - return result.([]byte), args.Error(1) + return mock } diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index ff4f784a86..bee46e0efb 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -9,9 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/Layr-Labs/eigenda/api/clients/v2/verification" "github.com/Layr-Labs/eigenda/common/geth" - verifiercontract "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDACertVerifier" core "github.com/Layr-Labs/eigenda/core/v2" - "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/consensys/gnark-crypto/ecc/bn254" gethcommon "github.com/ethereum/go-ethereum/common" @@ -26,7 +24,7 @@ type RelayPayloadRetriever struct { // Not all methods on Rand are guaranteed goroutine safe: if additional usages of random are added, they // must be evaluated for thread safety. random *rand.Rand - config RelayPayloadRetrieverConfig + config PayloadRetrieverConfig codec codecs.BlobCodec relayClient RelayClient g1Srs []bn254.G1Affine @@ -38,7 +36,7 @@ var _ PayloadRetriever = &RelayPayloadRetriever{} // BuildRelayPayloadRetriever builds a RelayPayloadRetriever from config structs. func BuildRelayPayloadRetriever( log logging.Logger, - relayPayloadRetrieverConfig RelayPayloadRetrieverConfig, + payloadRetrieverConfig PayloadRetrieverConfig, ethConfig geth.EthClientConfig, relayClientConfig *RelayClientConfig, g1Srs []bn254.G1Affine) (*RelayPayloadRetriever, error) { @@ -53,14 +51,14 @@ func BuildRelayPayloadRetriever( return nil, fmt.Errorf("new eth client: %w", err) } - certVerifier, err := verification.NewCertVerifier(*ethClient, relayPayloadRetrieverConfig.EigenDACertVerifierAddr) + certVerifier, err := verification.NewCertVerifier(*ethClient, payloadRetrieverConfig.EigenDACertVerifierAddr) if err != nil { return nil, fmt.Errorf("new cert verifier: %w", err) } codec, err := codecs.CreateCodec( - relayPayloadRetrieverConfig.PayloadPolynomialForm, - relayPayloadRetrieverConfig.BlobEncodingVersion) + payloadRetrieverConfig.PayloadPolynomialForm, + payloadRetrieverConfig.BlobEncodingVersion) if err != nil { return nil, err } @@ -68,7 +66,7 @@ func BuildRelayPayloadRetriever( return NewRelayPayloadRetriever( log, rand.New(rand.NewSource(rand.Int63())), - relayPayloadRetrieverConfig, + payloadRetrieverConfig, relayClient, certVerifier, codec, @@ -79,13 +77,13 @@ func BuildRelayPayloadRetriever( func NewRelayPayloadRetriever( log logging.Logger, random *rand.Rand, - relayPayloadRetrieverConfig RelayPayloadRetrieverConfig, + payloadRetrieverConfig PayloadRetrieverConfig, relayClient RelayClient, certVerifier verification.ICertVerifier, codec codecs.BlobCodec, g1Srs []bn254.G1Affine) (*RelayPayloadRetriever, error) { - err := relayPayloadRetrieverConfig.checkAndSetDefaults() + err := payloadRetrieverConfig.checkAndSetDefaults() if err != nil { return nil, fmt.Errorf("check and set RelayPayloadRetrieverConfig config: %w", err) } @@ -93,7 +91,7 @@ func NewRelayPayloadRetriever( return &RelayPayloadRetriever{ log: log, random: random, - config: relayPayloadRetrieverConfig, + config: payloadRetrieverConfig, codec: codec, relayClient: relayClient, certVerifier: certVerifier, @@ -126,7 +124,7 @@ func (pr *RelayPayloadRetriever) GetPayload( return nil, errors.New("relay key count is zero") } - blobCommitments, err := blobCommitmentsBindingToInternal( + blobCommitments, err := verification.BlobCommitmentsBindingToInternal( &eigenDACert.BlobInclusionInfo.BlobCertificate.BlobHeader.Commitment) if err != nil { @@ -154,11 +152,14 @@ func (pr *RelayPayloadRetriever) GetPayload( continue } - err = pr.verifyBlobAgainstCert(blobKey, relayKey, blob, blobCommitments.Commitment, blobCommitments.Length) - - // An honest relay should never send a blob which doesn't verify against the cert + err = verification.CheckBlobLength(blob, blobCommitments.Length) + if err != nil { + pr.log.Warn("check blob length", "blobKey", blobKey.Hex(), "relayKey", relayKey, "error", err) + continue + } + err = verification.VerifyBlobAgainstCert(blobKey, blob, blobCommitments.Commitment, pr.g1Srs) if err != nil { - pr.log.Warn("verify blob from relay against cert: %w", err) + pr.log.Warn("verify blob against cert", "blobKey", blobKey.Hex(), "relayKey", relayKey, "error", err) continue } @@ -180,65 +181,13 @@ func (pr *RelayPayloadRetriever) GetPayload( return nil, fmt.Errorf("unable to retrieve blob %v from any relay. relay count: %d", blobKey.Hex(), relayKeyCount) } -// verifyBlobAgainstCert verifies the blob received from a relay against the certificate. -// -// The following verifications are performed in this method: -// 1. Verify that the blob isn't empty -// 2. Verify the blob against the cert's kzg commitment -// 3. Verify that the blob length is less than or equal to the cert's blob length -// -// If all verifications succeed, the method returns nil. Otherwise, it returns an error. -func (pr *RelayPayloadRetriever) verifyBlobAgainstCert( - blobKey *core.BlobKey, - relayKey core.RelayKey, - blob []byte, - kzgCommitment *encoding.G1Commitment, - blobLength uint) error { - - // An honest relay should never send an empty blob - if len(blob) == 0 { - return fmt.Errorf("blob %v received from relay %v had length 0", blobKey.Hex(), relayKey) - } - - // TODO: in the future, this will be optimized to use fiat shamir transformation for verification, rather than - // regenerating the commitment: https://github.com/Layr-Labs/eigenda/issues/1037 - valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blob, kzgCommitment) - if err != nil { - return fmt.Errorf( - "generate and compare commitment for blob %v received from relay %v: %w", - blobKey.Hex(), - relayKey, - err) - } - - if !valid { - return fmt.Errorf("commitment for blob %v is invalid for bytes received from relay %v", blobKey.Hex(), relayKey) - } - - // Checking that the length returned by the relay is <= the length claimed in the BlobCommitments is sufficient - // here: it isn't necessary to verify the length proof itself, since this will have been done by DA nodes prior to - // signing for availability. - // - // Note that the length in the commitment is the length of the blob in symbols - if uint(len(blob)) > blobLength*encoding.BYTES_PER_SYMBOL { - return fmt.Errorf( - "length for blob %v (%d bytes) received from relay %v is greater than claimed blob length (%d bytes)", - blobKey.Hex(), - len(blob), - relayKey, - blobLength*encoding.BYTES_PER_SYMBOL) - } - - return nil -} - -// getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.RelayTimeout +// getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.FetchTimeout func (pr *RelayPayloadRetriever) getBlobWithTimeout( ctx context.Context, relayKey core.RelayKey, blobKey *core.BlobKey) ([]byte, error) { - timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.RelayTimeout) + timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.FetchTimeout) defer cancel() return pr.relayClient.GetBlob(timeoutCtx, relayKey, *blobKey) @@ -271,19 +220,3 @@ func (pr *RelayPayloadRetriever) Close() error { return nil } - -// blobCommitmentsBindingToInternal converts a blob commitment from an eigenDA cert into the internal -// encoding.BlobCommitments type -func blobCommitmentsBindingToInternal( - blobCommitmentBinding *verifiercontract.BlobCommitment, -) (*encoding.BlobCommitments, error) { - - blobCommitment, err := encoding.BlobCommitmentsFromProtobuf( - verification.BlobCommitmentBindingToProto(blobCommitmentBinding)) - - if err != nil { - return nil, fmt.Errorf("blob commitments from protobuf: %w", err) - } - - return blobCommitment, nil -} diff --git a/api/clients/v2/retrieval_client.go b/api/clients/v2/retrieval_client.go index 8304274d42..16a1c9faf0 100644 --- a/api/clients/v2/retrieval_client.go +++ b/api/clients/v2/retrieval_client.go @@ -21,7 +21,14 @@ import ( // To retrieve a blob from the relay, use RelayClient instead. type RetrievalClient interface { // GetBlob downloads chunks of a blob from operator network and reconstructs the blob. - GetBlob(ctx context.Context, blobHeader *corev2.BlobHeader, referenceBlockNumber uint64, quorumID core.QuorumID) ([]byte, error) + GetBlob( + ctx context.Context, + blobKey corev2.BlobKey, + blobVersion corev2.BlobVersion, + blobCommitments encoding.BlobCommitments, + referenceBlockNumber uint64, + quorumID core.QuorumID, + ) ([]byte, error) } type retrievalClient struct { @@ -32,6 +39,8 @@ type retrievalClient struct { numConnections int } +var _ RetrievalClient = &retrievalClient{} + // NewRetrievalClient creates a new retrieval client. func NewRetrievalClient( logger logging.Logger, @@ -49,18 +58,17 @@ func NewRetrievalClient( } } -func (r *retrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHeader, referenceBlockNumber uint64, quorumID core.QuorumID) ([]byte, error) { - if blobHeader == nil { - return nil, errors.New("blob header is nil") - } - - blobKey, err := blobHeader.BlobKey() - if err != nil { - return nil, err - } +func (r *retrievalClient) GetBlob( + ctx context.Context, + blobKey corev2.BlobKey, + blobVersion corev2.BlobVersion, + blobCommitments encoding.BlobCommitments, + referenceBlockNumber uint64, + quorumID core.QuorumID, +) ([]byte, error) { - commitmentBatch := []encoding.BlobCommitments{blobHeader.BlobCommitments} - err = r.verifier.VerifyCommitEquivalenceBatch(commitmentBatch) + commitmentBatch := []encoding.BlobCommitments{blobCommitments} + err := r.verifier.VerifyCommitEquivalenceBatch(commitmentBatch) if err != nil { return nil, err } @@ -79,12 +87,12 @@ func (r *retrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHe return nil, err } - blobParam, ok := blobVersions[blobHeader.BlobVersion] + blobParam, ok := blobVersions[blobVersion] if !ok { - return nil, fmt.Errorf("invalid blob version %d", blobHeader.BlobVersion) + return nil, fmt.Errorf("invalid blob version %d", blobVersion) } - encodingParams, err := blobHeader.GetEncodingParams(blobParam) + encodingParams, err := corev2.GetEncodingParams(blobCommitments.Length, blobParam) if err != nil { return nil, err } @@ -124,7 +132,7 @@ func (r *retrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHe assignmentIndices[i] = uint(index) } - err = r.verifier.VerifyFrames(reply.Chunks, assignmentIndices, blobHeader.BlobCommitments, encodingParams) + err = r.verifier.VerifyFrames(reply.Chunks, assignmentIndices, blobCommitments, encodingParams) if err != nil { r.logger.Warn("failed to verify chunks from operator", "operator", reply.OperatorID.Hex(), "err", err) continue @@ -144,7 +152,7 @@ func (r *retrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHe chunks, indices, encodingParams, - uint64(blobHeader.BlobCommitments.Length)*encoding.BYTES_PER_SYMBOL, + uint64(blobCommitments.Length)*encoding.BYTES_PER_SYMBOL, ) } diff --git a/api/clients/v2/test/relay_payload_retriever_test.go b/api/clients/v2/test/relay_payload_retriever_test.go index 3b5fa3ceb5..970daccb0b 100644 --- a/api/clients/v2/test/relay_payload_retriever_test.go +++ b/api/clients/v2/test/relay_payload_retriever_test.go @@ -50,9 +50,9 @@ func buildRelayPayloadRetrieverTester(t *testing.T) RelayPayloadRetrieverTester EigenDACertVerifierAddr: "y", } - clientConfig := clients.RelayPayloadRetrieverConfig{ + clientConfig := clients.PayloadRetrieverConfig{ PayloadClientConfig: payloadClientConfig, - RelayTimeout: 50 * time.Millisecond, + FetchTimeout: 50 * time.Millisecond, } mockRelayClient := clientsmock.MockRelayClient{} diff --git a/api/clients/v2/verification/blob_verifier.go b/api/clients/v2/verification/blob_verifier.go new file mode 100644 index 0000000000..003f378b45 --- /dev/null +++ b/api/clients/v2/verification/blob_verifier.go @@ -0,0 +1,55 @@ +package verification + +import ( + "errors" + "fmt" + + core "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/consensys/gnark-crypto/ecc/bn254" +) + +// VerifyBlobAgainstCert verifies the blob against a kzg commitment +// +// If verification succeeds, the method returns nil. Otherwise, it returns an error. +// +// TODO: in the future, this will be optimized to use fiat shamir transformation for verification, rather than +// +// regenerating the commitment: https://github.com/Layr-Labs/eigenda/issues/1037 +func VerifyBlobAgainstCert( + blobKey *core.BlobKey, + blobBytes []byte, + kzgCommitment *encoding.G1Commitment, + g1Srs []bn254.G1Affine) error { + + valid, err := GenerateAndCompareBlobCommitment(g1Srs, blobBytes, kzgCommitment) + if err != nil { + return fmt.Errorf("generate and compare commitment for blob %v: %w", blobKey.Hex(), err) + } + + if !valid { + return fmt.Errorf("commitment for blob %v is invalid", blobKey.Hex()) + } + + return nil +} + +// CheckBlobLength accepts bytes representing a blob, and a claimed length in symbols. Note that claimed length is an +// upper bound, not a precise length. Two length checks are performed: +// +// 1. Blob doesn't have length 0 +// 2. Blob length is <= the claimed blob length. Claimed blob length is from the BlobCommitment +func CheckBlobLength(blobBytes []byte, claimedBlobLength uint) error { + if len(blobBytes) == 0 { + return errors.New("blob has length 0") + } + + if uint(len(blobBytes)) > claimedBlobLength*encoding.BYTES_PER_SYMBOL { + return fmt.Errorf( + "length (%d bytes) is greater than claimed blob length (%d bytes)", + len(blobBytes), + claimedBlobLength*encoding.BYTES_PER_SYMBOL) + } + + return nil +} diff --git a/api/clients/v2/verification/conversion_utils.go b/api/clients/v2/verification/conversion_utils.go index 256772cb00..d4b61fe72a 100644 --- a/api/clients/v2/verification/conversion_utils.go +++ b/api/clients/v2/verification/conversion_utils.go @@ -10,6 +10,7 @@ import ( disperserv2 "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2" contractEigenDACertVerifier "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDACertVerifier" "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/encoding" "github.com/consensys/gnark-crypto/ecc/bn254" "github.com/consensys/gnark-crypto/ecc/bn254/fp" ) @@ -272,3 +273,18 @@ func repeatedBytesToBN254G1Points(repeatedBytes [][]byte) ([]contractEigenDACert return outputPoints, nil } + +// BlobCommitmentsBindingToInternal converts a blob commitment from an eigenDA cert into the internal +// encoding.BlobCommitments type +func BlobCommitmentsBindingToInternal( + blobCommitmentBinding *contractEigenDACertVerifier.BlobCommitment, +) (*encoding.BlobCommitments, error) { + + blobCommitment, err := encoding.BlobCommitmentsFromProtobuf(BlobCommitmentBindingToProto(blobCommitmentBinding)) + + if err != nil { + return nil, fmt.Errorf("blob commitments from protobuf: %w", err) + } + + return blobCommitment, nil +} diff --git a/api/clients/v2/verification/eigenda_cert.go b/api/clients/v2/verification/eigenda_cert.go index 3ba05afddc..eac3a1c0b6 100644 --- a/api/clients/v2/verification/eigenda_cert.go +++ b/api/clients/v2/verification/eigenda_cert.go @@ -6,7 +6,6 @@ import ( disperser "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2" contractEigenDACertVerifier "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDACertVerifier" v2 "github.com/Layr-Labs/eigenda/core/v2" - "github.com/Layr-Labs/eigenda/encoding" ) // EigenDACert contains all data necessary to retrieve and validate a blob @@ -45,8 +44,7 @@ func BuildEigenDACert( func (c *EigenDACert) ComputeBlobKey() (*v2.BlobKey, error) { blobHeader := c.BlobInclusionInfo.BlobCertificate.BlobHeader - blobCommitmentsProto := BlobCommitmentBindingToProto(&blobHeader.Commitment) - blobCommitments, err := encoding.BlobCommitmentsFromProtobuf(blobCommitmentsProto) + blobCommitments, err := BlobCommitmentsBindingToInternal(&blobHeader.Commitment) if err != nil { return nil, fmt.Errorf("blob commitments from protobuf: %w", err) } diff --git a/core/v2/core_test.go b/core/v2/core_test.go index 9c18cef74c..d3cfa9c16b 100644 --- a/core/v2/core_test.go +++ b/core/v2/core_test.go @@ -146,7 +146,7 @@ func prepareBlobs( blob := blobs[z] header := cert.BlobHeader - params, err := header.GetEncodingParams(blobParams) + params, err := v2.GetEncodingParams(header.BlobCommitments.Length, blobParams) require.NoError(t, err) chunks, err := p.GetFrames(blob, params) require.NoError(t, err) diff --git a/core/v2/types.go b/core/v2/types.go index 8985cb31de..584be3e38a 100644 --- a/core/v2/types.go +++ b/core/v2/types.go @@ -147,8 +147,8 @@ func (b *BlobHeader) ToProtobuf() (*commonpb.BlobHeader, error) { }, nil } -func (b *BlobHeader) GetEncodingParams(blobParams *core.BlobVersionParameters) (encoding.EncodingParams, error) { - length, err := GetChunkLength(uint32(b.BlobCommitments.Length), blobParams) +func GetEncodingParams(blobLength uint, blobParams *core.BlobVersionParameters) (encoding.EncodingParams, error) { + length, err := GetChunkLength(uint32(blobLength), blobParams) if err != nil { return encoding.EncodingParams{}, err } diff --git a/core/v2/validator.go b/core/v2/validator.go index c165280e3b..5338bec09a 100644 --- a/core/v2/validator.go +++ b/core/v2/validator.go @@ -150,10 +150,7 @@ func (v *shardValidator) ValidateBlobs(ctx context.Context, blobs []*BlobShard, } // TODO: Define params for the blob - params, err := blob.BlobHeader.GetEncodingParams(blobParams) - if err != nil { - return err - } + params, err := GetEncodingParams(blob.BlobHeader.BlobCommitments.Length, blobParams) if err != nil { return err diff --git a/disperser/controller/encoding_manager.go b/disperser/controller/encoding_manager.go index 480d8b1adb..8529aaf5e8 100644 --- a/disperser/controller/encoding_manager.go +++ b/disperser/controller/encoding_manager.go @@ -283,7 +283,7 @@ func (e *EncodingManager) encodeBlob(ctx context.Context, blobKey corev2.BlobKey }) ctx = metadata.NewOutgoingContext(ctx, md) - encodingParams, err := blob.BlobHeader.GetEncodingParams(blobParams) + encodingParams, err := corev2.GetEncodingParams(blob.BlobHeader.BlobCommitments.Length, blobParams) if err != nil { return nil, fmt.Errorf("failed to get encoding params: %w", err) } diff --git a/inabox/tests/integration_v2_test.go b/inabox/tests/integration_v2_test.go index 8f8c31743d..558467c6d2 100644 --- a/inabox/tests/integration_v2_test.go +++ b/inabox/tests/integration_v2_test.go @@ -240,20 +240,50 @@ var _ = Describe("Inabox v2 Integration", func() { } } + blob1Key, err := blobCert1.BlobHeader.BlobKey() + Expect(err).To(BeNil()) + + blob2Key, err := blobCert2.BlobHeader.BlobKey() + Expect(err).To(BeNil()) + // Test retrieval from DA network - b, err := retrievalClientV2.GetBlob(ctx, blobCert1.BlobHeader, batchHeader1.ReferenceBlockNumber, 0) + b, err := retrievalClientV2.GetBlob( + ctx, + blob1Key, + blobCert1.BlobHeader.BlobVersion, + blobCert1.BlobHeader.BlobCommitments, + batchHeader1.ReferenceBlockNumber, + 0) Expect(err).To(BeNil()) restored := bytes.TrimRight(b, "\x00") Expect(restored).To(Equal(paddedData1)) - b, err = retrievalClientV2.GetBlob(ctx, blobCert1.BlobHeader, batchHeader1.ReferenceBlockNumber, 1) + b, err = retrievalClientV2.GetBlob( + ctx, + blob1Key, + blobCert1.BlobHeader.BlobVersion, + blobCert1.BlobHeader.BlobCommitments, + batchHeader1.ReferenceBlockNumber, + 1) restored = bytes.TrimRight(b, "\x00") Expect(err).To(BeNil()) Expect(restored).To(Equal(paddedData1)) - b, err = retrievalClientV2.GetBlob(ctx, blobCert2.BlobHeader, batchHeader2.ReferenceBlockNumber, 0) + b, err = retrievalClientV2.GetBlob( + ctx, + blob2Key, + blobCert2.BlobHeader.BlobVersion, + blobCert2.BlobHeader.BlobCommitments, + batchHeader2.ReferenceBlockNumber, + 0) restored = bytes.TrimRight(b, "\x00") Expect(err).To(BeNil()) Expect(restored).To(Equal(paddedData2)) - b, err = retrievalClientV2.GetBlob(ctx, blobCert2.BlobHeader, batchHeader2.ReferenceBlockNumber, 1) + b, err = retrievalClientV2.GetBlob( + ctx, + blob2Key, + blobCert2.BlobHeader.BlobVersion, + blobCert2.BlobHeader.BlobCommitments, + batchHeader2.ReferenceBlockNumber, + 1) restored = bytes.TrimRight(b, "\x00") Expect(err).To(BeNil()) Expect(restored).To(Equal(paddedData2)) diff --git a/retriever/v2/server.go b/retriever/v2/server.go index f212a68cef..07105a51ec 100644 --- a/retriever/v2/server.go +++ b/retriever/v2/server.go @@ -72,7 +72,14 @@ func (s *Server) RetrieveBlob(ctx context.Context, req *pb.BlobRequest) (*pb.Blo ctxWithTimeout, cancel := context.WithTimeout(ctx, s.config.Timeout) defer cancel() - data, err := s.retrievalClient.GetBlob(ctxWithTimeout, blobHeader, uint64(req.GetReferenceBlockNumber()), core.QuorumID(req.GetQuorumId())) + + data, err := s.retrievalClient.GetBlob( + ctxWithTimeout, + blobKey, + blobHeader.BlobVersion, + blobHeader.BlobCommitments, + uint64(req.GetReferenceBlockNumber()), + core.QuorumID(req.GetQuorumId())) if err != nil { return nil, err } diff --git a/test/v2/test_client.go b/test/v2/test_client.go index 29b2bec450..abceb59afa 100644 --- a/test/v2/test_client.go +++ b/test/v2/test_client.go @@ -3,13 +3,14 @@ package v2 import ( "context" "fmt" - "github.com/docker/go-units" "os" "path" "strings" "testing" "time" + "github.com/docker/go-units" + "github.com/Layr-Labs/eigenda/api/clients/v2" commonv2 "github.com/Layr-Labs/eigenda/api/grpc/common/v2" v2 "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2" @@ -299,7 +300,16 @@ func (c *TestClient) ReadBlobFromValidators( header, err := corev2.BlobHeaderFromProtobuf(blobCert.BlobHeader) require.NoError(c.t, err) - retrievedBlob, err := c.RetrievalClient.GetBlob(ctx, header, uint64(currentBlockNumber), quorumID) + blobKey, err := header.BlobKey() + require.NoError(c.t, err) + + retrievedBlob, err := c.RetrievalClient.GetBlob( + ctx, + blobKey, + header.BlobVersion, + header.BlobCommitments, + uint64(currentBlockNumber), + quorumID) require.NoError(c.t, err) retrievedPayload := codec.RemoveEmptyByteFromPaddedBytes(retrievedBlob) From d3ef6deaf8df212f590b452c7771bcadf08fd67e Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:06:04 -0500 Subject: [PATCH 02/15] Fix test Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- retriever/v2/server_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/retriever/v2/server_test.go b/retriever/v2/server_test.go index aa0dfda566..a310d6ede6 100644 --- a/retriever/v2/server_test.go +++ b/retriever/v2/server_test.go @@ -17,10 +17,10 @@ import ( "github.com/Layr-Labs/eigenda/encoding/kzg/prover" "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" "github.com/Layr-Labs/eigenda/encoding/utils/codec" - "github.com/Layr-Labs/eigenda/retriever/mock" retriever "github.com/Layr-Labs/eigenda/retriever/v2" "github.com/consensys/gnark-crypto/ecc/bn254" "github.com/consensys/gnark-crypto/ecc/bn254/fp" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -29,7 +29,6 @@ const numOperators = 10 var ( indexedChainState core.IndexedChainState retrievalClient *clientsmock.MockRetrievalClient - chainClient *mock.MockChainClient gettysburgAddressBytes = []byte("Fourscore and seven years ago our fathers brought forth, on this continent, a new nation, conceived in liberty, and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived, and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting-place for those who here gave their lives, that that nation might live. It is altogether fitting and proper that we should do this. But, in a larger sense, we cannot dedicate, we cannot consecrate—we cannot hallow—this ground. The brave men, living and dead, who struggled here, have consecrated it far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us—that from these honored dead we take increased devotion to that cause for which they here gave the last full measure of devotion—that we here highly resolve that these dead shall not have died in vain—that this nation, under God, shall have a new birth of freedom, and that government of the people, by the people, for the people, shall not perish from the earth.") ) @@ -74,14 +73,20 @@ func newTestServer(t *testing.T) *retriever.Server { require.NoError(t, err) retrievalClient = &clientsmock.MockRetrievalClient{} - chainClient = mock.NewMockChainClient() return retriever.NewServer(config, logger, retrievalClient, indexedChainState) } func TestRetrieveBlob(t *testing.T) { server := newTestServer(t) data := codec.ConvertByPaddingEmptyByte(gettysburgAddressBytes) - retrievalClient.On("GetBlob").Return(data, nil) + retrievalClient.On( + "GetBlob", + mock.Anything, + mock.Anything, + mock.Anything, + mock.Anything, + mock.Anything, + mock.Anything).Return(data, nil) var X1, Y1 fp.Element X1 = *X1.SetBigInt(big.NewInt(1)) From 64492557b0d109e1fda53fd64175c5a7a640ddec Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:06:22 -0500 Subject: [PATCH 03/15] Add builder Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 81 ++++++++++++++++--- .../v2/distributed_payload_retriever.go | 63 ++++++++++++++- api/clients/v2/relay_payload_retriever.go | 24 +++--- .../v2/test/relay_payload_retriever_test.go | 4 +- api/clients/v2/verification/blob_verifier.go | 3 +- 5 files changed, 144 insertions(+), 31 deletions(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index b0f77f2d97..f6b96666d4 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -45,13 +45,31 @@ type PayloadClientConfig struct { BlobVersion v2.BlobVersion } -// PayloadRetrieverConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed -// by a PayloadRetriever -type PayloadRetrieverConfig struct { +// RelayPayloadRetrieverConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed +// by a RelayPayloadRetriever +type RelayPayloadRetrieverConfig struct { PayloadClientConfig - // The timeout duration for calls to retrieve blobs. - FetchTimeout time.Duration + // The timeout duration for relay calls to retrieve blobs. + RelayTimeout time.Duration +} + +// DistributedPayloadRetrieverConfig contains an embedded PayloadClientConfig, plus all additional configuration values +// needed by a DistributedPayloadRetriever +type DistributedPayloadRetrieverConfig struct { + PayloadClientConfig + + // The timeout duration for calls to retrieve blobs from a given quorum. + RetrievalTimeout time.Duration + + // The address of the BlsOperatorStateRetriever contract + BlsOperatorStateRetrieverAddr string + + // The address of the EigenDAServiceManager contract + EigenDAServiceManagerAddr string + + // The number of simultaneous connections to use when fetching chunks during distributed retrieval + ConnectionCount uint } // PayloadDisperserConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed @@ -125,13 +143,47 @@ func (cc *PayloadClientConfig) checkAndSetDefaults() error { return nil } -// GetDefaultPayloadRetrieverConfig creates a GetDefaultPayloadRetrieverConfig with default values +// GetDefaultRelayPayloadRetrieverConfig creates a RelayPayloadRetrieverConfig with default values // // NOTE: EthRpcUrl and EigenDACertVerifierAddr do not have defined defaults. These must always be specifically configured. -func GetDefaultPayloadRetrieverConfig() *PayloadRetrieverConfig { - return &PayloadRetrieverConfig{ +func GetDefaultRelayPayloadRetrieverConfig() *RelayPayloadRetrieverConfig { + return &RelayPayloadRetrieverConfig{ + PayloadClientConfig: *getDefaultPayloadClientConfig(), + RelayTimeout: 5 * time.Second, + } +} + +// checkAndSetDefaults checks an existing config struct and performs the following actions: +// +// 1. If a config value is 0, and a 0 value makes sense, do nothing. +// 2. If a config value is 0, but a 0 value doesn't make sense and a default value is defined, then set it to the default. +// 3. If a config value is 0, but a 0 value doesn't make sense and a default value isn't defined, return an error. +func (rc *RelayPayloadRetrieverConfig) checkAndSetDefaults() error { + err := rc.PayloadClientConfig.checkAndSetDefaults() + if err != nil { + return err + } + + defaultConfig := GetDefaultRelayPayloadRetrieverConfig() + if rc.RelayTimeout == 0 { + rc.RelayTimeout = defaultConfig.RelayTimeout + } + + return nil +} + +// GetDefaultDistributedPayloadRetrieverConfig creates a DistributedPayloadRetrieverConfig with default values +// +// NOTE: The following fields do not have defined defaults and must always be specifically configured: +// - EthRpcUrl +// - EigenDACertVerifierAddr +// - BlsOperatorStateRetrieverAddr +// - EigenDAServiceManagerAddr +func GetDefaultDistributedPayloadRetrieverConfig() *DistributedPayloadRetrieverConfig { + return &DistributedPayloadRetrieverConfig{ PayloadClientConfig: *getDefaultPayloadClientConfig(), - FetchTimeout: 5 * time.Second, + RetrievalTimeout: 20 * time.Second, + ConnectionCount: 10, } } @@ -140,15 +192,18 @@ func GetDefaultPayloadRetrieverConfig() *PayloadRetrieverConfig { // 1. If a config value is 0, and a 0 value makes sense, do nothing. // 2. If a config value is 0, but a 0 value doesn't make sense and a default value is defined, then set it to the default. // 3. If a config value is 0, but a 0 value doesn't make sense and a default value isn't defined, return an error. -func (rc *PayloadRetrieverConfig) checkAndSetDefaults() error { +func (rc *DistributedPayloadRetrieverConfig) checkAndSetDefaults() error { err := rc.PayloadClientConfig.checkAndSetDefaults() if err != nil { return err } - defaultConfig := GetDefaultPayloadRetrieverConfig() - if rc.FetchTimeout == 0 { - rc.FetchTimeout = defaultConfig.FetchTimeout + defaultConfig := GetDefaultDistributedPayloadRetrieverConfig() + if rc.RetrievalTimeout == 0 { + rc.RetrievalTimeout = defaultConfig.RetrievalTimeout + } + if rc.ConnectionCount == 0 { + rc.ConnectionCount = defaultConfig.ConnectionCount } return nil diff --git a/api/clients/v2/distributed_payload_retriever.go b/api/clients/v2/distributed_payload_retriever.go index 528435fd2a..f3f8e8bdba 100644 --- a/api/clients/v2/distributed_payload_retriever.go +++ b/api/clients/v2/distributed_payload_retriever.go @@ -6,11 +6,17 @@ import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/Layr-Labs/eigenda/api/clients/v2/verification" + "github.com/Layr-Labs/eigenda/common/geth" "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/core/eth" + "github.com/Layr-Labs/eigenda/core/thegraph" corev2 "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/kzg" + "github.com/Layr-Labs/eigenda/encoding/kzg/verifier" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/consensys/gnark-crypto/ecc/bn254" + gethcommon "github.com/ethereum/go-ethereum/common" ) // DistributedPayloadRetriever provides the ability to get payloads from the EigenDA nodes directly @@ -18,7 +24,7 @@ import ( // This struct is goroutine safe. type DistributedPayloadRetriever struct { logger logging.Logger - config PayloadRetrieverConfig + config DistributedPayloadRetrieverConfig codec codecs.BlobCodec retrievalClient RetrievalClient g1Srs []bn254.G1Affine @@ -26,6 +32,59 @@ type DistributedPayloadRetriever struct { var _ PayloadRetriever = &DistributedPayloadRetriever{} +// TODO: add basic constructor + +// BuildDistributedPayloadRetriever builds a DistributedPayloadRetriever from config structs. +func BuildDistributedPayloadRetriever( + logger logging.Logger, + distributedPayloadRetrieverConfig DistributedPayloadRetrieverConfig, + ethConfig geth.EthClientConfig, + thegraphConfig thegraph.Config, + kzgConfig kzg.KzgConfig, +) (*DistributedPayloadRetriever, error) { + + ethClient, err := geth.NewClient(ethConfig, gethcommon.Address{}, 0, logger) + if err != nil { + return nil, fmt.Errorf("new eth client: %w", err) + } + + reader, err := eth.NewReader( + logger, + ethClient, + distributedPayloadRetrieverConfig.BlsOperatorStateRetrieverAddr, + distributedPayloadRetrieverConfig.EigenDAServiceManagerAddr) + + chainState := eth.NewChainState(reader, ethClient) + indexedChainState := thegraph.MakeIndexedChainState(thegraphConfig, chainState, logger) + + kzgVerifier, err := verifier.NewVerifier(&kzgConfig, nil) + if err != nil { + return nil, fmt.Errorf("new verifier: %w", err) + } + + retrievalClient := NewRetrievalClient( + logger, + reader, + indexedChainState, + kzgVerifier, + int(distributedPayloadRetrieverConfig.ConnectionCount)) + + codec, err := codecs.CreateCodec( + distributedPayloadRetrieverConfig.PayloadPolynomialForm, + distributedPayloadRetrieverConfig.BlobEncodingVersion) + if err != nil { + return nil, err + } + + return &DistributedPayloadRetriever{ + logger: logger, + config: distributedPayloadRetrieverConfig, + codec: codec, + retrievalClient: retrievalClient, + g1Srs: kzgVerifier.Srs.G1, + }, nil +} + // GetPayload iteratively attempts to retrieve a given blob from the quorums listed in the EigenDACert. // // If the blob is successfully retrieved, then the blob is verified. If the verification succeeds, the blob is decoded @@ -103,7 +162,7 @@ func (pr *DistributedPayloadRetriever) getBlobWithTimeout( referenceBlockNumber uint32, quorumID core.QuorumID) ([]byte, error) { - timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.FetchTimeout) + timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.RetrievalTimeout) defer cancel() return pr.retrievalClient.GetBlob( diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index bee46e0efb..1ae715671d 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -24,7 +24,7 @@ type RelayPayloadRetriever struct { // Not all methods on Rand are guaranteed goroutine safe: if additional usages of random are added, they // must be evaluated for thread safety. random *rand.Rand - config PayloadRetrieverConfig + config RelayPayloadRetrieverConfig codec codecs.BlobCodec relayClient RelayClient g1Srs []bn254.G1Affine @@ -36,7 +36,7 @@ var _ PayloadRetriever = &RelayPayloadRetriever{} // BuildRelayPayloadRetriever builds a RelayPayloadRetriever from config structs. func BuildRelayPayloadRetriever( log logging.Logger, - payloadRetrieverConfig PayloadRetrieverConfig, + relayPayloadRetrieverConfig RelayPayloadRetrieverConfig, ethConfig geth.EthClientConfig, relayClientConfig *RelayClientConfig, g1Srs []bn254.G1Affine) (*RelayPayloadRetriever, error) { @@ -51,14 +51,14 @@ func BuildRelayPayloadRetriever( return nil, fmt.Errorf("new eth client: %w", err) } - certVerifier, err := verification.NewCertVerifier(*ethClient, payloadRetrieverConfig.EigenDACertVerifierAddr) + certVerifier, err := verification.NewCertVerifier(*ethClient, relayPayloadRetrieverConfig.EigenDACertVerifierAddr) if err != nil { return nil, fmt.Errorf("new cert verifier: %w", err) } codec, err := codecs.CreateCodec( - payloadRetrieverConfig.PayloadPolynomialForm, - payloadRetrieverConfig.BlobEncodingVersion) + relayPayloadRetrieverConfig.PayloadPolynomialForm, + relayPayloadRetrieverConfig.BlobEncodingVersion) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func BuildRelayPayloadRetriever( return NewRelayPayloadRetriever( log, rand.New(rand.NewSource(rand.Int63())), - payloadRetrieverConfig, + relayPayloadRetrieverConfig, relayClient, certVerifier, codec, @@ -77,21 +77,21 @@ func BuildRelayPayloadRetriever( func NewRelayPayloadRetriever( log logging.Logger, random *rand.Rand, - payloadRetrieverConfig PayloadRetrieverConfig, + relayPayloadRetrieverConfig RelayPayloadRetrieverConfig, relayClient RelayClient, certVerifier verification.ICertVerifier, codec codecs.BlobCodec, g1Srs []bn254.G1Affine) (*RelayPayloadRetriever, error) { - err := payloadRetrieverConfig.checkAndSetDefaults() + err := relayPayloadRetrieverConfig.checkAndSetDefaults() if err != nil { - return nil, fmt.Errorf("check and set RelayPayloadRetrieverConfig config: %w", err) + return nil, fmt.Errorf("check and set RelayRelayPayloadRetrieverConfig config: %w", err) } return &RelayPayloadRetriever{ log: log, random: random, - config: payloadRetrieverConfig, + config: relayPayloadRetrieverConfig, codec: codec, relayClient: relayClient, certVerifier: certVerifier, @@ -187,7 +187,7 @@ func (pr *RelayPayloadRetriever) getBlobWithTimeout( relayKey core.RelayKey, blobKey *core.BlobKey) ([]byte, error) { - timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.FetchTimeout) + timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.RelayTimeout) defer cancel() return pr.relayClient.GetBlob(timeoutCtx, relayKey, *blobKey) @@ -195,7 +195,7 @@ func (pr *RelayPayloadRetriever) getBlobWithTimeout( // verifyCertWithTimeout verifies an EigenDACert by making a call to VerifyCertV2. // -// This method times out after the duration configured in relayPayloadRetrieverConfig.ContractCallTimeout +// This method times out after the duration configured in RelayPayloadRetrieverConfig.ContractCallTimeout func (pr *RelayPayloadRetriever) verifyCertWithTimeout( ctx context.Context, eigenDACert *verification.EigenDACert, diff --git a/api/clients/v2/test/relay_payload_retriever_test.go b/api/clients/v2/test/relay_payload_retriever_test.go index 970daccb0b..3b5fa3ceb5 100644 --- a/api/clients/v2/test/relay_payload_retriever_test.go +++ b/api/clients/v2/test/relay_payload_retriever_test.go @@ -50,9 +50,9 @@ func buildRelayPayloadRetrieverTester(t *testing.T) RelayPayloadRetrieverTester EigenDACertVerifierAddr: "y", } - clientConfig := clients.PayloadRetrieverConfig{ + clientConfig := clients.RelayPayloadRetrieverConfig{ PayloadClientConfig: payloadClientConfig, - FetchTimeout: 50 * time.Millisecond, + RelayTimeout: 50 * time.Millisecond, } mockRelayClient := clientsmock.MockRelayClient{} diff --git a/api/clients/v2/verification/blob_verifier.go b/api/clients/v2/verification/blob_verifier.go index 003f378b45..969812a3e3 100644 --- a/api/clients/v2/verification/blob_verifier.go +++ b/api/clients/v2/verification/blob_verifier.go @@ -14,8 +14,7 @@ import ( // If verification succeeds, the method returns nil. Otherwise, it returns an error. // // TODO: in the future, this will be optimized to use fiat shamir transformation for verification, rather than -// -// regenerating the commitment: https://github.com/Layr-Labs/eigenda/issues/1037 +// regenerating the commitment: https://github.com/Layr-Labs/eigenda/issues/1037 func VerifyBlobAgainstCert( blobKey *core.BlobKey, blobBytes []byte, From 62f139c09376208518aa92ce8914c97e6ff7d002 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:11:44 -0500 Subject: [PATCH 04/15] Clean up Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 2 +- api/clients/v2/distributed_payload_retriever.go | 5 ++++- api/clients/v2/relay_payload_retriever.go | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index f6b96666d4..f9da6a4b7f 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -149,7 +149,7 @@ func (cc *PayloadClientConfig) checkAndSetDefaults() error { func GetDefaultRelayPayloadRetrieverConfig() *RelayPayloadRetrieverConfig { return &RelayPayloadRetrieverConfig{ PayloadClientConfig: *getDefaultPayloadClientConfig(), - RelayTimeout: 5 * time.Second, + RelayTimeout: 5 * time.Second, } } diff --git a/api/clients/v2/distributed_payload_retriever.go b/api/clients/v2/distributed_payload_retriever.go index f3f8e8bdba..7301e93368 100644 --- a/api/clients/v2/distributed_payload_retriever.go +++ b/api/clients/v2/distributed_payload_retriever.go @@ -53,6 +53,9 @@ func BuildDistributedPayloadRetriever( ethClient, distributedPayloadRetrieverConfig.BlsOperatorStateRetrieverAddr, distributedPayloadRetrieverConfig.EigenDAServiceManagerAddr) + if err != nil { + return nil, fmt.Errorf("new reader: %w", err) + } chainState := eth.NewChainState(reader, ethClient) indexedChainState := thegraph.MakeIndexedChainState(thegraphConfig, chainState, logger) @@ -73,7 +76,7 @@ func BuildDistributedPayloadRetriever( distributedPayloadRetrieverConfig.PayloadPolynomialForm, distributedPayloadRetrieverConfig.BlobEncodingVersion) if err != nil { - return nil, err + return nil, fmt.Errorf("create codec: %w", err) } return &DistributedPayloadRetriever{ diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index 1ae715671d..dde1f113eb 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -85,7 +85,7 @@ func NewRelayPayloadRetriever( err := relayPayloadRetrieverConfig.checkAndSetDefaults() if err != nil { - return nil, fmt.Errorf("check and set RelayRelayPayloadRetrieverConfig config: %w", err) + return nil, fmt.Errorf("check and set RelayPayloadRetrieverConfig config: %w", err) } return &RelayPayloadRetriever{ From f087501b1e80906e7e1cf238d322da8117abb5e8 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:33:16 -0500 Subject: [PATCH 05/15] Add basic constructor Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 8 +++++ .../v2/distributed_payload_retriever.go | 30 +++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index f9da6a4b7f..e95baa4d3a 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -198,6 +198,14 @@ func (rc *DistributedPayloadRetrieverConfig) checkAndSetDefaults() error { return err } + if rc.BlsOperatorStateRetrieverAddr == "" { + return fmt.Errorf("BlsOperatorStateRetrieverAddr is required") + } + + if rc.EigenDAServiceManagerAddr == "" { + return fmt.Errorf("EigenDAServiceManagerAddr is required") + } + defaultConfig := GetDefaultDistributedPayloadRetrieverConfig() if rc.RetrievalTimeout == 0 { rc.RetrievalTimeout = defaultConfig.RetrievalTimeout diff --git a/api/clients/v2/distributed_payload_retriever.go b/api/clients/v2/distributed_payload_retriever.go index 7301e93368..72fbcac4fc 100644 --- a/api/clients/v2/distributed_payload_retriever.go +++ b/api/clients/v2/distributed_payload_retriever.go @@ -24,7 +24,7 @@ import ( // This struct is goroutine safe. type DistributedPayloadRetriever struct { logger logging.Logger - config DistributedPayloadRetrieverConfig + config DistributedPayloadRetrieverConfig codec codecs.BlobCodec retrievalClient RetrievalClient g1Srs []bn254.G1Affine @@ -32,8 +32,6 @@ type DistributedPayloadRetriever struct { var _ PayloadRetriever = &DistributedPayloadRetriever{} -// TODO: add basic constructor - // BuildDistributedPayloadRetriever builds a DistributedPayloadRetriever from config structs. func BuildDistributedPayloadRetriever( logger logging.Logger, @@ -42,6 +40,10 @@ func BuildDistributedPayloadRetriever( thegraphConfig thegraph.Config, kzgConfig kzg.KzgConfig, ) (*DistributedPayloadRetriever, error) { + err := distributedPayloadRetrieverConfig.checkAndSetDefaults() + if err != nil { + return nil, fmt.Errorf("check and set config defaults: %w", err) + } ethClient, err := geth.NewClient(ethConfig, gethcommon.Address{}, 0, logger) if err != nil { @@ -88,6 +90,28 @@ func BuildDistributedPayloadRetriever( }, nil } +// NewDistributedPayloadRetriever creates a new DistributedPayloadRetriever from already constructed objects +func NewDistributedPayloadRetriever( + logger logging.Logger, + config DistributedPayloadRetrieverConfig, + codec codecs.BlobCodec, + retrievalClient RetrievalClient, + g1Srs []bn254.G1Affine, +) (*DistributedPayloadRetriever, error) { + err := config.checkAndSetDefaults() + if err != nil { + return nil, fmt.Errorf("check and set config defaults: %w", err) + } + + return &DistributedPayloadRetriever{ + logger: logger, + config: config, + codec: codec, + retrievalClient: retrievalClient, + g1Srs: g1Srs, + }, nil +} + // GetPayload iteratively attempts to retrieve a given blob from the quorums listed in the EigenDACert. // // If the blob is successfully retrieved, then the blob is verified. If the verification succeeds, the blob is decoded From 634fb5f3a0e01a5b0cb033f927452b942e51e4c3 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:44:03 -0500 Subject: [PATCH 06/15] Simplify commit verification Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- .../v2/distributed_payload_retriever.go | 11 ++++++-- api/clients/v2/relay_payload_retriever.go | 11 ++++++-- api/clients/v2/verification/blob_verifier.go | 26 ------------------- 3 files changed, 18 insertions(+), 30 deletions(-) diff --git a/api/clients/v2/distributed_payload_retriever.go b/api/clients/v2/distributed_payload_retriever.go index 72fbcac4fc..0b1053a1c0 100644 --- a/api/clients/v2/distributed_payload_retriever.go +++ b/api/clients/v2/distributed_payload_retriever.go @@ -156,9 +156,16 @@ func (pr *DistributedPayloadRetriever) GetPayload( pr.logger.Warn("check blob length", "blobKey", blobKey.Hex(), "quorumID", quorumID, "error", err) continue } - err = verification.VerifyBlobAgainstCert(blobKey, blobBytes, convertedCommitment.Commitment, pr.g1Srs) + + valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blobBytes, convertedCommitment.Commitment) if err != nil { - pr.logger.Warn("verify blob against cert", "blobKey", blobKey.Hex(), "quorumID", quorumID, "error", err) + pr.logger.Warn( + "generate and compare blob commitment", + "blobKey", blobKey.Hex(), "quorumID", quorumID, "error", err) + continue + } + if !valid { + pr.logger.Warn("cert is invalid", "blobKey", blobKey.Hex(), "quorumID", quorumID) continue } diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index dde1f113eb..1f7cb0bb90 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -157,9 +157,16 @@ func (pr *RelayPayloadRetriever) GetPayload( pr.log.Warn("check blob length", "blobKey", blobKey.Hex(), "relayKey", relayKey, "error", err) continue } - err = verification.VerifyBlobAgainstCert(blobKey, blob, blobCommitments.Commitment, pr.g1Srs) + + valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blob, blobCommitments.Commitment) if err != nil { - pr.log.Warn("verify blob against cert", "blobKey", blobKey.Hex(), "relayKey", relayKey, "error", err) + pr.log.Warn( + "generate and compare blob commitment", + "blobKey", blobKey.Hex(), "relayKey", relayKey, "error", err) + continue + } + if !valid { + pr.log.Warn("cert is invalid", "blobKey", blobKey.Hex(), "relayKey", relayKey) continue } diff --git a/api/clients/v2/verification/blob_verifier.go b/api/clients/v2/verification/blob_verifier.go index 969812a3e3..dfcade8ad2 100644 --- a/api/clients/v2/verification/blob_verifier.go +++ b/api/clients/v2/verification/blob_verifier.go @@ -4,35 +4,9 @@ import ( "errors" "fmt" - core "github.com/Layr-Labs/eigenda/core/v2" "github.com/Layr-Labs/eigenda/encoding" - "github.com/consensys/gnark-crypto/ecc/bn254" ) -// VerifyBlobAgainstCert verifies the blob against a kzg commitment -// -// If verification succeeds, the method returns nil. Otherwise, it returns an error. -// -// TODO: in the future, this will be optimized to use fiat shamir transformation for verification, rather than -// regenerating the commitment: https://github.com/Layr-Labs/eigenda/issues/1037 -func VerifyBlobAgainstCert( - blobKey *core.BlobKey, - blobBytes []byte, - kzgCommitment *encoding.G1Commitment, - g1Srs []bn254.G1Affine) error { - - valid, err := GenerateAndCompareBlobCommitment(g1Srs, blobBytes, kzgCommitment) - if err != nil { - return fmt.Errorf("generate and compare commitment for blob %v: %w", blobKey.Hex(), err) - } - - if !valid { - return fmt.Errorf("commitment for blob %v is invalid", blobKey.Hex()) - } - - return nil -} - // CheckBlobLength accepts bytes representing a blob, and a claimed length in symbols. Note that claimed length is an // upper bound, not a precise length. Two length checks are performed: // From 185501fc731c7c2fca94fa47362523d7689fd53b Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:57:36 -0500 Subject: [PATCH 07/15] Do minor cleanup Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 10 +++++----- api/clients/v2/distributed_payload_retriever.go | 15 ++++++++------- api/clients/v2/relay_payload_retriever.go | 5 +++-- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index e95baa4d3a..f2a693633c 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -1,7 +1,7 @@ package clients import ( - "fmt" + "errors" "time" "github.com/Layr-Labs/eigenda/api/clients/codecs" @@ -123,11 +123,11 @@ func (cc *PayloadClientConfig) checkAndSetDefaults() error { // BlobEncodingVersion may be 0, so don't do anything if cc.EthRpcUrl == "" { - return fmt.Errorf("EthRpcUrl is required") + return errors.New("EthRpcUrl is required") } if cc.EigenDACertVerifierAddr == "" { - return fmt.Errorf("EigenDACertVerifierAddr is required") + return errors.New("EigenDACertVerifierAddr is required") } // Nothing to do for PayloadPolynomialForm @@ -199,11 +199,11 @@ func (rc *DistributedPayloadRetrieverConfig) checkAndSetDefaults() error { } if rc.BlsOperatorStateRetrieverAddr == "" { - return fmt.Errorf("BlsOperatorStateRetrieverAddr is required") + return errors.New("BlsOperatorStateRetrieverAddr is required") } if rc.EigenDAServiceManagerAddr == "" { - return fmt.Errorf("EigenDAServiceManagerAddr is required") + return errors.New("EigenDAServiceManagerAddr is required") } defaultConfig := GetDefaultDistributedPayloadRetrieverConfig() diff --git a/api/clients/v2/distributed_payload_retriever.go b/api/clients/v2/distributed_payload_retriever.go index 0b1053a1c0..9c22ae46a1 100644 --- a/api/clients/v2/distributed_payload_retriever.go +++ b/api/clients/v2/distributed_payload_retriever.go @@ -114,8 +114,9 @@ func NewDistributedPayloadRetriever( // GetPayload iteratively attempts to retrieve a given blob from the quorums listed in the EigenDACert. // -// If the blob is successfully retrieved, then the blob is verified. If the verification succeeds, the blob is decoded -// to yield the payload (the original user data, with no padding or any modification), and the payload is returned. +// If the blob is successfully retrieved, then the blob verified against the EigenDACert. If the verification succeeds, +// the blob is decoded to yield the payload (the original user data, with no padding or any modification), and the +// payload is returned. func (pr *DistributedPayloadRetriever) GetPayload( ctx context.Context, eigenDACert *verification.EigenDACert, @@ -127,7 +128,7 @@ func (pr *DistributedPayloadRetriever) GetPayload( } blobHeader := eigenDACert.BlobInclusionInfo.BlobCertificate.BlobHeader - convertedCommitment, err := verification.BlobCommitmentsBindingToInternal(&blobHeader.Commitment) + commitment, err := verification.BlobCommitmentsBindingToInternal(&blobHeader.Commitment) if err != nil { return nil, fmt.Errorf("convert commitments binding to internal: %w", err) } @@ -138,7 +139,7 @@ func (pr *DistributedPayloadRetriever) GetPayload( ctx, *blobKey, blobHeader.Version, - *convertedCommitment, + *commitment, eigenDACert.BatchHeader.ReferenceBlockNumber, quorumID) @@ -151,13 +152,13 @@ func (pr *DistributedPayloadRetriever) GetPayload( continue } - err = verification.CheckBlobLength(blobBytes, convertedCommitment.Length) + err = verification.CheckBlobLength(blobBytes, commitment.Length) if err != nil { pr.logger.Warn("check blob length", "blobKey", blobKey.Hex(), "quorumID", quorumID, "error", err) continue } - valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blobBytes, convertedCommitment.Commitment) + valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blobBytes, commitment.Commitment) if err != nil { pr.logger.Warn( "generate and compare blob commitment", @@ -187,7 +188,7 @@ func (pr *DistributedPayloadRetriever) GetPayload( return nil, fmt.Errorf("unable to retrieve payload from quorums %v", blobHeader.QuorumNumbers) } -// getBlobWithTimeout attempts to get a blob from a given quorum, and times out based on config.FetchTimeout +// getBlobWithTimeout attempts to get a blob from a given quorum, and times out based on config.RetrievalTimeout func (pr *DistributedPayloadRetriever) getBlobWithTimeout( ctx context.Context, blobKey corev2.BlobKey, diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index 1f7cb0bb90..bee938a71a 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -102,8 +102,9 @@ func NewRelayPayloadRetriever( // GetPayload iteratively attempts to fetch a given blob with key blobKey from relays that have it, as claimed by the // blob certificate. The relays are attempted in random order. // -// If the blob is successfully retrieved, then the blob is verified. If the verification succeeds, the blob is decoded -// to yield the payload (the original user data, with no padding or any modification), and the payload is returned. +// If the blob is successfully retrieved, then the blob verified against the EigenDACert. If the verification succeeds, +// the blob is decoded to yield the payload (the original user data, with no padding or any modification), and the +// payload is returned. func (pr *RelayPayloadRetriever) GetPayload( ctx context.Context, eigenDACert *verification.EigenDACert) ([]byte, error) { From db338cd787368663147b821795360091e15b7c71 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 3 Feb 2025 08:50:44 -0500 Subject: [PATCH 08/15] Improve error message Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/distributed_payload_retriever.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/clients/v2/distributed_payload_retriever.go b/api/clients/v2/distributed_payload_retriever.go index 9c22ae46a1..4bd73aee73 100644 --- a/api/clients/v2/distributed_payload_retriever.go +++ b/api/clients/v2/distributed_payload_retriever.go @@ -166,7 +166,9 @@ func (pr *DistributedPayloadRetriever) GetPayload( continue } if !valid { - pr.logger.Warn("cert is invalid", "blobKey", blobKey.Hex(), "quorumID", quorumID) + pr.logger.Warn( + "generated commitment doesn't match cert commitment", + "blobKey", blobKey.Hex(), "quorumID", quorumID) continue } From d905069ba5af2ab8a4208abcf9046c9d9bce751b Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 3 Feb 2025 13:29:47 -0500 Subject: [PATCH 09/15] Rename to ValidatorPayloadRetriever Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 18 ++++---- ...ever.go => validator_payload_retriever.go} | 46 +++++++++---------- 2 files changed, 32 insertions(+), 32 deletions(-) rename api/clients/v2/{distributed_payload_retriever.go => validator_payload_retriever.go} (80%) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index f2a693633c..b85f481658 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -54,9 +54,9 @@ type RelayPayloadRetrieverConfig struct { RelayTimeout time.Duration } -// DistributedPayloadRetrieverConfig contains an embedded PayloadClientConfig, plus all additional configuration values -// needed by a DistributedPayloadRetriever -type DistributedPayloadRetrieverConfig struct { +// ValidatorPayloadRetrieverConfig contains an embedded PayloadClientConfig, plus all additional configuration values +// needed by a ValidatorPayloadRetriever +type ValidatorPayloadRetrieverConfig struct { PayloadClientConfig // The timeout duration for calls to retrieve blobs from a given quorum. @@ -68,7 +68,7 @@ type DistributedPayloadRetrieverConfig struct { // The address of the EigenDAServiceManager contract EigenDAServiceManagerAddr string - // The number of simultaneous connections to use when fetching chunks during distributed retrieval + // The number of simultaneous connections to use when fetching chunks during validtor retrieval ConnectionCount uint } @@ -172,15 +172,15 @@ func (rc *RelayPayloadRetrieverConfig) checkAndSetDefaults() error { return nil } -// GetDefaultDistributedPayloadRetrieverConfig creates a DistributedPayloadRetrieverConfig with default values +// GetDefaultValidatorPayloadRetrieverConfig creates a ValidatorPayloadRetrieverConfig with default values // // NOTE: The following fields do not have defined defaults and must always be specifically configured: // - EthRpcUrl // - EigenDACertVerifierAddr // - BlsOperatorStateRetrieverAddr // - EigenDAServiceManagerAddr -func GetDefaultDistributedPayloadRetrieverConfig() *DistributedPayloadRetrieverConfig { - return &DistributedPayloadRetrieverConfig{ +func GetDefaultValidatorPayloadRetrieverConfig() *ValidatorPayloadRetrieverConfig { + return &ValidatorPayloadRetrieverConfig{ PayloadClientConfig: *getDefaultPayloadClientConfig(), RetrievalTimeout: 20 * time.Second, ConnectionCount: 10, @@ -192,7 +192,7 @@ func GetDefaultDistributedPayloadRetrieverConfig() *DistributedPayloadRetrieverC // 1. If a config value is 0, and a 0 value makes sense, do nothing. // 2. If a config value is 0, but a 0 value doesn't make sense and a default value is defined, then set it to the default. // 3. If a config value is 0, but a 0 value doesn't make sense and a default value isn't defined, return an error. -func (rc *DistributedPayloadRetrieverConfig) checkAndSetDefaults() error { +func (rc *ValidatorPayloadRetrieverConfig) checkAndSetDefaults() error { err := rc.PayloadClientConfig.checkAndSetDefaults() if err != nil { return err @@ -206,7 +206,7 @@ func (rc *DistributedPayloadRetrieverConfig) checkAndSetDefaults() error { return errors.New("EigenDAServiceManagerAddr is required") } - defaultConfig := GetDefaultDistributedPayloadRetrieverConfig() + defaultConfig := GetDefaultValidatorPayloadRetrieverConfig() if rc.RetrievalTimeout == 0 { rc.RetrievalTimeout = defaultConfig.RetrievalTimeout } diff --git a/api/clients/v2/distributed_payload_retriever.go b/api/clients/v2/validator_payload_retriever.go similarity index 80% rename from api/clients/v2/distributed_payload_retriever.go rename to api/clients/v2/validator_payload_retriever.go index 4bd73aee73..275695399a 100644 --- a/api/clients/v2/distributed_payload_retriever.go +++ b/api/clients/v2/validator_payload_retriever.go @@ -19,28 +19,28 @@ import ( gethcommon "github.com/ethereum/go-ethereum/common" ) -// DistributedPayloadRetriever provides the ability to get payloads from the EigenDA nodes directly +// ValidatorPayloadRetriever provides the ability to get payloads from the EigenDA validator nodes directly // // This struct is goroutine safe. -type DistributedPayloadRetriever struct { +type ValidatorPayloadRetriever struct { logger logging.Logger - config DistributedPayloadRetrieverConfig + config ValidatorPayloadRetrieverConfig codec codecs.BlobCodec retrievalClient RetrievalClient g1Srs []bn254.G1Affine } -var _ PayloadRetriever = &DistributedPayloadRetriever{} +var _ PayloadRetriever = &ValidatorPayloadRetriever{} -// BuildDistributedPayloadRetriever builds a DistributedPayloadRetriever from config structs. -func BuildDistributedPayloadRetriever( +// BuildValidatorPayloadRetriever builds a ValidatorPayloadRetriever from config structs. +func BuildValidatorPayloadRetriever( logger logging.Logger, - distributedPayloadRetrieverConfig DistributedPayloadRetrieverConfig, + validatorPayloadRetrieverConfig ValidatorPayloadRetrieverConfig, ethConfig geth.EthClientConfig, thegraphConfig thegraph.Config, kzgConfig kzg.KzgConfig, -) (*DistributedPayloadRetriever, error) { - err := distributedPayloadRetrieverConfig.checkAndSetDefaults() +) (*ValidatorPayloadRetriever, error) { + err := validatorPayloadRetrieverConfig.checkAndSetDefaults() if err != nil { return nil, fmt.Errorf("check and set config defaults: %w", err) } @@ -53,8 +53,8 @@ func BuildDistributedPayloadRetriever( reader, err := eth.NewReader( logger, ethClient, - distributedPayloadRetrieverConfig.BlsOperatorStateRetrieverAddr, - distributedPayloadRetrieverConfig.EigenDAServiceManagerAddr) + validatorPayloadRetrieverConfig.BlsOperatorStateRetrieverAddr, + validatorPayloadRetrieverConfig.EigenDAServiceManagerAddr) if err != nil { return nil, fmt.Errorf("new reader: %w", err) } @@ -72,38 +72,38 @@ func BuildDistributedPayloadRetriever( reader, indexedChainState, kzgVerifier, - int(distributedPayloadRetrieverConfig.ConnectionCount)) + int(validatorPayloadRetrieverConfig.ConnectionCount)) codec, err := codecs.CreateCodec( - distributedPayloadRetrieverConfig.PayloadPolynomialForm, - distributedPayloadRetrieverConfig.BlobEncodingVersion) + validatorPayloadRetrieverConfig.PayloadPolynomialForm, + validatorPayloadRetrieverConfig.BlobEncodingVersion) if err != nil { return nil, fmt.Errorf("create codec: %w", err) } - return &DistributedPayloadRetriever{ + return &ValidatorPayloadRetriever{ logger: logger, - config: distributedPayloadRetrieverConfig, + config: validatorPayloadRetrieverConfig, codec: codec, retrievalClient: retrievalClient, g1Srs: kzgVerifier.Srs.G1, }, nil } -// NewDistributedPayloadRetriever creates a new DistributedPayloadRetriever from already constructed objects -func NewDistributedPayloadRetriever( +// NewValidatorPayloadRetriever creates a new ValidatorPayloadRetriever from already constructed objects +func NewValidatorPayloadRetriever( logger logging.Logger, - config DistributedPayloadRetrieverConfig, + config ValidatorPayloadRetrieverConfig, codec codecs.BlobCodec, retrievalClient RetrievalClient, g1Srs []bn254.G1Affine, -) (*DistributedPayloadRetriever, error) { +) (*ValidatorPayloadRetriever, error) { err := config.checkAndSetDefaults() if err != nil { return nil, fmt.Errorf("check and set config defaults: %w", err) } - return &DistributedPayloadRetriever{ + return &ValidatorPayloadRetriever{ logger: logger, config: config, codec: codec, @@ -117,7 +117,7 @@ func NewDistributedPayloadRetriever( // If the blob is successfully retrieved, then the blob verified against the EigenDACert. If the verification succeeds, // the blob is decoded to yield the payload (the original user data, with no padding or any modification), and the // payload is returned. -func (pr *DistributedPayloadRetriever) GetPayload( +func (pr *ValidatorPayloadRetriever) GetPayload( ctx context.Context, eigenDACert *verification.EigenDACert, ) ([]byte, error) { @@ -191,7 +191,7 @@ func (pr *DistributedPayloadRetriever) GetPayload( } // getBlobWithTimeout attempts to get a blob from a given quorum, and times out based on config.RetrievalTimeout -func (pr *DistributedPayloadRetriever) getBlobWithTimeout( +func (pr *ValidatorPayloadRetriever) getBlobWithTimeout( ctx context.Context, blobKey corev2.BlobKey, blobVersion corev2.BlobVersion, From b78dc9f578b5ef61226865be7bda8fa4049cd16c Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Mon, 3 Feb 2025 13:37:54 -0500 Subject: [PATCH 10/15] Improve docs Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index b85f481658..2af555d173 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -68,7 +68,7 @@ type ValidatorPayloadRetrieverConfig struct { // The address of the EigenDAServiceManager contract EigenDAServiceManagerAddr string - // The number of simultaneous connections to use when fetching chunks during validtor retrieval + // The number of simultaneous connections to use when fetching chunks during validator retrieval ConnectionCount uint } @@ -114,11 +114,11 @@ func getDefaultPayloadClientConfig() *PayloadClientConfig { } } -// checkAndSetDefaults checks an existing config struct and performs the following actions: +// checkAndSetDefaults checks an existing config struct. It performs one of the following actions for any contained 0 values: // -// 1. If a config value is 0, and a 0 value makes sense, do nothing. -// 2. If a config value is 0, but a 0 value doesn't make sense and a default value is defined, then set it to the default. -// 3. If a config value is 0, but a 0 value doesn't make sense and a default value isn't defined, return an error. +// 1. If 0 is an acceptable value for the field, do nothing. +// 2. If 0 is NOT an acceptable value for the field, and a default value is defined, then set it to the default. +// 3. If 0 is NOT an acceptable value for the field, and a default value is NOT defined, return an error. func (cc *PayloadClientConfig) checkAndSetDefaults() error { // BlobEncodingVersion may be 0, so don't do anything @@ -153,11 +153,11 @@ func GetDefaultRelayPayloadRetrieverConfig() *RelayPayloadRetrieverConfig { } } -// checkAndSetDefaults checks an existing config struct and performs the following actions: +// checkAndSetDefaults checks an existing config struct. It performs one of the following actions for any contained 0 values: // -// 1. If a config value is 0, and a 0 value makes sense, do nothing. -// 2. If a config value is 0, but a 0 value doesn't make sense and a default value is defined, then set it to the default. -// 3. If a config value is 0, but a 0 value doesn't make sense and a default value isn't defined, return an error. +// 1. If 0 is an acceptable value for the field, do nothing. +// 2. If 0 is NOT an acceptable value for the field, and a default value is defined, then set it to the default. +// 3. If 0 is NOT an acceptable value for the field, and a default value is NOT defined, return an error. func (rc *RelayPayloadRetrieverConfig) checkAndSetDefaults() error { err := rc.PayloadClientConfig.checkAndSetDefaults() if err != nil { @@ -187,11 +187,11 @@ func GetDefaultValidatorPayloadRetrieverConfig() *ValidatorPayloadRetrieverConfi } } -// checkAndSetDefaults checks an existing config struct and performs the following actions: +// checkAndSetDefaults checks an existing config struct. It performs one of the following actions for any contained 0 values: // -// 1. If a config value is 0, and a 0 value makes sense, do nothing. -// 2. If a config value is 0, but a 0 value doesn't make sense and a default value is defined, then set it to the default. -// 3. If a config value is 0, but a 0 value doesn't make sense and a default value isn't defined, return an error. +// 1. If 0 is an acceptable value for the field, do nothing. +// 2. If 0 is NOT an acceptable value for the field, and a default value is defined, then set it to the default. +// 3. If 0 is NOT an acceptable value for the field, and a default value is NOT defined, return an error. func (rc *ValidatorPayloadRetrieverConfig) checkAndSetDefaults() error { err := rc.PayloadClientConfig.checkAndSetDefaults() if err != nil { @@ -230,11 +230,11 @@ func GetDefaultPayloadDisperserConfig() *PayloadDisperserConfig { } } -// checkAndSetDefaults checks an existing config struct and performs the following actions: +// checkAndSetDefaults checks an existing config struct. It performs one of the following actions for any contained 0 values: // -// 1. If a config value is 0, and a 0 value makes sense, do nothing. -// 2. If a config value is 0, but a 0 value doesn't make sense and a default value is defined, then set it to the default. -// 3. If a config value is 0, but a 0 value doesn't make sense and a default value isn't defined, return an error. +// 1. If 0 is an acceptable value for the field, do nothing. +// 2. If 0 is NOT an acceptable value for the field, and a default value is defined, then set it to the default. +// 3. If 0 is NOT an acceptable value for the field, and a default value is NOT defined, return an error. func (dc *PayloadDisperserConfig) checkAndSetDefaults() error { err := dc.PayloadClientConfig.checkAndSetDefaults() if err != nil { From f9e79b8d4ed4e8f0f95f82e5934b2c3dff8239eb Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 5 Feb 2025 12:38:06 -0500 Subject: [PATCH 11/15] Make verification error messages consistent Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/relay_payload_retriever.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index d43efe851c..4a147fac65 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -154,7 +154,9 @@ func (pr *RelayPayloadRetriever) GetPayload( continue } if !valid { - pr.log.Warn("cert is invalid", "blobKey", blobKey.Hex(), "relayKey", relayKey) + pr.log.Warn( + "generated commitment doesn't match cert commitment", + "blobKey", blobKey.Hex(), "relayKey", relayKey) continue } From 8be41aa7b9f639e44e894f4d44669aa441910ccd Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 5 Feb 2025 13:12:21 -0500 Subject: [PATCH 12/15] Improve doc clarity Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index d8c5aedd32..97cab0f0d9 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -66,7 +66,8 @@ type RelayPayloadRetrieverConfig struct { type ValidatorPayloadRetrieverConfig struct { PayloadClientConfig - // The timeout duration for calls to retrieve blobs from a given quorum. + // The timeout duration for retrieving chunks from a given quorum, and reassembling the chunks into a blob. + // Once this timeout triggers, the retriever will give up on the quorum, and retry with the next quorum (if one exists) RetrievalTimeout time.Duration // The address of the BlsOperatorStateRetriever contract From 3f6f092c87fa167c8b5d1462e2e6ffaa4c456785 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Wed, 5 Feb 2025 13:15:38 -0500 Subject: [PATCH 13/15] Rename to MaxConnectionCount Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 10 +++++----- api/clients/v2/validator_payload_retriever.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index 97cab0f0d9..326fd56842 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -76,8 +76,8 @@ type ValidatorPayloadRetrieverConfig struct { // The address of the EigenDAServiceManager contract EigenDAServiceManagerAddr string - // The number of simultaneous connections to use when fetching chunks during validator retrieval - ConnectionCount uint + // The maximum number of simultaneous connections to use when fetching chunks during validator retrieval + MaxConnectionCount uint } // PayloadDisperserConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed @@ -194,7 +194,7 @@ func GetDefaultValidatorPayloadRetrieverConfig() *ValidatorPayloadRetrieverConfi return &ValidatorPayloadRetrieverConfig{ PayloadClientConfig: *getDefaultPayloadClientConfig(), RetrievalTimeout: 20 * time.Second, - ConnectionCount: 10, + MaxConnectionCount: 10, } } @@ -221,8 +221,8 @@ func (rc *ValidatorPayloadRetrieverConfig) checkAndSetDefaults() error { if rc.RetrievalTimeout == 0 { rc.RetrievalTimeout = defaultConfig.RetrievalTimeout } - if rc.ConnectionCount == 0 { - rc.ConnectionCount = defaultConfig.ConnectionCount + if rc.MaxConnectionCount == 0 { + rc.MaxConnectionCount = defaultConfig.MaxConnectionCount } return nil diff --git a/api/clients/v2/validator_payload_retriever.go b/api/clients/v2/validator_payload_retriever.go index 275695399a..a0e552ba27 100644 --- a/api/clients/v2/validator_payload_retriever.go +++ b/api/clients/v2/validator_payload_retriever.go @@ -72,7 +72,7 @@ func BuildValidatorPayloadRetriever( reader, indexedChainState, kzgVerifier, - int(validatorPayloadRetrieverConfig.ConnectionCount)) + int(validatorPayloadRetrieverConfig.MaxConnectionCount)) codec, err := codecs.CreateCodec( validatorPayloadRetrieverConfig.PayloadPolynomialForm, From 6c41942cd9d4ac866967727f9fc849de2e538ec3 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 6 Feb 2025 08:35:31 -0500 Subject: [PATCH 14/15] Tweak default config values Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index 326fd56842..97e32914c6 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -193,8 +193,8 @@ func (rc *RelayPayloadRetrieverConfig) checkAndSetDefaults() error { func GetDefaultValidatorPayloadRetrieverConfig() *ValidatorPayloadRetrieverConfig { return &ValidatorPayloadRetrieverConfig{ PayloadClientConfig: *getDefaultPayloadClientConfig(), - RetrievalTimeout: 20 * time.Second, - MaxConnectionCount: 10, + RetrievalTimeout: 30 * time.Second, + MaxConnectionCount: 200, } } From 09f2836d185e1430a5eabf472ac04f3352766fad Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Thu, 6 Feb 2025 08:37:10 -0500 Subject: [PATCH 15/15] Tweak default config values again Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index 97e32914c6..d3ce257d2f 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -194,7 +194,7 @@ func GetDefaultValidatorPayloadRetrieverConfig() *ValidatorPayloadRetrieverConfi return &ValidatorPayloadRetrieverConfig{ PayloadClientConfig: *getDefaultPayloadClientConfig(), RetrievalTimeout: 30 * time.Second, - MaxConnectionCount: 200, + MaxConnectionCount: 100, } }