From 1026ea2d9443204fbb2656e98c4cebaf101402e5 Mon Sep 17 00:00:00 2001 From: litt3 <102969658+litt3@users.noreply.github.com> Date: Fri, 31 Jan 2025 11:05:21 -0500 Subject: [PATCH] Draft distributed payload retrieval Signed-off-by: litt3 <102969658+litt3@users.noreply.github.com> --- api/clients/v2/config.go | 26 ++-- .../v2/distributed_payload_retriever.go | 116 ++++++++++++++++++ api/clients/v2/mock/retrieval_client.go | 54 ++++++-- api/clients/v2/relay_payload_retriever.go | 105 +++------------- api/clients/v2/retrieval_client.go | 42 ++++--- .../v2/test/relay_payload_retriever_test.go | 4 +- api/clients/v2/verification/blob_verifier.go | 55 +++++++++ .../v2/verification/conversion_utils.go | 16 +++ api/clients/v2/verification/eigenda_cert.go | 4 +- core/v2/core_test.go | 2 +- core/v2/types.go | 4 +- core/v2/validator.go | 5 +- disperser/controller/encoding_manager.go | 2 +- inabox/tests/integration_v2_test.go | 38 +++++- retriever/v2/server.go | 9 +- test/v2/test_client.go | 14 ++- 16 files changed, 350 insertions(+), 146 deletions(-) create mode 100644 api/clients/v2/distributed_payload_retriever.go create mode 100644 api/clients/v2/verification/blob_verifier.go diff --git a/api/clients/v2/config.go b/api/clients/v2/config.go index 9dae41ed08..b0f77f2d97 100644 --- a/api/clients/v2/config.go +++ b/api/clients/v2/config.go @@ -45,13 +45,13 @@ type PayloadClientConfig struct { BlobVersion v2.BlobVersion } -// RelayPayloadRetrieverConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed -// by a RelayPayloadRetriever -type RelayPayloadRetrieverConfig struct { +// PayloadRetrieverConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed +// by a PayloadRetriever +type PayloadRetrieverConfig struct { PayloadClientConfig - // The timeout duration for relay calls to retrieve blobs. - RelayTimeout time.Duration + // The timeout duration for calls to retrieve blobs. + FetchTimeout time.Duration } // PayloadDisperserConfig contains an embedded PayloadClientConfig, plus all additional configuration values needed @@ -125,13 +125,13 @@ func (cc *PayloadClientConfig) checkAndSetDefaults() error { return nil } -// GetDefaultRelayPayloadRetrieverConfig creates a RelayPayloadRetrieverConfig with default values +// GetDefaultPayloadRetrieverConfig creates a GetDefaultPayloadRetrieverConfig with default values // // NOTE: EthRpcUrl and EigenDACertVerifierAddr do not have defined defaults. These must always be specifically configured. -func GetDefaultRelayPayloadRetrieverConfig() *RelayPayloadRetrieverConfig { - return &RelayPayloadRetrieverConfig{ +func GetDefaultPayloadRetrieverConfig() *PayloadRetrieverConfig { + return &PayloadRetrieverConfig{ PayloadClientConfig: *getDefaultPayloadClientConfig(), - RelayTimeout: 5 * time.Second, + FetchTimeout: 5 * time.Second, } } @@ -140,15 +140,15 @@ func GetDefaultRelayPayloadRetrieverConfig() *RelayPayloadRetrieverConfig { // 1. If a config value is 0, and a 0 value makes sense, do nothing. // 2. If a config value is 0, but a 0 value doesn't make sense and a default value is defined, then set it to the default. // 3. If a config value is 0, but a 0 value doesn't make sense and a default value isn't defined, return an error. -func (rc *RelayPayloadRetrieverConfig) checkAndSetDefaults() error { +func (rc *PayloadRetrieverConfig) checkAndSetDefaults() error { err := rc.PayloadClientConfig.checkAndSetDefaults() if err != nil { return err } - defaultConfig := GetDefaultRelayPayloadRetrieverConfig() - if rc.RelayTimeout == 0 { - rc.RelayTimeout = defaultConfig.RelayTimeout + defaultConfig := GetDefaultPayloadRetrieverConfig() + if rc.FetchTimeout == 0 { + rc.FetchTimeout = defaultConfig.FetchTimeout } return nil diff --git a/api/clients/v2/distributed_payload_retriever.go b/api/clients/v2/distributed_payload_retriever.go new file mode 100644 index 0000000000..528435fd2a --- /dev/null +++ b/api/clients/v2/distributed_payload_retriever.go @@ -0,0 +1,116 @@ +package clients + +import ( + "context" + "fmt" + + "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/api/clients/v2/verification" + "github.com/Layr-Labs/eigenda/core" + corev2 "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigensdk-go/logging" + "github.com/consensys/gnark-crypto/ecc/bn254" +) + +// DistributedPayloadRetriever provides the ability to get payloads from the EigenDA nodes directly +// +// This struct is goroutine safe. +type DistributedPayloadRetriever struct { + logger logging.Logger + config PayloadRetrieverConfig + codec codecs.BlobCodec + retrievalClient RetrievalClient + g1Srs []bn254.G1Affine +} + +var _ PayloadRetriever = &DistributedPayloadRetriever{} + +// GetPayload iteratively attempts to retrieve a given blob from the quorums listed in the EigenDACert. +// +// If the blob is successfully retrieved, then the blob is verified. If the verification succeeds, the blob is decoded +// to yield the payload (the original user data, with no padding or any modification), and the payload is returned. +func (pr *DistributedPayloadRetriever) GetPayload( + ctx context.Context, + eigenDACert *verification.EigenDACert, +) ([]byte, error) { + + blobKey, err := eigenDACert.ComputeBlobKey() + if err != nil { + return nil, fmt.Errorf("compute blob key: %w", err) + } + + blobHeader := eigenDACert.BlobInclusionInfo.BlobCertificate.BlobHeader + convertedCommitment, err := verification.BlobCommitmentsBindingToInternal(&blobHeader.Commitment) + if err != nil { + return nil, fmt.Errorf("convert commitments binding to internal: %w", err) + } + + // TODO (litt3): Add a feature which keeps chunks from previous quorums, and just fills in gaps + for _, quorumID := range blobHeader.QuorumNumbers { + blobBytes, err := pr.getBlobWithTimeout( + ctx, + *blobKey, + blobHeader.Version, + *convertedCommitment, + eigenDACert.BatchHeader.ReferenceBlockNumber, + quorumID) + + if err != nil { + pr.logger.Warn( + "blob couldn't be retrieved from quorum", + "blobKey", blobKey.Hex(), + "quorumId", quorumID, + "error", err) + continue + } + + err = verification.CheckBlobLength(blobBytes, convertedCommitment.Length) + if err != nil { + pr.logger.Warn("check blob length", "blobKey", blobKey.Hex(), "quorumID", quorumID, "error", err) + continue + } + err = verification.VerifyBlobAgainstCert(blobKey, blobBytes, convertedCommitment.Commitment, pr.g1Srs) + if err != nil { + pr.logger.Warn("verify blob against cert", "blobKey", blobKey.Hex(), "quorumID", quorumID, "error", err) + continue + } + + payload, err := pr.codec.DecodeBlob(blobBytes) + if err != nil { + pr.logger.Error( + `Cert verification was successful, but decode blob failed! + This is likely a problem with the local blob codec configuration, + but could potentially indicate a maliciously generated blob certificate. + It should not be possible for an honestly generated certificate to verify + for an invalid blob!`, + "blobKey", blobKey.Hex(), "quorumID", quorumID, "eigenDACert", eigenDACert, "error", err) + return nil, fmt.Errorf("decode blob: %w", err) + } + + return payload, nil + } + + return nil, fmt.Errorf("unable to retrieve payload from quorums %v", blobHeader.QuorumNumbers) +} + +// getBlobWithTimeout attempts to get a blob from a given quorum, and times out based on config.FetchTimeout +func (pr *DistributedPayloadRetriever) getBlobWithTimeout( + ctx context.Context, + blobKey corev2.BlobKey, + blobVersion corev2.BlobVersion, + blobCommitments encoding.BlobCommitments, + referenceBlockNumber uint32, + quorumID core.QuorumID) ([]byte, error) { + + timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.FetchTimeout) + defer cancel() + + return pr.retrievalClient.GetBlob( + timeoutCtx, + blobKey, + blobVersion, + blobCommitments, + uint64(referenceBlockNumber), + quorumID) +} diff --git a/api/clients/v2/mock/retrieval_client.go b/api/clients/v2/mock/retrieval_client.go index 6f0e23b723..66c8653383 100644 --- a/api/clients/v2/mock/retrieval_client.go +++ b/api/clients/v2/mock/retrieval_client.go @@ -1,27 +1,61 @@ +// Code generated by mockery v2.50.0. DO NOT EDIT. + package mock import ( "context" - "github.com/Layr-Labs/eigenda/api/clients/v2" - "github.com/Layr-Labs/eigenda/core" - corev2 "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigenda/encoding" "github.com/stretchr/testify/mock" + + v2 "github.com/Layr-Labs/eigenda/core/v2" ) +// MockRetrievalClient is an autogenerated mock type for the MockRetrievalClient type type MockRetrievalClient struct { mock.Mock } -var _ clients.RetrievalClient = (*MockRetrievalClient)(nil) +// GetBlob provides a mock function with given fields: ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID +func (_m *MockRetrievalClient) GetBlob(ctx context.Context, blobKey v2.BlobKey, blobVersion uint16, blobCommitments encoding.BlobCommitments, referenceBlockNumber uint64, quorumID uint8) ([]byte, error) { + ret := _m.Called(ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID) + + if len(ret) == 0 { + panic("no return value specified for GetBlob") + } -func NewRetrievalClient() *MockRetrievalClient { - return &MockRetrievalClient{} + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, v2.BlobKey, uint16, encoding.BlobCommitments, uint64, uint8) ([]byte, error)); ok { + return rf(ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID) + } + if rf, ok := ret.Get(0).(func(context.Context, v2.BlobKey, uint16, encoding.BlobCommitments, uint64, uint8) []byte); ok { + r0 = rf(ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, v2.BlobKey, uint16, encoding.BlobCommitments, uint64, uint8) error); ok { + r1 = rf(ctx, blobKey, blobVersion, blobCommitments, referenceBlockNumber, quorumID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -func (c *MockRetrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHeader, referenceBlockNumber uint64, quorumID core.QuorumID) ([]byte, error) { - args := c.Called() +// NewMockRetrievalClient creates a new instance of MockRetrievalClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockRetrievalClient(t interface { + mock.TestingT + Cleanup(func()) +}) *MockRetrievalClient { + mock := &MockRetrievalClient{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) - result := args.Get(0) - return result.([]byte), args.Error(1) + return mock } diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index ff4f784a86..bee46e0efb 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -9,9 +9,7 @@ import ( "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/Layr-Labs/eigenda/api/clients/v2/verification" "github.com/Layr-Labs/eigenda/common/geth" - verifiercontract "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDACertVerifier" core "github.com/Layr-Labs/eigenda/core/v2" - "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/consensys/gnark-crypto/ecc/bn254" gethcommon "github.com/ethereum/go-ethereum/common" @@ -26,7 +24,7 @@ type RelayPayloadRetriever struct { // Not all methods on Rand are guaranteed goroutine safe: if additional usages of random are added, they // must be evaluated for thread safety. random *rand.Rand - config RelayPayloadRetrieverConfig + config PayloadRetrieverConfig codec codecs.BlobCodec relayClient RelayClient g1Srs []bn254.G1Affine @@ -38,7 +36,7 @@ var _ PayloadRetriever = &RelayPayloadRetriever{} // BuildRelayPayloadRetriever builds a RelayPayloadRetriever from config structs. func BuildRelayPayloadRetriever( log logging.Logger, - relayPayloadRetrieverConfig RelayPayloadRetrieverConfig, + payloadRetrieverConfig PayloadRetrieverConfig, ethConfig geth.EthClientConfig, relayClientConfig *RelayClientConfig, g1Srs []bn254.G1Affine) (*RelayPayloadRetriever, error) { @@ -53,14 +51,14 @@ func BuildRelayPayloadRetriever( return nil, fmt.Errorf("new eth client: %w", err) } - certVerifier, err := verification.NewCertVerifier(*ethClient, relayPayloadRetrieverConfig.EigenDACertVerifierAddr) + certVerifier, err := verification.NewCertVerifier(*ethClient, payloadRetrieverConfig.EigenDACertVerifierAddr) if err != nil { return nil, fmt.Errorf("new cert verifier: %w", err) } codec, err := codecs.CreateCodec( - relayPayloadRetrieverConfig.PayloadPolynomialForm, - relayPayloadRetrieverConfig.BlobEncodingVersion) + payloadRetrieverConfig.PayloadPolynomialForm, + payloadRetrieverConfig.BlobEncodingVersion) if err != nil { return nil, err } @@ -68,7 +66,7 @@ func BuildRelayPayloadRetriever( return NewRelayPayloadRetriever( log, rand.New(rand.NewSource(rand.Int63())), - relayPayloadRetrieverConfig, + payloadRetrieverConfig, relayClient, certVerifier, codec, @@ -79,13 +77,13 @@ func BuildRelayPayloadRetriever( func NewRelayPayloadRetriever( log logging.Logger, random *rand.Rand, - relayPayloadRetrieverConfig RelayPayloadRetrieverConfig, + payloadRetrieverConfig PayloadRetrieverConfig, relayClient RelayClient, certVerifier verification.ICertVerifier, codec codecs.BlobCodec, g1Srs []bn254.G1Affine) (*RelayPayloadRetriever, error) { - err := relayPayloadRetrieverConfig.checkAndSetDefaults() + err := payloadRetrieverConfig.checkAndSetDefaults() if err != nil { return nil, fmt.Errorf("check and set RelayPayloadRetrieverConfig config: %w", err) } @@ -93,7 +91,7 @@ func NewRelayPayloadRetriever( return &RelayPayloadRetriever{ log: log, random: random, - config: relayPayloadRetrieverConfig, + config: payloadRetrieverConfig, codec: codec, relayClient: relayClient, certVerifier: certVerifier, @@ -126,7 +124,7 @@ func (pr *RelayPayloadRetriever) GetPayload( return nil, errors.New("relay key count is zero") } - blobCommitments, err := blobCommitmentsBindingToInternal( + blobCommitments, err := verification.BlobCommitmentsBindingToInternal( &eigenDACert.BlobInclusionInfo.BlobCertificate.BlobHeader.Commitment) if err != nil { @@ -154,11 +152,14 @@ func (pr *RelayPayloadRetriever) GetPayload( continue } - err = pr.verifyBlobAgainstCert(blobKey, relayKey, blob, blobCommitments.Commitment, blobCommitments.Length) - - // An honest relay should never send a blob which doesn't verify against the cert + err = verification.CheckBlobLength(blob, blobCommitments.Length) + if err != nil { + pr.log.Warn("check blob length", "blobKey", blobKey.Hex(), "relayKey", relayKey, "error", err) + continue + } + err = verification.VerifyBlobAgainstCert(blobKey, blob, blobCommitments.Commitment, pr.g1Srs) if err != nil { - pr.log.Warn("verify blob from relay against cert: %w", err) + pr.log.Warn("verify blob against cert", "blobKey", blobKey.Hex(), "relayKey", relayKey, "error", err) continue } @@ -180,65 +181,13 @@ func (pr *RelayPayloadRetriever) GetPayload( return nil, fmt.Errorf("unable to retrieve blob %v from any relay. relay count: %d", blobKey.Hex(), relayKeyCount) } -// verifyBlobAgainstCert verifies the blob received from a relay against the certificate. -// -// The following verifications are performed in this method: -// 1. Verify that the blob isn't empty -// 2. Verify the blob against the cert's kzg commitment -// 3. Verify that the blob length is less than or equal to the cert's blob length -// -// If all verifications succeed, the method returns nil. Otherwise, it returns an error. -func (pr *RelayPayloadRetriever) verifyBlobAgainstCert( - blobKey *core.BlobKey, - relayKey core.RelayKey, - blob []byte, - kzgCommitment *encoding.G1Commitment, - blobLength uint) error { - - // An honest relay should never send an empty blob - if len(blob) == 0 { - return fmt.Errorf("blob %v received from relay %v had length 0", blobKey.Hex(), relayKey) - } - - // TODO: in the future, this will be optimized to use fiat shamir transformation for verification, rather than - // regenerating the commitment: https://github.com/Layr-Labs/eigenda/issues/1037 - valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blob, kzgCommitment) - if err != nil { - return fmt.Errorf( - "generate and compare commitment for blob %v received from relay %v: %w", - blobKey.Hex(), - relayKey, - err) - } - - if !valid { - return fmt.Errorf("commitment for blob %v is invalid for bytes received from relay %v", blobKey.Hex(), relayKey) - } - - // Checking that the length returned by the relay is <= the length claimed in the BlobCommitments is sufficient - // here: it isn't necessary to verify the length proof itself, since this will have been done by DA nodes prior to - // signing for availability. - // - // Note that the length in the commitment is the length of the blob in symbols - if uint(len(blob)) > blobLength*encoding.BYTES_PER_SYMBOL { - return fmt.Errorf( - "length for blob %v (%d bytes) received from relay %v is greater than claimed blob length (%d bytes)", - blobKey.Hex(), - len(blob), - relayKey, - blobLength*encoding.BYTES_PER_SYMBOL) - } - - return nil -} - -// getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.RelayTimeout +// getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.FetchTimeout func (pr *RelayPayloadRetriever) getBlobWithTimeout( ctx context.Context, relayKey core.RelayKey, blobKey *core.BlobKey) ([]byte, error) { - timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.RelayTimeout) + timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.FetchTimeout) defer cancel() return pr.relayClient.GetBlob(timeoutCtx, relayKey, *blobKey) @@ -271,19 +220,3 @@ func (pr *RelayPayloadRetriever) Close() error { return nil } - -// blobCommitmentsBindingToInternal converts a blob commitment from an eigenDA cert into the internal -// encoding.BlobCommitments type -func blobCommitmentsBindingToInternal( - blobCommitmentBinding *verifiercontract.BlobCommitment, -) (*encoding.BlobCommitments, error) { - - blobCommitment, err := encoding.BlobCommitmentsFromProtobuf( - verification.BlobCommitmentBindingToProto(blobCommitmentBinding)) - - if err != nil { - return nil, fmt.Errorf("blob commitments from protobuf: %w", err) - } - - return blobCommitment, nil -} diff --git a/api/clients/v2/retrieval_client.go b/api/clients/v2/retrieval_client.go index 8304274d42..16a1c9faf0 100644 --- a/api/clients/v2/retrieval_client.go +++ b/api/clients/v2/retrieval_client.go @@ -21,7 +21,14 @@ import ( // To retrieve a blob from the relay, use RelayClient instead. type RetrievalClient interface { // GetBlob downloads chunks of a blob from operator network and reconstructs the blob. - GetBlob(ctx context.Context, blobHeader *corev2.BlobHeader, referenceBlockNumber uint64, quorumID core.QuorumID) ([]byte, error) + GetBlob( + ctx context.Context, + blobKey corev2.BlobKey, + blobVersion corev2.BlobVersion, + blobCommitments encoding.BlobCommitments, + referenceBlockNumber uint64, + quorumID core.QuorumID, + ) ([]byte, error) } type retrievalClient struct { @@ -32,6 +39,8 @@ type retrievalClient struct { numConnections int } +var _ RetrievalClient = &retrievalClient{} + // NewRetrievalClient creates a new retrieval client. func NewRetrievalClient( logger logging.Logger, @@ -49,18 +58,17 @@ func NewRetrievalClient( } } -func (r *retrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHeader, referenceBlockNumber uint64, quorumID core.QuorumID) ([]byte, error) { - if blobHeader == nil { - return nil, errors.New("blob header is nil") - } - - blobKey, err := blobHeader.BlobKey() - if err != nil { - return nil, err - } +func (r *retrievalClient) GetBlob( + ctx context.Context, + blobKey corev2.BlobKey, + blobVersion corev2.BlobVersion, + blobCommitments encoding.BlobCommitments, + referenceBlockNumber uint64, + quorumID core.QuorumID, +) ([]byte, error) { - commitmentBatch := []encoding.BlobCommitments{blobHeader.BlobCommitments} - err = r.verifier.VerifyCommitEquivalenceBatch(commitmentBatch) + commitmentBatch := []encoding.BlobCommitments{blobCommitments} + err := r.verifier.VerifyCommitEquivalenceBatch(commitmentBatch) if err != nil { return nil, err } @@ -79,12 +87,12 @@ func (r *retrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHe return nil, err } - blobParam, ok := blobVersions[blobHeader.BlobVersion] + blobParam, ok := blobVersions[blobVersion] if !ok { - return nil, fmt.Errorf("invalid blob version %d", blobHeader.BlobVersion) + return nil, fmt.Errorf("invalid blob version %d", blobVersion) } - encodingParams, err := blobHeader.GetEncodingParams(blobParam) + encodingParams, err := corev2.GetEncodingParams(blobCommitments.Length, blobParam) if err != nil { return nil, err } @@ -124,7 +132,7 @@ func (r *retrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHe assignmentIndices[i] = uint(index) } - err = r.verifier.VerifyFrames(reply.Chunks, assignmentIndices, blobHeader.BlobCommitments, encodingParams) + err = r.verifier.VerifyFrames(reply.Chunks, assignmentIndices, blobCommitments, encodingParams) if err != nil { r.logger.Warn("failed to verify chunks from operator", "operator", reply.OperatorID.Hex(), "err", err) continue @@ -144,7 +152,7 @@ func (r *retrievalClient) GetBlob(ctx context.Context, blobHeader *corev2.BlobHe chunks, indices, encodingParams, - uint64(blobHeader.BlobCommitments.Length)*encoding.BYTES_PER_SYMBOL, + uint64(blobCommitments.Length)*encoding.BYTES_PER_SYMBOL, ) } diff --git a/api/clients/v2/test/relay_payload_retriever_test.go b/api/clients/v2/test/relay_payload_retriever_test.go index 3b5fa3ceb5..970daccb0b 100644 --- a/api/clients/v2/test/relay_payload_retriever_test.go +++ b/api/clients/v2/test/relay_payload_retriever_test.go @@ -50,9 +50,9 @@ func buildRelayPayloadRetrieverTester(t *testing.T) RelayPayloadRetrieverTester EigenDACertVerifierAddr: "y", } - clientConfig := clients.RelayPayloadRetrieverConfig{ + clientConfig := clients.PayloadRetrieverConfig{ PayloadClientConfig: payloadClientConfig, - RelayTimeout: 50 * time.Millisecond, + FetchTimeout: 50 * time.Millisecond, } mockRelayClient := clientsmock.MockRelayClient{} diff --git a/api/clients/v2/verification/blob_verifier.go b/api/clients/v2/verification/blob_verifier.go new file mode 100644 index 0000000000..003f378b45 --- /dev/null +++ b/api/clients/v2/verification/blob_verifier.go @@ -0,0 +1,55 @@ +package verification + +import ( + "errors" + "fmt" + + core "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/consensys/gnark-crypto/ecc/bn254" +) + +// VerifyBlobAgainstCert verifies the blob against a kzg commitment +// +// If verification succeeds, the method returns nil. Otherwise, it returns an error. +// +// TODO: in the future, this will be optimized to use fiat shamir transformation for verification, rather than +// +// regenerating the commitment: https://github.com/Layr-Labs/eigenda/issues/1037 +func VerifyBlobAgainstCert( + blobKey *core.BlobKey, + blobBytes []byte, + kzgCommitment *encoding.G1Commitment, + g1Srs []bn254.G1Affine) error { + + valid, err := GenerateAndCompareBlobCommitment(g1Srs, blobBytes, kzgCommitment) + if err != nil { + return fmt.Errorf("generate and compare commitment for blob %v: %w", blobKey.Hex(), err) + } + + if !valid { + return fmt.Errorf("commitment for blob %v is invalid", blobKey.Hex()) + } + + return nil +} + +// CheckBlobLength accepts bytes representing a blob, and a claimed length in symbols. Note that claimed length is an +// upper bound, not a precise length. Two length checks are performed: +// +// 1. Blob doesn't have length 0 +// 2. Blob length is <= the claimed blob length. Claimed blob length is from the BlobCommitment +func CheckBlobLength(blobBytes []byte, claimedBlobLength uint) error { + if len(blobBytes) == 0 { + return errors.New("blob has length 0") + } + + if uint(len(blobBytes)) > claimedBlobLength*encoding.BYTES_PER_SYMBOL { + return fmt.Errorf( + "length (%d bytes) is greater than claimed blob length (%d bytes)", + len(blobBytes), + claimedBlobLength*encoding.BYTES_PER_SYMBOL) + } + + return nil +} diff --git a/api/clients/v2/verification/conversion_utils.go b/api/clients/v2/verification/conversion_utils.go index 256772cb00..d4b61fe72a 100644 --- a/api/clients/v2/verification/conversion_utils.go +++ b/api/clients/v2/verification/conversion_utils.go @@ -10,6 +10,7 @@ import ( disperserv2 "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2" contractEigenDACertVerifier "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDACertVerifier" "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/encoding" "github.com/consensys/gnark-crypto/ecc/bn254" "github.com/consensys/gnark-crypto/ecc/bn254/fp" ) @@ -272,3 +273,18 @@ func repeatedBytesToBN254G1Points(repeatedBytes [][]byte) ([]contractEigenDACert return outputPoints, nil } + +// BlobCommitmentsBindingToInternal converts a blob commitment from an eigenDA cert into the internal +// encoding.BlobCommitments type +func BlobCommitmentsBindingToInternal( + blobCommitmentBinding *contractEigenDACertVerifier.BlobCommitment, +) (*encoding.BlobCommitments, error) { + + blobCommitment, err := encoding.BlobCommitmentsFromProtobuf(BlobCommitmentBindingToProto(blobCommitmentBinding)) + + if err != nil { + return nil, fmt.Errorf("blob commitments from protobuf: %w", err) + } + + return blobCommitment, nil +} diff --git a/api/clients/v2/verification/eigenda_cert.go b/api/clients/v2/verification/eigenda_cert.go index 3ba05afddc..eac3a1c0b6 100644 --- a/api/clients/v2/verification/eigenda_cert.go +++ b/api/clients/v2/verification/eigenda_cert.go @@ -6,7 +6,6 @@ import ( disperser "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2" contractEigenDACertVerifier "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDACertVerifier" v2 "github.com/Layr-Labs/eigenda/core/v2" - "github.com/Layr-Labs/eigenda/encoding" ) // EigenDACert contains all data necessary to retrieve and validate a blob @@ -45,8 +44,7 @@ func BuildEigenDACert( func (c *EigenDACert) ComputeBlobKey() (*v2.BlobKey, error) { blobHeader := c.BlobInclusionInfo.BlobCertificate.BlobHeader - blobCommitmentsProto := BlobCommitmentBindingToProto(&blobHeader.Commitment) - blobCommitments, err := encoding.BlobCommitmentsFromProtobuf(blobCommitmentsProto) + blobCommitments, err := BlobCommitmentsBindingToInternal(&blobHeader.Commitment) if err != nil { return nil, fmt.Errorf("blob commitments from protobuf: %w", err) } diff --git a/core/v2/core_test.go b/core/v2/core_test.go index 9c18cef74c..d3cfa9c16b 100644 --- a/core/v2/core_test.go +++ b/core/v2/core_test.go @@ -146,7 +146,7 @@ func prepareBlobs( blob := blobs[z] header := cert.BlobHeader - params, err := header.GetEncodingParams(blobParams) + params, err := v2.GetEncodingParams(header.BlobCommitments.Length, blobParams) require.NoError(t, err) chunks, err := p.GetFrames(blob, params) require.NoError(t, err) diff --git a/core/v2/types.go b/core/v2/types.go index 8985cb31de..584be3e38a 100644 --- a/core/v2/types.go +++ b/core/v2/types.go @@ -147,8 +147,8 @@ func (b *BlobHeader) ToProtobuf() (*commonpb.BlobHeader, error) { }, nil } -func (b *BlobHeader) GetEncodingParams(blobParams *core.BlobVersionParameters) (encoding.EncodingParams, error) { - length, err := GetChunkLength(uint32(b.BlobCommitments.Length), blobParams) +func GetEncodingParams(blobLength uint, blobParams *core.BlobVersionParameters) (encoding.EncodingParams, error) { + length, err := GetChunkLength(uint32(blobLength), blobParams) if err != nil { return encoding.EncodingParams{}, err } diff --git a/core/v2/validator.go b/core/v2/validator.go index c165280e3b..5338bec09a 100644 --- a/core/v2/validator.go +++ b/core/v2/validator.go @@ -150,10 +150,7 @@ func (v *shardValidator) ValidateBlobs(ctx context.Context, blobs []*BlobShard, } // TODO: Define params for the blob - params, err := blob.BlobHeader.GetEncodingParams(blobParams) - if err != nil { - return err - } + params, err := GetEncodingParams(blob.BlobHeader.BlobCommitments.Length, blobParams) if err != nil { return err diff --git a/disperser/controller/encoding_manager.go b/disperser/controller/encoding_manager.go index 480d8b1adb..8529aaf5e8 100644 --- a/disperser/controller/encoding_manager.go +++ b/disperser/controller/encoding_manager.go @@ -283,7 +283,7 @@ func (e *EncodingManager) encodeBlob(ctx context.Context, blobKey corev2.BlobKey }) ctx = metadata.NewOutgoingContext(ctx, md) - encodingParams, err := blob.BlobHeader.GetEncodingParams(blobParams) + encodingParams, err := corev2.GetEncodingParams(blob.BlobHeader.BlobCommitments.Length, blobParams) if err != nil { return nil, fmt.Errorf("failed to get encoding params: %w", err) } diff --git a/inabox/tests/integration_v2_test.go b/inabox/tests/integration_v2_test.go index 8f8c31743d..558467c6d2 100644 --- a/inabox/tests/integration_v2_test.go +++ b/inabox/tests/integration_v2_test.go @@ -240,20 +240,50 @@ var _ = Describe("Inabox v2 Integration", func() { } } + blob1Key, err := blobCert1.BlobHeader.BlobKey() + Expect(err).To(BeNil()) + + blob2Key, err := blobCert2.BlobHeader.BlobKey() + Expect(err).To(BeNil()) + // Test retrieval from DA network - b, err := retrievalClientV2.GetBlob(ctx, blobCert1.BlobHeader, batchHeader1.ReferenceBlockNumber, 0) + b, err := retrievalClientV2.GetBlob( + ctx, + blob1Key, + blobCert1.BlobHeader.BlobVersion, + blobCert1.BlobHeader.BlobCommitments, + batchHeader1.ReferenceBlockNumber, + 0) Expect(err).To(BeNil()) restored := bytes.TrimRight(b, "\x00") Expect(restored).To(Equal(paddedData1)) - b, err = retrievalClientV2.GetBlob(ctx, blobCert1.BlobHeader, batchHeader1.ReferenceBlockNumber, 1) + b, err = retrievalClientV2.GetBlob( + ctx, + blob1Key, + blobCert1.BlobHeader.BlobVersion, + blobCert1.BlobHeader.BlobCommitments, + batchHeader1.ReferenceBlockNumber, + 1) restored = bytes.TrimRight(b, "\x00") Expect(err).To(BeNil()) Expect(restored).To(Equal(paddedData1)) - b, err = retrievalClientV2.GetBlob(ctx, blobCert2.BlobHeader, batchHeader2.ReferenceBlockNumber, 0) + b, err = retrievalClientV2.GetBlob( + ctx, + blob2Key, + blobCert2.BlobHeader.BlobVersion, + blobCert2.BlobHeader.BlobCommitments, + batchHeader2.ReferenceBlockNumber, + 0) restored = bytes.TrimRight(b, "\x00") Expect(err).To(BeNil()) Expect(restored).To(Equal(paddedData2)) - b, err = retrievalClientV2.GetBlob(ctx, blobCert2.BlobHeader, batchHeader2.ReferenceBlockNumber, 1) + b, err = retrievalClientV2.GetBlob( + ctx, + blob2Key, + blobCert2.BlobHeader.BlobVersion, + blobCert2.BlobHeader.BlobCommitments, + batchHeader2.ReferenceBlockNumber, + 1) restored = bytes.TrimRight(b, "\x00") Expect(err).To(BeNil()) Expect(restored).To(Equal(paddedData2)) diff --git a/retriever/v2/server.go b/retriever/v2/server.go index f212a68cef..07105a51ec 100644 --- a/retriever/v2/server.go +++ b/retriever/v2/server.go @@ -72,7 +72,14 @@ func (s *Server) RetrieveBlob(ctx context.Context, req *pb.BlobRequest) (*pb.Blo ctxWithTimeout, cancel := context.WithTimeout(ctx, s.config.Timeout) defer cancel() - data, err := s.retrievalClient.GetBlob(ctxWithTimeout, blobHeader, uint64(req.GetReferenceBlockNumber()), core.QuorumID(req.GetQuorumId())) + + data, err := s.retrievalClient.GetBlob( + ctxWithTimeout, + blobKey, + blobHeader.BlobVersion, + blobHeader.BlobCommitments, + uint64(req.GetReferenceBlockNumber()), + core.QuorumID(req.GetQuorumId())) if err != nil { return nil, err } diff --git a/test/v2/test_client.go b/test/v2/test_client.go index 29b2bec450..abceb59afa 100644 --- a/test/v2/test_client.go +++ b/test/v2/test_client.go @@ -3,13 +3,14 @@ package v2 import ( "context" "fmt" - "github.com/docker/go-units" "os" "path" "strings" "testing" "time" + "github.com/docker/go-units" + "github.com/Layr-Labs/eigenda/api/clients/v2" commonv2 "github.com/Layr-Labs/eigenda/api/grpc/common/v2" v2 "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2" @@ -299,7 +300,16 @@ func (c *TestClient) ReadBlobFromValidators( header, err := corev2.BlobHeaderFromProtobuf(blobCert.BlobHeader) require.NoError(c.t, err) - retrievedBlob, err := c.RetrievalClient.GetBlob(ctx, header, uint64(currentBlockNumber), quorumID) + blobKey, err := header.BlobKey() + require.NoError(c.t, err) + + retrievedBlob, err := c.RetrievalClient.GetBlob( + ctx, + blobKey, + header.BlobVersion, + header.BlobCommitments, + uint64(currentBlockNumber), + quorumID) require.NoError(c.t, err) retrievedPayload := codec.RemoveEmptyByteFromPaddedBytes(retrievedBlob)