From d7fb87f273d3a2e1319bfb41bd487966d6a6833e Mon Sep 17 00:00:00 2001 From: Ian Shim <100327837+ian-shim@users.noreply.github.com> Date: Fri, 31 Jan 2025 08:44:25 -0800 Subject: [PATCH] Update v2 blob statuses (#1185) --- api/clients/v2/payload_disperser.go | 4 +- api/docs/disperser_v2.html | 35 ++++-- api/docs/disperser_v2.md | 16 ++- api/docs/eigenda-protos.html | 35 ++++-- api/docs/eigenda-protos.md | 16 ++- api/grpc/disperser/v2/disperser_v2.pb.go | 113 ++++++++++-------- api/proto/disperser/v2/disperser_v2.proto | 35 ++++-- disperser/apiserver/get_blob_status_v2.go | 15 +-- disperser/apiserver/server_v2_test.go | 8 +- disperser/common/v2/blob.go | 28 ++--- .../v2/blobstore/dynamo_metadata_store.go | 11 +- .../blobstore/dynamo_metadata_store_test.go | 10 +- disperser/controller/dispatcher.go | 10 +- disperser/controller/dispatcher_metrics.go | 9 +- disperser/controller/dispatcher_test.go | 12 +- disperser/dataapi/docs/v1/V1_docs.go | 18 +++ disperser/dataapi/docs/v1/V1_swagger.json | 18 +++ disperser/dataapi/docs/v1/V1_swagger.yaml | 12 ++ disperser/dataapi/docs/v2/V2_docs.go | 39 +++++- disperser/dataapi/docs/v2/V2_swagger.json | 39 +++++- disperser/dataapi/docs/v2/V2_swagger.yaml | 26 +++- inabox/tests/integration_v2_test.go | 2 +- test/v2/test_client.go | 17 +-- 23 files changed, 348 insertions(+), 180 deletions(-) diff --git a/api/clients/v2/payload_disperser.go b/api/clients/v2/payload_disperser.go index 3c8923a455..e285cb61bc 100644 --- a/api/clients/v2/payload_disperser.go +++ b/api/clients/v2/payload_disperser.go @@ -213,7 +213,7 @@ func (pd *PayloadDisperser) pollBlobStatusUntilCertified( case <-ctx.Done(): return nil, fmt.Errorf( "timed out waiting for %v blob status, final status was %v: %w", - dispgrpc.BlobStatus_CERTIFIED.Descriptor(), + dispgrpc.BlobStatus_COMPLETE.Descriptor(), previousStatus.Descriptor(), ctx.Err()) case <-ticker.C: @@ -237,7 +237,7 @@ func (pd *PayloadDisperser) pollBlobStatusUntilCertified( // TODO: we'll need to add more in-depth response status processing to derive failover errors switch newStatus { - case dispgrpc.BlobStatus_CERTIFIED: + case dispgrpc.BlobStatus_COMPLETE: return blobStatusReply, nil case dispgrpc.BlobStatus_QUEUED, dispgrpc.BlobStatus_ENCODED: continue diff --git a/api/docs/disperser_v2.html b/api/docs/disperser_v2.html index 8b88e4b9f5..0c11c08c87 100644 --- a/api/docs/disperser_v2.html +++ b/api/docs/disperser_v2.html @@ -428,7 +428,14 @@

BlobStatusReply

signed_batch SignedBatch -

The signed batch. Unset if the status is not CERTIFIED.

+

The signed batch. Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. +signed_batch and blob_inclusion_info are only set if the blob status is GATHERING_SIGNATURES or COMPLETE. +When blob is in GATHERING_SIGNATURES status, the attestation object in signed_batch contains attestation information +at the point in time. As it gathers more signatures, attestation object will be updated according to the latest attestation status. +The client can use this intermediate attestation to verify a blob if it has gathered enough signatures. +Otherwise, it should should poll the GetBlobStatus API until the desired level of attestation has been gathered or status is COMPLETE. +When blob is in COMPLETE status, the attestation object in signed_batch contains the final attestation information. +If the final attestation does not meet the client's requirement, the client should try a new dispersal.

@@ -436,7 +443,7 @@

BlobStatusReply

BlobInclusionInfo

BlobInclusionInfo is the information needed to verify the inclusion of a blob in a batch. -Unset if the status is not CERTIFIED.

+Only set if the blob status is GATHERING_SIGNATURES or COMPLETE.

@@ -812,7 +819,7 @@

SignedBatch

BlobStatus

-

BlobStatus represents the status of a blob.

The status of a blob is updated as the blob is processed by the disperser.

The status of a blob can be queried by the client using the GetBlobStatus API.

Intermediate states are states that the blob can be in while being processed, and it can be updated to a different state:

- QUEUED

- ENCODED

Terminal states are states that will not be updated to a different state:

- UNKNOWN

- CERTIFIED

- FAILED

- INSUFFICIENT_SIGNATURES

+

BlobStatus represents the status of a blob.

The status of a blob is updated as the blob is processed by the disperser.

The status of a blob can be queried by the client using the GetBlobStatus API.

Intermediate states are states that the blob can be in while being processed, and it can be updated to a different state:

- QUEUED

- ENCODED

- GATHERING_SIGNATURES

Terminal states are states that will not be updated to a different state:

- UNKNOWN

- COMPLETE

- FAILED

@@ -844,25 +851,27 @@

BlobStatus

- + - + - + - + - + - + diff --git a/api/docs/disperser_v2.md b/api/docs/disperser_v2.md index 9bf70fff52..e889825aad 100644 --- a/api/docs/disperser_v2.md +++ b/api/docs/disperser_v2.md @@ -111,8 +111,8 @@ BlobStatusReply is the reply to a BlobStatusRequest. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | | status | [BlobStatus](#disperser-v2-BlobStatus) | | The status of the blob. | -| signed_batch | [SignedBatch](#disperser-v2-SignedBatch) | | The signed batch. Unset if the status is not CERTIFIED. | -| blob_inclusion_info | [BlobInclusionInfo](#disperser-v2-BlobInclusionInfo) | | BlobInclusionInfo is the information needed to verify the inclusion of a blob in a batch. Unset if the status is not CERTIFIED. | +| signed_batch | [SignedBatch](#disperser-v2-SignedBatch) | | The signed batch. Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. signed_batch and blob_inclusion_info are only set if the blob status is GATHERING_SIGNATURES or COMPLETE. When blob is in GATHERING_SIGNATURES status, the attestation object in signed_batch contains attestation information at the point in time. As it gathers more signatures, attestation object will be updated according to the latest attestation status. The client can use this intermediate attestation to verify a blob if it has gathered enough signatures. Otherwise, it should should poll the GetBlobStatus API until the desired level of attestation has been gathered or status is COMPLETE. When blob is in COMPLETE status, the attestation object in signed_batch contains the final attestation information. If the final attestation does not meet the client's requirement, the client should try a new dispersal. | +| blob_inclusion_info | [BlobInclusionInfo](#disperser-v2-BlobInclusionInfo) | | BlobInclusionInfo is the information needed to verify the inclusion of a blob in a batch. Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. | @@ -294,11 +294,11 @@ The status of a blob can be queried by the client using the GetBlobStatus API. Intermediate states are states that the blob can be in while being processed, and it can be updated to a different state: - QUEUED - ENCODED +- GATHERING_SIGNATURES Terminal states are states that will not be updated to a different state: - UNKNOWN -- CERTIFIED +- COMPLETE - FAILED -- INSUFFICIENT_SIGNATURES | Name | Number | Description | | ---- | ------ | ----------- | @@ -307,11 +307,9 @@ Terminal states are states that will not be updated to a different state: This status is functionally equivalent to FAILED, but is used to indicate that the failure is due to an unanticipated bug. | | QUEUED | 1 | QUEUED means that the blob has been queued by the disperser for processing. The DisperseBlob API is asynchronous, meaning that after request validation, but before any processing, the blob is stored in a queue of some sort, and a response immediately returned to the client. | | ENCODED | 2 | ENCODED means that the blob has been Reed-Solomon encoded into chunks and is ready to be dispersed to DA Nodes. | -| CERTIFIED | 3 | CERTIFIED means the blob has been dispersed and attested by the DA nodes. | -| FAILED | 4 | FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to retry the blob, the client must submit the blob again with different salt (blob key is required to be unique). | -| INSUFFICIENT_SIGNATURES | 5 | INSUFFICIENT_SIGNATURES means that the blob has failed to gather sufficient attestation. - -This status is functionally equivalent to FAILED, but is used to indicate that the failure is due to an an inability to gather sufficient signatures. | +| GATHERING_SIGNATURES | 3 | GATHERING_SIGNATURES means that the blob chunks are currently actively being transmitted to validators,requesting for signatures. Requests that timeout or receive errors are resubmitted to DA nodes for some period of time set by the disperser, after which the BlobStatus becomes COMPLETE. This status is not currently implemented, and is a placeholder for future functionality. | +| COMPLETE | 4 | COMPLETE means the blob has been dispersed to DA nodes, and the GATHERING_SIGNATURES period of time has completed. This status does not guarantee any signer percentage, so a client should check that the signature has met its required threshold, and resubmit a new blob dispersal request if not. | +| FAILED | 5 | FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to retry the blob, the client must submit the blob again with different salt (blob key is required to be unique). | diff --git a/api/docs/eigenda-protos.html b/api/docs/eigenda-protos.html index 3ac96ce284..8f2bc1742d 100644 --- a/api/docs/eigenda-protos.html +++ b/api/docs/eigenda-protos.html @@ -2129,7 +2129,14 @@

BlobStatusReply

- + @@ -2137,7 +2144,7 @@

BlobStatusReply

+Only set if the blob status is GATHERING_SIGNATURES or COMPLETE.

@@ -2513,7 +2520,7 @@

SignedBatch

BlobStatus

-

BlobStatus represents the status of a blob.

The status of a blob is updated as the blob is processed by the disperser.

The status of a blob can be queried by the client using the GetBlobStatus API.

Intermediate states are states that the blob can be in while being processed, and it can be updated to a different state:

- QUEUED

- ENCODED

Terminal states are states that will not be updated to a different state:

- UNKNOWN

- CERTIFIED

- FAILED

- INSUFFICIENT_SIGNATURES

+

BlobStatus represents the status of a blob.

The status of a blob is updated as the blob is processed by the disperser.

The status of a blob can be queried by the client using the GetBlobStatus API.

Intermediate states are states that the blob can be in while being processed, and it can be updated to a different state:

- QUEUED

- ENCODED

- GATHERING_SIGNATURES

Terminal states are states that will not be updated to a different state:

- UNKNOWN

- COMPLETE

- FAILED

NameNumberDescription
CERTIFIEDGATHERING_SIGNATURES 3

CERTIFIED means the blob has been dispersed and attested by the DA nodes.

GATHERING_SIGNATURES means that the blob chunks are currently actively being transmitted to validators,requesting for signatures. +Requests that timeout or receive errors are resubmitted to DA nodes for some period of time set by the disperser, +after which the BlobStatus becomes COMPLETE. +This status is not currently implemented, and is a placeholder for future functionality.

FAILEDCOMPLETE 4

FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to -retry the blob, the client must submit the blob again with different salt (blob key is required to be unique).

COMPLETE means the blob has been dispersed to DA nodes, and the GATHERING_SIGNATURES period of time has completed. +This status does not guarantee any signer percentage, so a client should check that the signature has met +its required threshold, and resubmit a new blob dispersal request if not.

INSUFFICIENT_SIGNATURESFAILED 5

INSUFFICIENT_SIGNATURES means that the blob has failed to gather sufficient attestation. - -This status is functionally equivalent to FAILED, but is used to indicate that the failure is due to an -an inability to gather sufficient signatures.

FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to +retry the blob, the client must submit the blob again with different salt (blob key is required to be unique).

signed_batch SignedBatch

The signed batch. Unset if the status is not CERTIFIED.

The signed batch. Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. +signed_batch and blob_inclusion_info are only set if the blob status is GATHERING_SIGNATURES or COMPLETE. +When blob is in GATHERING_SIGNATURES status, the attestation object in signed_batch contains attestation information +at the point in time. As it gathers more signatures, attestation object will be updated according to the latest attestation status. +The client can use this intermediate attestation to verify a blob if it has gathered enough signatures. +Otherwise, it should should poll the GetBlobStatus API until the desired level of attestation has been gathered or status is COMPLETE. +When blob is in COMPLETE status, the attestation object in signed_batch contains the final attestation information. +If the final attestation does not meet the client's requirement, the client should try a new dispersal.

BlobInclusionInfo

BlobInclusionInfo is the information needed to verify the inclusion of a blob in a batch. -Unset if the status is not CERTIFIED.

@@ -2545,25 +2552,27 @@

BlobStatus

- + - + - + - + - + - + diff --git a/api/docs/eigenda-protos.md b/api/docs/eigenda-protos.md index 8c1428a9a3..67db464447 100644 --- a/api/docs/eigenda-protos.md +++ b/api/docs/eigenda-protos.md @@ -814,8 +814,8 @@ BlobStatusReply is the reply to a BlobStatusRequest. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | | status | [BlobStatus](#disperser-v2-BlobStatus) | | The status of the blob. | -| signed_batch | [SignedBatch](#disperser-v2-SignedBatch) | | The signed batch. Unset if the status is not CERTIFIED. | -| blob_inclusion_info | [BlobInclusionInfo](#disperser-v2-BlobInclusionInfo) | | BlobInclusionInfo is the information needed to verify the inclusion of a blob in a batch. Unset if the status is not CERTIFIED. | +| signed_batch | [SignedBatch](#disperser-v2-SignedBatch) | | The signed batch. Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. signed_batch and blob_inclusion_info are only set if the blob status is GATHERING_SIGNATURES or COMPLETE. When blob is in GATHERING_SIGNATURES status, the attestation object in signed_batch contains attestation information at the point in time. As it gathers more signatures, attestation object will be updated according to the latest attestation status. The client can use this intermediate attestation to verify a blob if it has gathered enough signatures. Otherwise, it should should poll the GetBlobStatus API until the desired level of attestation has been gathered or status is COMPLETE. When blob is in COMPLETE status, the attestation object in signed_batch contains the final attestation information. If the final attestation does not meet the client's requirement, the client should try a new dispersal. | +| blob_inclusion_info | [BlobInclusionInfo](#disperser-v2-BlobInclusionInfo) | | BlobInclusionInfo is the information needed to verify the inclusion of a blob in a batch. Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. | @@ -997,11 +997,11 @@ The status of a blob can be queried by the client using the GetBlobStatus API. Intermediate states are states that the blob can be in while being processed, and it can be updated to a different state: - QUEUED - ENCODED +- GATHERING_SIGNATURES Terminal states are states that will not be updated to a different state: - UNKNOWN -- CERTIFIED +- COMPLETE - FAILED -- INSUFFICIENT_SIGNATURES | Name | Number | Description | | ---- | ------ | ----------- | @@ -1010,11 +1010,9 @@ Terminal states are states that will not be updated to a different state: This status is functionally equivalent to FAILED, but is used to indicate that the failure is due to an unanticipated bug. | | QUEUED | 1 | QUEUED means that the blob has been queued by the disperser for processing. The DisperseBlob API is asynchronous, meaning that after request validation, but before any processing, the blob is stored in a queue of some sort, and a response immediately returned to the client. | | ENCODED | 2 | ENCODED means that the blob has been Reed-Solomon encoded into chunks and is ready to be dispersed to DA Nodes. | -| CERTIFIED | 3 | CERTIFIED means the blob has been dispersed and attested by the DA nodes. | -| FAILED | 4 | FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to retry the blob, the client must submit the blob again with different salt (blob key is required to be unique). | -| INSUFFICIENT_SIGNATURES | 5 | INSUFFICIENT_SIGNATURES means that the blob has failed to gather sufficient attestation. - -This status is functionally equivalent to FAILED, but is used to indicate that the failure is due to an an inability to gather sufficient signatures. | +| GATHERING_SIGNATURES | 3 | GATHERING_SIGNATURES means that the blob chunks are currently actively being transmitted to validators,requesting for signatures. Requests that timeout or receive errors are resubmitted to DA nodes for some period of time set by the disperser, after which the BlobStatus becomes COMPLETE. This status is not currently implemented, and is a placeholder for future functionality. | +| COMPLETE | 4 | COMPLETE means the blob has been dispersed to DA nodes, and the GATHERING_SIGNATURES period of time has completed. This status does not guarantee any signer percentage, so a client should check that the signature has met its required threshold, and resubmit a new blob dispersal request if not. | +| FAILED | 5 | FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to retry the blob, the client must submit the blob again with different salt (blob key is required to be unique). | diff --git a/api/grpc/disperser/v2/disperser_v2.pb.go b/api/grpc/disperser/v2/disperser_v2.pb.go index 4de93e86be..70d37cf84e 100644 --- a/api/grpc/disperser/v2/disperser_v2.pb.go +++ b/api/grpc/disperser/v2/disperser_v2.pb.go @@ -28,11 +28,11 @@ const ( // Intermediate states are states that the blob can be in while being processed, and it can be updated to a different state: // - QUEUED // - ENCODED +// - GATHERING_SIGNATURES // Terminal states are states that will not be updated to a different state: // - UNKNOWN -// - CERTIFIED +// - COMPLETE // - FAILED -// - INSUFFICIENT_SIGNATURES type BlobStatus int32 const ( @@ -48,16 +48,18 @@ const ( BlobStatus_QUEUED BlobStatus = 1 // ENCODED means that the blob has been Reed-Solomon encoded into chunks and is ready to be dispersed to DA Nodes. BlobStatus_ENCODED BlobStatus = 2 - // CERTIFIED means the blob has been dispersed and attested by the DA nodes. - BlobStatus_CERTIFIED BlobStatus = 3 + // GATHERING_SIGNATURES means that the blob chunks are currently actively being transmitted to validators,requesting for signatures. + // Requests that timeout or receive errors are resubmitted to DA nodes for some period of time set by the disperser, + // after which the BlobStatus becomes COMPLETE. + // This status is not currently implemented, and is a placeholder for future functionality. + BlobStatus_GATHERING_SIGNATURES BlobStatus = 3 + // COMPLETE means the blob has been dispersed to DA nodes, and the GATHERING_SIGNATURES period of time has completed. + // This status does not guarantee any signer percentage, so a client should check that the signature has met + // its required threshold, and resubmit a new blob dispersal request if not. + BlobStatus_COMPLETE BlobStatus = 4 // FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to // retry the blob, the client must submit the blob again with different salt (blob key is required to be unique). - BlobStatus_FAILED BlobStatus = 4 - // INSUFFICIENT_SIGNATURES means that the blob has failed to gather sufficient attestation. - // - // This status is functionally equivalent to FAILED, but is used to indicate that the failure is due to an - // an inability to gather sufficient signatures. - BlobStatus_INSUFFICIENT_SIGNATURES BlobStatus = 5 + BlobStatus_FAILED BlobStatus = 5 ) // Enum value maps for BlobStatus. @@ -66,17 +68,17 @@ var ( 0: "UNKNOWN", 1: "QUEUED", 2: "ENCODED", - 3: "CERTIFIED", - 4: "FAILED", - 5: "INSUFFICIENT_SIGNATURES", + 3: "GATHERING_SIGNATURES", + 4: "COMPLETE", + 5: "FAILED", } BlobStatus_value = map[string]int32{ - "UNKNOWN": 0, - "QUEUED": 1, - "ENCODED": 2, - "CERTIFIED": 3, - "FAILED": 4, - "INSUFFICIENT_SIGNATURES": 5, + "UNKNOWN": 0, + "QUEUED": 1, + "ENCODED": 2, + "GATHERING_SIGNATURES": 3, + "COMPLETE": 4, + "FAILED": 5, } ) @@ -310,10 +312,17 @@ type BlobStatusReply struct { // The status of the blob. Status BlobStatus `protobuf:"varint,1,opt,name=status,proto3,enum=disperser.v2.BlobStatus" json:"status,omitempty"` - // The signed batch. Unset if the status is not CERTIFIED. + // The signed batch. Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. + // signed_batch and blob_inclusion_info are only set if the blob status is GATHERING_SIGNATURES or COMPLETE. + // When blob is in GATHERING_SIGNATURES status, the attestation object in signed_batch contains attestation information + // at the point in time. As it gathers more signatures, attestation object will be updated according to the latest attestation status. + // The client can use this intermediate attestation to verify a blob if it has gathered enough signatures. + // Otherwise, it should should poll the GetBlobStatus API until the desired level of attestation has been gathered or status is COMPLETE. + // When blob is in COMPLETE status, the attestation object in signed_batch contains the final attestation information. + // If the final attestation does not meet the client's requirement, the client should try a new dispersal. SignedBatch *SignedBatch `protobuf:"bytes,2,opt,name=signed_batch,json=signedBatch,proto3" json:"signed_batch,omitempty"` // BlobInclusionInfo is the information needed to verify the inclusion of a blob in a batch. - // Unset if the status is not CERTIFIED. + // Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. BlobInclusionInfo *BlobInclusionInfo `protobuf:"bytes,3,opt,name=blob_inclusion_info,json=blobInclusionInfo,proto3" json:"blob_inclusion_info,omitempty"` } @@ -1202,40 +1211,40 @@ var file_disperser_v2_disperser_v2_proto_rawDesc = []byte{ 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x14, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, - 0x65, 0x2a, 0x6a, 0x0a, 0x0a, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x65, 0x2a, 0x66, 0x0a, 0x0a, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x51, 0x55, 0x45, 0x55, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x45, 0x4e, 0x43, 0x4f, - 0x44, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x45, 0x52, 0x54, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, - 0x12, 0x1b, 0x0a, 0x17, 0x49, 0x4e, 0x53, 0x55, 0x46, 0x46, 0x49, 0x43, 0x49, 0x45, 0x4e, 0x54, - 0x5f, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x53, 0x10, 0x05, 0x32, 0xf2, 0x02, - 0x0a, 0x09, 0x44, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x12, 0x54, 0x0a, 0x0c, 0x44, - 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x21, 0x2e, 0x64, 0x69, - 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x44, 0x69, 0x73, 0x70, 0x65, - 0x72, 0x73, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, - 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x44, 0x69, - 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, - 0x00, 0x12, 0x51, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, - 0x32, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, - 0x76, 0x32, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x43, - 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x23, 0x2e, 0x64, 0x69, 0x73, 0x70, - 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x6f, 0x6d, - 0x6d, 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, + 0x44, 0x45, 0x44, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x47, 0x41, 0x54, 0x48, 0x45, 0x52, 0x49, + 0x4e, 0x47, 0x5f, 0x53, 0x49, 0x47, 0x4e, 0x41, 0x54, 0x55, 0x52, 0x45, 0x53, 0x10, 0x03, 0x12, + 0x0c, 0x0a, 0x08, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x04, 0x12, 0x0a, 0x0a, + 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x05, 0x32, 0xf2, 0x02, 0x0a, 0x09, 0x44, 0x69, + 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x12, 0x54, 0x0a, 0x0c, 0x44, 0x69, 0x73, 0x70, 0x65, + 0x72, 0x73, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x21, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, + 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x44, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x42, + 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x64, 0x69, 0x73, + 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x44, 0x69, 0x73, 0x70, 0x65, 0x72, + 0x73, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x51, 0x0a, + 0x0d, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x6c, - 0x6f, 0x62, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6c, - 0x79, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x24, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, - 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x64, - 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x47, 0x65, 0x74, 0x50, - 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x22, 0x00, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x4c, 0x61, 0x79, 0x72, 0x2d, 0x4c, 0x61, 0x62, 0x73, 0x2f, 0x65, 0x69, 0x67, 0x65, 0x6e, - 0x64, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x69, 0x73, 0x70, - 0x65, 0x72, 0x73, 0x65, 0x72, 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x42, + 0x6c, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, + 0x12, 0x5d, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x6f, 0x6d, 0x6d, 0x69, + 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x23, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, + 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x6d, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x64, 0x69, 0x73, + 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x6f, + 0x6d, 0x6d, 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, + 0x5d, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x24, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, + 0x32, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x64, 0x69, 0x73, 0x70, 0x65, + 0x72, 0x73, 0x65, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x34, + 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x4c, 0x61, 0x79, + 0x72, 0x2d, 0x4c, 0x61, 0x62, 0x73, 0x2f, 0x65, 0x69, 0x67, 0x65, 0x6e, 0x64, 0x61, 0x2f, 0x61, + 0x70, 0x69, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x64, 0x69, 0x73, 0x70, 0x65, 0x72, 0x73, 0x65, + 0x72, 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/proto/disperser/v2/disperser_v2.proto b/api/proto/disperser/v2/disperser_v2.proto index e521171705..9bbd34f134 100644 --- a/api/proto/disperser/v2/disperser_v2.proto +++ b/api/proto/disperser/v2/disperser_v2.proto @@ -73,10 +73,17 @@ message BlobStatusRequest { message BlobStatusReply { // The status of the blob. BlobStatus status = 1; - // The signed batch. Unset if the status is not CERTIFIED. + // The signed batch. Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. + // signed_batch and blob_inclusion_info are only set if the blob status is GATHERING_SIGNATURES or COMPLETE. + // When blob is in GATHERING_SIGNATURES status, the attestation object in signed_batch contains attestation information + // at the point in time. As it gathers more signatures, attestation object will be updated according to the latest attestation status. + // The client can use this intermediate attestation to verify a blob if it has gathered enough signatures. + // Otherwise, it should should poll the GetBlobStatus API until the desired level of attestation has been gathered or status is COMPLETE. + // When blob is in COMPLETE status, the attestation object in signed_batch contains the final attestation information. + // If the final attestation does not meet the client's requirement, the client should try a new dispersal. SignedBatch signed_batch = 2; // BlobInclusionInfo is the information needed to verify the inclusion of a blob in a batch. - // Unset if the status is not CERTIFIED. + // Only set if the blob status is GATHERING_SIGNATURES or COMPLETE. BlobInclusionInfo blob_inclusion_info = 3; } @@ -123,11 +130,11 @@ message GetPaymentStateReply { // Intermediate states are states that the blob can be in while being processed, and it can be updated to a different state: // - QUEUED // - ENCODED +// - GATHERING_SIGNATURES // Terminal states are states that will not be updated to a different state: // - UNKNOWN -// - CERTIFIED +// - COMPLETE // - FAILED -// - INSUFFICIENT_SIGNATURES enum BlobStatus { // UNKNOWN means that the status of the blob is unknown. // This is a catch all and should not be encountered absent a bug. @@ -144,18 +151,20 @@ enum BlobStatus { // ENCODED means that the blob has been Reed-Solomon encoded into chunks and is ready to be dispersed to DA Nodes. ENCODED = 2; - // CERTIFIED means the blob has been dispersed and attested by the DA nodes. - CERTIFIED = 3; + // GATHERING_SIGNATURES means that the blob chunks are currently actively being transmitted to validators,requesting for signatures. + // Requests that timeout or receive errors are resubmitted to DA nodes for some period of time set by the disperser, + // after which the BlobStatus becomes COMPLETE. + // This status is not currently implemented, and is a placeholder for future functionality. + GATHERING_SIGNATURES = 3; + + // COMPLETE means the blob has been dispersed to DA nodes, and the GATHERING_SIGNATURES period of time has completed. + // This status does not guarantee any signer percentage, so a client should check that the signature has met + // its required threshold, and resubmit a new blob dispersal request if not. + COMPLETE = 4; // FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to // retry the blob, the client must submit the blob again with different salt (blob key is required to be unique). - FAILED = 4; - - // INSUFFICIENT_SIGNATURES means that the blob has failed to gather sufficient attestation. - // - // This status is functionally equivalent to FAILED, but is used to indicate that the failure is due to an - // an inability to gather sufficient signatures. - INSUFFICIENT_SIGNATURES = 5; + FAILED = 5; } // SignedBatch is a batch of blobs with a signature. diff --git a/disperser/apiserver/get_blob_status_v2.go b/disperser/apiserver/get_blob_status_v2.go index e5f540ba7d..145c28bb98 100644 --- a/disperser/apiserver/get_blob_status_v2.go +++ b/disperser/apiserver/get_blob_status_v2.go @@ -34,7 +34,8 @@ func (s *DispersalServerV2) GetBlobStatus(ctx context.Context, req *pb.BlobStatu return nil, api.NewErrorInternal(fmt.Sprintf("failed to get blob metadata: %s", err.Error())) } - if metadata.BlobStatus != dispv2.Certified { + // If the blob is not complete or gathering signatures, return the status without the signed batch + if metadata.BlobStatus != dispv2.Complete && metadata.BlobStatus != dispv2.GatheringSignatures { return &pb.BlobStatusReply{ Status: metadata.BlobStatus.ToProfobuf(), }, nil @@ -42,27 +43,27 @@ func (s *DispersalServerV2) GetBlobStatus(ctx context.Context, req *pb.BlobStatu cert, _, err := s.blobMetadataStore.GetBlobCertificate(ctx, blobKey) if err != nil { - s.logger.Error("failed to get blob certificate for certified blob", "err", err, "blobKey", blobKey.Hex()) + s.logger.Error("failed to get blob certificate for blob in GatheringSignatures/Complete status", "err", err, "blobKey", blobKey.Hex()) if errors.Is(err, dispcommon.ErrMetadataNotFound) { return nil, api.NewErrorNotFound("no such blob certificate found") } return nil, api.NewErrorInternal(fmt.Sprintf("failed to get blob certificate: %s", err.Error())) } - // For certified blobs, include signed batch and blob inclusion info + // For blobs in GatheringSignatures/Complete status, include signed batch and blob inclusion info blobInclusionInfos, err := s.blobMetadataStore.GetBlobInclusionInfos(ctx, blobKey) if err != nil { - s.logger.Error("failed to get blob inclusion info for certified blob", "err", err, "blobKey", blobKey.Hex()) + s.logger.Error("failed to get blob inclusion info for blob", "err", err, "blobKey", blobKey.Hex()) return nil, api.NewErrorInternal(fmt.Sprintf("failed to get blob inclusion info: %s", err.Error())) } if len(blobInclusionInfos) == 0 { - s.logger.Error("no blob inclusion info found for certified blob", "blobKey", blobKey.Hex()) + s.logger.Error("no blob inclusion info found for blob", "blobKey", blobKey.Hex()) return nil, api.NewErrorInternal("no blob inclusion info found") } if len(blobInclusionInfos) > 1 { - s.logger.Warn("multiple inclusion info found for certified blob", "blobKey", blobKey.Hex()) + s.logger.Warn("multiple inclusion info found for blob", "blobKey", blobKey.Hex()) } for _, inclusionInfo := range blobInclusionInfos { @@ -101,6 +102,6 @@ func (s *DispersalServerV2) GetBlobStatus(ctx context.Context, req *pb.BlobStatu }, nil } - s.logger.Error("no signed batch found for certified blob", "blobKey", blobKey.Hex()) + s.logger.Error("no signed batch found for blob", "blobKey", blobKey.Hex()) return nil, api.NewErrorInternal("no signed batch found") } diff --git a/disperser/apiserver/server_v2_test.go b/disperser/apiserver/server_v2_test.go index 8503e55cf8..8dc267ac4d 100644 --- a/disperser/apiserver/server_v2_test.go +++ b/disperser/apiserver/server_v2_test.go @@ -329,7 +329,7 @@ func TestV2GetBlobStatus(t *testing.T) { err = c.BlobMetadataStore.PutBlobCertificate(ctx, blobCert, nil) require.NoError(t, err) - // Non ceritified blob status + // Queued/Encoded blob status status, err := c.DispersalServerV2.GetBlobStatus(ctx, &pbv2.BlobStatusRequest{ BlobKey: blobKey[:], }) @@ -343,8 +343,8 @@ func TestV2GetBlobStatus(t *testing.T) { require.NoError(t, err) require.Equal(t, pbv2.BlobStatus_ENCODED, status.Status) - // Certified blob status - err = c.BlobMetadataStore.UpdateBlobStatus(ctx, blobKey, dispv2.Certified) + // Complete blob status + err = c.BlobMetadataStore.UpdateBlobStatus(ctx, blobKey, dispv2.Complete) require.NoError(t, err) batchHeader := &corev2.BatchHeader{ BatchRoot: [32]byte{1, 2, 3}, @@ -384,7 +384,7 @@ func TestV2GetBlobStatus(t *testing.T) { BlobKey: blobKey[:], }) require.NoError(t, err) - require.Equal(t, pbv2.BlobStatus_CERTIFIED, reply.GetStatus()) + require.Equal(t, pbv2.BlobStatus_COMPLETE, reply.GetStatus()) blobHeaderProto, err := blobHeader.ToProtobuf() require.NoError(t, err) blobCertProto, err := blobCert.ToProtobuf() diff --git a/disperser/common/v2/blob.go b/disperser/common/v2/blob.go index 5882ed84db..95716ced25 100644 --- a/disperser/common/v2/blob.go +++ b/disperser/common/v2/blob.go @@ -13,9 +13,9 @@ type BlobStatus uint const ( Queued BlobStatus = iota Encoded - Certified + GatheringSignatures + Complete Failed - InsufficientSignatures ) func (s BlobStatus) String() string { @@ -24,12 +24,12 @@ func (s BlobStatus) String() string { return "Queued" case Encoded: return "Encoded" - case Certified: - return "Certified" + case GatheringSignatures: + return "Gathering Signatures" + case Complete: + return "Complete" case Failed: return "Failed" - case InsufficientSignatures: - return "Insufficient Signatures" default: return "Unknown" } @@ -41,12 +41,12 @@ func (s BlobStatus) ToProfobuf() pb.BlobStatus { return pb.BlobStatus_QUEUED case Encoded: return pb.BlobStatus_ENCODED - case Certified: - return pb.BlobStatus_CERTIFIED + case GatheringSignatures: + return pb.BlobStatus_GATHERING_SIGNATURES + case Complete: + return pb.BlobStatus_COMPLETE case Failed: return pb.BlobStatus_FAILED - case InsufficientSignatures: - return pb.BlobStatus_INSUFFICIENT_SIGNATURES default: return pb.BlobStatus_UNKNOWN } @@ -58,12 +58,12 @@ func BlobStatusFromProtobuf(s pb.BlobStatus) (BlobStatus, error) { return Queued, nil case pb.BlobStatus_ENCODED: return Encoded, nil - case pb.BlobStatus_CERTIFIED: - return Certified, nil + case pb.BlobStatus_GATHERING_SIGNATURES: + return GatheringSignatures, nil + case pb.BlobStatus_COMPLETE: + return Complete, nil case pb.BlobStatus_FAILED: return Failed, nil - case pb.BlobStatus_INSUFFICIENT_SIGNATURES: - return InsufficientSignatures, nil default: return 0, fmt.Errorf("unknown blob status: %v", s) } diff --git a/disperser/common/v2/blobstore/dynamo_metadata_store.go b/disperser/common/v2/blobstore/dynamo_metadata_store.go index 793f4bddcf..776f4b0c80 100644 --- a/disperser/common/v2/blobstore/dynamo_metadata_store.go +++ b/disperser/common/v2/blobstore/dynamo_metadata_store.go @@ -59,11 +59,12 @@ const ( var ( statusUpdatePrecondition = map[v2.BlobStatus][]v2.BlobStatus{ - v2.Queued: {}, - v2.Encoded: {v2.Queued}, - v2.Certified: {v2.Encoded}, - v2.Failed: {v2.Queued, v2.Encoded}, - v2.InsufficientSignatures: {v2.Encoded}, + v2.Queued: {}, + v2.Encoded: {v2.Queued}, + v2.GatheringSignatures: {v2.Encoded}, + // TODO: when GatheringSignatures is fully supported, remove v2.Encoded from below + v2.Complete: {v2.Encoded, v2.GatheringSignatures}, + v2.Failed: {v2.Queued, v2.Encoded, v2.GatheringSignatures}, } ErrInvalidStateTransition = errors.New("invalid state transition") ) diff --git a/disperser/common/v2/blobstore/dynamo_metadata_store_test.go b/disperser/common/v2/blobstore/dynamo_metadata_store_test.go index d01110c35b..456caa83f5 100644 --- a/disperser/common/v2/blobstore/dynamo_metadata_store_test.go +++ b/disperser/common/v2/blobstore/dynamo_metadata_store_test.go @@ -201,7 +201,7 @@ func TestBlobMetadataStoreOperations(t *testing.T) { metadata2 := &v2.BlobMetadata{ BlobHeader: blobHeader2, Signature: []byte{4, 5, 6}, - BlobStatus: v2.Certified, + BlobStatus: v2.Complete, Expiry: uint64(now.Add(time.Hour).Unix()), NumRetries: 0, UpdatedAt: uint64(now.UnixNano()), @@ -227,10 +227,10 @@ func TestBlobMetadataStoreOperations(t *testing.T) { assert.NoError(t, err) assert.Len(t, queued, 0) - certified, err := blobMetadataStore.GetBlobMetadataByStatus(ctx, v2.Certified, 0) + complete, err := blobMetadataStore.GetBlobMetadataByStatus(ctx, v2.Complete, 0) assert.NoError(t, err) - assert.Len(t, certified, 1) - assert.Equal(t, metadata2, certified[0]) + assert.Len(t, complete, 1) + assert.Equal(t, metadata2, complete[0]) queuedCount, err := blobMetadataStore.GetBlobMetadataCountByStatus(ctx, v2.Queued) assert.NoError(t, err) @@ -914,7 +914,7 @@ func TestBlobMetadataStoreUpdateBlobStatus(t *testing.T) { assert.NoError(t, err) // Update the blob status to invalid status - err = blobMetadataStore.UpdateBlobStatus(ctx, blobKey, v2.Certified) + err = blobMetadataStore.UpdateBlobStatus(ctx, blobKey, v2.Complete) assert.ErrorIs(t, err, blobstore.ErrInvalidStateTransition) // Update the blob status to a valid status diff --git a/disperser/controller/dispatcher.go b/disperser/controller/dispatcher.go index 31c9396268..a653eed600 100644 --- a/disperser/controller/dispatcher.go +++ b/disperser/controller/dispatcher.go @@ -560,24 +560,24 @@ func (d *Dispatcher) updateBatchStatus(ctx context.Context, batch *batchData, qu } if failed { - err := d.blobMetadataStore.UpdateBlobStatus(ctx, blobKey, v2.InsufficientSignatures) + err := d.blobMetadataStore.UpdateBlobStatus(ctx, blobKey, v2.Failed) if err != nil { multierr = multierror.Append(multierr, fmt.Errorf("failed to update blob status for blob %s to failed: %w", blobKey.Hex(), err)) } if metadata, ok := batch.Metadata[blobKey]; ok { - d.metrics.reportCompletedBlob(int(metadata.BlobSize), v2.InsufficientSignatures) + d.metrics.reportCompletedBlob(int(metadata.BlobSize), v2.Failed) } continue } - err := d.blobMetadataStore.UpdateBlobStatus(ctx, blobKey, v2.Certified) + err := d.blobMetadataStore.UpdateBlobStatus(ctx, blobKey, v2.Complete) if err != nil { - multierr = multierror.Append(multierr, fmt.Errorf("failed to update blob status for blob %s to certified: %w", blobKey.Hex(), err)) + multierr = multierror.Append(multierr, fmt.Errorf("failed to update blob status for blob %s to complete: %w", blobKey.Hex(), err)) } if metadata, ok := batch.Metadata[blobKey]; ok { requestedAt := time.Unix(0, int64(metadata.RequestedAt)) d.metrics.reportE2EDispersalLatency(time.Since(requestedAt)) - d.metrics.reportCompletedBlob(int(metadata.BlobSize), v2.Certified) + d.metrics.reportCompletedBlob(int(metadata.BlobSize), v2.Complete) } } diff --git a/disperser/controller/dispatcher_metrics.go b/disperser/controller/dispatcher_metrics.go index 1c6aaf34d5..d27f58cbd6 100644 --- a/disperser/controller/dispatcher_metrics.go +++ b/disperser/controller/dispatcher_metrics.go @@ -368,15 +368,12 @@ func (m *dispatcherMetrics) reportE2EDispersalLatency(duration time.Duration) { func (m *dispatcherMetrics) reportCompletedBlob(size int, status dispv2.BlobStatus) { switch status { - case dispv2.Certified: - m.completedBlobs.WithLabelValues("certified", "number").Inc() - m.completedBlobs.WithLabelValues("certified", "size").Add(float64(size)) + case dispv2.Complete: + m.completedBlobs.WithLabelValues("complete", "number").Inc() + m.completedBlobs.WithLabelValues("complete", "size").Add(float64(size)) case dispv2.Failed: m.completedBlobs.WithLabelValues("failed", "number").Inc() m.completedBlobs.WithLabelValues("failed", "size").Add(float64(size)) - case dispv2.InsufficientSignatures: - m.completedBlobs.WithLabelValues("insufficient_signature", "number").Inc() - m.completedBlobs.WithLabelValues("insufficient_signature", "size").Add(float64(size)) default: return } diff --git a/disperser/controller/dispatcher_test.go b/disperser/controller/dispatcher_test.go index 6a499c84dd..183ced987c 100644 --- a/disperser/controller/dispatcher_test.go +++ b/disperser/controller/dispatcher_test.go @@ -98,10 +98,10 @@ func TestDispatcherHandleBatch(t *testing.T) { // Test that the blob metadata status are updated bm0, err := components.BlobMetadataStore.GetBlobMetadata(ctx, objs.blobKeys[0]) require.NoError(t, err) - require.Equal(t, v2.Certified, bm0.BlobStatus) + require.Equal(t, v2.Complete, bm0.BlobStatus) bm1, err := components.BlobMetadataStore.GetBlobMetadata(ctx, objs.blobKeys[1]) require.NoError(t, err) - require.Equal(t, v2.Certified, bm1.BlobStatus) + require.Equal(t, v2.Complete, bm1.BlobStatus) // Get batch header vis, err := components.BlobMetadataStore.GetBlobInclusionInfos(ctx, objs.blobKeys[0]) @@ -173,12 +173,12 @@ func TestDispatcherInsufficientSignatures(t *testing.T) { for _, blobKey := range failedObjs.blobKeys { bm, err := components.BlobMetadataStore.GetBlobMetadata(ctx, blobKey) require.NoError(t, err) - require.Equal(t, v2.InsufficientSignatures, bm.BlobStatus) + require.Equal(t, v2.Failed, bm.BlobStatus) } for _, blobKey := range successfulObjs.blobKeys { bm, err := components.BlobMetadataStore.GetBlobMetadata(ctx, blobKey) require.NoError(t, err) - require.Equal(t, v2.Certified, bm.BlobStatus) + require.Equal(t, v2.Complete, bm.BlobStatus) } // Get batch header @@ -245,12 +245,12 @@ func TestDispatcherInsufficientSignatures2(t *testing.T) { for _, blobKey := range objsInBothQuorum.blobKeys { bm, err := components.BlobMetadataStore.GetBlobMetadata(ctx, blobKey) require.NoError(t, err) - require.Equal(t, v2.InsufficientSignatures, bm.BlobStatus) + require.Equal(t, v2.Failed, bm.BlobStatus) } for _, blobKey := range objsInQuorum1.blobKeys { bm, err := components.BlobMetadataStore.GetBlobMetadata(ctx, blobKey) require.NoError(t, err) - require.Equal(t, v2.InsufficientSignatures, bm.BlobStatus) + require.Equal(t, v2.Failed, bm.BlobStatus) } // Get batch header diff --git a/disperser/dataapi/docs/v1/V1_docs.go b/disperser/dataapi/docs/v1/V1_docs.go index 73e7cbf182..f12eef0b3e 100644 --- a/disperser/dataapi/docs/v1/V1_docs.go +++ b/disperser/dataapi/docs/v1/V1_docs.go @@ -1141,6 +1141,12 @@ const docTemplateV1 = `{ "items": { "type": "integer" } + }, + "y": { + "type": "array", + "items": { + "type": "integer" + } } } }, @@ -1149,6 +1155,9 @@ const docTemplateV1 = `{ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -1157,6 +1166,9 @@ const docTemplateV1 = `{ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -1187,6 +1199,12 @@ const docTemplateV1 = `{ "items": { "type": "integer" } + }, + "a1": { + "type": "array", + "items": { + "type": "integer" + } } } }, diff --git a/disperser/dataapi/docs/v1/V1_swagger.json b/disperser/dataapi/docs/v1/V1_swagger.json index 5f1ae1cc37..3e7c93d39b 100644 --- a/disperser/dataapi/docs/v1/V1_swagger.json +++ b/disperser/dataapi/docs/v1/V1_swagger.json @@ -1137,6 +1137,12 @@ "items": { "type": "integer" } + }, + "y": { + "type": "array", + "items": { + "type": "integer" + } } } }, @@ -1145,6 +1151,9 @@ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -1153,6 +1162,9 @@ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -1183,6 +1195,12 @@ "items": { "type": "integer" } + }, + "a1": { + "type": "array", + "items": { + "type": "integer" + } } } }, diff --git a/disperser/dataapi/docs/v1/V1_swagger.yaml b/disperser/dataapi/docs/v1/V1_swagger.yaml index b845673e11..c76b46042d 100644 --- a/disperser/dataapi/docs/v1/V1_swagger.yaml +++ b/disperser/dataapi/docs/v1/V1_swagger.yaml @@ -263,16 +263,24 @@ definitions: items: type: integer type: array + "y": + items: + type: integer + type: array type: object encoding.G2Commitment: properties: x: $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' + "y": + $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' type: object encoding.LengthProof: properties: x: $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' + "y": + $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' type: object github_com_Layr-Labs_eigenda_disperser.BlobStatus: enum: @@ -296,6 +304,10 @@ definitions: items: type: integer type: array + a1: + items: + type: integer + type: array type: object semver.SemverMetrics: properties: diff --git a/disperser/dataapi/docs/v2/V2_docs.go b/disperser/dataapi/docs/v2/V2_docs.go index 7995fda7f2..9f59e14556 100644 --- a/disperser/dataapi/docs/v2/V2_docs.go +++ b/disperser/dataapi/docs/v2/V2_docs.go @@ -663,6 +663,12 @@ const docTemplateV2 = `{ "items": { "type": "integer" } + }, + "y": { + "type": "array", + "items": { + "type": "integer" + } } } }, @@ -671,6 +677,9 @@ const docTemplateV2 = `{ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -703,6 +712,12 @@ const docTemplateV2 = `{ "items": { "type": "integer" } + }, + "y": { + "type": "array", + "items": { + "type": "integer" + } } } }, @@ -731,6 +746,12 @@ const docTemplateV2 = `{ "items": { "type": "integer" } + }, + "y": { + "type": "array", + "items": { + "type": "integer" + } } } }, @@ -739,6 +760,9 @@ const docTemplateV2 = `{ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -747,6 +771,9 @@ const docTemplateV2 = `{ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -925,9 +952,9 @@ const docTemplateV2 = `{ "x-enum-varnames": [ "Queued", "Encoded", - "Certified", - "Failed", - "InsufficientSignatures" + "GatheringSignatures", + "Complete", + "Failed" ] }, "github_com_Layr-Labs_eigenda_disperser_dataapi_v2.SignedBatch": { @@ -949,6 +976,12 @@ const docTemplateV2 = `{ "items": { "type": "integer" } + }, + "a1": { + "type": "array", + "items": { + "type": "integer" + } } } }, diff --git a/disperser/dataapi/docs/v2/V2_swagger.json b/disperser/dataapi/docs/v2/V2_swagger.json index b36b6ba2d1..aaf1d3ff33 100644 --- a/disperser/dataapi/docs/v2/V2_swagger.json +++ b/disperser/dataapi/docs/v2/V2_swagger.json @@ -660,6 +660,12 @@ "items": { "type": "integer" } + }, + "y": { + "type": "array", + "items": { + "type": "integer" + } } } }, @@ -668,6 +674,9 @@ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -700,6 +709,12 @@ "items": { "type": "integer" } + }, + "y": { + "type": "array", + "items": { + "type": "integer" + } } } }, @@ -728,6 +743,12 @@ "items": { "type": "integer" } + }, + "y": { + "type": "array", + "items": { + "type": "integer" + } } } }, @@ -736,6 +757,9 @@ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -744,6 +768,9 @@ "properties": { "x": { "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" + }, + "y": { + "$ref": "#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2" } } }, @@ -922,9 +949,9 @@ "x-enum-varnames": [ "Queued", "Encoded", - "Certified", - "Failed", - "InsufficientSignatures" + "GatheringSignatures", + "Complete", + "Failed" ] }, "github_com_Layr-Labs_eigenda_disperser_dataapi_v2.SignedBatch": { @@ -946,6 +973,12 @@ "items": { "type": "integer" } + }, + "a1": { + "type": "array", + "items": { + "type": "integer" + } } } }, diff --git a/disperser/dataapi/docs/v2/V2_swagger.yaml b/disperser/dataapi/docs/v2/V2_swagger.yaml index 3fdaf7fa61..c06ebdb8c2 100644 --- a/disperser/dataapi/docs/v2/V2_swagger.yaml +++ b/disperser/dataapi/docs/v2/V2_swagger.yaml @@ -51,11 +51,17 @@ definitions: items: type: integer type: array + "y": + items: + type: integer + type: array type: object core.G2Point: properties: x: $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' + "y": + $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' type: object core.PaymentMetadata: properties: @@ -78,6 +84,10 @@ definitions: items: type: integer type: array + "y": + items: + type: integer + type: array type: object encoding.BlobCommitments: properties: @@ -96,16 +106,24 @@ definitions: items: type: integer type: array + "y": + items: + type: integer + type: array type: object encoding.G2Commitment: properties: x: $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' + "y": + $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' type: object encoding.LengthProof: properties: x: $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' + "y": + $ref: '#/definitions/github_com_consensys_gnark-crypto_ecc_bn254_internal_fptower.E2' type: object github_com_Layr-Labs_eigenda_core_v2.Attestation: properties: @@ -238,9 +256,9 @@ definitions: x-enum-varnames: - Queued - Encoded - - Certified + - GatheringSignatures + - Complete - Failed - - InsufficientSignatures github_com_Layr-Labs_eigenda_disperser_dataapi_v2.SignedBatch: properties: attestation: @@ -254,6 +272,10 @@ definitions: items: type: integer type: array + a1: + items: + type: integer + type: array type: object semver.SemverMetrics: properties: diff --git a/inabox/tests/integration_v2_test.go b/inabox/tests/integration_v2_test.go index 8f8c31743d..e4e539b0fd 100644 --- a/inabox/tests/integration_v2_test.go +++ b/inabox/tests/integration_v2_test.go @@ -93,7 +93,7 @@ var _ = Describe("Inabox v2 Integration", func() { status2, err := dispv2.BlobStatusFromProtobuf(reply2.GetStatus()) Expect(err).To(BeNil()) - if status1 != dispv2.Certified || status2 != dispv2.Certified { + if status1 != dispv2.Complete || status2 != dispv2.Complete { continue } diff --git a/test/v2/test_client.go b/test/v2/test_client.go index 29b2bec450..c1fade82f8 100644 --- a/test/v2/test_client.go +++ b/test/v2/test_client.go @@ -3,13 +3,14 @@ package v2 import ( "context" "fmt" - "github.com/docker/go-units" "os" "path" "strings" "testing" "time" + "github.com/docker/go-units" + "github.com/Layr-Labs/eigenda/api/clients/v2" commonv2 "github.com/Layr-Labs/eigenda/api/grpc/common/v2" v2 "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2" @@ -190,7 +191,7 @@ func (c *TestClient) DisperseAndVerify( if err != nil { return err } - blobCert := c.WaitForCertification(ctx, key) + blobCert := c.WaitForCompletion(ctx, key) // Unpad the payload unpaddedPayload := codec.RemoveEmptyByteFromPaddedBytes(payload) @@ -215,8 +216,9 @@ func (c *TestClient) DispersePayload( return key, err } -// WaitForCertification waits for a blob to be certified. Returns the blob certificate. -func (c *TestClient) WaitForCertification(ctx context.Context, key corev2.BlobKey) *commonv2.BlobCertificate { +// WaitForCompletion waits for a blob to be complete. Returns the blob certificate. +// TODO: When GATHERING_SIGNATURES is added, this function should be updated to validate it first moves to GATHERING_SIGNATURES before moving to COMPLETE status. +func (c *TestClient) WaitForCompletion(ctx context.Context, key corev2.BlobKey) *commonv2.BlobCertificate { var status *v2.BlobStatus = nil ticker := time.NewTicker(time.Second) start := time.Now() @@ -227,11 +229,11 @@ func (c *TestClient) WaitForCertification(ctx context.Context, key corev2.BlobKe reply, err := c.DisperserClient.GetBlobStatus(ctx, key) require.NoError(c.t, err) - if reply.Status == v2.BlobStatus_CERTIFIED { + if reply.Status == v2.BlobStatus_COMPLETE { elapsed := time.Since(statusStart) totalElapsed := time.Since(start) fmt.Printf( - "Blob is certified (spent %0.1fs in prior status, total time %0.1fs)\n", + "Blob is complete (spent %0.1fs in prior status, total time %0.1fs)\n", elapsed.Seconds(), totalElapsed.Seconds()) @@ -253,8 +255,7 @@ func (c *TestClient) WaitForCertification(ctx context.Context, key corev2.BlobKe status = &reply.Status if reply.Status == v2.BlobStatus_FAILED || - reply.Status == v2.BlobStatus_UNKNOWN || - reply.Status == v2.BlobStatus_INSUFFICIENT_SIGNATURES { + reply.Status == v2.BlobStatus_UNKNOWN { require.Fail( c.t, "Blob status is in a terminal non-successful state.",
NameNumberDescription
CERTIFIEDGATHERING_SIGNATURES 3

CERTIFIED means the blob has been dispersed and attested by the DA nodes.

GATHERING_SIGNATURES means that the blob chunks are currently actively being transmitted to validators,requesting for signatures. +Requests that timeout or receive errors are resubmitted to DA nodes for some period of time set by the disperser, +after which the BlobStatus becomes COMPLETE. +This status is not currently implemented, and is a placeholder for future functionality.

FAILEDCOMPLETE 4

FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to -retry the blob, the client must submit the blob again with different salt (blob key is required to be unique).

COMPLETE means the blob has been dispersed to DA nodes, and the GATHERING_SIGNATURES period of time has completed. +This status does not guarantee any signer percentage, so a client should check that the signature has met +its required threshold, and resubmit a new blob dispersal request if not.

INSUFFICIENT_SIGNATURESFAILED 5

INSUFFICIENT_SIGNATURES means that the blob has failed to gather sufficient attestation. - -This status is functionally equivalent to FAILED, but is used to indicate that the failure is due to an -an inability to gather sufficient signatures.

FAILED means that the blob has failed permanently. Note that this is a terminal state, and in order to +retry the blob, the client must submit the blob again with different salt (blob key is required to be unique).