Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

peerDAS: Decouple network subnets from das-core. #14784

Open
wants to merge 1 commit into
base: peerDAS
Choose a base branch
from
Open

peerDAS: Decouple network subnets from das-core. #14784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 21 additions & 11 deletions beacon-chain/blockchain/process_block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -652,16 +652,20 @@ func uint64MapToSortedSlice(input map[uint64]bool) []uint64 {
}

func (s *Service) areDataColumnsAvailable(ctx context.Context, root [32]byte, signed interfaces.ReadOnlySignedBeaconBlock) error {
if signed.Version() < version.Deneb {
if signed.Version() < version.Fulu {
return nil
}

block := signed.Block()
if block == nil {
return errors.New("invalid nil beacon block")
}

// We are only required to check within MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
if !params.WithinDAPeriod(slots.ToEpoch(block.Slot()), slots.ToEpoch(s.CurrentSlot())) {
blockSlot, currentSlot := block.Slot(), s.CurrentSlot()
blockEpoch, currentEpoch := slots.ToEpoch(blockSlot), slots.ToEpoch(currentSlot)

if !params.WithinDAPeriod(blockEpoch, currentEpoch) {
return nil
}

Expand All @@ -681,20 +685,26 @@ func (s *Service) areDataColumnsAvailable(ctx context.Context, root [32]byte, si
}

// All columns to sample need to be available for the block to be considered available.
// https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/das-core.md#subnet-sampling
// https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/fulu/das-core.md#custody-sampling
nodeID := s.cfg.P2P.NodeID()
subnetSamplingSize := peerdas.SubnetSamplingSize()
custodyGroupSamplingSize := peerdas.CustodyGroupSamplingSize()

colMap, err := peerdas.CustodyColumns(nodeID, subnetSamplingSize)
custodyGroups, err := peerdas.CustodyGroups(nodeID, custodyGroupSamplingSize)
if err != nil {
return errors.Wrap(err, "custody columns")
return errors.Wrap(err, "custody groups")
}

// colMap represents the data columnns a node is expected to custody.
if len(colMap) == 0 {
// Exit early if the node is not expected to custody any data columns.
if len(custodyGroups) == 0 {
return nil
}

// Get the custody columns from the groups.
columnsMap, err := peerdas.CustodyColumns(custodyGroups)
if err != nil {
return errors.Wrap(err, "custody columns")
}

// Subscribe to newsly data columns stored in the database.
rootIndexChan := make(chan filesystem.RootIndexPair)
subscription := s.blobStorage.DataColumnFeed.Subscribe(rootIndexChan)
Expand All @@ -715,7 +725,7 @@ func (s *Service) areDataColumnsAvailable(ctx context.Context, root [32]byte, si
}

// Get a map of data column indices that are not currently available.
missingMap, err := missingDataColumns(s.blobStorage, root, colMap)
missingMap, err := missingDataColumns(s.blobStorage, root, columnsMap)
if err != nil {
return err
}
Expand Down Expand Up @@ -743,10 +753,10 @@ func (s *Service) areDataColumnsAvailable(ctx context.Context, root [32]byte, si
)

numberOfColumns := params.BeaconConfig().NumberOfColumns
colMapCount := uint64(len(colMap))
colMapCount := uint64(len(columnsMap))

if colMapCount < numberOfColumns {
expected = uint64MapToSortedSlice(colMap)
expected = uint64MapToSortedSlice(columnsMap)
}

if missingMapCount < numberOfColumns {
Expand Down
206 changes: 126 additions & 80 deletions beacon-chain/core/peerdas/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,41 +30,41 @@ import (
)

const (
CustodySubnetCountEnrKey = "csc"
CustodyGroupCountEnrKey = "cgc"
)

// https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/p2p-interface.md#the-discovery-domain-discv5
type Csc uint64
// https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/fulu/p2p-interface.md#the-discovery-domain-discv5
type Cgc uint64

func (Csc) ENRKey() string { return CustodySubnetCountEnrKey }
func (Cgc) ENRKey() string { return CustodyGroupCountEnrKey }

var (
// Custom errors
errCustodySubnetCountTooLarge = errors.New("custody subnet count larger than data column sidecar subnet count")
errIndexTooLarge = errors.New("column index is larger than the specified columns count")
errMismatchLength = errors.New("mismatch in the length of the commitments and proofs")
errRecordNil = errors.New("record is nil")
errCannotLoadCustodySubnetCount = errors.New("cannot load the custody subnet count from peer")
errCustodyGroupCountTooLarge = errors.New("custody group count too large")
errWrongComputedCustodyGroupCount = errors.New("wrong computed custody group count, should never happen")
errIndexTooLarge = errors.New("column index is larger than the specified columns count")
errMismatchLength = errors.New("mismatch in the length of the commitments and proofs")
errRecordNil = errors.New("record is nil")
errCannotLoadCustodyGroupCount = errors.New("cannot load the custody group count from peer")

// maxUint256 is the maximum value of a uint256.
maxUint256 = &uint256.Int{math.MaxUint64, math.MaxUint64, math.MaxUint64, math.MaxUint64}
)

// CustodyColumnSubnets computes the subnets the node should participate in for custody.
func CustodyColumnSubnets(nodeId enode.ID, custodySubnetCount uint64) (map[uint64]bool, error) {
dataColumnSidecarSubnetCount := params.BeaconConfig().DataColumnSidecarSubnetCount
// CustodyGroups computes the custody groups the node should participate in for custody.
// https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/fulu/das-core.md#get_custody_groups
func CustodyGroups(nodeId enode.ID, custodyGroupCount uint64) (map[uint64]bool, error) {
numberOfCustodyGroup := params.BeaconConfig().NumberOfCustodyGroups

// Check if the custody subnet count is larger than the data column sidecar subnet count.
if custodySubnetCount > dataColumnSidecarSubnetCount {
return nil, errCustodySubnetCountTooLarge
// Check if the custody group count is larger than the number of custody groups.
if custodyGroupCount > numberOfCustodyGroup {
return nil, errCustodyGroupCountTooLarge
}

// First, compute the subnet IDs that the node should participate in.
subnetIds := make(map[uint64]bool, custodySubnetCount)

custodyGroups := make(map[uint64]bool, custodyGroupCount)
one := uint256.NewInt(1)

for currentId := new(uint256.Int).SetBytes(nodeId.Bytes()); uint64(len(subnetIds)) < custodySubnetCount; currentId.Add(currentId, one) {
for currentId := new(uint256.Int).SetBytes(nodeId.Bytes()); uint64(len(custodyGroups)) < custodyGroupCount; currentId.Add(currentId, one) {
// Convert to big endian bytes.
currentIdBytesBigEndian := currentId.Bytes32()

Expand All @@ -74,49 +74,112 @@ func CustodyColumnSubnets(nodeId enode.ID, custodySubnetCount uint64) (map[uint6
// Hash the result.
hashedCurrentId := hash.Hash(currentIdBytesLittleEndian)

// Get the subnet ID.
subnetId := binary.LittleEndian.Uint64(hashedCurrentId[:8]) % dataColumnSidecarSubnetCount
// Get the custody group ID.
custodyGroupId := binary.LittleEndian.Uint64(hashedCurrentId[:8]) % numberOfCustodyGroup

// Add the subnet to the map.
subnetIds[subnetId] = true
// Add the custody group to the map.
custodyGroups[custodyGroupId] = true

// Overflow prevention.
if currentId.Cmp(maxUint256) == 0 {
currentId = uint256.NewInt(0)
}
}

return subnetIds, nil
// Final check.
if uint64(len(custodyGroups)) != custodyGroupCount {
return nil, errWrongComputedCustodyGroupCount
}

return custodyGroups, nil
}

// CustodyColumns computes the columns the node should custody.
// https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/das-core.md#helper-functions
func CustodyColumns(nodeId enode.ID, custodySubnetCount uint64) (map[uint64]bool, error) {
dataColumnSidecarSubnetCount := params.BeaconConfig().DataColumnSidecarSubnetCount
// ComputeColumnsForCustodyGroup computes the columns for a given custody group.
// https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/fulu/das-core.md#compute_columns_for_custody_group
func ComputeColumnsForCustodyGroup(custodyGroup uint64) ([]uint64, error) {
beaconConfig := params.BeaconConfig()
numberOfCustodyGroup := beaconConfig.NumberOfCustodyGroups

// Compute the custody subnets.
subnetIds, err := CustodyColumnSubnets(nodeId, custodySubnetCount)
if err != nil {
return nil, errors.Wrap(err, "custody subnets")
if custodyGroup > numberOfCustodyGroup {
return nil, errCustodyGroupCountTooLarge
}

numberOfColumns := beaconConfig.NumberOfColumns

columnsPerGroup := numberOfColumns / numberOfCustodyGroup

columns := make([]uint64, 0, columnsPerGroup)
for i := range columnsPerGroup {
column := numberOfCustodyGroup*i + custodyGroup
columns = append(columns, column)
}

columnsPerSubnet := fieldparams.NumberOfColumns / dataColumnSidecarSubnetCount
return columns, nil
}

// ComputeCustodyGroupForColumn computes the custody group for a given column.
// It is the reciprocal function of ComputeColumnsForCustodyGroup.
func ComputeCustodyGroupForColumn(columnIndex uint64) (uint64, error) {
beaconConfig := params.BeaconConfig()
numberOfColumns := beaconConfig.NumberOfColumns

if columnIndex >= numberOfColumns {
return 0, errIndexTooLarge
}

numberOfCustodyGroups := beaconConfig.NumberOfCustodyGroups
columnsPerGroup := numberOfColumns / numberOfCustodyGroups

return columnIndex / columnsPerGroup, nil
}

// ComputeSubnetForDataColumnSidecar computes the subnet for a data column sidecar.
// https://github.com/ethereum/consensus-specs/blob/dev/specs/fulu/p2p-interface.md#compute_subnet_for_data_column_sidecar
func ComputeSubnetForDataColumnSidecar(columnIndex uint64) uint64 {
dataColumnSidecarSubnetCount := params.BeaconConfig().DataColumnSidecarSubnetCount
return columnIndex % dataColumnSidecarSubnetCount
}

// Knowing the subnet ID and the number of columns per subnet, select all the columns the node should custody.
// Columns belonging to the same subnet are contiguous.
columnIndices := make(map[uint64]bool, custodySubnetCount*columnsPerSubnet)
for i := uint64(0); i < columnsPerSubnet; i++ {
for subnetId := range subnetIds {
columnIndex := dataColumnSidecarSubnetCount*i + subnetId
columnIndices[columnIndex] = true
// CustodyColumns computes the columns the node should custody.
func CustodyColumns(custodyGroups map[uint64]bool) (map[uint64]bool, error) {
numberOfCustodyGroups := params.BeaconConfig().NumberOfCustodyGroups

custodyGroupCount := len(custodyGroups)

// Compute the columns for each custody group.
columns := make(map[uint64]bool, custodyGroupCount)
for group := range custodyGroups {
if group >= numberOfCustodyGroups {
return nil, errCustodyGroupCountTooLarge
}

groupColumns, err := ComputeColumnsForCustodyGroup(group)
if err != nil {
return nil, errors.Wrap(err, "compute columns for custody group")
}

for _, column := range groupColumns {
columns[column] = true
}
}

return columnIndices, nil
return columns, nil
}

// DataColumnSubnets computes the subnets for the data columns.
func DataColumnSubnets(dataColumns map[uint64]bool) map[uint64]bool {
subnets := make(map[uint64]bool, len(dataColumns))

for column := range dataColumns {
subnet := ComputeSubnetForDataColumnSidecar(column)
subnets[subnet] = true
}

return subnets
}

// DataColumnSidecars computes the data column sidecars from the signed block and blobs.
// https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/das-core.md#recover_matrix
// https://github.com/ethereum/consensus-specs/blob/dev/specs/fulu/das-core.md#get_data_column_sidecars
func DataColumnSidecars(signedBlock interfaces.ReadOnlySignedBeaconBlock, blobs []kzg.Blob) ([]*ethpb.DataColumnSidecar, error) {
startTime := time.Now()
blobsCount := len(blobs)
Expand Down Expand Up @@ -454,39 +517,22 @@ func VerifyDataColumnsSidecarKZGProofs(sidecars []blocks.RODataColumn) (bool, er
return verified, nil
}

// CustodySubnetCount returns the number of subnets the node should participate in for custody.
func CustodySubnetCount() uint64 {
// CustodyGroupCount returns the number of groups the node should participate in for custody.
func CustodyGroupCount() uint64 {
if flags.Get().SubscribeToAllSubnets {
return params.BeaconConfig().DataColumnSidecarSubnetCount
return params.BeaconConfig().NumberOfCustodyGroups
}

return params.BeaconConfig().CustodyRequirement
}

// SubnetSamplingSize returns the number of subnets the node should sample from.
// https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/das-core.md#subnet-sampling
func SubnetSamplingSize() uint64 {
// CustodyGroupSamplingSize returns the number of custody groups the node should sample from.
// https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/fulu/das-core.md#custody-sampling
func CustodyGroupSamplingSize() uint64 {
samplesPerSlot := params.BeaconConfig().SamplesPerSlot
custodySubnetCount := CustodySubnetCount()

return max(samplesPerSlot, custodySubnetCount)
}

// CustodyColumnCount returns the number of columns the node should custody.
func CustodyColumnCount() uint64 {
// Get the number of subnets.
dataColumnSidecarSubnetCount := params.BeaconConfig().DataColumnSidecarSubnetCount

// Compute the number of columns per subnet.
columnsPerSubnet := fieldparams.NumberOfColumns / dataColumnSidecarSubnetCount

// Get the number of subnets we custody
custodySubnetCount := CustodySubnetCount()

// Finally, compute the number of columns we should custody.
custodyColumnCount := custodySubnetCount * columnsPerSubnet
custodyGroupCount := CustodyGroupCount()

return custodyColumnCount
return max(samplesPerSlot, custodyGroupCount)
}

// HypergeomCDF computes the hypergeometric cumulative distribution function.
Expand Down Expand Up @@ -538,27 +584,27 @@ func ExtendedSampleCount(samplesPerSlot, allowedFailures uint64) uint64 {
return sampleCount
}

func CustodyCountFromRecord(record *enr.Record) (uint64, error) {
// By default, we assume the peer custodies the minimum number of subnets.
// CustodyGroupCountFromRecord extracts the custody group count from an ENR record.
func CustodyGroupCountFromRecord(record *enr.Record) (uint64, error) {
if record == nil {
return 0, errRecordNil
}

// Load the `custody_subnet_count`
var csc Csc
if err := record.Load(&csc); err != nil {
return 0, errCannotLoadCustodySubnetCount
// Load the `cgc`
var cgc Cgc
if cgc := record.Load(&cgc); cgc != nil {
return 0, errCannotLoadCustodyGroupCount
}

return uint64(csc), nil
return uint64(cgc), nil
}

func CanSelfReconstruct(numCol uint64) bool {
total := params.BeaconConfig().NumberOfColumns
// if total is odd, then we need total / 2 + 1 columns to reconstruct
// if total is even, then we need total / 2 columns to reconstruct
columnsNeeded := total/2 + total%2
return numCol >= columnsNeeded
func CanSelfReconstruct(custodyGroupCount uint64) bool {
total := params.BeaconConfig().NumberOfCustodyGroups
// If total is odd, then we need total / 2 + 1 columns to reconstruct.
// If total is even, then we need total / 2 columns to reconstruct.
custodyGroupsNeeded := total/2 + total%2
return custodyGroupCount >= custodyGroupsNeeded
}

// RecoverCellsAndProofs recovers the cells and proofs from the data column sidecars.
Expand Down
Loading
Loading