Validate each level parents (#1827)

* Create BlockParentBuilder.

* Implement BuildParents.

* Explictly set level 0 blocks to be the same as direct parents.

* Add checkIndirectParents to validateBlockHeaderInContext.

* Fix test_block_builder.go and BlockLevelParents::Equal.

* Don't check indirect parents for blocks with trusted data.

* Handle pruned blocks when building block level parents.

* Fix bad deletions from unprocessedXxxParents.

* Fix merge errors.

* Fix bad pruning point parent replaces.

* Fix duplicates in newBlockLevelParents.

* Skip checkIndirectParents

* Get rid of staging constant IDs

* Fix BuildParents

* Fix tests

* Add comments

* Change order of directParentHashes

* Get rid of maybeAddDirectParentParents

* Add comments

* Add blockToReferences type

* Use ParentsAtLevel

Co-authored-by: stasatdaglabs <[email protected]>

domain/consensus/datastructures/acceptancedatastore/acceptance_data_staging_shard.go

@@ -12,7 +12,7 @@ type acceptanceDataStagingShard struct {
 }
 
 func (ads *acceptanceDataStore) stagingShard(stagingArea *model.StagingArea) *acceptanceDataStagingShard {
-	return stagingArea.GetOrCreateShard(model.StagingShardIDAcceptanceData, func() model.StagingShard {
+	return stagingArea.GetOrCreateShard(ads.shardID, func() model.StagingShard {
 		return &acceptanceDataStagingShard{
 			store:    ads,
 			toAdd:    make(map[externalapi.DomainHash]externalapi.AcceptanceData),

domain/consensus/datastructures/acceptancedatastore/acceptance_data_store.go

@@ -1,28 +1,29 @@
 package acceptancedatastore
 
 import (
-	"github.com/kaspanet/kaspad/domain/consensus/database"
 	"github.com/kaspanet/kaspad/domain/consensus/database/serialization"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/lrucache"
-	"github.com/kaspanet/kaspad/domain/prefixmanager/prefix"
+	"github.com/kaspanet/kaspad/util/staging"
 	"google.golang.org/protobuf/proto"
 )
 
 var bucketName = []byte("acceptance-data")
 
 // acceptanceDataStore represents a store of AcceptanceData
 type acceptanceDataStore struct {
-	cache  *lrucache.LRUCache
-	bucket model.DBBucket
+	shardID model.StagingShardID
+	cache   *lrucache.LRUCache
+	bucket  model.DBBucket
 }
 
 // New instantiates a new AcceptanceDataStore
-func New(prefix *prefix.Prefix, cacheSize int, preallocate bool) model.AcceptanceDataStore {
+func New(prefixBucket model.DBBucket, cacheSize int, preallocate bool) model.AcceptanceDataStore {
 	return &acceptanceDataStore{
-		cache:  lrucache.New(cacheSize, preallocate),
-		bucket: database.MakeBucket(prefix.Serialize()).Bucket(bucketName),
+		shardID: staging.GenerateShardingID(),
+		cache:   lrucache.New(cacheSize, preallocate),
+		bucket:  prefixBucket.Bucket(bucketName),
 	}
 }
 

domain/consensus/datastructures/blockheaderstore/block_header_staging_shard.go

@@ -12,7 +12,7 @@ type blockHeaderStagingShard struct {
 }
 
 func (bhs *blockHeaderStore) stagingShard(stagingArea *model.StagingArea) *blockHeaderStagingShard {
-	return stagingArea.GetOrCreateShard(model.StagingShardIDBlockHeader, func() model.StagingShard {
+	return stagingArea.GetOrCreateShard(bhs.shardID, func() model.StagingShard {
 		return &blockHeaderStagingShard{
 			store:    bhs,
 			toAdd:    make(map[externalapi.DomainHash]externalapi.BlockHeader),

domain/consensus/datastructures/blockheaderstore/block_header_store.go

@@ -2,31 +2,32 @@ package blockheaderstore
 
 import (
 	"github.com/golang/protobuf/proto"
-	"github.com/kaspanet/kaspad/domain/consensus/database"
 	"github.com/kaspanet/kaspad/domain/consensus/database/serialization"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/lrucache"
-	"github.com/kaspanet/kaspad/domain/prefixmanager/prefix"
+	"github.com/kaspanet/kaspad/util/staging"
 )
 
 var bucketName = []byte("block-headers")
 var countKeyName = []byte("block-headers-count")
 
 // blockHeaderStore represents a store of blocks
 type blockHeaderStore struct {
+	shardID     model.StagingShardID
 	cache       *lrucache.LRUCache
 	countCached uint64
 	bucket      model.DBBucket
 	countKey    model.DBKey
 }
 
 // New instantiates a new BlockHeaderStore
-func New(dbContext model.DBReader, prefix *prefix.Prefix, cacheSize int, preallocate bool) (model.BlockHeaderStore, error) {
+func New(dbContext model.DBReader, prefixBucket model.DBBucket, cacheSize int, preallocate bool) (model.BlockHeaderStore, error) {
 	blockHeaderStore := &blockHeaderStore{
+		shardID:  staging.GenerateShardingID(),
 		cache:    lrucache.New(cacheSize, preallocate),
-		bucket:   database.MakeBucket(prefix.Serialize()).Bucket(bucketName),
-		countKey: database.MakeBucket(prefix.Serialize()).Key(countKeyName),
+		bucket:   prefixBucket.Bucket(bucketName),
+		countKey: prefixBucket.Key(countKeyName),
 	}
 
 	err := blockHeaderStore.initializeCount(dbContext)

domain/consensus/datastructures/blockrelationstore/block_relation_staging_shard.go

@@ -11,7 +11,7 @@ type blockRelationStagingShard struct {
 }
 
 func (brs *blockRelationStore) stagingShard(stagingArea *model.StagingArea) *blockRelationStagingShard {
-	return stagingArea.GetOrCreateShard(model.StagingShardIDBlockRelation, func() model.StagingShard {
+	return stagingArea.GetOrCreateShard(brs.shardID, func() model.StagingShard {
 		return &blockRelationStagingShard{
 			store: brs,
 			toAdd: make(map[externalapi.DomainHash]*model.BlockRelations),

domain/consensus/datastructures/blockrelationstore/block_relation_store.go

@@ -2,27 +2,28 @@ package blockrelationstore
 
 import (
 	"github.com/golang/protobuf/proto"
-	"github.com/kaspanet/kaspad/domain/consensus/database"
 	"github.com/kaspanet/kaspad/domain/consensus/database/serialization"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/lrucache"
-	"github.com/kaspanet/kaspad/domain/prefixmanager/prefix"
+	"github.com/kaspanet/kaspad/util/staging"
 )
 
 var bucketName = []byte("block-relations")
 
 // blockRelationStore represents a store of BlockRelations
 type blockRelationStore struct {
-	cache  *lrucache.LRUCache
-	bucket model.DBBucket
+	shardID model.StagingShardID
+	cache   *lrucache.LRUCache
+	bucket  model.DBBucket
 }
 
 // New instantiates a new BlockRelationStore
-func New(prefix *prefix.Prefix, cacheSize int, preallocate bool) model.BlockRelationStore {
+func New(prefixBucket model.DBBucket, cacheSize int, preallocate bool) model.BlockRelationStore {
 	return &blockRelationStore{
-		cache:  lrucache.New(cacheSize, preallocate),
-		bucket: database.MakeBucket(prefix.Serialize()).Bucket(bucketName),
+		shardID: staging.GenerateShardingID(),
+		cache:   lrucache.New(cacheSize, preallocate),
+		bucket:  prefixBucket.Bucket(bucketName),
 	}
 }
 

domain/consensus/datastructures/blockstatusstore/block_status_staging_shard.go

@@ -11,7 +11,7 @@ type blockStatusStagingShard struct {
 }
 
 func (bss *blockStatusStore) stagingShard(stagingArea *model.StagingArea) *blockStatusStagingShard {
-	return stagingArea.GetOrCreateShard(model.StagingShardIDBlockStatus, func() model.StagingShard {
+	return stagingArea.GetOrCreateShard(bss.shardID, func() model.StagingShard {
 		return &blockStatusStagingShard{
 			store: bss,
 			toAdd: make(map[externalapi.DomainHash]externalapi.BlockStatus),

domain/consensus/datastructures/blockstatusstore/block_status_store.go

@@ -2,27 +2,28 @@ package blockstatusstore
 
 import (
 	"github.com/golang/protobuf/proto"
-	"github.com/kaspanet/kaspad/domain/consensus/database"
 	"github.com/kaspanet/kaspad/domain/consensus/database/serialization"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/lrucache"
-	"github.com/kaspanet/kaspad/domain/prefixmanager/prefix"
+	"github.com/kaspanet/kaspad/util/staging"
 )
 
 var bucketName = []byte("block-statuses")
 
 // blockStatusStore represents a store of BlockStatuses
 type blockStatusStore struct {
-	cache  *lrucache.LRUCache
-	bucket model.DBBucket
+	shardID model.StagingShardID
+	cache   *lrucache.LRUCache
+	bucket  model.DBBucket
 }
 
 // New instantiates a new BlockStatusStore
-func New(prefix *prefix.Prefix, cacheSize int, preallocate bool) model.BlockStatusStore {
+func New(prefixBucket model.DBBucket, cacheSize int, preallocate bool) model.BlockStatusStore {
 	return &blockStatusStore{
-		cache:  lrucache.New(cacheSize, preallocate),
-		bucket: database.MakeBucket(prefix.Serialize()).Bucket(bucketName),
+		shardID: staging.GenerateShardingID(),
+		cache:   lrucache.New(cacheSize, preallocate),
+		bucket:  prefixBucket.Bucket(bucketName),
 	}
 }
 

domain/consensus/datastructures/blockstore/block_staging_shard.go

@@ -12,7 +12,7 @@ type blockStagingShard struct {
 }
 
 func (bs *blockStore) stagingShard(stagingArea *model.StagingArea) *blockStagingShard {
-	return stagingArea.GetOrCreateShard(model.StagingShardIDBlock, func() model.StagingShard {
+	return stagingArea.GetOrCreateShard(bs.shardID, func() model.StagingShard {
 		return &blockStagingShard{
 			store:    bs,
 			toAdd:    make(map[externalapi.DomainHash]*externalapi.DomainBlock),

domain/consensus/datastructures/blockstore/block_store.go

@@ -2,31 +2,32 @@ package blockstore
 
 import (
 	"github.com/golang/protobuf/proto"
-	"github.com/kaspanet/kaspad/domain/consensus/database"
 	"github.com/kaspanet/kaspad/domain/consensus/database/serialization"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/lrucache"
-	"github.com/kaspanet/kaspad/domain/prefixmanager/prefix"
+	"github.com/kaspanet/kaspad/util/staging"
 	"github.com/pkg/errors"
 )
 
 var bucketName = []byte("blocks")
 
 // blockStore represents a store of blocks
 type blockStore struct {
+	shardID     model.StagingShardID
 	cache       *lrucache.LRUCache
 	countCached uint64
 	bucket      model.DBBucket
 	countKey    model.DBKey
 }
 
 // New instantiates a new BlockStore
-func New(dbContext model.DBReader, prefix *prefix.Prefix, cacheSize int, preallocate bool) (model.BlockStore, error) {
+func New(dbContext model.DBReader, prefixBucket model.DBBucket, cacheSize int, preallocate bool) (model.BlockStore, error) {
 	blockStore := &blockStore{
+		shardID:  staging.GenerateShardingID(),
 		cache:    lrucache.New(cacheSize, preallocate),
-		bucket:   database.MakeBucket(prefix.Serialize()).Bucket(bucketName),
-		countKey: database.MakeBucket(prefix.Serialize()).Key([]byte("blocks-count")),
+		bucket:   prefixBucket.Bucket(bucketName),
+		countKey: prefixBucket.Key([]byte("blocks-count")),
 	}
 
 	err := blockStore.initializeCount(dbContext)

domain/consensus/datastructures/consensusstatestore/consensus_state_staging_shard.go

@@ -12,7 +12,7 @@ type consensusStateStagingShard struct {
 }
 
 func (bs *consensusStateStore) stagingShard(stagingArea *model.StagingArea) *consensusStateStagingShard {
-	return stagingArea.GetOrCreateShard(model.StagingShardIDConsensusState, func() model.StagingShard {
+	return stagingArea.GetOrCreateShard(bs.shardID, func() model.StagingShard {
 		return &consensusStateStagingShard{
 			store:                  bs,
 			tipsStaging:            nil,

domain/consensus/datastructures/consensusstatestore/consensus_state_store.go

@@ -1,17 +1,17 @@
 package consensusstatestore
 
 import (
-	"github.com/kaspanet/kaspad/domain/consensus/database"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/utxolrucache"
-	"github.com/kaspanet/kaspad/domain/prefixmanager/prefix"
+	"github.com/kaspanet/kaspad/util/staging"
 )
 
 var importingPruningPointUTXOSetKeyName = []byte("importing-pruning-point-utxo-set")
 
 // consensusStateStore represents a store for the current consensus state
 type consensusStateStore struct {
+	shardID                         model.StagingShardID
 	virtualUTXOSetCache             *utxolrucache.LRUCache
 	tipsCache                       []*externalapi.DomainHash
 	tipsKey                         model.DBKey
@@ -20,12 +20,13 @@ type consensusStateStore struct {
 }
 
 // New instantiates a new ConsensusStateStore
-func New(prefix *prefix.Prefix, utxoSetCacheSize int, preallocate bool) model.ConsensusStateStore {
+func New(prefixBucket model.DBBucket, utxoSetCacheSize int, preallocate bool) model.ConsensusStateStore {
 	return &consensusStateStore{
+		shardID:                         staging.GenerateShardingID(),
 		virtualUTXOSetCache:             utxolrucache.New(utxoSetCacheSize, preallocate),
-		tipsKey:                         database.MakeBucket(prefix.Serialize()).Key(tipsKeyName),
-		importingPruningPointUTXOSetKey: database.MakeBucket(prefix.Serialize()).Key(importingPruningPointUTXOSetKeyName),
-		utxoSetBucket:                   database.MakeBucket(prefix.Serialize()).Bucket(utxoSetBucketName),
+		tipsKey:                         prefixBucket.Key(tipsKeyName),
+		importingPruningPointUTXOSetKey: prefixBucket.Key(importingPruningPointUTXOSetKeyName),
+		utxoSetBucket:                   prefixBucket.Bucket(utxoSetBucketName),
 	}
 }
 

domain/consensus/datastructures/daablocksstore/daa_blocks_staging_shard.go

@@ -15,7 +15,7 @@ type daaBlocksStagingShard struct {
 }
 
 func (daas *daaBlocksStore) stagingShard(stagingArea *model.StagingArea) *daaBlocksStagingShard {
-	return stagingArea.GetOrCreateShard(model.StagingShardIDDAABlocks, func() model.StagingShard {
+	return stagingArea.GetOrCreateShard(daas.shardID, func() model.StagingShard {
 		return &daaBlocksStagingShard{
 			store:                  daas,
 			daaScoreToAdd:          make(map[externalapi.DomainHash]uint64),

domain/consensus/datastructures/daablocksstore/daa_blocks_store.go

@@ -1,32 +1,33 @@
 package daablocksstore
 
 import (
-	"github.com/kaspanet/kaspad/domain/consensus/database"
 	"github.com/kaspanet/kaspad/domain/consensus/database/binaryserialization"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/lrucache"
-	"github.com/kaspanet/kaspad/domain/prefixmanager/prefix"
+	"github.com/kaspanet/kaspad/util/staging"
 )
 
 var daaScoreBucketName = []byte("daa-score")
 var daaAddedBlocksBucketName = []byte("daa-added-blocks")
 
 // daaBlocksStore represents a store of DAABlocksStore
 type daaBlocksStore struct {
+	shardID                model.StagingShardID
 	daaScoreLRUCache       *lrucache.LRUCache
 	daaAddedBlocksLRUCache *lrucache.LRUCache
 	daaScoreBucket         model.DBBucket
 	daaAddedBlocksBucket   model.DBBucket
 }
 
 // New instantiates a new DAABlocksStore
-func New(prefix *prefix.Prefix, daaScoreCacheSize int, daaAddedBlocksCacheSize int, preallocate bool) model.DAABlocksStore {
+func New(prefixBucket model.DBBucket, daaScoreCacheSize int, daaAddedBlocksCacheSize int, preallocate bool) model.DAABlocksStore {
 	return &daaBlocksStore{
+		shardID:                staging.GenerateShardingID(),
 		daaScoreLRUCache:       lrucache.New(daaScoreCacheSize, preallocate),
 		daaAddedBlocksLRUCache: lrucache.New(daaAddedBlocksCacheSize, preallocate),
-		daaScoreBucket:         database.MakeBucket(prefix.Serialize()).Bucket(daaScoreBucketName),
-		daaAddedBlocksBucket:   database.MakeBucket(prefix.Serialize()).Bucket(daaAddedBlocksBucketName),
+		daaScoreBucket:         prefixBucket.Bucket(daaScoreBucketName),
+		daaAddedBlocksBucket:   prefixBucket.Bucket(daaAddedBlocksBucketName),
 	}
 }
 

domain/consensus/datastructures/daawindowstore/daa_window_staging_shard.go

@@ -25,7 +25,7 @@ type daaWindowStagingShard struct {
 }
 
 func (daaws *daaWindowStore) stagingShard(stagingArea *model.StagingArea) *daaWindowStagingShard {
-	return stagingArea.GetOrCreateShard(model.StagingShardIDDAAWindow, func() model.StagingShard {
+	return stagingArea.GetOrCreateShard(daaws.shardID, func() model.StagingShard {
 		return &daaWindowStagingShard{
 			store: daaws,
 			toAdd: make(map[dbKey]*externalapi.BlockGHOSTDAGDataHashPair),

domain/consensus/datastructures/daawindowstore/daa_window_store.go

@@ -3,26 +3,27 @@ package daawindowstore
 import (
 	"encoding/binary"
 	"github.com/golang/protobuf/proto"
-	"github.com/kaspanet/kaspad/domain/consensus/database"
 	"github.com/kaspanet/kaspad/domain/consensus/database/serialization"
 	"github.com/kaspanet/kaspad/domain/consensus/model"
 	"github.com/kaspanet/kaspad/domain/consensus/model/externalapi"
 	"github.com/kaspanet/kaspad/domain/consensus/utils/lrucachehashpairtoblockghostdagdatahashpair"
-	"github.com/kaspanet/kaspad/domain/prefixmanager/prefix"
+	"github.com/kaspanet/kaspad/util/staging"
 )
 
 var bucketName = []byte("daa-window")
 
 type daaWindowStore struct {
-	cache  *lrucachehashpairtoblockghostdagdatahashpair.LRUCache
-	bucket model.DBBucket
+	shardID model.StagingShardID
+	cache   *lrucachehashpairtoblockghostdagdatahashpair.LRUCache
+	bucket  model.DBBucket
 }
 
 // New instantiates a new BlocksWithTrustedDataDAAWindowStore
-func New(prefix *prefix.Prefix, cacheSize int, preallocate bool) model.BlocksWithTrustedDataDAAWindowStore {
+func New(prefixBucket model.DBBucket, cacheSize int, preallocate bool) model.BlocksWithTrustedDataDAAWindowStore {
 	return &daaWindowStore{
-		cache:  lrucachehashpairtoblockghostdagdatahashpair.New(cacheSize, preallocate),
-		bucket: database.MakeBucket(prefix.Serialize()).Bucket(bucketName),
+		shardID: staging.GenerateShardingID(),
+		cache:   lrucachehashpairtoblockghostdagdatahashpair.New(cacheSize, preallocate),
+		bucket:  prefixBucket.Bucket(bucketName),
 	}
 }
 

domain/consensus/datastructures/finalitystore/finality_staging_shard.go

@@ -11,7 +11,7 @@ type finalityStagingShard struct {
 }
 
 func (fs *finalityStore) stagingShard(stagingArea *model.StagingArea) *finalityStagingShard {
-	return stagingArea.GetOrCreateShard(model.StagingShardIDFinality, func() model.StagingShard {
+	return stagingArea.GetOrCreateShard(fs.shardID, func() model.StagingShard {
 		return &finalityStagingShard{
 			store: fs,
 			toAdd: make(map[externalapi.DomainHash]*externalapi.DomainHash),