Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Store each relation child in its own key #325

Merged
merged 31 commits into from
Dec 5, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
c6c5d59
Store each relation child in its own key
someone235 Nov 15, 2023
d16f049
Database version upgrade logic
someone235 Nov 18, 2023
89494a7
Improve staging relations perf
someone235 Nov 28, 2023
493b4fa
Implement CachedDbSetAccess
someone235 Nov 29, 2023
2ea8e53
Use BlockHasher for children store
someone235 Nov 29, 2023
3432558
Pass children readlock
someone235 Nov 29, 2023
b0c8f0f
clippy fix
someone235 Nov 29, 2023
198e097
Use default Debug impl for ReadLock
someone235 Nov 30, 2023
35e3122
Address review comments
someone235 Nov 30, 2023
0a4c5dd
Remove relations rwlock
someone235 Nov 30, 2023
e8f61b1
Get rid of relations service
someone235 Nov 30, 2023
b54a91c
Use RefCell instead of Mutex in MemoryRelationsStore and StagingRelat…
someone235 Nov 30, 2023
e41e34c
fix clippy warnings
michaelsutton Nov 30, 2023
dd1c892
fix simpa for low delay values
michaelsutton Nov 30, 2023
ed71486
Improve delete_children n StagingRelationsStore
someone235 Dec 1, 2023
9ca1b91
Suggestion for removing the need for `RefCell` (#4)
michaelsutton Dec 1, 2023
1f9dcf6
flatten staging (semantic change only)
michaelsutton Dec 1, 2023
7852fc5
unify deletions
michaelsutton Dec 1, 2023
48a635b
use correct prefix
michaelsutton Dec 1, 2023
de00e4f
bug fix: add to child deletions even if not removed from insertions
michaelsutton Dec 1, 2023
0dea3ce
remove unused API method
michaelsutton Dec 1, 2023
08c904f
fix user msg
michaelsutton Dec 1, 2023
0977ae3
add simpa as test
michaelsutton Dec 1, 2023
8401add
Revert "Get rid of relations service"
someone235 Dec 5, 2023
0fad12f
Revert "Remove relations rwlock"
someone235 Dec 5, 2023
26302c7
Remove redundant ChildKey
someone235 Dec 5, 2023
64a5317
set access:
michaelsutton Dec 5, 2023
fe1e82f
bug fix: make sure to propagate key not found err if staging has no d…
michaelsutton Dec 5, 2023
8336c93
clean
michaelsutton Dec 5, 2023
f118dcd
Merge branch 'master' into relations-children-prefix
someone235 Dec 5, 2023
5292b76
Remove redundant comment
someone235 Dec 5, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion components/consensusmanager/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ impl ConsensusSessionOwned {
self.clone().spawn_blocking(move |c| c.get_ghostdag_data(hash)).await
}

pub async fn async_get_block_children(&self, hash: Hash) -> Option<Arc<Vec<Hash>>> {
pub async fn async_get_block_children(&self, hash: Hash) -> Option<Vec<Hash>> {
self.clone().spawn_blocking(move |c| c.get_block_children(hash)).await
}

Expand Down
2 changes: 1 addition & 1 deletion consensus/core/src/api/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ pub trait ConsensusApi: Send + Sync {
unimplemented!()
}

fn get_block_children(&self, hash: Hash) -> Option<Arc<Vec<Hash>>> {
fn get_block_children(&self, hash: Hash) -> Option<Vec<Hash>> {
unimplemented!()
}

Expand Down
24 changes: 23 additions & 1 deletion consensus/src/consensus/factory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ pub enum ConsensusEntryType {
New(ConsensusEntry),
}

#[derive(Serialize, Deserialize, Clone, Default)]
#[derive(Serialize, Deserialize, Clone)]
pub struct MultiConsensusMetadata {
current_consensus_key: Option<u64>,
staging_consensus_key: Option<u64>,
Expand All @@ -54,6 +54,20 @@ pub struct MultiConsensusMetadata {
version: u32,
}

const LATEST_DB_VERSION: u32 = 1;
impl Default for MultiConsensusMetadata {
fn default() -> Self {
Self {
current_consensus_key: Default::default(),
someone235 marked this conversation as resolved.
Show resolved Hide resolved
staging_consensus_key: Default::default(),
max_key_used: Default::default(),
is_archival_node: Default::default(),
props: Default::default(),
version: LATEST_DB_VERSION,
}
}
}

#[derive(Clone)]
pub struct MultiConsensusManagementStore {
db: Arc<DB>,
Expand Down Expand Up @@ -199,6 +213,14 @@ impl MultiConsensusManagementStore {
self.metadata.write(BatchDbWriter::new(&mut batch), &metadata).unwrap();
}
}

pub fn should_upgrade(&self) -> StoreResult<bool> {
match self.metadata.read() {
Ok(data) => Ok(data.version != LATEST_DB_VERSION),
Err(StoreError::KeyNotFound(_)) => Ok(false),
Err(err) => Err(err),
}
}
}

pub struct Factory {
Expand Down
8 changes: 6 additions & 2 deletions consensus/src/consensus/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -759,8 +759,12 @@ impl ConsensusApi for Consensus {
Ok((&*ghostdag).into())
}

fn get_block_children(&self, hash: Hash) -> Option<Arc<Vec<Hash>>> {
self.services.relations_service.get_children(hash).unwrap_option()
fn get_block_children(&self, hash: Hash) -> Option<Vec<Hash>> {
self.services
.relations_service
.get_children(hash)
.unwrap_option()
.map(|children| children.read().iter().copied().collect_vec())
}

fn get_block_parents(&self, hash: Hash) -> Option<Arc<Vec<Hash>>> {
Expand Down
5 changes: 3 additions & 2 deletions consensus/src/model/services/relations.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::model::stores::relations::RelationsStoreReader;
use kaspa_database::prelude::StoreError;
use kaspa_consensus_core::BlockHashSet;
use kaspa_database::prelude::{ReadLock, StoreError, StoreResult};
use kaspa_hashes::Hash;
use parking_lot::RwLock;
use std::sync::Arc;
Expand All @@ -22,7 +23,7 @@ impl<T: RelationsStoreReader> RelationsStoreReader for MTRelationsService<T> {
self.store.read()[self.level].get_parents(hash)
}

fn get_children(&self, hash: Hash) -> Result<kaspa_consensus_core::blockhash::BlockHashes, StoreError> {
fn get_children(&self, hash: Hash) -> StoreResult<ReadLock<BlockHashSet>> {
self.store.read()[self.level].get_children(hash)
}

Expand Down
79 changes: 79 additions & 0 deletions consensus/src/model/stores/children.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
use kaspa_consensus_core::BlockHashSet;
use kaspa_consensus_core::BlockHasher;
use kaspa_consensus_core::BlockLevel;
use kaspa_database::prelude::BatchDbWriter;
use kaspa_database::prelude::CachedDbSetAccess;
use kaspa_database::prelude::DbWriter;
use kaspa_database::prelude::ReadLock;
use kaspa_database::prelude::StoreError;
use kaspa_database::prelude::StoreResult;
use kaspa_database::prelude::DB;
use kaspa_database::registry::DatabaseStorePrefixes;
use kaspa_hashes::Hash;
use rocksdb::WriteBatch;
use std::sync::Arc;

pub trait ChildrenStoreReader {
fn get(&self, hash: Hash) -> StoreResult<ReadLock<BlockHashSet>>;
}

pub trait ChildrenStore {
fn insert_child(&mut self, writer: impl DbWriter, parent: Hash, child: Hash) -> Result<(), StoreError>;
fn delete_child(&mut self, writer: impl DbWriter, parent: Hash, child: Hash) -> Result<(), StoreError>;
}

/// A DB + cache implementation of `DbChildrenStore` trait, with concurrency support.
#[derive(Clone)]
pub struct DbChildrenStore {
db: Arc<DB>,
access: CachedDbSetAccess<Hash, Hash, BlockHasher, BlockHasher>,
}

impl DbChildrenStore {
pub fn new(db: Arc<DB>, level: BlockLevel, cache_size: u64) -> Self {
let lvl_bytes = level.to_le_bytes();
Self {
db: Arc::clone(&db),
access: CachedDbSetAccess::new(
db,
cache_size,
DatabaseStorePrefixes::RelationsChildren.into_iter().chain(lvl_bytes).collect(),
),
}
}

pub fn with_prefix(db: Arc<DB>, prefix: &[u8], cache_size: u64) -> Self {
let db_prefix = prefix.iter().copied().chain(DatabaseStorePrefixes::RelationsChildren).collect();
Self { db: Arc::clone(&db), access: CachedDbSetAccess::new(db, cache_size, db_prefix) }
}

pub fn insert_batch(&self, batch: &mut WriteBatch, parent: Hash, child: Hash) -> Result<(), StoreError> {
self.access.write(BatchDbWriter::new(batch), parent, child)?;
Ok(())
}

pub(crate) fn delete_children(&self, mut writer: impl DbWriter, parent: Hash) -> Result<(), StoreError> {
self.access.delete_bucket(&mut writer, parent)
}

pub(crate) fn prefix(&self) -> &[u8] {
self.access.prefix()
}
}

impl ChildrenStoreReader for DbChildrenStore {
fn get(&self, parent: Hash) -> StoreResult<ReadLock<BlockHashSet>> {
self.access.read(parent)
}
}

impl ChildrenStore for DbChildrenStore {
fn insert_child(&mut self, writer: impl DbWriter, parent: Hash, child: Hash) -> Result<(), StoreError> {
self.access.write(writer, parent, child)?;
Ok(())
}

fn delete_child(&mut self, writer: impl DbWriter, parent: Hash, child: Hash) -> Result<(), StoreError> {
self.access.delete(writer, parent, child)
}
}
1 change: 1 addition & 0 deletions consensus/src/model/stores/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
pub mod acceptance_data;
pub mod block_transactions;
pub mod block_window_cache;
pub mod children;
pub mod daa;
pub mod selected_chain;
use std::{fmt::Display, mem::size_of};
Expand Down
Loading