Skip to content

Commit

Permalink
rename Mnemosyne && apply Mnemosyne when num_point_reads == 0 in Mnmo…
Browse files Browse the repository at this point in the history 
…syne+
  • Loading branch information
@littlepig2013
littlepig2013 committed Nov 30, 2024
1 parent 6c0d2ed commit 9ec8126
Show file tree
Hide file tree
Showing 9 changed files with 471 additions and 429 deletions.
776 changes: 393 additions & 383 deletions db/bpk_alloc_helper.cc
View file

Large diffs are not rendered by default.

39 changes: 26 additions & 13 deletions db/bpk_alloc_helper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ class BitsPerKeyAllocHelper {
explicit BitsPerKeyAllocHelper(const ImmutableOptions* immutable_options,
const VersionStorageInfo* vstorage)
: ioptions_(immutable_options), vstorage_(vstorage) {
bpk_optimization_prepared_flag_ = false;
mnemosyne_bpk_optimization_prepared_flag_ = false;
mnemosyne_plus_bpk_optimization_prepared_flag_ = false;
num_bits_for_filter_to_be_removed_ = 0;
num_entries_in_compaction_ = 0;
}
Expand All @@ -29,33 +30,45 @@ class BitsPerKeyAllocHelper {
bool IfNeedAllocateBitsPerKey(const FileMetaData& meta,
uint64_t num_entries_in_output_level,
double* bits_per_key,
bool is_last_level = false);
bool is_last_level = false,
const Compaction* compaction = nullptr);
// pre-process the statistics to prepare Mnemosyne-based bits-per-key
// allocation
void PrepareForMnemosyne(const Compaction* compaction);
// pre-process the statistics to prepare Mnemosyne-plus-based bits-per-key
// allocation
void PrepareForMnemosynePlus(const Compaction* compaction);

double avg_curr_bits_per_key = 0.0;
uint64_t agg_filter_size_ = 0;

const ImmutableOptions* ioptions_;
const VersionStorageInfo* vstorage_;
BitsPerKeyAllocationType bpk_alloc_type_ =
BitsPerKeyAllocationType::kDefaultBpkAlloc;
bool flush_flag_ = false;
double naive_monkey_bpk = 0.0;
bool bpk_optimization_prepared_flag_ = false;
bool mnemosyne_bpk_optimization_prepared_flag_ = false;
bool mnemosyne_plus_bpk_optimization_prepared_flag_ = false;
bool no_filter_optimize_for_level0_ = false;
double workload_aware_bpk_weight_threshold_ =
double mnemosyne_plus_bpk_weight_threshold_ =
std::numeric_limits<double>::max();
uint64_t dynamic_monkey_bpk_num_entries_threshold_ =
uint64_t mnemosyne_bpk_num_entries_threshold_ =
std::numeric_limits<uint64_t>::max();
uint64_t dynamic_monkey_num_entries_ = 0;
uint64_t workload_aware_num_entries_ = 0;
uint64_t workload_aware_num_entries_with_empty_queries_ = 0;
double temp_sum_in_bpk_optimization_ = 0;
double common_constant_in_bpk_optimization_ = 0;
uint64_t total_empty_queries_ = 0;
uint64_t mnemosyne_num_entries_ = 0;
uint64_t mnemosyne_plus_num_entries_ = 0;
uint64_t mnemosyne_plus_num_entries_with_empty_queries_ = 0;
double mnemosyne_plus_temp_sum_in_bpk_optimization_ = 0;
double mnemosyne_plus_common_constant_in_bpk_optimization_ = 0;
uint64_t mnemosyne_plus_total_empty_queries_ = 0;
double total_bits_for_filter_ = 0.0;

double overall_bits_per_key_ = 0.0;
uint64_t num_entries_in_compaction_ = 0;
uint64_t num_bits_for_filter_to_be_removed_ = 0;

double mnemosyne_temp_sum_in_bpk_optimization_ = 0;
double mnemosyne_common_constant_in_bpk_optimization_ = 0;
double max_bits_per_key_ = 100;
struct LevelState {
int level;
Expand Down Expand Up @@ -90,7 +103,7 @@ class BitsPerKeyAllocHelper {
}
};
};
// used by kMonkey
// used by kMnemosyne
std::priority_queue<LevelState> level_states_pq_;

struct FileWorkloadState {
Expand All @@ -113,7 +126,7 @@ class BitsPerKeyAllocHelper {
return weight < tmp.weight;
}
};
// used by kWorkloadAware
// used by kMnemosynePlus
std::priority_queue<FileWorkloadState> file_workload_state_pq_;
};

Expand Down
3 changes: 2 additions & 1 deletion db/builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,8 @@ Status BuildTable(
}
version->storage_info()->SetBpkCommonConstant(
bpk_alloc_helper.bpk_alloc_type_,
bpk_alloc_helper.common_constant_in_bpk_optimization_);
bpk_alloc_helper
.mnemosyne_plus_common_constant_in_bpk_optimization_);
if (ioptions.point_reads_track_method == kDynamicCompactionAwareTrack) {
// adjusted to the estimated number of point reads that may be tracked
// when it is as an SST file
Expand Down
11 changes: 4 additions & 7 deletions db/compaction/compaction.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -451,13 +451,10 @@ Compaction::Compaction(
}
}

if (input_vstorage_->GetBitsPerKeyAllocationType() ==
BitsPerKeyAllocationType::kDynamicMonkeyBpkAlloc) {
for (const FileMetaData* meta :
input_vstorage_->LevelFiles(output_level_)) {
max_num_entries_in_output_level_ +=
meta->num_entries - meta->num_range_deletions;
}
for (const FileMetaData* meta :
input_vstorage_->LevelFiles(output_level_)) {
max_num_entries_in_output_level_ +=
meta->num_entries - meta->num_range_deletions;
}

if (immutable_options_.point_reads_track_method == kNaiiveTrack) {
Expand Down
2 changes: 1 addition & 1 deletion db/compaction/compaction_job.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -638,7 +638,7 @@ Status CompactionJob::Run() {
bpk_alloc_helper_->PrepareBpkAllocation(compact_->compaction);
compact_->compaction->input_vstorage()->SetBpkCommonConstant(
bpk_alloc_helper_->bpk_alloc_type_,
bpk_alloc_helper_->common_constant_in_bpk_optimization_);
bpk_alloc_helper_->mnemosyne_plus_common_constant_in_bpk_optimization_);
}

// Launch a thread for each of subcompactions 1...num_threads-1
Expand Down
37 changes: 26 additions & 11 deletions db/version_set.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2251,6 +2251,7 @@ VersionStorageInfo::VersionStorageInfo(
accumulated_num_empty_point_reads_by_file_(0),
point_reads_num_when_last_flush_(0),
num_flushes_(0),
current_total_filter_size_(0),
bits_per_key_alloc_type_(BitsPerKeyAllocationType::kDefaultBpkAlloc),
common_constant_in_bpk_optimization_(0),
accumulated_file_size_(0),
Expand All @@ -2267,8 +2268,7 @@ VersionStorageInfo::VersionStorageInfo(
finalized_(false),
force_consistency_checks_(_force_consistency_checks),
epoch_number_requirement_(epoch_number_requirement),
offpeak_time_option_(std::move(offpeak_time_option)),
current_total_filter_size_(0) {
offpeak_time_option_(std::move(offpeak_time_option)) {
if (ref_vstorage != nullptr) {
accumulated_num_point_reads_.store(
ref_vstorage->accumulated_num_point_reads_.load(
Expand Down Expand Up @@ -2297,12 +2297,15 @@ VersionStorageInfo::VersionStorageInfo(
accumulated_num_non_deletions_ =
ref_vstorage->accumulated_num_non_deletions_;
accumulated_num_deletions_ = ref_vstorage->accumulated_num_deletions_;
skipped_filter_size_ = ref_vstorage->GetSkippedFilterSize();
// skipped_filter_size_ = ref_vstorage->GetSkippedFilterSize();
current_num_non_deletions_ = ref_vstorage->current_num_non_deletions_;
current_num_deletions_ = ref_vstorage->current_num_deletions_;
current_num_samples_ = ref_vstorage->current_num_samples_;
oldest_snapshot_seqnum_ = ref_vstorage->oldest_snapshot_seqnum_;
current_total_filter_size_ = ref_vstorage->current_total_filter_size_;
current_total_filter_size_.store(
ref_vstorage->current_total_filter_size_.load(
std::memory_order_relaxed));
;
compact_cursor_ = ref_vstorage->compact_cursor_;
compact_cursor_.resize(num_levels_);
}
Expand Down Expand Up @@ -3325,11 +3328,11 @@ bool Version::IsFilterSkipped(int level, bool is_file_last_in_level,
if (max_accessed_modulars) *max_accessed_modulars = max_modulars_;
return false;
} else if (storage_info_.GetBitsPerKeyAllocationType() ==
BitsPerKeyAllocationType::kDynamicMonkeyBpkAlloc) {
BitsPerKeyAllocationType::kMnemosyneBpkAlloc) {
if (max_accessed_modulars) *max_accessed_modulars = max_modulars_;
return storage_info_.IsFilterSkippedWithEmptyBpkInDynamicMonkey(level);
return storage_info_.IsFilterSkippedWithEmptyBpkInMnemosyne(level);
} else if (storage_info_.GetBitsPerKeyAllocationType() ==
BitsPerKeyAllocationType::kWorkloadAwareBpkAlloc) {
BitsPerKeyAllocationType::kMnemosynePlusBpkAlloc) {
*max_accessed_modulars = 0;
uint64_t current_global_point_read_number =
storage_info_.GetAccumulatedNumPointReads();
Expand Down Expand Up @@ -3545,7 +3548,13 @@ void VersionStorageInfo::UpdateAccumulatedStats(FileMetaData* file_meta) {
file_meta->num_entries - file_meta->num_deletions;
current_num_deletions_ += file_meta->num_deletions;
current_num_samples_++;
current_total_filter_size_ += file_meta->filter_size;
// uint64_t temp_num_point_reads = file_meta->stats.GetNumPointReads();
// uint64_t temp_num_existing_point_reads =
// file_meta->stats.GetNumExistingPointReads(); if (temp_num_point_reads == 0
// || (temp_num_point_reads > 0 && temp_num_point_reads >
// temp_num_existing_point_reads)) {
// current_total_filter_size_ += file_meta->filter_size;
// }
}

void VersionStorageInfo::RemoveCurrentStats(FileMetaData* file_meta) {
Expand All @@ -3554,9 +3563,15 @@ void VersionStorageInfo::RemoveCurrentStats(FileMetaData* file_meta) {
file_meta->num_entries - file_meta->num_deletions;
current_num_deletions_ -= file_meta->num_deletions;
current_num_samples_--;
if (current_total_filter_size_ >= file_meta->filter_size) {
current_total_filter_size_ -= file_meta->filter_size;
}
// if (current_total_filter_size_ >= file_meta->filter_size) {
// uint64_t temp_num_point_reads = file_meta->stats.GetNumPointReads();
// uint64_t temp_num_existing_point_reads =
// file_meta->stats.GetNumExistingPointReads(); if (temp_num_point_reads
// == 0 || (temp_num_point_reads > 0 && temp_num_point_reads >
// temp_num_existing_point_reads)) {
// current_total_filter_size_ -= file_meta->filter_size;
// }
// }
}
}

Expand Down
23 changes: 14 additions & 9 deletions db/version_set.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,13 @@ class VersionStorageInfo {
num_flushes_++;
}

void UpdateSkippedFilterSize(uint64_t _skipped_filter_size) const {
skipped_filter_size_.store(_skipped_filter_size, std::memory_order_relaxed);
// void UpdateSkippedFilterSize(uint64_t _skipped_filter_size) const {
// skipped_filter_size_.store(_skipped_filter_size,
// std::memory_order_relaxed);
// }
void UpdateCurrentTotalFilterSize(uint64_t _current_total_filter_size) const {
current_total_filter_size_.store(_current_total_filter_size,
std::memory_order_relaxed);
}

void UpdateNumEmptyPointReads(
Expand Down Expand Up @@ -694,12 +699,12 @@ class VersionStorageInfo {
return common_constant_in_bpk_optimization_;
}

void SetLevelIDsWithEmptyBpkInDynamicMonkey(
void SetLevelIDsWithEmptyBpkInMnemosyne(
const std::unordered_set<size_t>& _levelIDs_with_bpk0_in_monkey) const {
levelIDs_with_bpk0_in_monkey_ = _levelIDs_with_bpk0_in_monkey;
}

bool IsFilterSkippedWithEmptyBpkInDynamicMonkey(size_t level) const {
bool IsFilterSkippedWithEmptyBpkInMnemosyne(size_t level) const {
return levelIDs_with_bpk0_in_monkey_.find(level) !=
levelIDs_with_bpk0_in_monkey_.end();
}
Expand All @@ -718,9 +723,9 @@ class VersionStorageInfo {
return current_total_filter_size_;
}

uint64_t GetSkippedFilterSize() const {
return skipped_filter_size_.load(std::memory_order_relaxed);
}
// uint64_t GetSkippedFilterSize() const {
// return skipped_filter_size_.load(std::memory_order_relaxed);
// }

uint64_t GetCurrentTotalNumEntries() const {
return current_num_deletions_ + current_num_non_deletions_;
Expand Down Expand Up @@ -857,7 +862,8 @@ class VersionStorageInfo {
mutable std::atomic<uint64_t> accumulated_num_empty_point_reads_by_file_;
mutable uint64_t point_reads_num_when_last_flush_ = 0;
mutable std::atomic<uint64_t> num_flushes_ = 0;
mutable std::atomic<uint64_t> skipped_filter_size_ = 0;
// mutable std::atomic<uint64_t> skipped_filter_size_ = 0;
mutable std::atomic<uint64_t> current_total_filter_size_ = 0;
mutable std::thread::id leader_thread_id_;
mutable std::mutex thread_ids_mutex_;

Expand Down Expand Up @@ -903,7 +909,6 @@ class VersionStorageInfo {

OffpeakTimeOption offpeak_time_option_;

uint64_t current_total_filter_size_;

friend class Version;
friend class VersionSet;
Expand Down
5 changes: 3 additions & 2 deletions include/rocksdb/table.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,9 @@ struct CacheUsageOptions {
enum BitsPerKeyAllocationType : char {
kDefaultBpkAlloc = 0x1,
kNaiveMonkeyBpkAlloc = 0x2,
kDynamicMonkeyBpkAlloc = 0x3,
kWorkloadAwareBpkAlloc = 0x4,
kMnemosyneBpkAlloc = 0x3,
// MnemosynePlus: workload-aware allocation, requires accurate statistics
kMnemosynePlusBpkAlloc = 0x4,
};

struct HashDigest {
Expand Down
4 changes: 2 additions & 2 deletions options/options_helper.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,8 +365,8 @@ std::unordered_map<std::string, BitsPerKeyAllocationType>
OptionsHelper::bpk_alloc_type_string_map = {
{"kDefaultBpkAlloc", kDefaultBpkAlloc},
{"kNaiveMonkeyBpkAlloc", kNaiveMonkeyBpkAlloc},
{"kDynamicMonkeyBpkAlloc", kDynamicMonkeyBpkAlloc},
{"kWorkloadAwareBpkAlloc", kWorkloadAwareBpkAlloc}};
{"kMnemosyneBpkAlloc", kMnemosyneBpkAlloc},
{"kMnemosynePlusBpkAlloc", kMnemosynePlusBpkAlloc}};

std::unordered_map<std::string, CompressionType>
OptionsHelper::compression_type_string_map = {
Expand Down

0 comments on commit 9ec8126

Please sign in to comment.