diff --git a/Cargo.toml b/Cargo.toml index e1f108df3f..4669a0358c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ itertools = "0.12.0" jemalloc-sys = { version = "0.5.3", features = ["disable_initial_exec_tls"], optional = true } lazy_static = "1.1" libc = "0.2" -log = { version = "0.4", features = ["max_level_trace", "release_max_level_off"] } +log = { version = "0.4", features = ["max_level_trace", "release_max_level_trace"] } memoffset = "0.9" mimalloc-sys = { version = "0.1.6", optional = true } # MMTk macros - we have to specify a version here in order to publish the crate, even though we use the dependency from a local path. @@ -152,6 +152,9 @@ malloc_counted_size = [] # Count the size of all live objects in GC count_live_bytes_in_gc = [] +# Dump memory stats about the plan (live bytes, live lines, live blocks, and used pages) +dump_memory_stats = [] + # Workaround a problem where bpftrace scripts (see tools/tracing/timeline/capture.bt) cannot # capture the type names of work packets. bpftrace_workaround = [] diff --git a/src/plan/global.rs b/src/plan/global.rs index 5b28c4b720..4e66a05d69 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -315,6 +315,10 @@ pub trait Plan: 'static + HasSpaces + Sync + Downcast { space.verify_side_metadata_sanity(&mut side_metadata_sanity_checker); }) } + + /// Dump memory stats for the plan + #[cfg(feature = "dump_memory_stats")] + fn dump_memory_stats(&self) {} } impl_downcast!(Plan assoc VM); diff --git a/src/plan/immix/global.rs b/src/plan/immix/global.rs index 0741ba3a99..571e1d31c7 100644 --- a/src/plan/immix/global.rs +++ b/src/plan/immix/global.rs @@ -116,6 +116,12 @@ impl Plan for Immix { fn common(&self) -> &CommonPlan { &self.common } + + #[cfg(feature = "dump_memory_stats")] + fn dump_memory_stats(&self) { + self.immix_space.dump_memory_stats(); + self.common.los.dump_memory_stats(); + } } impl Immix { diff --git a/src/plan/sticky/immix/global.rs b/src/plan/sticky/immix/global.rs index f913e2e4b8..ead55e1ebe 100644 --- a/src/plan/sticky/immix/global.rs +++ b/src/plan/sticky/immix/global.rs @@ -80,6 +80,12 @@ impl Plan for StickyImmix { self.immix.common() } + #[cfg(feature = "dump_memory_stats")] + fn dump_memory_stats(&self) { + self.immix.immix_space.dump_memory_stats(); + self.common().los.dump_memory_stats(); + } + fn schedule_collection(&'static self, scheduler: &crate::scheduler::GCWorkScheduler) { let is_full_heap = self.requires_full_heap_collection(); self.gc_full_heap.store(is_full_heap, Ordering::SeqCst); @@ -266,7 +272,7 @@ impl crate::plan::generational::global::GenerationalPlanExt f ); self.immix .immix_space - .trace_object_without_moving(queue, object) + .trace_object_without_moving(queue, object, false) }; return object; diff --git a/src/policy/immix/block.rs b/src/policy/immix/block.rs index 7f52a1e9ee..6561cb84ce 100644 --- a/src/policy/immix/block.rs +++ b/src/policy/immix/block.rs @@ -70,9 +70,9 @@ pub struct Block(Address); impl Region for Block { #[cfg(not(feature = "immix_smaller_block"))] - const LOG_BYTES: usize = 15; + const LOG_BYTES: usize = 14; #[cfg(feature = "immix_smaller_block")] - const LOG_BYTES: usize = 13; + const LOG_BYTES: usize = 14; fn from_aligned_address(address: Address) -> Self { debug_assert!(address.is_aligned_to(Self::BYTES)); diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index bf72394c01..1010509e41 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -33,6 +33,80 @@ use std::sync::{atomic::AtomicU8, atomic::AtomicUsize, Arc}; pub(crate) const TRACE_KIND_FAST: TraceKind = 0; pub(crate) const TRACE_KIND_DEFRAG: TraceKind = 1; +#[cfg(feature = "dump_memory_stats")] +#[derive(Default)] +/// Keeping track of the number of traced/copied/tpinned objects and live bytes +struct ImmixSpaceStats { + live_bytes: AtomicUsize, + traced_objects: AtomicUsize, + pinned_objects: AtomicUsize, + tpinned_objects: AtomicUsize, + copied_objects: AtomicUsize, +} + +#[cfg(feature = "dump_memory_stats")] +impl ImmixSpaceStats { + pub fn get_live_bytes(&self) -> usize { + self.live_bytes.load(Ordering::SeqCst) + } + + pub fn set_live_bytes(&self, size: usize) { + self.live_bytes.store(size, Ordering::SeqCst) + } + + pub fn increase_live_bytes(&self, size: usize) { + self.live_bytes.fetch_add(size, Ordering::SeqCst); + } + + pub fn get_traced_objects(&self) -> usize { + self.traced_objects.load(Ordering::SeqCst) + } + + pub fn set_traced_objects(&self, size: usize) { + self.traced_objects.store(size, Ordering::SeqCst) + } + + pub fn increase_traced_objects(&self, size: usize) { + self.traced_objects.fetch_add(size, Ordering::SeqCst); + } + + pub fn get_copied_objects(&self) -> usize { + self.copied_objects.load(Ordering::SeqCst) + } + + pub fn set_copied_objects(&self, size: usize) { + self.copied_objects.store(size, Ordering::SeqCst) + } + + pub fn increase_copied_objects(&self, size: usize) { + self.copied_objects.fetch_add(size, Ordering::SeqCst); + } + + pub fn get_pinned_objects(&self) -> usize { + self.pinned_objects.load(Ordering::SeqCst) + } + + pub fn set_pinned_objects(&self, size: usize) { + self.pinned_objects.store(size, Ordering::SeqCst) + } + + pub fn increase_pinned_objects(&self, size: usize) { + self.pinned_objects.fetch_add(size, Ordering::SeqCst); + } + + pub fn get_tpinned_objects(&self) -> usize { + self.tpinned_objects.load(Ordering::SeqCst) + } + + pub fn set_tpinned_objects(&self, size: usize) { + self.tpinned_objects.store(size, Ordering::SeqCst) + } + + pub fn increase_tpinned_objects(&self, size: usize) { + self.tpinned_objects.fetch_add(size, Ordering::SeqCst); + } +} + pub struct ImmixSpace { common: CommonSpace, pr: BlockPageResource, @@ -54,6 +128,9 @@ pub struct ImmixSpace { scheduler: Arc>, /// Some settings for this space space_args: ImmixSpaceArgs, + /// Keeping track of immix stats + #[cfg(feature = "dump_memory_stats")] + immix_stats: ImmixSpaceStats, } /// Some arguments for Immix Space. @@ -186,7 +263,7 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace ) -> ObjectReference { debug_assert!(!object.is_null()); if KIND == TRACE_KIND_TRANSITIVE_PIN { - self.trace_object_without_moving(queue, object) + self.trace_object_without_moving(queue, object, true) } else if KIND == TRACE_KIND_DEFRAG { if Block::containing::(object).is_defrag_source() { debug_assert!(self.in_defrag()); @@ -203,10 +280,10 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace false, ) } else { - self.trace_object_without_moving(queue, object) + self.trace_object_without_moving(queue, object, false) } } else if KIND == TRACE_KIND_FAST { - self.trace_object_without_moving(queue, object) + self.trace_object_without_moving(queue, object, false) } else { unreachable!() } @@ -217,6 +294,14 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace debug_assert!(self.in_space(object)); self.mark_lines(object); } + + // count the bytes for each object + #[cfg(feature = "dump_memory_stats")] + self.immix_stats.increase_live_bytes(VM::VMObjectModel::get_current_size(object)); + + // increase the number of objects scanned + #[cfg(feature = "dump_memory_stats")] + self.immix_stats.increase_traced_objects(1); } fn may_move_objects() -> bool { @@ -315,6 +400,8 @@ impl ImmixSpace { mark_state: Self::MARKED_STATE, scheduler: scheduler.clone(), space_args, + #[cfg(feature = "dump_memory_stats")] + immix_stats: Default::default() } } @@ -436,6 +523,15 @@ impl ImmixSpace { self.scheduler.work_buckets[WorkBucketStage::ClearVOBits].bulk_add(work_packets); } } + + #[cfg(feature = "dump_memory_stats")] + { + self.immix_stats.set_live_bytes(0); + self.immix_stats.set_traced_objects(0); + self.immix_stats.set_copied_objects(0); + self.immix_stats.set_tpinned_objects(0); + self.immix_stats.set_pinned_objects(0); + } } /// Release for the immix space. This is called when a GC finished. @@ -467,6 +563,86 @@ impl ImmixSpace { did_defrag } + #[cfg(feature = "dump_memory_stats")] + pub(crate) fn dump_memory_stats(&self) { + use std::time::{SystemTime, UNIX_EPOCH}; + + #[derive(Default)] + struct Dist { + live_blocks: usize, + live_lines: usize, + } + let mut dist = Dist::default(); + + for chunk in self.chunk_map.all_chunks() { + if !self.address_in_space(chunk.start()) { + continue; + } + + for block in chunk + .iter_region::() + .filter(|b| b.get_state() != BlockState::Unallocated) + { + dist.live_blocks += 1; + + let line_mark_state = self.line_mark_state.load(Ordering::Acquire); + let mut live_lines_in_table = 0; + let mut live_lines_from_block_state = 0; + + for line in block.lines() { + if line.is_marked(line_mark_state) { + live_lines_in_table += 1; + } + } + + match block.get_state() { + BlockState::Marked => { + panic!("At this point the block should have been swept already"); + } + BlockState::Unmarked => { + // Block is unmarked and cannot be reused (has no holes) + dist.live_lines += Block::LINES; + live_lines_from_block_state += Block::LINES; + } + BlockState::Reusable { unavailable_lines } => { + dist.live_lines += unavailable_lines as usize; + live_lines_from_block_state += unavailable_lines as usize; + } + BlockState::Unallocated => {} + } + + assert_eq!(live_lines_in_table, live_lines_from_block_state); + } + } + + let start = SystemTime::now(); + let since_the_epoch = start + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); + + println!("{:?} mmtk_immixspace", since_the_epoch.as_millis()); + println!("\t#Live objects = {}", self.immix_stats.get_traced_objects()); + println!("\t#Copied objects = {}", self.immix_stats.get_copied_objects()); + println!("\t#Pinned objects = {}", self.immix_stats.get_pinned_objects()); + println!("\t#Transitively pinned objects = {}", self.immix_stats.get_tpinned_objects()); + println!("\tLive bytes = {}", self.immix_stats.get_live_bytes()); + println!("\tReserved pages = {}", self.reserved_pages()); + println!( + "\tReserved pages (bytes) = {}", + self.reserved_pages() << LOG_BYTES_IN_PAGE + ); + println!("\tLive blocks = {}", dist.live_blocks); + println!( + "\tLive blocks (bytes) = {}", + dist.live_blocks << Block::LOG_BYTES + ); + println!("\tLive lines = {}", dist.live_lines); + println!( + "\tLive lines (bytes) = {}", + dist.live_lines << Line::LOG_BYTES + ); + } + /// Generate chunk sweep tasks fn generate_sweep_tasks(&self) -> Vec>> { self.defrag.mark_histograms.lock().clear(); @@ -543,6 +719,7 @@ impl ImmixSpace { &self, queue: &mut impl ObjectQueue, object: ObjectReference, + _is_tpinned: bool ) -> ObjectReference { #[cfg(feature = "vo_bit")] vo_bit::helper::on_trace_object::(object); @@ -563,6 +740,14 @@ impl ImmixSpace { // Visit node queue.enqueue(object); self.unlog_object_if_needed(object); + + #[cfg(feature = "dump_memory_stats")] + if _is_tpinned { + // increase the number of objects being tpinned + #[cfg(feature = "dump_memory_stats")] + self.immix_stats.increase_tpinned_objects(1); + } + return object; } object @@ -617,6 +802,10 @@ impl ImmixSpace { object_forwarding::clear_forwarding_bits::(object); object } else { + #[cfg(feature = "dump_memory_stats")] + if self.is_pinned(object) { + self.immix_stats.increase_pinned_objects(1); + } // We won the forwarding race; actually forward and copy the object if it is not pinned // and we have sufficient space in our copy allocator let new_object = if self.is_pinned(object) @@ -639,6 +828,10 @@ impl ImmixSpace { let new_object = object_forwarding::forward_object::(object, semantics, copy_context); + // increase the number of objects being moved + #[cfg(feature = "dump_memory_stats")] + self.immix_stats.increase_copied_objects(1); + #[cfg(feature = "vo_bit")] vo_bit::helper::on_object_forwarded::(new_object); diff --git a/src/policy/immix/mod.rs b/src/policy/immix/mod.rs index 7870a6b003..6746dbbe16 100644 --- a/src/policy/immix/mod.rs +++ b/src/policy/immix/mod.rs @@ -32,15 +32,15 @@ pub const DEFRAG: bool = !cfg!(feature = "immix_non_moving"); // defrag if we ar // | `DEFRAG_HEADROOM_PERCENT` | stress | `50` | Reserve enough headroom to copy all objects. 50% is like SemiSpace. | /// Make every GC a defragment GC. (for debugging) -pub const STRESS_DEFRAG: bool = false; +pub const STRESS_DEFRAG: bool = true; /// Mark every allocated block as defragmentation source before GC. (for debugging) -pub const DEFRAG_EVERY_BLOCK: bool = false; +pub const DEFRAG_EVERY_BLOCK: bool = true; /// Percentage of heap size reserved for defragmentation. /// According to [this paper](https://doi.org/10.1145/1375581.1375586), Immix works well with /// headroom between 1% to 3% of the heap size. -pub const DEFRAG_HEADROOM_PERCENT: usize = 2; +pub const DEFRAG_HEADROOM_PERCENT: usize = 50; /// If Immix is used as a nursery space, do we prefer copy? pub const PREFER_COPY_ON_NURSERY_GC: bool = diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index 64a13c8f37..0c263cfb1d 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -6,6 +6,8 @@ use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::space::{CommonSpace, Space}; use crate::util::constants::BYTES_IN_PAGE; +#[cfg(feature = "dump_memory_stats")] +use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::heap::{FreeListPageResource, PageResource}; use crate::util::metadata; use crate::util::opaque_pointer::*; @@ -13,6 +15,8 @@ use crate::util::treadmill::TreadMill; use crate::util::{Address, ObjectReference}; use crate::vm::ObjectModel; use crate::vm::VMBinding; +#[cfg(feature = "dump_memory_stats")] +use std::sync::atomic::AtomicUsize; #[allow(unused)] const PAGE_MASK: usize = !(BYTES_IN_PAGE - 1); @@ -28,6 +32,9 @@ pub struct LargeObjectSpace { mark_state: u8, in_nursery_gc: bool, treadmill: TreadMill, + /// Keeping track of live bytes + #[cfg(feature = "dump_memory_stats")] + live_bytes: AtomicUsize, } impl SFT for LargeObjectSpace { @@ -162,6 +169,8 @@ impl LargeObjectSpace { mark_state: 0, in_nursery_gc: false, treadmill: TreadMill::new(), + #[cfg(feature = "dump_memory_stats")] + live_bytes: AtomicUsize::new(0), } } @@ -172,6 +181,8 @@ impl LargeObjectSpace { } self.treadmill.flip(full_heap); self.in_nursery_gc = !full_heap; + #[cfg(feature = "dump_memory_stats")] + self.set_live_bytes(0); } pub fn release(&mut self, full_heap: bool) { @@ -282,6 +293,8 @@ impl LargeObjectSpace { break; } } + #[cfg(feature = "dump_memory_stats")] + self.increase_live_bytes(VM::VMObjectModel::get_current_size(object)); true } @@ -303,6 +316,39 @@ impl LargeObjectSpace { ) & NURSERY_BIT == NURSERY_BIT } + + #[cfg(feature = "dump_memory_stats")] + pub fn get_live_bytes(&self) -> usize { + self.live_bytes.load(Ordering::SeqCst) + } + + #[cfg(feature = "dump_memory_stats")] + pub fn set_live_bytes(&self, size: usize) { + self.live_bytes.store(size, Ordering::SeqCst) + } + + #[cfg(feature = "dump_memory_stats")] + pub fn increase_live_bytes(&self, size: usize) { + self.live_bytes.fetch_add(size, Ordering::SeqCst); + } + + #[cfg(feature = "dump_memory_stats")] + pub(crate) fn dump_memory_stats(&self) { + use std::time::{SystemTime, UNIX_EPOCH}; + + let start = SystemTime::now(); + let since_the_epoch = start + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); + + println!("{} mmtk_los", since_the_epoch.as_millis()); + println!("\tLive bytes = {}", self.get_live_bytes()); + println!("\tReserved pages = {}", self.reserved_pages()); + println!( + "\tReserved pages (bytes) = {}", + self.reserved_pages() << LOG_BYTES_IN_PAGE + ); + } } fn get_super_page(cell: Address) -> Address { diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 8cf4c74bee..3c51e96cb1 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -244,6 +244,9 @@ impl GCWork for EndOfGC { ); } + #[cfg(feature = "dump_memory_stats")] + mmtk.get_plan().dump_memory_stats(); + // We assume this is the only running work packet that accesses plan at the point of execution let plan_mut: &mut dyn Plan = unsafe { mmtk.get_plan_mut() }; plan_mut.end_of_gc(worker.tls);