Adding feature to query the fragmentation of immixspace

Cargo.toml

@@ -34,7 +34,7 @@ itertools = "0.12.0"
 jemalloc-sys = { version = "0.5.3", features = ["disable_initial_exec_tls"], optional = true }
 lazy_static = "1.1"
 libc = "0.2"
-log = { version = "0.4", features = ["max_level_trace", "release_max_level_off"] }
+log = { version = "0.4", features = ["max_level_trace", "release_max_level_trace"] }
 memoffset = "0.9"
 mimalloc-sys = { version = "0.1.6", optional = true }
 # MMTk macros - we have to specify a version here in order to publish the crate, even though we use the dependency from a local path.
@@ -152,6 +152,9 @@ malloc_counted_size = []
 # Count the size of all live objects in GC
 count_live_bytes_in_gc = []
 
+# Dump memory stats about the plan (live bytes, live lines, live blocks, and used pages)
+dump_memory_stats = []
+
 # Workaround a problem where bpftrace scripts (see tools/tracing/timeline/capture.bt) cannot
 # capture the type names of work packets.
 bpftrace_workaround = []

src/plan/global.rs

@@ -315,6 +315,10 @@ pub trait Plan: 'static + HasSpaces + Sync + Downcast {
             space.verify_side_metadata_sanity(&mut side_metadata_sanity_checker);
         })
     }
+
+    /// Dump memory stats for the plan
+    #[cfg(feature = "dump_memory_stats")]
+    fn dump_memory_stats(&self) {}
 }
 
 impl_downcast!(Plan assoc VM);

src/plan/immix/global.rs

@@ -116,6 +116,12 @@ impl<VM: VMBinding> Plan for Immix<VM> {
     fn common(&self) -> &CommonPlan<VM> {
         &self.common
     }
+
+    #[cfg(feature = "dump_memory_stats")]
+    fn dump_memory_stats(&self) {
+        self.immix_space.dump_memory_stats();
+        self.common.los.dump_memory_stats();
+    }
 }
 
 impl<VM: VMBinding> Immix<VM> {

src/plan/sticky/immix/global.rs

@@ -80,6 +80,12 @@ impl<VM: VMBinding> Plan for StickyImmix<VM> {
         self.immix.common()
     }
 
+    #[cfg(feature = "dump_memory_stats")]
+    fn dump_memory_stats(&self) {
+        self.immix.immix_space.dump_memory_stats();
+        self.common().los.dump_memory_stats();
+    }
+
     fn schedule_collection(&'static self, scheduler: &crate::scheduler::GCWorkScheduler<Self::VM>) {
         let is_full_heap = self.requires_full_heap_collection();
         self.gc_full_heap.store(is_full_heap, Ordering::SeqCst);
@@ -266,7 +272,7 @@ impl<VM: VMBinding> crate::plan::generational::global::GenerationalPlanExt<VM> f
                     );
                     self.immix
                         .immix_space
-                        .trace_object_without_moving(queue, object)
+                        .trace_object_without_moving(queue, object, false)
                 };
 
                 return object;

src/policy/immix/block.rs

@@ -70,9 +70,9 @@ pub struct Block(Address);
 
 impl Region for Block {
     #[cfg(not(feature = "immix_smaller_block"))]
-    const LOG_BYTES: usize = 15;
+    const LOG_BYTES: usize = 14;
     #[cfg(feature = "immix_smaller_block")]
-    const LOG_BYTES: usize = 13;
+    const LOG_BYTES: usize = 14;
 
     fn from_aligned_address(address: Address) -> Self {
         debug_assert!(address.is_aligned_to(Self::BYTES));

src/policy/immix/immixspace.rs

@@ -33,6 +33,80 @@ use std::sync::{atomic::AtomicU8, atomic::AtomicUsize, Arc};
 pub(crate) const TRACE_KIND_FAST: TraceKind = 0;
 pub(crate) const TRACE_KIND_DEFRAG: TraceKind = 1;
 
+#[cfg(feature = "dump_memory_stats")]
+#[derive(Default)]
+/// Keeping track of the number of traced/copied/tpinned objects and live bytes
+struct ImmixSpaceStats {
+    live_bytes: AtomicUsize,
+    traced_objects: AtomicUsize,
+    pinned_objects: AtomicUsize,
+    tpinned_objects: AtomicUsize,
+    copied_objects: AtomicUsize,
+}
+
+#[cfg(feature = "dump_memory_stats")]
+impl ImmixSpaceStats {
+    pub fn get_live_bytes(&self) -> usize {
+        self.live_bytes.load(Ordering::SeqCst)
+    }
+
+    pub fn set_live_bytes(&self, size: usize) {
+        self.live_bytes.store(size, Ordering::SeqCst)
+    }
+
+    pub fn increase_live_bytes(&self, size: usize) {
+        self.live_bytes.fetch_add(size, Ordering::SeqCst);
+    }
+
+    pub fn get_traced_objects(&self) -> usize {
+        self.traced_objects.load(Ordering::SeqCst)
+    }
+
+    pub fn set_traced_objects(&self, size: usize) {
+        self.traced_objects.store(size, Ordering::SeqCst)
+    }
+
+    pub fn increase_traced_objects(&self, size: usize) {
+        self.traced_objects.fetch_add(size, Ordering::SeqCst);
+    }
+
+    pub fn get_copied_objects(&self) -> usize {
+        self.copied_objects.load(Ordering::SeqCst)
+    }
+
+    pub fn set_copied_objects(&self, size: usize) {
+        self.copied_objects.store(size, Ordering::SeqCst)
+    }
+
+    pub fn increase_copied_objects(&self, size: usize) {
+        self.copied_objects.fetch_add(size, Ordering::SeqCst);
+    }
+
+    pub fn get_pinned_objects(&self) -> usize {
+        self.pinned_objects.load(Ordering::SeqCst)
+    }
+
+    pub fn set_pinned_objects(&self, size: usize) {
+        self.pinned_objects.store(size, Ordering::SeqCst)
+    }
+
+    pub fn increase_pinned_objects(&self, size: usize) {
+        self.pinned_objects.fetch_add(size, Ordering::SeqCst);
+    }
+
+    pub fn get_tpinned_objects(&self) -> usize {
+        self.tpinned_objects.load(Ordering::SeqCst)
+    }
+
+    pub fn set_tpinned_objects(&self, size: usize) {
+        self.tpinned_objects.store(size, Ordering::SeqCst)
+    }
+
+    pub fn increase_tpinned_objects(&self, size: usize) {
+        self.tpinned_objects.fetch_add(size, Ordering::SeqCst);
+    }
+}
+
 pub struct ImmixSpace<VM: VMBinding> {
     common: CommonSpace<VM>,
     pr: BlockPageResource<VM, Block>,
@@ -54,6 +128,9 @@ pub struct ImmixSpace<VM: VMBinding> {
     scheduler: Arc<GCWorkScheduler<VM>>,
     /// Some settings for this space
     space_args: ImmixSpaceArgs,
+    /// Keeping track of immix stats
+    #[cfg(feature = "dump_memory_stats")]
+    immix_stats: ImmixSpaceStats,
 }
 
 /// Some arguments for Immix Space.
@@ -186,7 +263,7 @@ impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for ImmixSpace
     ) -> ObjectReference {
         debug_assert!(!object.is_null());
         if KIND == TRACE_KIND_TRANSITIVE_PIN {
-            self.trace_object_without_moving(queue, object)
+            self.trace_object_without_moving(queue, object, true)
         } else if KIND == TRACE_KIND_DEFRAG {
             if Block::containing::<VM>(object).is_defrag_source() {
                 debug_assert!(self.in_defrag());
@@ -203,10 +280,10 @@ impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for ImmixSpace
                     false,
                 )
             } else {
-                self.trace_object_without_moving(queue, object)
+                self.trace_object_without_moving(queue, object, false)
             }
         } else if KIND == TRACE_KIND_FAST {
-            self.trace_object_without_moving(queue, object)
+            self.trace_object_without_moving(queue, object, false)
         } else {
             unreachable!()
         }
@@ -217,6 +294,14 @@ impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for ImmixSpace
             debug_assert!(self.in_space(object));
             self.mark_lines(object);
         }
+
+        // count the bytes for each object
+        #[cfg(feature = "dump_memory_stats")]
+        self.immix_stats.increase_live_bytes(VM::VMObjectModel::get_current_size(object));
+
+        // increase the number of objects scanned
+        #[cfg(feature = "dump_memory_stats")]
+        self.immix_stats.increase_traced_objects(1);
     }
 
     fn may_move_objects<const KIND: TraceKind>() -> bool {
@@ -315,6 +400,8 @@ impl<VM: VMBinding> ImmixSpace<VM> {
             mark_state: Self::MARKED_STATE,
             scheduler: scheduler.clone(),
             space_args,
+            #[cfg(feature = "dump_memory_stats")]
+            immix_stats: Default::default()
         }
     }
 
@@ -436,6 +523,15 @@ impl<VM: VMBinding> ImmixSpace<VM> {
                 self.scheduler.work_buckets[WorkBucketStage::ClearVOBits].bulk_add(work_packets);
             }
         }
+
+        #[cfg(feature = "dump_memory_stats")]
+        {
+            self.immix_stats.set_live_bytes(0);
+            self.immix_stats.set_traced_objects(0);
+            self.immix_stats.set_copied_objects(0);
+            self.immix_stats.set_tpinned_objects(0);
+            self.immix_stats.set_pinned_objects(0);
+        }
     }
 
     /// Release for the immix space. This is called when a GC finished.
@@ -467,6 +563,86 @@ impl<VM: VMBinding> ImmixSpace<VM> {
         did_defrag
     }
 
+    #[cfg(feature = "dump_memory_stats")]
+    pub(crate) fn dump_memory_stats(&self) {
+        use std::time::{SystemTime, UNIX_EPOCH};
+
+        #[derive(Default)]
+        struct Dist {
+            live_blocks: usize,
+            live_lines: usize,
+        }
+        let mut dist = Dist::default();
+
+        for chunk in self.chunk_map.all_chunks() {
+            if !self.address_in_space(chunk.start()) {
+                continue;
+            }
+
+            for block in chunk
+                .iter_region::<Block>()
+                .filter(|b| b.get_state() != BlockState::Unallocated)
+            {
+                dist.live_blocks += 1;
+
+                let line_mark_state = self.line_mark_state.load(Ordering::Acquire);
+                let mut live_lines_in_table = 0;
+                let mut live_lines_from_block_state = 0;
+
+                for line in block.lines() {
+                    if line.is_marked(line_mark_state) {
+                        live_lines_in_table += 1;
+                    }
+                }
+
+                match block.get_state() {
+                    BlockState::Marked => {
+                        panic!("At this point the block should have been swept already");
+                    }
+                    BlockState::Unmarked => {
+                        // Block is unmarked and cannot be reused (has no holes)
+                        dist.live_lines += Block::LINES;
+                        live_lines_from_block_state += Block::LINES;
+                    }
+                    BlockState::Reusable { unavailable_lines } => {
+                        dist.live_lines += unavailable_lines as usize;
+                        live_lines_from_block_state += unavailable_lines as usize;
+                    }
+                    BlockState::Unallocated => {}
+                }
+
+                assert_eq!(live_lines_in_table, live_lines_from_block_state);
+            }
+        }
+
+        let start = SystemTime::now();
+        let since_the_epoch = start
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards");
+
+        println!("{:?} mmtk_immixspace", since_the_epoch.as_millis());
+        println!("\t#Live objects = {}", self.immix_stats.get_traced_objects());
+        println!("\t#Copied objects = {}", self.immix_stats.get_copied_objects());
+        println!("\t#Pinned objects = {}", self.immix_stats.get_pinned_objects());
+        println!("\t#Transitively pinned objects = {}", self.immix_stats.get_tpinned_objects());
+        println!("\tLive bytes = {}", self.immix_stats.get_live_bytes());
+        println!("\tReserved pages = {}", self.reserved_pages());
+        println!(
+            "\tReserved pages (bytes) = {}",
+            self.reserved_pages() << LOG_BYTES_IN_PAGE
+        );
+        println!("\tLive blocks = {}", dist.live_blocks);
+        println!(
+            "\tLive blocks (bytes) = {}",
+            dist.live_blocks << Block::LOG_BYTES
+        );
+        println!("\tLive lines = {}", dist.live_lines);
+        println!(
+            "\tLive lines (bytes) = {}",
+            dist.live_lines << Line::LOG_BYTES
+        );
+    }
+
     /// Generate chunk sweep tasks
     fn generate_sweep_tasks(&self) -> Vec<Box<dyn GCWork<VM>>> {
         self.defrag.mark_histograms.lock().clear();
@@ -543,6 +719,7 @@ impl<VM: VMBinding> ImmixSpace<VM> {
         &self,
         queue: &mut impl ObjectQueue,
         object: ObjectReference,
+        _is_tpinned: bool
     ) -> ObjectReference {
         #[cfg(feature = "vo_bit")]
         vo_bit::helper::on_trace_object::<VM>(object);
@@ -563,6 +740,14 @@ impl<VM: VMBinding> ImmixSpace<VM> {
             // Visit node
             queue.enqueue(object);
             self.unlog_object_if_needed(object);
+
+            #[cfg(feature = "dump_memory_stats")]
+            if _is_tpinned {
+                // increase the number of objects being tpinned
+                #[cfg(feature = "dump_memory_stats")]
+                self.immix_stats.increase_tpinned_objects(1);
+            }
+
             return object;
         }
         object
@@ -617,6 +802,10 @@ impl<VM: VMBinding> ImmixSpace<VM> {
             object_forwarding::clear_forwarding_bits::<VM>(object);
             object
         } else {
+            #[cfg(feature = "dump_memory_stats")]
+            if self.is_pinned(object) {
+                self.immix_stats.increase_pinned_objects(1);
+            }
             // We won the forwarding race; actually forward and copy the object if it is not pinned
             // and we have sufficient space in our copy allocator
             let new_object = if self.is_pinned(object)
@@ -639,6 +828,10 @@ impl<VM: VMBinding> ImmixSpace<VM> {
                 let new_object =
                     object_forwarding::forward_object::<VM>(object, semantics, copy_context);
 
+                // increase the number of objects being moved
+                #[cfg(feature = "dump_memory_stats")]
+                self.immix_stats.increase_copied_objects(1);
+
                 #[cfg(feature = "vo_bit")]
                 vo_bit::helper::on_object_forwarded::<VM>(new_object);
 

src/policy/immix/mod.rs

@@ -32,15 +32,15 @@ pub const DEFRAG: bool = !cfg!(feature = "immix_non_moving"); // defrag if we ar
 // | `DEFRAG_HEADROOM_PERCENT` | stress  | `50`    | Reserve enough headroom to copy all objects.  50% is like SemiSpace. |
 
 /// Make every GC a defragment GC. (for debugging)
-pub const STRESS_DEFRAG: bool = false;
+pub const STRESS_DEFRAG: bool = true;
 
 /// Mark every allocated block as defragmentation source before GC. (for debugging)
-pub const DEFRAG_EVERY_BLOCK: bool = false;
+pub const DEFRAG_EVERY_BLOCK: bool = true;
 
 /// Percentage of heap size reserved for defragmentation.
 /// According to [this paper](https://doi.org/10.1145/1375581.1375586), Immix works well with
 /// headroom between 1% to 3% of the heap size.
-pub const DEFRAG_HEADROOM_PERCENT: usize = 2;
+pub const DEFRAG_HEADROOM_PERCENT: usize = 50;
 
 /// If Immix is used as a nursery space, do we prefer copy?
 pub const PREFER_COPY_ON_NURSERY_GC: bool =