From 1e7964656eb0988e3fa4f216c5f8dfb1dba7b31f Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 8 Jul 2021 16:19:05 +1200 Subject: [PATCH] Object model specs refactoring (#367) * Many funcitons take Metadata args as references * Use specific types for ObjectModel. Remove vm_metadata_used() from Space. * add comments for side metadata offsets to objectmodel Co-authored-by: Javad Amiri --- src/plan/barriers.rs | 2 +- src/plan/gencopy/gc_work.rs | 2 +- src/plan/gencopy/global.rs | 2 +- src/plan/gencopy/mutator.rs | 2 +- src/policy/copyspace.rs | 4 +- src/policy/immortalspace.rs | 10 +- src/policy/largeobjectspace.rs | 18 +- src/policy/mallocspace/global.rs | 2 +- src/policy/mallocspace/metadata.rs | 12 +- src/scheduler/gc_work.rs | 2 +- src/util/metadata/global.rs | 15 +- src/util/metadata/header_metadata.rs | 10 +- src/util/metadata/side_metadata/constants.rs | 26 +++ src/util/metadata/side_metadata/global.rs | 18 +- src/util/metadata/side_metadata/helpers.rs | 16 +- src/util/metadata/side_metadata/helpers_32.rs | 14 +- src/util/metadata/side_metadata/sanity.rs | 10 +- .../side_metadata/side_metadata_tests.rs | 130 +++++++------ src/util/object_forwarding.rs | 25 +-- src/vm/mod.rs | 1 + src/vm/object_model.rs | 171 ++++++++++++------ vmbindings/dummyvm/src/object_model.rs | 29 ++- 22 files changed, 312 insertions(+), 209 deletions(-) diff --git a/src/plan/barriers.rs b/src/plan/barriers.rs index d27876405b..b1f9177e84 100644 --- a/src/plan/barriers.rs +++ b/src/plan/barriers.rs @@ -53,7 +53,7 @@ impl ObjectRememberingBarrier { #[inline(always)] fn enqueue_node(&mut self, obj: ObjectReference) { if compare_exchange_metadata::( - self.meta, + &self.meta, obj, 0b1, 0b0, diff --git a/src/plan/gencopy/gc_work.rs b/src/plan/gencopy/gc_work.rs index df5ccc3163..2ab51b4d6d 100644 --- a/src/plan/gencopy/gc_work.rs +++ b/src/plan/gencopy/gc_work.rs @@ -58,7 +58,7 @@ impl CopyContext for GenCopyCopyContext { object_forwarding::clear_forwarding_bits::(obj); if !super::NO_SLOW && super::ACTIVE_BARRIER == BarrierSelector::ObjectBarrier { store_metadata::( - VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC, + &VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC, obj, 0b1, None, diff --git a/src/plan/gencopy/global.rs b/src/plan/gencopy/global.rs index 481043ed34..250cb4b05b 100644 --- a/src/plan/gencopy/global.rs +++ b/src/plan/gencopy/global.rs @@ -200,7 +200,7 @@ impl GenCopy { ) -> Self { let mut heap = HeapMeta::new(HEAP_START, HEAP_END); let gencopy_specs = if super::ACTIVE_BARRIER == BarrierSelector::ObjectBarrier { - metadata::extract_side_metadata(&[VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC]) + metadata::extract_side_metadata(&[*VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC]) } else { vec![] }; diff --git a/src/plan/gencopy/mutator.rs b/src/plan/gencopy/mutator.rs index 637c4902af..93ff0a2129 100644 --- a/src/plan/gencopy/mutator.rs +++ b/src/plan/gencopy/mutator.rs @@ -59,7 +59,7 @@ pub fn create_gencopy_mutator( allocators: Allocators::::new(mutator_tls, &*mmtk.plan, &config.space_mapping), barrier: box ObjectRememberingBarrier::>::new( mmtk, - VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC, + *VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC, ), mutator_tls, config, diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 7f947544ce..a02b8017ad 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -77,8 +77,8 @@ impl CopySpace { heap: &mut HeapMeta, ) -> Self { let local_specs = extract_side_metadata(&[ - VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, - VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC, + *VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, + *VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC, ]); let common = CommonSpace::new( SpaceOptions { diff --git a/src/policy/immortalspace.rs b/src/policy/immortalspace.rs index 1eef8ac88d..a76932da50 100644 --- a/src/policy/immortalspace.rs +++ b/src/policy/immortalspace.rs @@ -46,14 +46,14 @@ impl SFT for ImmortalSpace { } fn initialize_object_metadata(&self, object: ObjectReference, _alloc: bool) { let old_value = load_metadata::( - VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, object, None, Some(Ordering::SeqCst), ); let new_value = (old_value & GC_MARK_BIT_MASK) | self.mark_state; store_metadata::( - VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, object, new_value, None, @@ -106,7 +106,7 @@ impl ImmortalSpace { side_metadata_specs: SideMetadataContext { global: global_side_metadata_specs, local: metadata::extract_side_metadata(&[ - VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, ]), }, }, @@ -133,7 +133,7 @@ impl ImmortalSpace { fn test_and_mark(object: ObjectReference, value: usize) -> bool { loop { let old_value = load_metadata::( - VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, object, None, Some(Ordering::SeqCst), @@ -143,7 +143,7 @@ impl ImmortalSpace { } if compare_exchange_metadata::( - VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, object, old_value, old_value ^ GC_MARK_BIT_MASK, diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index 6e3ec712e8..c419769fff 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -52,7 +52,7 @@ impl SFT for LargeObjectSpace { } fn initialize_object_metadata(&self, object: ObjectReference, alloc: bool) { let old_value = load_metadata::( - VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + &VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, object, None, Some(Ordering::SeqCst), @@ -62,7 +62,7 @@ impl SFT for LargeObjectSpace { new_value |= NURSERY_BIT; } store_metadata::( - VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + &VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, object, new_value, None, @@ -118,7 +118,7 @@ impl LargeObjectSpace { side_metadata_specs: SideMetadataContext { global: global_side_metadata_specs, local: metadata::extract_side_metadata(&[ - VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + *VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, ]), }, }, @@ -208,7 +208,7 @@ impl LargeObjectSpace { MARK_BIT }; let old_value = load_metadata::( - VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + &VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, object, None, Some(Ordering::SeqCst), @@ -218,7 +218,7 @@ impl LargeObjectSpace { return false; } if compare_exchange_metadata::( - VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + &VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, object, old_value, old_value & !LOS_BIT_MASK | value, @@ -234,7 +234,7 @@ impl LargeObjectSpace { fn test_mark_bit(&self, object: ObjectReference, value: usize) -> bool { load_metadata::( - VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + &VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, object, None, Some(Ordering::SeqCst), @@ -245,7 +245,7 @@ impl LargeObjectSpace { /// Check if a given object is in nursery fn is_in_nursery(&self, object: ObjectReference) -> bool { load_metadata::( - VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + &VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, object, None, Some(Ordering::Relaxed), @@ -257,14 +257,14 @@ impl LargeObjectSpace { fn clear_nursery(&self, object: ObjectReference) { loop { let old_val = load_metadata::( - VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + &VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, object, None, Some(Ordering::Relaxed), ); let new_val = old_val & !NURSERY_BIT; if compare_exchange_metadata::( - VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + &VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, object, old_val, new_val, diff --git a/src/policy/mallocspace/global.rs b/src/policy/mallocspace/global.rs index 5fb4fc1589..9ea340680a 100644 --- a/src/policy/mallocspace/global.rs +++ b/src/policy/mallocspace/global.rs @@ -149,7 +149,7 @@ impl MallocSpace { global: global_side_metadata_specs, local: metadata::extract_side_metadata(&[ MetadataSpec::OnSide(ALLOC_SIDE_METADATA_SPEC), - VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, ]), }, #[cfg(debug_assertions)] diff --git a/src/policy/mallocspace/metadata.rs b/src/policy/mallocspace/metadata.rs index e3501c62b9..87de8eaaf2 100644 --- a/src/policy/mallocspace/metadata.rs +++ b/src/policy/mallocspace/metadata.rs @@ -73,12 +73,12 @@ pub fn is_alloced(object: ObjectReference) -> bool { } pub fn is_alloced_object(address: Address) -> bool { - side_metadata::load_atomic(ALLOC_SIDE_METADATA_SPEC, address, Ordering::SeqCst) == 1 + side_metadata::load_atomic(&ALLOC_SIDE_METADATA_SPEC, address, Ordering::SeqCst) == 1 } pub fn is_marked(object: ObjectReference) -> bool { load_metadata::( - VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, object, None, Some(Ordering::SeqCst), @@ -87,7 +87,7 @@ pub fn is_marked(object: ObjectReference) -> bool { pub fn set_alloc_bit(object: ObjectReference) { side_metadata::store_atomic( - ALLOC_SIDE_METADATA_SPEC, + &ALLOC_SIDE_METADATA_SPEC, object.to_address(), 1, Ordering::SeqCst, @@ -96,7 +96,7 @@ pub fn set_alloc_bit(object: ObjectReference) { pub fn set_mark_bit(object: ObjectReference) { store_metadata::( - VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, object, 1, None, @@ -106,7 +106,7 @@ pub fn set_mark_bit(object: ObjectReference) { pub fn unset_alloc_bit(object: ObjectReference) { side_metadata::store_atomic( - ALLOC_SIDE_METADATA_SPEC, + &ALLOC_SIDE_METADATA_SPEC, object.to_address(), 0, Ordering::SeqCst, @@ -115,7 +115,7 @@ pub fn unset_alloc_bit(object: ObjectReference) { pub fn unset_mark_bit(object: ObjectReference) { store_metadata::( - VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + &VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, object, 0, None, diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 03107d3de6..ce67116a1d 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -497,7 +497,7 @@ impl GCWork for ProcessModBuf { if !self.modbuf.is_empty() { for obj in &self.modbuf { compare_exchange_metadata::( - self.meta, + &self.meta, *obj, 0b0, 0b1, diff --git a/src/util/metadata/global.rs b/src/util/metadata/global.rs index 6108d71ed7..975ecbeb36 100644 --- a/src/util/metadata/global.rs +++ b/src/util/metadata/global.rs @@ -1,10 +1,9 @@ use crate::util::metadata::side_metadata; -use crate::vm::ObjectModel; -use atomic::Ordering; - use crate::util::metadata::side_metadata::SideMetadataSpec; use crate::util::ObjectReference; +use crate::vm::ObjectModel; use crate::vm::VMBinding; +use atomic::Ordering; use super::header_metadata::HeaderMetadataSpec; @@ -39,7 +38,7 @@ impl MetadataSpec { /// #[inline(always)] pub fn load_metadata( - metadata_spec: MetadataSpec, + metadata_spec: &MetadataSpec, object: ObjectReference, mask: Option, atomic_ordering: Option, @@ -70,7 +69,7 @@ pub fn load_metadata( /// #[inline(always)] pub fn store_metadata( - metadata_spec: MetadataSpec, + metadata_spec: &MetadataSpec, object: ObjectReference, val: usize, mask: Option, @@ -108,7 +107,7 @@ pub fn store_metadata( /// #[inline(always)] pub fn compare_exchange_metadata( - metadata_spec: MetadataSpec, + metadata_spec: &MetadataSpec, object: ObjectReference, old_val: usize, new_val: usize, @@ -150,7 +149,7 @@ pub fn compare_exchange_metadata( /// #[inline(always)] pub fn fetch_add_metadata( - metadata_spec: MetadataSpec, + metadata_spec: &MetadataSpec, object: ObjectReference, val: usize, order: Ordering, @@ -178,7 +177,7 @@ pub fn fetch_add_metadata( /// #[inline(always)] pub fn fetch_sub_metadata( - metadata_spec: MetadataSpec, + metadata_spec: &MetadataSpec, object: ObjectReference, val: usize, order: Ordering, diff --git a/src/util/metadata/header_metadata.rs b/src/util/metadata/header_metadata.rs index 9ec6731bca..a2f0f26d43 100644 --- a/src/util/metadata/header_metadata.rs +++ b/src/util/metadata/header_metadata.rs @@ -44,7 +44,7 @@ impl fmt::Debug for HeaderMetadataSpec { /// This function provides a default implementation for the `load_metadata` method from the `ObjectModel` trait. #[inline(always)] pub fn load_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, optional_mask: Option, atomic_ordering: Option, @@ -151,7 +151,7 @@ pub fn load_metadata( /// This function provides a default implementation for the `store_metadata` method from the `ObjectModel` trait. #[inline(always)] pub fn store_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, val: usize, optional_mask: Option, @@ -353,7 +353,7 @@ pub fn store_metadata( /// This function provides a default implementation for the `compare_exchange_metadata` method from the `ObjectModel` trait. #[inline(always)] pub fn compare_exchange_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, old_metadata: usize, new_metadata: usize, @@ -510,7 +510,7 @@ pub fn compare_exchange_metadata( /// This function provides a default implementation for the `fetch_add_metadata` method from the `ObjectModel` trait. #[inline(always)] pub fn fetch_add_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, val: usize, order: Ordering, @@ -602,7 +602,7 @@ pub fn fetch_add_metadata( /// This function provides a default implementation for the `fetch_sub_metadata` method from the `ObjectModel` trait. #[inline(always)] pub fn fetch_sub_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, val: usize, order: Ordering, diff --git a/src/util/metadata/side_metadata/constants.rs b/src/util/metadata/side_metadata/constants.rs index f135526250..91fcdb2efc 100644 --- a/src/util/metadata/side_metadata/constants.rs +++ b/src/util/metadata/side_metadata/constants.rs @@ -51,10 +51,36 @@ pub(super) const CHUNK_MASK: usize = (1 << LOG_BYTES_IN_CHUNK) - 1; pub(super) const LOCAL_SIDE_METADATA_PER_CHUNK: usize = BYTES_IN_CHUNK >> LOG_LOCAL_SIDE_METADATA_WORST_CASE_RATIO; +// The constants _VM_BASE_ADDRESS depends on the side metadata we use inside mmtk-core. +// If we add any new side metadata internal to mmtk-core, we need to update this accordingly. + +// TODO: We should think if it is possible to update this when we create a new side metadata spec. +// One issue is that these need to be constants. Possibly we need to use macros or custom build scripts. + +// -------------------------------------------------- +// +// Global Metadata +// +// MMTk reserved Global side metadata offsets: +// [currently empty] +// +// -------------------------------------------------- + /// The base address for the global side metadata space available to VM bindings, to be used for the per-object metadata. /// VM bindings must use this to avoid overlap with core internal global side metadata. pub const GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: Address = GLOBAL_SIDE_METADATA_BASE_ADDRESS; +// -------------------------------------------------- +// PolicySpecific Metadata +// +// MMTk reserved PolicySpecific side metadata offsets: +// +// 1 - MarkSweep Alloc bit: +// - Offset `0x0` on 32-bits +// - Offset `LOCAL_SIDE_METADATA_BASE_ADDRESS` on 64-bits +// +// -------------------------------------------------- + /// The base address for the local side metadata space available to VM bindings, to be used for the per-object metadata. /// VM bindings must use this to avoid overlap with core internal local side metadata. #[cfg(target_pointer_width = "64")] diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index 2c4836b3b4..1b80f5f287 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -205,7 +205,7 @@ impl SideMetadataContext { // Used only for debugging // Panics in the required metadata for data_addr is not mapped -pub fn ensure_metadata_is_mapped(metadata_spec: SideMetadataSpec, data_addr: Address) { +pub fn ensure_metadata_is_mapped(metadata_spec: &SideMetadataSpec, data_addr: Address) { let meta_start = address_to_meta_address(metadata_spec, data_addr).align_down(BYTES_IN_PAGE); debug!( @@ -217,7 +217,7 @@ pub fn ensure_metadata_is_mapped(metadata_spec: SideMetadataSpec, data_addr: Add } #[inline(always)] -pub fn load_atomic(metadata_spec: SideMetadataSpec, data_addr: Address, order: Ordering) -> usize { +pub fn load_atomic(metadata_spec: &SideMetadataSpec, data_addr: Address, order: Ordering) -> usize { #[cfg(feature = "extreme_assertions")] let _lock = sanity::SANITY_LOCK.lock().unwrap(); @@ -255,7 +255,7 @@ pub fn load_atomic(metadata_spec: SideMetadataSpec, data_addr: Address, order: O #[inline(always)] pub fn store_atomic( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, metadata: usize, order: Ordering, @@ -306,7 +306,7 @@ pub fn store_atomic( #[inline(always)] pub fn compare_exchange_atomic( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, old_metadata: usize, new_metadata: usize, @@ -408,7 +408,7 @@ pub fn compare_exchange_atomic( // same as Rust atomics, this wraps around on overflow #[inline(always)] pub fn fetch_add_atomic( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, val: usize, order: Ordering, @@ -467,7 +467,7 @@ pub fn fetch_add_atomic( // same as Rust atomics, this wraps around on overflow #[inline(always)] pub fn fetch_sub_atomic( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, val: usize, order: Ordering, @@ -533,7 +533,7 @@ pub fn fetch_sub_atomic( /// 2. Interleaving Non-atomic and atomic operations is undefined behaviour. /// #[inline(always)] -pub unsafe fn load(metadata_spec: SideMetadataSpec, data_addr: Address) -> usize { +pub unsafe fn load(metadata_spec: &SideMetadataSpec, data_addr: Address) -> usize { #[cfg(feature = "extreme_assertions")] let _lock = sanity::SANITY_LOCK.lock().unwrap(); @@ -580,7 +580,7 @@ pub unsafe fn load(metadata_spec: SideMetadataSpec, data_addr: Address) -> usize /// 2. Interleaving Non-atomic and atomic operations is undefined behaviour. /// #[inline(always)] -pub unsafe fn store(metadata_spec: SideMetadataSpec, data_addr: Address, metadata: usize) { +pub unsafe fn store(metadata_spec: &SideMetadataSpec, data_addr: Address, metadata: usize) { #[cfg(feature = "extreme_assertions")] let _lock = sanity::SANITY_LOCK.lock().unwrap(); @@ -626,7 +626,7 @@ pub unsafe fn store(metadata_spec: SideMetadataSpec, data_addr: Address, metadat /// /// * `chunk_start` - The starting address of the chunk whose metadata is being zeroed. /// -pub fn bzero_metadata(metadata_spec: SideMetadataSpec, start: Address, size: usize) { +pub fn bzero_metadata(metadata_spec: &SideMetadataSpec, start: Address, size: usize) { #[cfg(feature = "extreme_assertions")] let _lock = sanity::SANITY_LOCK.lock().unwrap(); diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index 7b7a6891cf..0905d1bb19 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -14,7 +14,7 @@ use std::io::Result; /// Performs address translation in contiguous metadata spaces (e.g. global and policy-specific in 64-bits, and global in 32-bits) #[inline(always)] pub(crate) fn address_to_contiguous_meta_address( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, ) -> Address { let log_bits_num = metadata_spec.log_num_of_bits as i32; @@ -53,10 +53,10 @@ pub(crate) fn ensure_munmap_contiguos_metadata_space( spec: &SideMetadataSpec, ) -> usize { // nearest page-aligned starting address - let mmap_start = address_to_meta_address(*spec, start).align_down(BYTES_IN_PAGE); + let mmap_start = address_to_meta_address(spec, start).align_down(BYTES_IN_PAGE); // nearest page-aligned ending address let mmap_size = - address_to_meta_address(*spec, start + size).align_up(BYTES_IN_PAGE) - mmap_start; + address_to_meta_address(spec, start + size).align_up(BYTES_IN_PAGE) - mmap_start; if mmap_size > 0 { ensure_munmap_metadata(mmap_start, mmap_size); } @@ -76,10 +76,10 @@ pub(crate) fn try_mmap_contiguous_metadata_space( debug_assert!(size % BYTES_IN_PAGE == 0); // nearest page-aligned starting address - let mmap_start = address_to_meta_address(*spec, start).align_down(BYTES_IN_PAGE); + let mmap_start = address_to_meta_address(spec, start).align_down(BYTES_IN_PAGE); // nearest page-aligned ending address let mmap_size = - address_to_meta_address(*spec, start + size).align_up(BYTES_IN_PAGE) - mmap_start; + address_to_meta_address(spec, start + size).align_up(BYTES_IN_PAGE) - mmap_start; if mmap_size > 0 { if !no_reserve { MMAPPER.ensure_mapped(mmap_start, mmap_size >> LOG_BYTES_IN_PAGE) @@ -95,7 +95,7 @@ pub(crate) fn try_mmap_contiguous_metadata_space( /// Performs the translation of data address (`data_addr`) to metadata address for the specified metadata (`metadata_spec`). #[inline(always)] pub(crate) fn address_to_meta_address( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, ) -> Address { #[cfg(target_pointer_width = "32")] @@ -133,7 +133,7 @@ pub const fn metadata_address_range_size(metadata_spec: &SideMetadataSpec) -> us } #[inline(always)] -pub(crate) fn meta_byte_lshift(metadata_spec: SideMetadataSpec, data_addr: Address) -> u8 { +pub(crate) fn meta_byte_lshift(metadata_spec: &SideMetadataSpec, data_addr: Address) -> u8 { let bits_num_log = metadata_spec.log_num_of_bits as i32; if bits_num_log >= 3 { return 0; @@ -144,7 +144,7 @@ pub(crate) fn meta_byte_lshift(metadata_spec: SideMetadataSpec, data_addr: Addre } #[inline(always)] -pub(crate) fn meta_byte_mask(metadata_spec: SideMetadataSpec) -> u8 { +pub(crate) fn meta_byte_mask(metadata_spec: &SideMetadataSpec) -> u8 { let bits_num_log = metadata_spec.log_num_of_bits; ((1usize << (1usize << bits_num_log)) - 1) as u8 } diff --git a/src/util/metadata/side_metadata/helpers_32.rs b/src/util/metadata/side_metadata/helpers_32.rs index 41ef56beb2..be77cf578c 100644 --- a/src/util/metadata/side_metadata/helpers_32.rs +++ b/src/util/metadata/side_metadata/helpers_32.rs @@ -18,7 +18,7 @@ use crate::MMAPPER; #[inline(always)] pub(super) fn address_to_chunked_meta_address( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, ) -> Address { let log_bits_num = metadata_spec.log_num_of_bits as i32; @@ -47,13 +47,13 @@ pub(crate) fn ensure_munmap_chunked_metadata_space( ) -> usize { use super::address_to_meta_address; use crate::util::constants::BYTES_IN_PAGE; - let meta_start = address_to_meta_address(*spec, start).align_down(BYTES_IN_PAGE); + let meta_start = address_to_meta_address(spec, start).align_down(BYTES_IN_PAGE); // per chunk policy-specific metadata for 32-bits targets let chunk_num = ((start + size - 1usize).align_down(BYTES_IN_CHUNK) - start.align_down(BYTES_IN_CHUNK)) / BYTES_IN_CHUNK; if chunk_num == 0 { - let size_to_unmap = address_to_meta_address(*spec, start + size) - meta_start; + let size_to_unmap = address_to_meta_address(spec, start + size) - meta_start; ensure_munmap_metadata(meta_start, size_to_unmap); size_to_unmap @@ -61,13 +61,13 @@ pub(crate) fn ensure_munmap_chunked_metadata_space( let mut total_unmapped = 0; let second_data_chunk = (start + 1usize).align_up(BYTES_IN_CHUNK); // unmap the first sub-chunk - let first_sub_chunk_size = address_to_meta_address(*spec, second_data_chunk) - meta_start; + let first_sub_chunk_size = address_to_meta_address(spec, second_data_chunk) - meta_start; ensure_munmap_metadata(meta_start, first_sub_chunk_size); total_unmapped += first_sub_chunk_size; let last_data_chunk = (start + size).align_down(BYTES_IN_CHUNK); - let last_meta_chunk = address_to_meta_address(*spec, last_data_chunk); - let last_sub_chunk_size = address_to_meta_address(*spec, start + size) - last_meta_chunk; + let last_meta_chunk = address_to_meta_address(spec, last_data_chunk); + let last_sub_chunk_size = address_to_meta_address(spec, start + size) - last_meta_chunk; // unmap the last sub-chunk ensure_munmap_metadata(last_meta_chunk, last_sub_chunk_size); total_unmapped += last_sub_chunk_size; @@ -76,7 +76,7 @@ pub(crate) fn ensure_munmap_chunked_metadata_space( // unmap all chunks in the middle while next_data_chunk != last_data_chunk { let to_unmap = metadata_bytes_per_chunk(spec.log_min_obj_size, spec.log_num_of_bits); - ensure_munmap_metadata(address_to_meta_address(*spec, next_data_chunk), to_unmap); + ensure_munmap_metadata(address_to_meta_address(spec, next_data_chunk), to_unmap); total_unmapped += to_unmap; next_data_chunk += BYTES_IN_CHUNK; } diff --git a/src/util/metadata/side_metadata/sanity.rs b/src/util/metadata/side_metadata/sanity.rs index 91e755cb5f..77148e9f1e 100644 --- a/src/util/metadata/side_metadata/sanity.rs +++ b/src/util/metadata/side_metadata/sanity.rs @@ -383,7 +383,7 @@ impl SideMetadataSanity { /// * `size`: size of the source data /// #[cfg(feature = "extreme_assertions")] -pub fn verify_bzero(metadata_spec: SideMetadataSpec, start: Address, size: usize) { +pub fn verify_bzero(metadata_spec: &SideMetadataSpec, start: Address, size: usize) { let sanity_map = &mut CONTENT_SANITY_MAP.write().unwrap(); match sanity_map.get_mut(&metadata_spec) { Some(spec_sanity_map) => { @@ -446,7 +446,7 @@ pub fn verify_load(metadata_spec: &SideMetadataSpec, data_addr: Address, actual_ /// * `metadata`: the metadata content to store /// #[cfg(feature = "extreme_assertions")] -pub fn verify_store(metadata_spec: SideMetadataSpec, data_addr: Address, metadata: usize) { +pub fn verify_store(metadata_spec: &SideMetadataSpec, data_addr: Address, metadata: usize) { let sanity_map = &mut CONTENT_SANITY_MAP.write().unwrap(); match sanity_map.get_mut(&metadata_spec) { Some(spec_sanity_map) => { @@ -461,7 +461,7 @@ pub fn verify_store(metadata_spec: SideMetadataSpec, data_addr: Address, metadat /// A helper function encapsulating the common parts of addition and subtraction #[cfg(feature = "extreme_assertions")] fn do_math( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, val: usize, math_op: MathOp, @@ -498,7 +498,7 @@ fn do_math( /// #[cfg(feature = "extreme_assertions")] pub fn verify_add( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, val_to_add: usize, actual_old_val: usize, @@ -529,7 +529,7 @@ pub fn verify_add( /// #[cfg(feature = "extreme_assertions")] pub fn verify_sub( - metadata_spec: SideMetadataSpec, + metadata_spec: &SideMetadataSpec, data_addr: Address, val_to_sub: usize, actual_old_val: usize, diff --git a/src/util/metadata/side_metadata/side_metadata_tests.rs b/src/util/metadata/side_metadata/side_metadata_tests.rs index 23ac65e88c..110b260af6 100644 --- a/src/util/metadata/side_metadata/side_metadata_tests.rs +++ b/src/util/metadata/side_metadata/side_metadata_tests.rs @@ -35,29 +35,29 @@ mod tests { }; assert_eq!( - address_to_meta_address(gspec, unsafe { Address::from_usize(0) }).as_usize(), + address_to_meta_address(&gspec, unsafe { Address::from_usize(0) }).as_usize(), GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize() ); assert_eq!( - address_to_meta_address(lspec, unsafe { Address::from_usize(0) }).as_usize(), + address_to_meta_address(&lspec, unsafe { Address::from_usize(0) }).as_usize(), LOCAL_SIDE_METADATA_BASE_ADDRESS.as_usize() ); assert_eq!( - address_to_meta_address(gspec, unsafe { Address::from_usize(7) }).as_usize(), + address_to_meta_address(&gspec, unsafe { Address::from_usize(7) }).as_usize(), GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize() ); assert_eq!( - address_to_meta_address(lspec, unsafe { Address::from_usize(7) }).as_usize(), + address_to_meta_address(&lspec, unsafe { Address::from_usize(7) }).as_usize(), LOCAL_SIDE_METADATA_BASE_ADDRESS.as_usize() ); assert_eq!( - address_to_meta_address(gspec, unsafe { Address::from_usize(27) }).as_usize(), + address_to_meta_address(&gspec, unsafe { Address::from_usize(27) }).as_usize(), GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 3 ); assert_eq!( - address_to_meta_address(lspec, unsafe { Address::from_usize(129) }).as_usize(), + address_to_meta_address(&lspec, unsafe { Address::from_usize(129) }).as_usize(), LOCAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 16 ); @@ -65,29 +65,29 @@ mod tests { lspec.log_min_obj_size = 1; assert_eq!( - address_to_meta_address(gspec, unsafe { Address::from_usize(0) }).as_usize(), + address_to_meta_address(&gspec, unsafe { Address::from_usize(0) }).as_usize(), GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize() ); assert_eq!( - address_to_meta_address(lspec, unsafe { Address::from_usize(0) }).as_usize(), + address_to_meta_address(&lspec, unsafe { Address::from_usize(0) }).as_usize(), LOCAL_SIDE_METADATA_BASE_ADDRESS.as_usize() ); assert_eq!( - address_to_meta_address(gspec, unsafe { Address::from_usize(32) }).as_usize(), + address_to_meta_address(&gspec, unsafe { Address::from_usize(32) }).as_usize(), GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 1 ); assert_eq!( - address_to_meta_address(lspec, unsafe { Address::from_usize(32) }).as_usize(), + address_to_meta_address(&lspec, unsafe { Address::from_usize(32) }).as_usize(), LOCAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 2 ); assert_eq!( - address_to_meta_address(gspec, unsafe { Address::from_usize(316) }).as_usize(), + address_to_meta_address(&gspec, unsafe { Address::from_usize(316) }).as_usize(), GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 9 ); assert_eq!( - address_to_meta_address(lspec, unsafe { Address::from_usize(316) }).as_usize(), + address_to_meta_address(&lspec, unsafe { Address::from_usize(316) }).as_usize(), LOCAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 19 ); @@ -95,29 +95,29 @@ mod tests { lspec.log_num_of_bits = 3; assert_eq!( - address_to_meta_address(gspec, unsafe { Address::from_usize(0) }).as_usize(), + address_to_meta_address(&gspec, unsafe { Address::from_usize(0) }).as_usize(), GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize() ); assert_eq!( - address_to_meta_address(lspec, unsafe { Address::from_usize(0) }).as_usize(), + address_to_meta_address(&lspec, unsafe { Address::from_usize(0) }).as_usize(), LOCAL_SIDE_METADATA_BASE_ADDRESS.as_usize() ); assert_eq!( - address_to_meta_address(gspec, unsafe { Address::from_usize(32) }).as_usize(), + address_to_meta_address(&gspec, unsafe { Address::from_usize(32) }).as_usize(), GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 2 ); assert_eq!( - address_to_meta_address(lspec, unsafe { Address::from_usize(32) }).as_usize(), + address_to_meta_address(&lspec, unsafe { Address::from_usize(32) }).as_usize(), LOCAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 16 ); assert_eq!( - address_to_meta_address(gspec, unsafe { Address::from_usize(316) }).as_usize(), + address_to_meta_address(&gspec, unsafe { Address::from_usize(316) }).as_usize(), GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 19 ); assert_eq!( - address_to_meta_address(lspec, unsafe { Address::from_usize(318) }).as_usize(), + address_to_meta_address(&lspec, unsafe { Address::from_usize(318) }).as_usize(), LOCAL_SIDE_METADATA_BASE_ADDRESS.as_usize() + 159 ); } @@ -131,14 +131,14 @@ mod tests { log_min_obj_size: 0, }; - assert_eq!(meta_byte_mask(spec), 1); + assert_eq!(meta_byte_mask(&spec), 1); spec.log_num_of_bits = 1; - assert_eq!(meta_byte_mask(spec), 3); + assert_eq!(meta_byte_mask(&spec), 3); spec.log_num_of_bits = 2; - assert_eq!(meta_byte_mask(spec), 15); + assert_eq!(meta_byte_mask(&spec), 15); spec.log_num_of_bits = 3; - assert_eq!(meta_byte_mask(spec), 255); + assert_eq!(meta_byte_mask(&spec), 255); } #[test] @@ -150,23 +150,35 @@ mod tests { log_min_obj_size: 0, }; - assert_eq!(meta_byte_lshift(spec, unsafe { Address::from_usize(0) }), 0); - assert_eq!(meta_byte_lshift(spec, unsafe { Address::from_usize(5) }), 5); assert_eq!( - meta_byte_lshift(spec, unsafe { Address::from_usize(15) }), + meta_byte_lshift(&spec, unsafe { Address::from_usize(0) }), + 0 + ); + assert_eq!( + meta_byte_lshift(&spec, unsafe { Address::from_usize(5) }), + 5 + ); + assert_eq!( + meta_byte_lshift(&spec, unsafe { Address::from_usize(15) }), 7 ); spec.log_num_of_bits = 2; - assert_eq!(meta_byte_lshift(spec, unsafe { Address::from_usize(0) }), 0); - assert_eq!(meta_byte_lshift(spec, unsafe { Address::from_usize(5) }), 4); assert_eq!( - meta_byte_lshift(spec, unsafe { Address::from_usize(15) }), + meta_byte_lshift(&spec, unsafe { Address::from_usize(0) }), + 0 + ); + assert_eq!( + meta_byte_lshift(&spec, unsafe { Address::from_usize(5) }), + 4 + ); + assert_eq!( + meta_byte_lshift(&spec, unsafe { Address::from_usize(15) }), 4 ); assert_eq!( - meta_byte_lshift(spec, unsafe { Address::from_usize(0x10010) }), + meta_byte_lshift(&spec, unsafe { Address::from_usize(0x10010) }), 0 ); } @@ -214,14 +226,14 @@ mod tests { ) .is_ok()); - ensure_metadata_is_mapped(gspec, vm_layout_constants::HEAP_START); - ensure_metadata_is_mapped(lspec, vm_layout_constants::HEAP_START); + ensure_metadata_is_mapped(&gspec, vm_layout_constants::HEAP_START); + ensure_metadata_is_mapped(&lspec, vm_layout_constants::HEAP_START); ensure_metadata_is_mapped( - gspec, + &gspec, vm_layout_constants::HEAP_START + constants::BYTES_IN_PAGE - 1, ); ensure_metadata_is_mapped( - lspec, + &lspec, vm_layout_constants::HEAP_START + constants::BYTES_IN_PAGE - 1, ); @@ -253,20 +265,20 @@ mod tests { .is_ok()); ensure_metadata_is_mapped( - gspec, + &gspec, vm_layout_constants::HEAP_START + vm_layout_constants::BYTES_IN_CHUNK, ); ensure_metadata_is_mapped( - lspec, + &lspec, vm_layout_constants::HEAP_START + vm_layout_constants::BYTES_IN_CHUNK, ); ensure_metadata_is_mapped( - gspec, + &gspec, vm_layout_constants::HEAP_START + vm_layout_constants::BYTES_IN_CHUNK * 2 - 8, ); ensure_metadata_is_mapped( - lspec, + &lspec, vm_layout_constants::HEAP_START + vm_layout_constants::BYTES_IN_CHUNK * 2 - 16, ); @@ -319,30 +331,30 @@ mod tests { .try_map_metadata_space(data_addr, constants::BYTES_IN_PAGE,) .is_ok()); - let zero = fetch_add_atomic(metadata_1_spec, data_addr, 5, Ordering::SeqCst); + let zero = fetch_add_atomic(&metadata_1_spec, data_addr, 5, Ordering::SeqCst); assert_eq!(zero, 0); - let five = load_atomic(metadata_1_spec, data_addr, Ordering::SeqCst); + let five = load_atomic(&metadata_1_spec, data_addr, Ordering::SeqCst); assert_eq!(five, 5); - let zero = fetch_add_atomic(metadata_2_spec, data_addr, 5, Ordering::SeqCst); + let zero = fetch_add_atomic(&metadata_2_spec, data_addr, 5, Ordering::SeqCst); assert_eq!(zero, 0); - let five = load_atomic(metadata_2_spec, data_addr, Ordering::SeqCst); + let five = load_atomic(&metadata_2_spec, data_addr, Ordering::SeqCst); assert_eq!(five, 5); let another_five = - fetch_sub_atomic(metadata_1_spec, data_addr, 2, Ordering::SeqCst); + fetch_sub_atomic(&metadata_1_spec, data_addr, 2, Ordering::SeqCst); assert_eq!(another_five, 5); - let three = load_atomic(metadata_1_spec, data_addr, Ordering::SeqCst); + let three = load_atomic(&metadata_1_spec, data_addr, Ordering::SeqCst); assert_eq!(three, 3); let another_five = - fetch_sub_atomic(metadata_2_spec, data_addr, 2, Ordering::SeqCst); + fetch_sub_atomic(&metadata_2_spec, data_addr, 2, Ordering::SeqCst); assert_eq!(another_five, 5); - let three = load_atomic(metadata_2_spec, data_addr, Ordering::SeqCst); + let three = load_atomic(&metadata_2_spec, data_addr, Ordering::SeqCst); assert_eq!(three, 3); metadata.ensure_unmap_metadata_space(data_addr, constants::BYTES_IN_PAGE); @@ -384,17 +396,17 @@ mod tests { .try_map_metadata_space(data_addr, constants::BYTES_IN_PAGE,) .is_ok()); - let zero = fetch_add_atomic(metadata_1_spec, data_addr, 2, Ordering::SeqCst); + let zero = fetch_add_atomic(&metadata_1_spec, data_addr, 2, Ordering::SeqCst); assert_eq!(zero, 0); - let two = load_atomic(metadata_1_spec, data_addr, Ordering::SeqCst); + let two = load_atomic(&metadata_1_spec, data_addr, Ordering::SeqCst); assert_eq!(two, 2); let another_two = - fetch_sub_atomic(metadata_1_spec, data_addr, 1, Ordering::SeqCst); + fetch_sub_atomic(&metadata_1_spec, data_addr, 1, Ordering::SeqCst); assert_eq!(another_two, 2); - let one = load_atomic(metadata_1_spec, data_addr, Ordering::SeqCst); + let one = load_atomic(&metadata_1_spec, data_addr, Ordering::SeqCst); assert_eq!(one, 1); metadata.ensure_unmap_metadata_space(data_addr, constants::BYTES_IN_PAGE); @@ -465,30 +477,30 @@ mod tests { .try_map_metadata_space(data_addr, constants::BYTES_IN_PAGE,) .is_ok()); - let zero = fetch_add_atomic(metadata_1_spec, data_addr, 5, Ordering::SeqCst); + let zero = fetch_add_atomic(&metadata_1_spec, data_addr, 5, Ordering::SeqCst); assert_eq!(zero, 0); - let five = load_atomic(metadata_1_spec, data_addr, Ordering::SeqCst); + let five = load_atomic(&metadata_1_spec, data_addr, Ordering::SeqCst); assert_eq!(five, 5); - let zero = fetch_add_atomic(metadata_2_spec, data_addr, 5, Ordering::SeqCst); + let zero = fetch_add_atomic(&metadata_2_spec, data_addr, 5, Ordering::SeqCst); assert_eq!(zero, 0); - let five = load_atomic(metadata_2_spec, data_addr, Ordering::SeqCst); + let five = load_atomic(&metadata_2_spec, data_addr, Ordering::SeqCst); assert_eq!(five, 5); - bzero_metadata(metadata_2_spec, data_addr, constants::BYTES_IN_PAGE); + bzero_metadata(&metadata_2_spec, data_addr, constants::BYTES_IN_PAGE); - let five = load_atomic(metadata_1_spec, data_addr, Ordering::SeqCst); + let five = load_atomic(&metadata_1_spec, data_addr, Ordering::SeqCst); assert_eq!(five, 5); - let five = load_atomic(metadata_2_spec, data_addr, Ordering::SeqCst); + let five = load_atomic(&metadata_2_spec, data_addr, Ordering::SeqCst); assert_eq!(five, 0); - bzero_metadata(metadata_1_spec, data_addr, constants::BYTES_IN_PAGE); + bzero_metadata(&metadata_1_spec, data_addr, constants::BYTES_IN_PAGE); - let five = load_atomic(metadata_1_spec, data_addr, Ordering::SeqCst); + let five = load_atomic(&metadata_1_spec, data_addr, Ordering::SeqCst); assert_eq!(five, 0); - let five = load_atomic(metadata_2_spec, data_addr, Ordering::SeqCst); + let five = load_atomic(&metadata_2_spec, data_addr, Ordering::SeqCst); assert_eq!(five, 0); metadata.ensure_unmap_metadata_space(data_addr, constants::BYTES_IN_PAGE); diff --git a/src/util/object_forwarding.rs b/src/util/object_forwarding.rs index fc4756b624..3203130992 100644 --- a/src/util/object_forwarding.rs +++ b/src/util/object_forwarding.rs @@ -31,14 +31,14 @@ const FORWARDING_POINTER_MASK: usize = 0xffff_fffc; pub fn attempt_to_forward(object: ObjectReference) -> usize { loop { let old_value = load_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, object, None, Some(Ordering::SeqCst), ); if old_value != FORWARDING_NOT_TRIGGERED_YET || compare_exchange_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, object, old_value, old_value | BEING_FORWARDED, @@ -68,7 +68,7 @@ pub fn spin_and_get_forwarded_object( let mut forwarding_bits = forwarding_bits; while forwarding_bits == BEING_FORWARDED { forwarding_bits = load_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, object, None, Some(Ordering::SeqCst), @@ -95,7 +95,7 @@ pub fn forward_object( let new_object = VM::VMObjectModel::copy(object, semantics, copy_context); if let Some(shift) = forwarding_bits_offset_in_forwarding_pointer::() { store_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC, object, new_object.to_address().as_usize() | (FORWARDED << shift), None, @@ -104,7 +104,7 @@ pub fn forward_object( } else { write_forwarding_pointer::(object, new_object); store_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, object, FORWARDED, None, @@ -116,7 +116,7 @@ pub fn forward_object( pub fn is_forwarded(object: ObjectReference) -> bool { load_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, object, None, Some(Ordering::SeqCst), @@ -125,7 +125,7 @@ pub fn is_forwarded(object: ObjectReference) -> bool { pub fn is_forwarded_or_being_forwarded(object: ObjectReference) -> bool { load_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, object, None, Some(Ordering::SeqCst), @@ -144,7 +144,7 @@ pub fn state_is_being_forwarded(forwarding_bits: usize) -> bool { /// This function is used on new objects. pub fn clear_forwarding_bits(object: ObjectReference) { store_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, object, 0, None, @@ -157,7 +157,7 @@ pub fn clear_forwarding_bits(object: ObjectReference) { pub fn read_forwarding_pointer(object: ObjectReference) -> ObjectReference { unsafe { Address::from_usize(load_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC, object, Some(FORWARDING_POINTER_MASK), Some(Ordering::SeqCst), @@ -174,7 +174,7 @@ pub fn write_forwarding_pointer( ) { trace!("GCForwardingWord::write({:#?}, {:x})\n", object, new_object); store_metadata::( - VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC, + &VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC, object, new_object.to_address().as_usize(), Some(FORWARDING_POINTER_MASK), @@ -191,10 +191,11 @@ pub fn write_forwarding_pointer( /// #[cfg(target_endian = "little")] pub(super) fn forwarding_bits_offset_in_forwarding_pointer() -> Option { + use std::ops::Deref; // if both forwarding bits and forwarding pointer are in-header match ( - VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC, - VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, + VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC.deref(), + VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC.deref(), ) { (MetadataSpec::InHeader(fp), MetadataSpec::InHeader(fb)) => { let maybe_shift = fb.bit_offset - fp.bit_offset; diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 8b5bc0264d..10b678ecac 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -24,6 +24,7 @@ mod reference_glue; mod scanning; pub use self::active_plan::ActivePlan; pub use self::collection::Collection; +pub use self::object_model::specs::*; pub use self::object_model::ObjectModel; pub use self::reference_glue::ReferenceGlue; pub use self::scanning::Scanning; diff --git a/src/vm/object_model.rs b/src/vm/object_model.rs index b004f02171..b2610d062d 100644 --- a/src/vm/object_model.rs +++ b/src/vm/object_model.rs @@ -1,67 +1,78 @@ use atomic::Ordering; +use self::specs::*; use crate::plan::AllocationSemantics; use crate::plan::CopyContext; -use crate::util::metadata::{header_metadata::HeaderMetadataSpec, MetadataSpec}; +use crate::util::metadata::header_metadata::HeaderMetadataSpec; use crate::util::{Address, ObjectReference}; use crate::vm::VMBinding; /// VM-specific methods for object model. /// -/// MMTk does not require but recommands using in-header per-object metadata for better performance. -/// MMTk requires VMs to announce whether they can provide certain per-object metadata in object headers by overriding the metadata related constants in the ObjectModel trait. +/// This trait includes 3 parts: +/// +/// 1. Specifications for per object metadata: a binding needs to specify the location for each per object metadata spec. +/// A binding can choose between `in_header()` or `side()`, e.g. `VMGlobalLogBitSpec::side()`. +/// * in_header: a binding needs to specify the bit offset to an object reference that can be used for the per object metadata spec. +/// The actual number of bits required for a spec can be obtained from the `num_bits()` method of the spec type. +/// * side: a binding does not need to provide any specific storage for metadata in the header. Instead, MMTk +/// will use side tables to store the metadata. A binding should use the offset from +/// [`GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS`] or [`LOCAL_SIDE_METADATA_VM_BASE_ADDRESS`], and lay out all the side specs one after +/// another (see the following section - Side Specs Layout). +/// 2. In header metadata access: A binding +/// need to further define the functions with suffix _metadata about how to access the bits in the header. A binding may use +/// functions in the [`header_metadata`] module if the bits are always available to MMTk, or they could implement their +/// own routines to access the bits if VM specific treatment is needed (e.g. some bits are not always available to MMTk). +/// 3. VM-specific object info needed by MMTk: MMTk does not know object info as it is VM specific. However, MMTk needs +/// some object information for GC. A binding needs to implement them correctly. /// /// Note that depending on the selected GC plan, only a subset of the methods provided here will be used. +/// +/// Side Specs Layout +/// +/// There are two types of side metadata layout in MMTk: +/// +/// 1. Contiguous layout: is the layout in which the whole metadata space for a SideMetadataSpec is contiguous. +/// 2. Chunked layout: is the layout in which the whole metadata memory space, that is shared between MMTk policies, is divided into metadata-chunks. Each metadata-chunk stores all of the metadata for all `SideMetadataSpec`s which apply to a source-data chunk. +/// +/// In 64-bits targets, both Global and PolicySpecific side metadata are contiguous. +/// Also, in 32-bits targets, the Global side metadata is contiguous. +/// This means if the starting address (variable named `offset`) of the metadata space for a SideMetadataSpec (`SPEC1`) is `BASE1`, the starting address (`offset`) of the next SideMetadataSpec (`SPEC2`) will be `BASE1 + total_metadata_space_size(SPEC1)`, which is located immediately after the end of the whole metadata space of `SPEC1`. +/// Now, if we add a third SideMetadataSpec (`SPEC3`), its starting address (`offset`) will be `BASE2 + total_metadata_space_size(SPEC2)`, which is located immediately after the end of the whole metadata space of `SPEC2`. +/// +/// In 32-bits targets, the PolicySpecific side metadata is chunked. +/// This means for each chunk (2^22 Bytes) of data, which, by definition, is managed by exactly one MMTk policy, there is a metadata chunk (2^22 * some_fixed_ratio Bytes) that contains all of its PolicySpecific metadata. +/// This means if a policy has one SideMetadataSpec (`LS1`), the `offset` of that spec will be `0` (= at the start of a metadata chunk). +/// If there is a second SideMetadataSpec (`LS2`) for this specific policy, the `offset` for that spec will be `0 + required_metadata_space_per_chunk(LS1)`, +/// and for a third SideMetadataSpec (`LS3`), the `offset` will be `BASE(LS2) + required_metadata_space_per_chunk(LS2)`. +/// +/// For all other policies, the `offset` starts from zero. This is safe because no two policies ever manage one chunk, so there will be no overlap. +/// +/// [`HeaderMetadataSpec`]: ../util/metadata/header_metadata/struct.HeaderMetadataSpec.html +/// [`SideMetadataSpec`]: ../util/metadata/side_metadata/strutc.SideMetadataSpec.html +/// [`header_metadata`]: ../util/metadata/header_metadata/index.html +/// [`GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS`]: ../util/metadata/side_metadata/constant.GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.html +/// [`LOCAL_SIDE_METADATA_VM_BASE_ADDRESS`]: ../util/metadata/side_metadata/constant.LOCAL_SIDE_METADATA_VM_BASE_ADDRESS.html pub trait ObjectModel { - // -------------------------------------------------- // Per-object Metadata Spec definitions go here // - // - // NOTE to mmtk binding developers: - // - // A number of Global and PolicySpecific side metadata specifications are already reserved by mmtk-core. - // These are mentioned in their related section as follows. - // - // Any side metadata offset calculation must consider these to prevent overlaps. - // - // - // NOTE to mmtk-core developers: - // - // Adding to the list of reserved side metadata specs must consider the offsets currently being used by mmtk bindings to prevent overlaps. - // - // -------------------------------------------------- + // Note a number of Global and PolicySpecific side metadata specifications are already reserved by mmtk-core. + // Any side metadata offset calculation must consider these to prevent overlaps. A binding should start their + // side metadata from GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS or LOCAL_SIDE_METADATA_VM_BASE_ADDRESS. - // -------------------------------------------------- - // - // Global Metadata - // - // MMTk reserved Global side metadata offsets: - // [currently empty] - // - // -------------------------------------------------- + /// The metadata specification of the global log bit. 1 bit. + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec; - /// The metadata specification of the global log bit. - const GLOBAL_LOG_BIT_SPEC: MetadataSpec; + /// The metadata specification for the forwarding pointer, used by copying plans. Word size. + const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec; + /// The metadata specification for the forwarding status bits, used by copying plans. 2 bits. + const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec; - // -------------------------------------------------- - // PolicySpecific Metadata - // - // MMTk reserved PolicySpecific side metadata offsets: - // - // 1 - MarkSweep Alloc bit: - // - Offset `0x0` on 32-bits - // - Offset `LOCAL_SIDE_METADATA_BASE_ADDRESS` on 64-bits - // - // -------------------------------------------------- + /// The metadata specification for the mark bit, used by most plans that need to mark live objects. 1 bit. + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec; - /// The metadata specification for the forwarding pointer, which is currently specific to the CopySpace policy. - const LOCAL_FORWARDING_POINTER_SPEC: MetadataSpec; - /// The metadata specification for the forwarding status bits, which is currently specific to the CopySpace policy. - const LOCAL_FORWARDING_BITS_SPEC: MetadataSpec; - /// The metadata specification for the mark bit, which is currently specific to the ImmortalSpace policy. - const LOCAL_MARK_BIT_SPEC: MetadataSpec; - /// The metadata specification for the mark-and-nursery bits, which is currently specific to the LargeObjectSpace policy. - const LOCAL_LOS_MARK_NURSERY_SPEC: MetadataSpec; + /// The metadata specification for the mark-and-nursery bits, used by most plans that has large object allocation. 2 bits. + const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec; /// A function to load the specified per-object metadata's content. /// @@ -75,7 +86,7 @@ pub trait ObjectModel { /// # Returns the metadata value as a word. If the metadata size is less than a word, the effective value is stored in the low-order bits of the word. /// fn load_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, mask: Option, atomic_ordering: Option, @@ -92,7 +103,7 @@ pub trait ObjectModel { /// * `atomic_ordering`: is an optional atomic ordering for the store operation. An input value of `None` means the store operation is not atomic, and an input value of `Some(Ordering::X)` means the atomic store operation will use the `Ordering::X`. /// fn store_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, val: usize, mask: Option, @@ -114,7 +125,7 @@ pub trait ObjectModel { /// # Returns `true` if the operation is successful, and `false` otherwise. /// fn compare_exchange_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, old_val: usize, new_val: usize, @@ -135,7 +146,7 @@ pub trait ObjectModel { /// # Returns the old metadata value as a word. /// fn fetch_add_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, val: usize, order: Ordering, @@ -153,7 +164,7 @@ pub trait ObjectModel { /// # Returns the old metadata value as a word. /// fn fetch_sub_metadata( - metadata_spec: HeaderMetadataSpec, + metadata_spec: &HeaderMetadataSpec, object: ObjectReference, val: usize, order: Ordering, @@ -228,3 +239,61 @@ pub trait ObjectModel { /// * `object`: The object to be dumped. fn dump_object(object: ObjectReference); } + +pub mod specs { + use crate::util::constants::LOG_BITS_IN_WORD; + use crate::util::constants::LOG_BYTES_IN_PAGE; + use crate::util::constants::LOG_MIN_OBJECT_SIZE; + use crate::util::metadata::{ + header_metadata::HeaderMetadataSpec, side_metadata::SideMetadataSpec, MetadataSpec, + }; + + macro_rules! define_vm_metadata_spec { + ($spec_name: ident, $log_num_bits: expr, $is_global: expr, $side_min_obj_size: expr) => { + pub struct $spec_name(MetadataSpec); + impl $spec_name { + const LOG_NUM_BITS: usize = $log_num_bits; + const IS_GLOBAL: bool = $is_global; + pub const fn in_header(bit_offset: isize) -> Self { + Self(MetadataSpec::InHeader(HeaderMetadataSpec { + bit_offset, + num_of_bits: 1 << Self::LOG_NUM_BITS, + })) + } + pub const fn side(offset: usize) -> Self { + Self(MetadataSpec::OnSide(SideMetadataSpec { + is_global: Self::IS_GLOBAL, + offset, + log_num_of_bits: Self::LOG_NUM_BITS, + log_min_obj_size: $side_min_obj_size as usize, + })) + } + pub const fn num_bits(&self) -> usize { + 1 << $log_num_bits + } + } + impl std::ops::Deref for $spec_name { + type Target = MetadataSpec; + fn deref(&self) -> &Self::Target { + &self.0 + } + } + }; + } + + // Log bit: 1 bit per object, global + define_vm_metadata_spec!(VMGlobalLogBitSpec, 0, true, LOG_MIN_OBJECT_SIZE); + // Forwarding pointer: word size per object, local + define_vm_metadata_spec!( + VMLocalForwardingPointerSpec, + LOG_BITS_IN_WORD, + false, + LOG_MIN_OBJECT_SIZE + ); + // Forwarding bits: 2 bits per object, local + define_vm_metadata_spec!(VMLocalForwardingBitsSpec, 1, false, LOG_MIN_OBJECT_SIZE); + // Mark bit: 1 bit per object, local + define_vm_metadata_spec!(VMLocalMarkBitSpec, 0, false, LOG_MIN_OBJECT_SIZE); + // Mark&nursery bits for LOS: 2 bit per page, local + define_vm_metadata_spec!(VMLocalLOSMarkNurserySpec, 1, false, LOG_BYTES_IN_PAGE); +} diff --git a/vmbindings/dummyvm/src/object_model.rs b/vmbindings/dummyvm/src/object_model.rs index 2659579ebc..0b48f9aba3 100644 --- a/vmbindings/dummyvm/src/object_model.rs +++ b/vmbindings/dummyvm/src/object_model.rs @@ -1,6 +1,6 @@ -use mmtk::util::metadata::{header_metadata::HeaderMetadataSpec, MetadataSpec}; +use mmtk::util::metadata::header_metadata::HeaderMetadataSpec; use mmtk::util::{Address, ObjectReference}; -use mmtk::vm::ObjectModel; +use mmtk::vm::*; use mmtk::AllocationSemantics; use mmtk::CopyContext; use std::sync::atomic::Ordering; @@ -8,20 +8,15 @@ use DummyVM; pub struct VMObjectModel {} -const DUMMY_METADATA: MetadataSpec = MetadataSpec::InHeader(HeaderMetadataSpec { - bit_offset: 0, - num_of_bits: 0, -}); - impl ObjectModel for VMObjectModel { - const GLOBAL_LOG_BIT_SPEC: MetadataSpec = DUMMY_METADATA; - const LOCAL_FORWARDING_POINTER_SPEC: MetadataSpec = DUMMY_METADATA; - const LOCAL_FORWARDING_BITS_SPEC: MetadataSpec = DUMMY_METADATA; - const LOCAL_MARK_BIT_SPEC: MetadataSpec = DUMMY_METADATA; - const LOCAL_LOS_MARK_NURSERY_SPEC: MetadataSpec = DUMMY_METADATA; + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::in_header(0); + const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = VMLocalForwardingPointerSpec::in_header(0); + const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = VMLocalForwardingBitsSpec::in_header(0); + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::in_header(0); + const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = VMLocalLOSMarkNurserySpec::in_header(0); fn load_metadata( - _metadata_spec: HeaderMetadataSpec, + _metadata_spec: &HeaderMetadataSpec, _object: ObjectReference, _mask: Option, _atomic_ordering: Option, @@ -30,7 +25,7 @@ impl ObjectModel for VMObjectModel { } fn store_metadata( - _metadata_spec: HeaderMetadataSpec, + _metadata_spec: &HeaderMetadataSpec, _object: ObjectReference, _val: usize, _mask: Option, @@ -40,7 +35,7 @@ impl ObjectModel for VMObjectModel { } fn compare_exchange_metadata( - _metadata_spec: HeaderMetadataSpec, + _metadata_spec: &HeaderMetadataSpec, _object: ObjectReference, _old_val: usize, _new_val: usize, @@ -52,7 +47,7 @@ impl ObjectModel for VMObjectModel { } fn fetch_add_metadata( - _metadata_spec: HeaderMetadataSpec, + _metadata_spec: &HeaderMetadataSpec, _object: ObjectReference, _val: usize, _order: Ordering, @@ -61,7 +56,7 @@ impl ObjectModel for VMObjectModel { } fn fetch_sub_metadata( - _metadata_spec: HeaderMetadataSpec, + _metadata_spec: &HeaderMetadataSpec, _object: ObjectReference, _val: usize, _order: Ordering,