From c04aad2b67f4ef170f1a6cdd4018e07797dbf698 Mon Sep 17 00:00:00 2001 From: Andrew Au Date: Wed, 22 May 2024 13:58:24 -0700 Subject: [PATCH] Commit Accounting related changes (#100380) --- src/coreclr/gc/gc.cpp | 531 ++++++++++++++++++++-------------------- src/coreclr/gc/gcpriv.h | 40 ++- 2 files changed, 288 insertions(+), 283 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 3ef9c640c2eef0..ced393436f555b 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2328,7 +2328,9 @@ uint64_t gc_heap::gc_last_ephemeral_decommit_time = 0; CLRCriticalSection gc_heap::check_commit_cs; +#ifdef COMMITTED_BYTES_SHADOW CLRCriticalSection gc_heap::decommit_lock; +#endif //COMMITTED_BYTES_SHADOW size_t gc_heap::current_total_committed = 0; @@ -5992,7 +5994,7 @@ gc_heap::get_segment (size_t size, gc_oh_num oh) return 0; } - result = make_heap_segment ((uint8_t*)mem, size, __this, (uoh_p ? max_generation : 0)); + result = make_heap_segment ((uint8_t*)mem, size, __this, (oh + max_generation)); if (result) { @@ -6018,7 +6020,18 @@ gc_heap::get_segment (size_t size, gc_oh_num oh) if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, uoh_p) != 0) { - virtual_free (mem, size); + // release_segment needs the flags to decrement the proper bucket + size_t flags = 0; + if (oh == poh) + { + flags = heap_segment_flags_poh; + } + else if (oh == loh) + { + flags = heap_segment_flags_loh; + } + result->flags |= flags; + release_segment (result); return 0; } } @@ -6052,7 +6065,19 @@ void gc_heap::release_segment (heap_segment* sg) { ptrdiff_t delta = 0; FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg)); - virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg, sg); + size_t reserved_size = (uint8_t*)heap_segment_reserved (sg) - (uint8_t*)sg; + reduce_committed_bytes ( + sg, + ((uint8_t*)heap_segment_committed (sg) - (uint8_t*)sg), + (int) heap_segment_oh (sg) +#ifdef MULTIPLE_HEAPS + , heap_segment_heap (sg)->heap_number +#else + , -1 +#endif + , true + ); + virtual_free (sg, reserved_size, sg); } BOOL gc_heap::set_ro_segment_in_range (heap_segment* seg) @@ -6969,7 +6994,11 @@ void gc_heap::gc_thread_function () uint32_t wait_time = DECOMMIT_TIME_STEP_MILLISECONDS; #ifdef DYNAMIC_HEAP_COUNT // background_running_p can only change from false to true during suspension. - if (!gc_heap::background_running_p () && dynamic_heap_count_data.should_change_heap_count) + if ( +#ifdef BACKGROUND_GC + !gc_heap::background_running_p () && +#endif + dynamic_heap_count_data.should_change_heap_count) { assert (dynamic_adaptation_mode == dynamic_adaptation_to_application_sizes); @@ -7004,9 +7033,13 @@ void gc_heap::gc_thread_function () if (gradual_decommit_in_progress_p) { +#ifdef COMMITTED_BYTES_SHADOW decommit_lock.Enter (); +#endif //COMMITTED_BYTES_SHADOW gradual_decommit_in_progress_p = decommit_step (DECOMMIT_TIME_STEP_MILLISECONDS); +#ifdef COMMITTED_BYTES_SHADOW decommit_lock.Leave (); +#endif //COMMITTED_BYTES_SHADOW } continue; } @@ -7212,7 +7245,13 @@ void gc_heap::gc_thread_function () // check if we should do some decommitting if (gradual_decommit_in_progress_p) { +#ifdef COMMITTED_BYTES_SHADOW + decommit_lock.Enter (); +#endif //COMMITTED_BYTES_SHADOW gradual_decommit_in_progress_p = decommit_step (DECOMMIT_TIME_STEP_MILLISECONDS); +#ifdef COMMITTED_BYTES_SHADOW + decommit_lock.Leave (); +#endif //COMMITTED_BYTES_SHADOW } } else @@ -7274,13 +7313,15 @@ bool gc_heap::virtual_commit (void* address, size_t size, int bucket, int h_numb assert(0 <= bucket && bucket < recorded_committed_bucket_counts); assert(bucket < total_oh_count || h_number == -1); +#ifdef USE_REGIONS assert(bucket != recorded_committed_free_bucket); +#endif //USE_REGIONS dprintf(3, ("commit-accounting: commit in %d [%p, %p) for heap %d", bucket, address, ((uint8_t*)address + size), h_number)); -#ifndef COMMITTED_BYTES_SHADOW - if (heap_hard_limit) -#endif //!COMMITTED_BYTES_SHADOW +#ifndef USE_REGIONS + if (bucket != recorded_committed_ignored_bucket) +#endif //USE_REGIONS { check_commit_cs.Enter(); bool exceeded_p = false; @@ -7303,20 +7344,19 @@ bool gc_heap::virtual_commit (void* address, size_t size, int bucket, int h_numb exceeded_p = true; } } -#ifdef COMMITTED_BYTES_SHADOW + if (!heap_hard_limit) { exceeded_p = false; } -#endif //COMMITTED_BYTES_SHADOW if (!exceeded_p) { -#if defined(_DEBUG) && defined(MULTIPLE_HEAPS) +#if defined(MULTIPLE_HEAPS) && defined(_DEBUG) if ((h_number != -1) && (bucket < total_oh_count)) { g_heaps[h_number]->committed_by_oh_per_heap[bucket] += size; } -#endif // _DEBUG && MULTIPLE_HEAPS +#endif // MULTIPLE_HEAPS && _DEBUG committed_by_oh[bucket] += size; current_total_committed += size; if (h_number < 0) @@ -7345,13 +7385,13 @@ bool gc_heap::virtual_commit (void* address, size_t size, int bucket, int h_numb { check_commit_cs.Enter(); committed_by_oh[bucket] -= size; -#if defined(_DEBUG) && defined(MULTIPLE_HEAPS) +#if defined(MULTIPLE_HEAPS) && defined(_DEBUG) if ((h_number != -1) && (bucket < total_oh_count)) { assert (g_heaps[h_number]->committed_by_oh_per_heap[bucket] >= size); g_heaps[h_number]->committed_by_oh_per_heap[bucket] -= size; } -#endif // _DEBUG && MULTIPLE_HEAPS +#endif // MULTIPLE_HEAPS && _DEBUG dprintf (1, ("commit failed, updating %zd to %zd", current_total_committed, (current_total_committed - size))); current_total_committed -= size; @@ -7366,41 +7406,28 @@ bool gc_heap::virtual_commit (void* address, size_t size, int bucket, int h_numb return commit_succeeded_p; } -bool gc_heap::virtual_decommit (void* address, size_t size, int bucket, int h_number) +void gc_heap::reduce_committed_bytes (void* address, size_t size, int bucket, int h_number, bool decommit_succeeded_p) { - /** - * Here are all possible cases for the decommits: - * - * Case 1: This is for a particular generation - the bucket will be one of the gc_oh_num != unknown, and the h_number will be the right heap - * Case 2: This is for bookkeeping - the bucket will be recorded_committed_bookkeeping_bucket, and the h_number will be -1 - * Case 3: This is for free - the bucket will be recorded_committed_free_bucket, and the h_number will be -1 - */ -#ifndef HOST_64BIT - assert (heap_hard_limit == 0); -#endif //!HOST_64BIT - assert(0 <= bucket && bucket < recorded_committed_bucket_counts); assert(bucket < total_oh_count || h_number == -1); - bool decommit_succeeded_p = ((bucket != recorded_committed_bookkeeping_bucket) && use_large_pages_p) ? true : GCToOSInterface::VirtualDecommit (address, size); - dprintf(3, ("commit-accounting: decommit in %d [%p, %p) for heap %d", bucket, address, ((uint8_t*)address + size), h_number)); +#ifndef USE_REGIONS + if (bucket != recorded_committed_ignored_bucket) +#endif if (decommit_succeeded_p) -#ifndef COMMITTED_BYTES_SHADOW - if (heap_hard_limit) -#endif //!COMMITTED_BYTES_SHADOW { check_commit_cs.Enter(); assert (committed_by_oh[bucket] >= size); committed_by_oh[bucket] -= size; -#if defined(_DEBUG) && defined(MULTIPLE_HEAPS) +#if defined(MULTIPLE_HEAPS) && defined(_DEBUG) if ((h_number != -1) && (bucket < total_oh_count)) { assert (g_heaps[h_number]->committed_by_oh_per_heap[bucket] >= size); g_heaps[h_number]->committed_by_oh_per_heap[bucket] -= size; } -#endif // _DEBUG && MULTIPLE_HEAPS +#endif // MULTIPLE_HEAPS && _DEBUG assert (current_total_committed >= size); current_total_committed -= size; if (bucket == recorded_committed_bookkeeping_bucket) @@ -7410,6 +7437,24 @@ bool gc_heap::virtual_decommit (void* address, size_t size, int bucket, int h_nu } check_commit_cs.Leave(); } +} + +bool gc_heap::virtual_decommit (void* address, size_t size, int bucket, int h_number) +{ + /** + * Here are all possible cases for the decommits: + * + * Case 1: This is for a particular generation - the bucket will be one of the gc_oh_num != unknown, and the h_number will be the right heap + * Case 2: This is for bookkeeping - the bucket will be recorded_committed_bookkeeping_bucket, and the h_number will be -1 + * Case 3: This is for free - the bucket will be recorded_committed_free_bucket, and the h_number will be -1 + */ +#ifndef HOST_64BIT + assert (heap_hard_limit == 0); +#endif //!HOST_64BIT + + bool decommit_succeeded_p = ((bucket != recorded_committed_bookkeeping_bucket) && use_large_pages_p) ? true : GCToOSInterface::VirtualDecommit (address, size); + + reduce_committed_bytes (address, size, bucket, h_number, decommit_succeeded_p); return decommit_succeeded_p; } @@ -9033,10 +9078,23 @@ void destroy_card_table (uint32_t* c_table) { // delete (uint32_t*)&card_table_refcount(c_table); - GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), card_table_size(c_table)); + size_t size = card_table_size(c_table); + gc_heap::destroy_card_table_helper (c_table); + GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), size); dprintf (2, ("Table Virtual Free : %zx", (size_t)&card_table_refcount(c_table))); } +void gc_heap::destroy_card_table_helper (uint32_t* c_table) +{ + uint8_t* lowest = card_table_lowest_address (c_table); + uint8_t* highest = card_table_highest_address (c_table); + get_card_table_element_layout(lowest, highest, card_table_element_layout); + size_t result = card_table_element_layout[seg_mapping_table_element + 1]; + gc_heap::reduce_committed_bytes (&card_table_refcount(c_table), result, recorded_committed_bookkeeping_bucket, -1, true); + + // If we don't put the mark array committed in the ignored bucket, then this is where to account for the decommit of it +} + void gc_heap::get_card_table_element_sizes (uint8_t* start, uint8_t* end, size_t sizes[total_bookkeeping_elements]) { memset (sizes, 0, sizeof(size_t) * total_bookkeeping_elements); @@ -11760,9 +11818,6 @@ void gc_heap::return_free_region (heap_segment* region) { gc_oh_num oh = heap_segment_oh (region); dprintf(3, ("commit-accounting: from %d to free [%p, %p) for heap %d", oh, get_region_start (region), heap_segment_committed (region), heap_number)); -#ifndef COMMITTED_BYTES_SHADOW - if (heap_hard_limit) -#endif //!COMMITTED_BYTES_SHADOW { size_t committed = heap_segment_committed (region) - get_region_start (region); if (committed > 0) @@ -11771,10 +11826,10 @@ void gc_heap::return_free_region (heap_segment* region) assert (committed_by_oh[oh] >= committed); committed_by_oh[oh] -= committed; committed_by_oh[recorded_committed_free_bucket] += committed; -#if defined(_DEBUG) && defined(MULTIPLE_HEAPS) +#if defined(MULTIPLE_HEAPS) && defined(_DEBUG) assert (committed_by_oh_per_heap[oh] >= committed); committed_by_oh_per_heap[oh] -= committed; -#endif // _DEBUG && MULTIPLE_HEAPS +#endif // MULTIPLE_HEAPS && _DEBUG check_commit_cs.Leave(); } } @@ -11857,9 +11912,6 @@ heap_segment* gc_heap::get_free_region (int gen_number, size_t size) gc_oh_num oh = gen_to_oh (gen_number); dprintf(3, ("commit-accounting: from free to %d [%p, %p) for heap %d", oh, get_region_start (region), heap_segment_committed (region), heap_number)); -#ifndef COMMITTED_BYTES_SHADOW - if (heap_hard_limit) -#endif //!COMMITTED_BYTES_SHADOW { size_t committed = heap_segment_committed (region) - get_region_start (region); if (committed > 0) @@ -11868,9 +11920,9 @@ heap_segment* gc_heap::get_free_region (int gen_number, size_t size) committed_by_oh[oh] += committed; assert (committed_by_oh[recorded_committed_free_bucket] >= committed); committed_by_oh[recorded_committed_free_bucket] -= committed; -#if defined(_DEBUG) && defined(MULTIPLE_HEAPS) +#if defined(MULTIPLE_HEAPS) && defined(_DEBUG) committed_by_oh_per_heap[oh] += committed; -#endif // _DEBUG && MULTIPLE_HEAPS +#endif // MULTIPLE_HEAPS && _DEBUG check_commit_cs.Leave(); } } @@ -14111,13 +14163,10 @@ HRESULT gc_heap::initialize_gc (size_t soh_segment_size, int number_of_heaps = 1; #endif //MULTIPLE_HEAPS -#ifndef COMMITTED_BYTES_SHADOW - if (heap_hard_limit) -#endif //!COMMITTED_BYTES_SHADOW - { - check_commit_cs.Initialize(); - } + check_commit_cs.Initialize(); +#ifdef COMMITTED_BYTES_SHADOW decommit_lock.Initialize(); +#endif //COMMITTED_BYTES_SHADOW #ifdef USE_REGIONS if (regions_range) @@ -14733,7 +14782,7 @@ gc_heap::init_gc_heap (int h_number) #ifdef MULTIPLE_HEAPS #ifdef _DEBUG memset (committed_by_oh_per_heap, 0, sizeof (committed_by_oh_per_heap)); -#endif +#endif //_DEBUG g_heaps [h_number] = this; @@ -22686,6 +22735,7 @@ void gc_heap::gc1() // compute max of gen0_must_clear_bricks over all heaps max_gen0_must_clear_bricks = max(max_gen0_must_clear_bricks, hp->gen0_must_clear_bricks); } + verify_committed_bytes_per_heap (); #ifdef USE_REGIONS initGCShadow(); distribute_free_regions(); @@ -22756,6 +22806,7 @@ void gc_heap::gc1() if (!(settings.concurrent)) { rearrange_uoh_segments(); + verify_committed_bytes_per_heap (); #ifdef USE_REGIONS initGCShadow(); distribute_free_regions(); @@ -24455,22 +24506,16 @@ heap_segment* gc_heap::unlink_first_rw_region (int gen_idx) assert (!heap_segment_read_only_p (region)); dprintf (REGIONS_LOG, ("unlink_first_rw_region on heap: %d gen: %d region: %p", heap_number, gen_idx, heap_segment_mem (region))); -#if defined(_DEBUG) && defined(HOST_64BIT) -#ifndef COMMITTED_BYTES_SHADOW - if (heap_hard_limit) -#endif //!COMMITTED_BYTES_SHADOW + int oh = heap_segment_oh (region); + dprintf(3, ("commit-accounting: from %d to temp [%p, %p) for heap %d", oh, get_region_start (region), heap_segment_committed (region), this->heap_number)); +#ifdef _DEBUG + size_t committed = heap_segment_committed (region) - get_region_start (region); + if (committed > 0) { - int old_oh = heap_segment_oh (region); - int old_heap = heap_segment_heap (region)->heap_number; - dprintf(3, ("commit-accounting: from %d to temp [%p, %p) for heap %d", old_oh, get_region_start (region), heap_segment_committed (region), old_heap)); - - size_t committed = heap_segment_committed (region) - get_region_start (region); - check_commit_cs.Enter(); - assert (g_heaps[old_heap]->committed_by_oh_per_heap[old_oh] >= committed); - g_heaps[old_heap]->committed_by_oh_per_heap[old_oh] -= committed; - check_commit_cs.Leave(); + assert (this->committed_by_oh_per_heap[oh] >= committed); + this->committed_by_oh_per_heap[oh] -= committed; } -#endif // _DEBUG && HOST_64BIT +#endif //_DEBUG set_heap_for_contained_basic_regions (region, nullptr); @@ -24498,22 +24543,13 @@ void gc_heap::thread_rw_region_front (int gen_idx, heap_segment* region) } dprintf (REGIONS_LOG, ("thread_rw_region_front on heap: %d gen: %d region: %p", heap_number, gen_idx, heap_segment_mem (region))); -#if defined(_DEBUG) && defined(HOST_64BIT) -#ifndef COMMITTED_BYTES_SHADOW - if (heap_hard_limit) -#endif //!COMMITTED_BYTES_SHADOW - { - int new_oh = gen_to_oh (gen_idx); - int new_heap = this->heap_number; - dprintf(3, ("commit-accounting: from temp to %d [%p, %p) for heap %d", new_oh, get_region_start (region), heap_segment_committed (region), new_heap)); - - size_t committed = heap_segment_committed (region) - get_region_start (region); - check_commit_cs.Enter(); - assert (heap_segment_heap (region) == nullptr); - g_heaps[new_heap]->committed_by_oh_per_heap[new_oh] += committed; - check_commit_cs.Leave(); - } -#endif // _DEBUG && HOST_64BIT + int oh = heap_segment_oh (region); + dprintf(3, ("commit-accounting: from temp to %d [%p, %p) for heap %d", oh, get_region_start (region), heap_segment_committed (region), this->heap_number)); +#ifdef _DEBUG + size_t committed = heap_segment_committed (region) - get_region_start (region); + assert (heap_segment_heap (region) == nullptr); + this->committed_by_oh_per_heap[oh] += committed; +#endif //_DEBUG set_heap_for_contained_basic_regions (region, this); } @@ -24646,6 +24682,14 @@ void gc_heap::equalize_promoted_bytes(int condemned_gen_number) assert (start_region); dprintf (3, ("making sure heap %d gen %d has at least one region by adding region %zx", start_region)); heap_segment_next (start_region) = nullptr; + + assert (heap_segment_heap (start_region) == nullptr && hp != nullptr); + int oh = heap_segment_oh (start_region); + size_t committed = heap_segment_committed (start_region) - get_region_start (start_region); + dprintf(3, ("commit-accounting: from temp to %d [%p, %p) for heap %d", oh, get_region_start (start_region), heap_segment_committed (start_region), hp->heap_number)); +#ifdef _DEBUG + g_heaps[hp->heap_number]->committed_by_oh_per_heap[oh] += committed; +#endif //_DEBUG set_heap_for_contained_basic_regions (start_region, hp); max_survived = max (max_survived, heap_segment_survived (start_region)); hp->thread_start_region (gen, start_region); @@ -24772,7 +24816,6 @@ void gc_heap::equalize_promoted_bytes(int condemned_gen_number) for (int i = 0; i < n_heaps; i++) { gc_heap* hp = g_heaps[i]; - hp->verify_regions (gen_idx, true, true); } #ifdef TRACE_GC @@ -25906,6 +25949,20 @@ bool gc_heap::change_heap_count (int new_n_heaps) for (heap_segment* region = start_region; region != nullptr; region = heap_segment_next(region)) { + assert ((hp != nullptr) && (hpd != nullptr) && (hp != hpd)); + + int oh = heap_segment_oh (region); + size_t committed = heap_segment_committed (region) - get_region_start (region); + if (committed > 0) + { + dprintf(3, ("commit-accounting: from %d to %d [%p, %p) for heap %d to heap %d", oh, oh, get_region_start (region), heap_segment_committed (region), i, dest_heap_number)); +#ifdef _DEBUG + assert (hp->committed_by_oh_per_heap[oh] >= committed); + hp->committed_by_oh_per_heap[oh] -= committed; + hpd->committed_by_oh_per_heap[oh] += committed; +#endif // _DEBUG + } + set_heap_for_contained_basic_regions (region, hpd); } if (tail_ro_region != nullptr) @@ -33775,62 +33832,9 @@ void gc_heap::plan_phase (int condemned_gen_number) } #endif //FEATURE_EVENT_TRACE -#if defined(_DEBUG) && defined(USE_REGIONS) - if (heap_hard_limit) - { - size_t committed = 0; - for (int i = 0; i < total_generation_count; i++) - { - int oh = i - max_generation; -#ifdef MULTIPLE_HEAPS - for (int hn = 0; hn < gc_heap::n_heaps; hn++) - { - gc_heap* hp = gc_heap::g_heaps[hn]; -#else - { - gc_heap* hp = pGenGCHeap; -#endif // MULTIPLE_HEAPS - heap_segment* region = generation_start_segment (hp->generation_of (i)); - while (region) - { - if (!heap_segment_read_only_p (region)) - { - committed += heap_segment_committed (region) - get_region_start (region); - } - region = heap_segment_next (region); - } -#ifdef BACKGROUND_GC - if (oh == soh) - { - heap_segment* freeable = hp->freeable_soh_segment; - while (freeable) - { - committed += (heap_segment_committed (freeable) - get_region_start (freeable)); - freeable = heap_segment_next (freeable); - } - } - else - { - heap_segment* freeable = hp->freeable_uoh_segment; - while (freeable) - { - if (heap_segment_oh (freeable) == oh) - { - committed += (heap_segment_committed (freeable) - get_region_start (freeable)); - } - freeable = heap_segment_next (freeable); - } - } -#endif //BACKGROUND_GC - } - if (i >= max_generation) - { - assert (committed_by_oh[oh] == committed); - committed = 0; - } - } - } -#endif // _DEBUG && USE_REGIONS +#if defined(_DEBUG) + verify_committed_bytes (); +#endif // _DEBUG #ifdef MULTIPLE_HEAPS //join all threads to make sure they are synchronized @@ -37608,7 +37612,7 @@ BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t size)); #endif //SIMPLE_DPRINTF - if (virtual_commit (commit_start, size, recorded_committed_bookkeeping_bucket)) + if (virtual_commit (commit_start, size, recorded_committed_mark_array_bucket)) { // We can only verify the mark array is cleared from begin to end, the first and the last // page aren't necessarily all cleared 'cause they could be used by other segments or @@ -37833,7 +37837,7 @@ void gc_heap::decommit_mark_array_by_seg (heap_segment* seg) if (decommit_start < decommit_end) { - if (!virtual_decommit (decommit_start, size, recorded_committed_bookkeeping_bucket)) + if (!virtual_decommit (decommit_start, size, recorded_committed_mark_array_bucket)) { dprintf (GC_TABLE_LOG, ("decommit on %p for %zd bytes failed", decommit_start, size)); @@ -47349,9 +47353,52 @@ gc_heap::verify_free_lists () } } -void gc_heap::verify_regions (int gen_number, bool can_verify_gen_num, bool can_verify_tail, size_t* p_total_committed) +void gc_heap::verify_committed_bytes_per_heap() +{ + size_t committed_bookkeeping = 0; // unused + for (int oh = soh; oh < total_oh_count; oh++) + { +#ifdef MULTIPLE_HEAPS + assert (committed_by_oh_per_heap[oh] == compute_committed_bytes_per_heap (oh, committed_bookkeeping)); +#else + assert (committed_by_oh[oh] == compute_committed_bytes_per_heap (oh, committed_bookkeeping)); +#endif //MULTIPLE_HEAPS + } +} + +void gc_heap::verify_committed_bytes() { + size_t total_committed = 0; + size_t committed_decommit; // unused + size_t committed_free; // unused + size_t committed_bookkeeping = 0; + size_t new_current_total_committed; + size_t new_current_total_committed_bookkeeping; + size_t new_committed_by_oh[recorded_committed_bucket_counts]; + compute_committed_bytes(total_committed, committed_decommit, committed_free, + committed_bookkeeping, new_current_total_committed, new_current_total_committed_bookkeeping, + new_committed_by_oh); +#ifdef MULTIPLE_HEAPS + for (int h = 0; h < n_heaps; h++) + { + for (int oh = soh; oh < total_oh_count; oh++) + { + assert (g_heaps[h]->committed_by_oh_per_heap[oh] == g_heaps[h]->committed_by_oh_per_heap_refresh[oh]); + } + } + for (int i = 0; i < recorded_committed_bucket_counts; i++) + { + assert (new_committed_by_oh[i] == committed_by_oh[i]); + } +#endif //MULTIPLE_HEAPS + assert (new_current_total_committed_bookkeeping == current_total_committed_bookkeeping); + assert (new_current_total_committed == current_total_committed); +} + #ifdef USE_REGIONS +void gc_heap::verify_regions (int gen_number, bool can_verify_gen_num, bool can_verify_tail) +{ +#ifdef _DEBUG // For the given generation, verify that // // 1) it has at least one region. @@ -47368,13 +47415,6 @@ void gc_heap::verify_regions (int gen_number, bool can_verify_gen_num, bool can_ while (seg_in_gen) { - if (p_total_committed) - { - if (!heap_segment_read_only_p (seg_in_gen)) - { - *p_total_committed += (heap_segment_committed (seg_in_gen) - get_region_start (seg_in_gen)); - } - } if (can_verify_gen_num) { if (heap_segment_gen_num (seg_in_gen) != min (gen_number, (int)max_generation)) @@ -47427,7 +47467,7 @@ void gc_heap::verify_regions (int gen_number, bool can_verify_gen_num, bool can_ prev_region_in_gen, heap_segment_mem (prev_region_in_gen))); FATAL_GC_ERROR(); } -#endif //USE_REGIONS +#endif // _DEBUG } inline bool is_user_alloc_gen (int gen_number) @@ -47437,61 +47477,22 @@ inline bool is_user_alloc_gen (int gen_number) void gc_heap::verify_regions (bool can_verify_gen_num, bool concurrent_p) { -#ifdef USE_REGIONS - size_t total_committed = 0; +#ifdef _DEBUG for (int i = 0; i < total_generation_count; i++) { bool can_verify_tail = (concurrent_p ? !is_user_alloc_gen (i) : true); - verify_regions (i, can_verify_gen_num, can_verify_tail, &total_committed); + verify_regions (i, can_verify_gen_num, can_verify_tail); if (can_verify_gen_num && can_verify_tail && - (i >= max_generation) && - heap_hard_limit) + (i >= max_generation)) { - int oh = i - max_generation; -#ifdef BACKGROUND_GC - if (oh == soh) - { - heap_segment* freeable = freeable_soh_segment; - while (freeable) - { - total_committed += (heap_segment_committed (freeable) - get_region_start (freeable)); - freeable = heap_segment_next (freeable); - } - } - else - { - heap_segment* freeable = freeable_uoh_segment; - while (freeable) - { - if (heap_segment_oh (freeable) == oh) - { - total_committed += (heap_segment_committed (freeable) - get_region_start (freeable)); - } - freeable = heap_segment_next (freeable); - } - } -#endif //BACKGROUND_GC -#ifdef MULTIPLE_HEAPS -#ifdef _DEBUG - size_t total_accounted = committed_by_oh_per_heap[i - max_generation]; -#else // _DEBUG - size_t total_accounted = total_committed; -#endif // _DEBUG -#else // MULTIPLE_HEAPS - size_t total_accounted = committed_by_oh[i - max_generation]; -#endif // MULTIPLE_HEAPS - if (total_committed != total_accounted) - { - FATAL_GC_ERROR(); - } - dprintf(3, ("commit-accounting: checkpoint for %d for heap %d", oh, heap_number)); - total_committed = 0; + verify_committed_bytes_per_heap (); } } -#endif //USE_REGIONS +#endif // _DEBUG } +#endif // USE_REGIONS BOOL gc_heap::check_need_card (uint8_t* child_obj, int gen_num_for_cards, uint8_t* low, uint8_t* high) @@ -47612,6 +47613,7 @@ void gc_heap::verify_heap (BOOL begin_gc_p) //verify that the generation structures makes sense { +#ifdef _DEBUG #ifdef USE_REGIONS verify_regions (true, settings.concurrent); #else //USE_REGIONS @@ -47638,6 +47640,7 @@ void gc_heap::verify_heap (BOOL begin_gc_p) gen_num--; } #endif //USE_REGIONS +#endif //_DEBUG } size_t total_objects_verified = 0; @@ -52661,7 +52664,35 @@ bool gc_heap::compute_memory_settings(bool is_initialization, uint32_t& nhp, uin return true; } +size_t gc_heap::compute_committed_bytes_per_heap(int oh, size_t& committed_bookkeeping) +{ #ifdef USE_REGIONS + int start_generation = (oh == 0) ? 0 : oh + max_generation; +#else + int start_generation = oh + max_generation; +#endif + int end_generation = oh + max_generation; + + size_t total_committed_per_heap = 0; + for (int gen = start_generation; gen <= end_generation; gen++) + { + accumulate_committed_bytes (generation_start_segment (generation_of (gen)), total_committed_per_heap, committed_bookkeeping); + } + +#ifdef BACKGROUND_GC + if (oh == soh) + { + accumulate_committed_bytes (freeable_soh_segment, total_committed_per_heap, committed_bookkeeping); + } + else +#endif //BACKGROUND_GC + { + accumulate_committed_bytes (freeable_uoh_segment, total_committed_per_heap, committed_bookkeeping, (gc_oh_num)oh); + } + + return total_committed_per_heap; +} + void gc_heap::compute_committed_bytes(size_t& total_committed, size_t& committed_decommit, size_t& committed_free, size_t& committed_bookkeeping, size_t& new_current_total_committed, size_t& new_current_total_committed_bookkeeping, size_t* new_committed_by_oh) @@ -52669,8 +52700,6 @@ void gc_heap::compute_committed_bytes(size_t& total_committed, size_t& committed // Accounting for the bytes committed for the regions for (int oh = soh; oh < total_oh_count; oh++) { - int start_generation = (oh == 0) ? 0 : oh + max_generation; - int end_generation = oh + max_generation; size_t total_committed_per_oh = 0; #ifdef MULTIPLE_HEAPS for (int h = 0; h < n_heaps; h++) @@ -52680,31 +52709,17 @@ void gc_heap::compute_committed_bytes(size_t& total_committed, size_t& committed { gc_heap* heap = pGenGCHeap; #endif //MULTIPLE_HEAPS - size_t total_committed_per_heap = 0; - for (int gen = start_generation; gen <= end_generation; gen++) - { - heap->accumulate_committed_bytes ( generation_start_segment (heap->generation_of (gen)), total_committed_per_heap, committed_bookkeeping); - } - -#ifdef BACKGROUND_GC - if (oh == soh) - { - heap->accumulate_committed_bytes (heap->freeable_soh_segment, total_committed_per_heap, committed_bookkeeping); - } - else -#endif //BACKGROUND_GC - { - heap->accumulate_committed_bytes (heap->freeable_uoh_segment, total_committed_per_heap, committed_bookkeeping, (gc_oh_num)oh); - } + size_t total_committed_per_heap = heap->compute_committed_bytes_per_heap (oh, committed_bookkeeping); #if defined(MULTIPLE_HEAPS) && defined(_DEBUG) heap->committed_by_oh_per_heap_refresh[oh] = total_committed_per_heap; -#endif //MULTIPLE_HEAPS && _DEBUG +#endif // MULTIPLE_HEAPS && _DEBUG total_committed_per_oh += total_committed_per_heap; } new_committed_by_oh[oh] = total_committed_per_oh; total_committed += total_committed_per_oh; } +#ifdef USE_REGIONS // Accounting for the bytes committed for the free lists size_t committed_old_free = 0; committed_free = 0; @@ -52765,10 +52780,25 @@ void gc_heap::compute_committed_bytes(size_t& total_committed, size_t& committed new_current_total_committed_bookkeeping = committed_bookkeeping; new_committed_by_oh[recorded_committed_bookkeeping_bucket] = committed_bookkeeping; +#else + new_committed_by_oh[recorded_committed_ignored_bucket] = committed_free = 0; + + uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))]; + while (ct) + { + uint8_t* lowest = card_table_lowest_address (ct); + uint8_t* highest = card_table_highest_address (ct); + get_card_table_element_layout(lowest, highest, card_table_element_layout); + size_t result = card_table_element_layout[seg_mapping_table_element + 1]; + committed_bookkeeping += result; + ct = card_table_next (ct); + } + // If we don't put the mark array committed in the ignored bucket, calculate the committed memory for mark array here + new_committed_by_oh[recorded_committed_bookkeeping_bucket] = new_current_total_committed_bookkeeping = committed_bookkeeping; +#endif //USE_REGIONS total_committed += committed_bookkeeping; new_current_total_committed = total_committed; } -#endif //USE_REGIONS int gc_heap::refresh_memory_limit() { @@ -52781,19 +52811,10 @@ int gc_heap::refresh_memory_limit() GCToEEInterface::SuspendEE(SUSPEND_FOR_GC); -#ifdef USE_REGIONS + +#ifdef COMMITTED_BYTES_SHADOW decommit_lock.Enter(); - size_t total_committed = 0; - size_t committed_decommit; // unused - size_t committed_free; // unused - size_t committed_bookkeeping = 0; - size_t new_current_total_committed; - size_t new_current_total_committed_bookkeeping; - size_t new_committed_by_oh[recorded_committed_bucket_counts]; - compute_committed_bytes(total_committed, committed_decommit, committed_free, - committed_bookkeeping, new_current_total_committed, new_current_total_committed_bookkeeping, - new_committed_by_oh); -#endif //USE_REGIONS +#endif //COMMITTED_BYTES_SHADOW uint32_t nhp_from_config = static_cast(GCConfig::GetHeapCount()); #ifdef MULTIPLE_HEAPS @@ -52828,7 +52849,7 @@ int gc_heap::refresh_memory_limit() size_t new_current_total_committed = 0; #endif //USE_REGIONS - if (succeed && !compute_memory_settings(false, nhp, nhp_from_config, seg_size_from_config, new_current_total_committed)) + if (succeed && !compute_memory_settings(false, nhp, nhp_from_config, seg_size_from_config, current_total_committed)) { succeed = false; status = refresh_hard_limit_too_low; @@ -52844,59 +52865,21 @@ int gc_heap::refresh_memory_limit() heap_hard_limit_oh[poh] = old_heap_hard_limit_poh; hard_limit_config_p = old_hard_limit_config_p; } +#ifdef COMMITTED_BYTES_SHADOW else -#ifndef COMMITTED_BYTES_SHADOW - if (!old_heap_hard_limit && heap_hard_limit) -#endif //COMMITTED_BYTES_SHADOW { -#ifdef USE_REGIONS - check_commit_cs.Initialize(); -#ifdef COMMITTED_BYTES_SHADOW - assert (new_current_total_committed == current_total_committed); - assert (new_current_total_committed_bookkeeping == current_total_committed_bookkeeping); -#else - current_total_committed = new_current_total_committed; - current_total_committed_bookkeeping = new_current_total_committed_bookkeeping; -#endif - for (int i = 0; i < recorded_committed_bucket_counts; i++) - { -#ifdef COMMITTED_BYTES_SHADOW - assert (new_committed_by_oh[i] == committed_by_oh[i]); -#else - committed_by_oh[i] = new_committed_by_oh[i]; -#endif - } -#ifdef MULTIPLE_HEAPS -#ifdef _DEBUG - for (int h = 0; h < n_heaps; h++) - { - for (int oh = soh; oh < total_oh_count; oh++) - { -#ifdef COMMITTED_BYTES_SHADOW - assert (g_heaps[h]->committed_by_oh_per_heap[oh] == g_heaps[h]->committed_by_oh_per_heap_refresh[oh]); -#else - g_heaps[h]->committed_by_oh_per_heap[oh] = g_heaps[h]->committed_by_oh_per_heap_refresh[oh]; -#endif - } - } -#endif //_DEBUG -#endif //MULTIPLE_HEAPS -#else - assert (!"NYI - Segments"); -#endif //USE_REGIONS + verify_committed_bytes (); } +#endif //COMMITTED_BYTES_SHADOW - -#ifdef USE_REGIONS + GCToEEInterface::RestartEE(TRUE); +#ifdef COMMITTED_BYTES_SHADOW decommit_lock.Leave(); #endif - GCToEEInterface::RestartEE(TRUE); return (int)status; } -#ifdef USE_REGIONS - void gc_heap::accumulate_committed_bytes(heap_segment* seg, size_t& committed_bytes, size_t& mark_array_committed_bytes, gc_oh_num oh) { seg = heap_segment_rw (seg); @@ -52904,13 +52887,21 @@ void gc_heap::accumulate_committed_bytes(heap_segment* seg, size_t& committed_by { if ((oh == unknown) || (heap_segment_oh (seg) == oh)) { + uint8_t* start; +#ifdef USE_REGIONS mark_array_committed_bytes += get_mark_array_size (seg); - committed_bytes += (heap_segment_committed (seg) - get_region_start (seg)); + start = get_region_start (seg); +#else + start = (uint8_t*)seg; +#endif + committed_bytes += (heap_segment_committed (seg) - start); } seg = heap_segment_next_rw (seg); } } +#ifdef USE_REGIONS + size_t gc_heap::get_mark_array_size (heap_segment* seg) { #ifdef BACKGROUND_GC diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index e69826e96b71ce..da6c2f17c07747 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -494,8 +494,15 @@ enum gc_oh_num }; const int total_oh_count = gc_oh_num::poh + 1; +#ifdef USE_REGIONS const int recorded_committed_free_bucket = total_oh_count; const int recorded_committed_bookkeeping_bucket = recorded_committed_free_bucket + 1; +const int recorded_committed_mark_array_bucket = recorded_committed_bookkeeping_bucket; +#else +const int recorded_committed_ignored_bucket = total_oh_count; +const int recorded_committed_bookkeeping_bucket = recorded_committed_ignored_bucket + 1; +const int recorded_committed_mark_array_bucket = recorded_committed_ignored_bucket; +#endif //USE_REGIONS const int recorded_committed_bucket_counts = recorded_committed_bookkeeping_bucket + 1; gc_oh_num gen_to_oh (int gen); @@ -1563,8 +1570,10 @@ class gc_heap #ifdef VERIFY_HEAP PER_HEAP_METHOD void verify_free_lists(); - PER_HEAP_METHOD void verify_regions (int gen_number, bool can_verify_gen_num, bool can_verify_tail, size_t* p_total_committed = nullptr); +#if defined (USE_REGIONS) + PER_HEAP_METHOD void verify_regions (int gen_number, bool can_verify_gen_num, bool can_verify_tail); PER_HEAP_METHOD void verify_regions (bool can_verify_gen_num, bool concurrent_p); +#endif //USE_REGIONS PER_HEAP_ISOLATED_METHOD void enter_gc_lock_for_verify_heap(); PER_HEAP_ISOLATED_METHOD void leave_gc_lock_for_verify_heap(); PER_HEAP_METHOD void verify_heap (BOOL begin_gc_p); @@ -1572,6 +1581,9 @@ class gc_heap uint8_t* low, uint8_t* high); #endif //VERIFY_HEAP + PER_HEAP_METHOD void verify_committed_bytes_per_heap(); + PER_HEAP_ISOLATED_METHOD void verify_committed_bytes(); + PER_HEAP_ISOLATED_METHOD void fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num); PER_HEAP_ISOLATED_METHOD void fire_pevents(); @@ -1667,9 +1679,6 @@ class gc_heap // Compute the size committed for the mark array for this region. PER_HEAP_METHOD size_t get_mark_array_size(heap_segment* seg); - // Accumulate the committed bytes for both the region and the mark array for this list of regions. - PER_HEAP_METHOD void accumulate_committed_bytes(heap_segment* seg, size_t& committed_bytes, size_t& mark_array_committed_bytes, gc_oh_num oh = unknown); - PER_HEAP_ISOLATED_METHOD void verify_region_to_generation_map(); PER_HEAP_ISOLATED_METHOD void compute_gc_and_ephemeral_range (int condemned_gen_number, bool end_of_gc_p); @@ -1678,6 +1687,9 @@ class gc_heap #endif //STRESS_REGIONS #endif //USE_REGIONS + // Accumulate the committed bytes for both the region and the mark array for this list of regions. + PER_HEAP_METHOD void accumulate_committed_bytes(heap_segment* seg, size_t& committed_bytes, size_t& mark_array_committed_bytes, gc_oh_num oh = unknown); + PER_HEAP_ISOLATED_METHOD gc_heap* make_gc_heap( #if defined (MULTIPLE_HEAPS) GCHeap* vm_heap, @@ -2378,6 +2390,9 @@ class gc_heap PER_HEAP_ISOLATED_METHOD bool virtual_alloc_commit_for_heap (void* addr, size_t size, int h_number); PER_HEAP_ISOLATED_METHOD bool virtual_commit (void* address, size_t size, int bucket, int h_number=-1, bool* hard_limit_exceeded_p=NULL); PER_HEAP_ISOLATED_METHOD bool virtual_decommit (void* address, size_t size, int bucket, int h_number=-1); + PER_HEAP_ISOLATED_METHOD void reduce_committed_bytes (void* address, size_t size, int bucket, int h_number, bool decommit_succeeded_p); + friend void destroy_card_table (uint32_t*); + PER_HEAP_ISOLATED_METHOD void destroy_card_table_helper (uint32_t* c_table); PER_HEAP_ISOLATED_METHOD void virtual_free (void* add, size_t size, heap_segment* sg=NULL); PER_HEAP_ISOLATED_METHOD void reset_memory(uint8_t* o, size_t sizeo); PER_HEAP_METHOD void clear_gen0_bricks(); @@ -3349,11 +3364,11 @@ class gc_heap PER_HEAP_ISOLATED_METHOD bool compute_memory_settings(bool is_initialization, uint32_t& nhp, uint32_t nhp_from_config, size_t& seg_size_from_config, size_t new_current_total_committed); -#ifdef USE_REGIONS - PER_HEAP_ISOLATED_METHOD void compute_committed_bytes(size_t& total_committed, size_t& committed_decommit, size_t& committed_free, - size_t& committed_bookkeeping, size_t& new_current_total_committed, size_t& new_current_total_committed_bookkeeping, + PER_HEAP_METHOD size_t compute_committed_bytes_per_heap(int oh, size_t& committed_bookkeeping); + + PER_HEAP_ISOLATED_METHOD void compute_committed_bytes(size_t& total_committed, size_t& committed_decommit, size_t& committed_free, + size_t& committed_bookkeeping, size_t& new_current_total_committed, size_t& new_current_total_committed_bookkeeping, size_t* new_committed_by_oh); -#endif PER_HEAP_METHOD void update_collection_counts (); @@ -3892,13 +3907,10 @@ class gc_heap PER_HEAP_FIELD_DIAG_ONLY int gchist_index_per_heap; PER_HEAP_FIELD_DIAG_ONLY gc_history gchist_per_heap[max_history_count]; -#ifdef MULTIPLE_HEAPS -#ifdef _DEBUG +#if defined(MULTIPLE_HEAPS) && defined(_DEBUG) PER_HEAP_FIELD_DIAG_ONLY size_t committed_by_oh_per_heap[total_oh_count]; PER_HEAP_FIELD_DIAG_ONLY size_t committed_by_oh_per_heap_refresh[total_oh_count]; -#endif //_DEBUG -#else //MULTIPLE_HEAPS -#endif //MULTIPLE_HEAPS +#endif // MULTIPLE_HEAPS && _DEBUG #ifdef BACKGROUND_GC PER_HEAP_FIELD_DIAG_ONLY gc_history_per_heap bgc_data_per_heap; @@ -5093,7 +5105,9 @@ class gc_heap // Used both in a GC and on the allocator code paths when heap_hard_limit is non zero PER_HEAP_ISOLATED_FIELD_INIT_ONLY CLRCriticalSection check_commit_cs; +#ifdef COMMITTED_BYTES_SHADOW PER_HEAP_ISOLATED_FIELD_INIT_ONLY CLRCriticalSection decommit_lock; +#endif // Indicate to use large pages. This only works if hardlimit is also enabled. PER_HEAP_ISOLATED_FIELD_INIT_ONLY bool use_large_pages_p;