From 75734b3a6f6725d05015a12b9b110ff011cec5f3 Mon Sep 17 00:00:00 2001 From: Oscar Smith Date: Sun, 23 Jul 2023 09:11:50 -0400 Subject: [PATCH] Page based heap size heuristics (#50144) This PR implements GC heuristics based on the amount of pages allocated instead of live objects like was done before. The heuristic for new heap target is based on https://dl.acm.org/doi/10.1145/3563323 (in summary it argues that the heap target should have square root behaviour). From my testing this fixes https://github.com/JuliaLang/julia/issues/49545 and https://github.com/JuliaLang/julia/issues/49761 (cherry picked from commit 32aa29f561a3dcaffad7b2595928a4aa41c51b81) --- NEWS.md | 1 + doc/src/devdocs/gc.md | 12 +- src/gc-debug.c | 17 ++- src/gc-pages.c | 4 + src/gc.c | 256 ++++++++++++++++++++++---------------- src/gc.h | 11 +- src/jl_exported_funcs.inc | 1 + src/julia_threads.h | 4 +- test/testenv.jl | 9 ++ 9 files changed, 203 insertions(+), 112 deletions(-) diff --git a/NEWS.md b/NEWS.md index 81f744158d58e..112b229c3cf7e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -31,6 +31,7 @@ Language changes Compiler/Runtime improvements ----------------------------- +* Updated GC heuristics to count allocated pages instead of individual objects ([#50144]). * The `@pure` macro is now deprecated. Use `Base.@assume_effects :foldable` instead ([#48682]). * The mark phase of the Garbage Collector is now multi-threaded ([#48600]). diff --git a/doc/src/devdocs/gc.md b/doc/src/devdocs/gc.md index c072912e77c3f..942535f426b34 100644 --- a/doc/src/devdocs/gc.md +++ b/doc/src/devdocs/gc.md @@ -67,6 +67,12 @@ This scheme eliminates the need of explicitly keeping a flag to indicate a full ## Heuristics GC heuristics tune the GC by changing the size of the allocation interval between garbage collections. -If a GC was unproductive, then we increase the size of the allocation interval to allow objects more time to die. -If a GC returns a lot of space we can shrink the interval. The goal is to find a steady state where we are -allocating just about the same amount as we are collecting. + +The GC heuristics measure how big the heap size is after a collection and set the next +collection according to the algorithm described by https://dl.acm.org/doi/10.1145/3563323, +in summary, it argues that the heap target should have a square root relationship with the live heap, and that it should also be scaled by how fast the GC is freeing objects and how fast the mutators are allocating. +The heuristics measure the heap size by counting the number of pages that are in use and the objects that use malloc. Previously we measured the heap size by counting +the alive objects, but that doesn't take into account fragmentation which could lead to bad decisions, that also meant that we used thread local information (allocations) to make +decisions about a process wide (when to GC), measuring pages means the decision is global. + +The GC will do full collections when the heap size reaches 80% of the maximum allowed size. diff --git a/src/gc-debug.c b/src/gc-debug.c index bab2c5b0fa607..6e1587b17a6d9 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -1,7 +1,10 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "gc.h" +#include "julia.h" #include +#include +#include #include // re-include assert.h without NDEBUG, @@ -1216,15 +1219,25 @@ JL_DLLEXPORT void jl_enable_gc_logging(int enable) { gc_logging_enabled = enable; } -void _report_gc_finished(uint64_t pause, uint64_t freed, int full, int recollect) JL_NOTSAFEPOINT { +void _report_gc_finished(uint64_t pause, uint64_t freed, int full, int recollect, int64_t live_bytes) JL_NOTSAFEPOINT { if (!gc_logging_enabled) { return; } jl_safe_printf("GC: pause %.2fms. collected %fMB. %s %s\n", - pause/1e6, freed/1e6, + pause/1e6, freed/(double)(1<<20), full ? "full" : "incr", recollect ? "recollect" : "" ); + + jl_safe_printf("Heap stats: bytes_mapped %.2f MB, bytes_resident %.2f MB, heap_size %.2f MB, heap_target %.2f MB, live_bytes %.2f MB\n, Fragmentation %.3f", + jl_atomic_load_relaxed(&gc_heap_stats.bytes_mapped)/(double)(1<<20), + jl_atomic_load_relaxed(&gc_heap_stats.bytes_resident)/(double)(1<<20), + jl_atomic_load_relaxed(&gc_heap_stats.heap_size)/(double)(1<<20), + jl_atomic_load_relaxed(&gc_heap_stats.heap_target)/(double)(1<<20), + live_bytes/(double)(1<<20), + (double)live_bytes/(double)jl_atomic_load_relaxed(&gc_heap_stats.heap_size) + ); + // Should fragmentation use bytes_resident instead of heap_size? } #ifdef __cplusplus diff --git a/src/gc-pages.c b/src/gc-pages.c index 682e76611f5d9..8d596f4a815ca 100644 --- a/src/gc-pages.c +++ b/src/gc-pages.c @@ -52,6 +52,8 @@ char *jl_gc_try_alloc_pages_(int pg_cnt) JL_NOTSAFEPOINT // round data pointer up to the nearest gc_page_data-aligned // boundary if mmap didn't already do so. mem = (char*)gc_page_data(mem + GC_PAGE_SZ - 1); + jl_atomic_fetch_add_relaxed(&gc_heap_stats.bytes_mapped, pages_sz); + jl_atomic_fetch_add_relaxed(&gc_heap_stats.bytes_resident, pages_sz); return mem; } @@ -115,6 +117,7 @@ NOINLINE jl_gc_pagemeta_t *jl_gc_alloc_page(void) JL_NOTSAFEPOINT // try to get page from `pool_freed` meta = pop_lf_page_metadata_back(&global_page_pool_freed); if (meta != NULL) { + jl_atomic_fetch_add_relaxed(&gc_heap_stats.bytes_resident, GC_PAGE_SZ); gc_alloc_map_set(meta->data, GC_PAGE_ALLOCATED); goto exit; } @@ -188,6 +191,7 @@ void jl_gc_free_page(jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT madvise(p, decommit_size, MADV_DONTNEED); #endif msan_unpoison(p, decommit_size); + jl_atomic_fetch_add_relaxed(&gc_heap_stats.bytes_resident, -decommit_size); } #ifdef __cplusplus diff --git a/src/gc.c b/src/gc.c index 0be48081d5687..41c0baf492507 100644 --- a/src/gc.c +++ b/src/gc.c @@ -42,6 +42,8 @@ static jl_gc_callback_list_t *gc_cblist_pre_gc; static jl_gc_callback_list_t *gc_cblist_post_gc; static jl_gc_callback_list_t *gc_cblist_notify_external_alloc; static jl_gc_callback_list_t *gc_cblist_notify_external_free; +static jl_gc_callback_list_t *gc_cblist_notify_gc_pressure; +typedef void (*jl_gc_cb_notify_gc_pressure_t)(void); #define gc_invoke_callbacks(ty, list, args) \ do { \ @@ -128,6 +130,14 @@ JL_DLLEXPORT void jl_gc_set_cb_notify_external_free(jl_gc_cb_notify_external_fre jl_gc_deregister_callback(&gc_cblist_notify_external_free, (jl_gc_cb_func_t)cb); } +JL_DLLEXPORT void jl_gc_set_cb_notify_gc_pressure(jl_gc_cb_notify_gc_pressure_t cb, int enable) +{ + if (enable) + jl_gc_register_callback(&gc_cblist_notify_gc_pressure, (jl_gc_cb_func_t)cb); + else + jl_gc_deregister_callback(&gc_cblist_notify_gc_pressure, (jl_gc_cb_func_t)cb); +} + // Protect all access to `finalizer_list_marked` and `to_finalize`. // For accessing `ptls->finalizers`, the lock is needed if a thread // is going to realloc the buffer (of its own list) or accessing the @@ -178,6 +188,8 @@ jl_gc_num_t gc_num = {0}; static size_t last_long_collect_interval; int gc_n_threads; jl_ptls_t* gc_all_tls_states; +gc_heapstatus_t gc_heap_stats = {0}; +int next_sweep_full = 0; const uint64_t _jl_buff_tag[3] = {0x4eadc0004eadc000ull, 0x4eadc0004eadc000ull, 0x4eadc0004eadc000ull}; // aka 0xHEADER00 JL_DLLEXPORT uintptr_t jl_get_buff_tag(void) { @@ -665,19 +677,26 @@ static int64_t last_gc_total_bytes = 0; #ifdef _P64 typedef uint64_t memsize_t; static const size_t default_collect_interval = 5600 * 1024 * sizeof(void*); -static const size_t max_collect_interval = 1250000000UL; static size_t total_mem; // We expose this to the user/ci as jl_gc_set_max_memory static memsize_t max_total_memory = (memsize_t) 2 * 1024 * 1024 * 1024 * 1024 * 1024; #else typedef uint32_t memsize_t; static const size_t default_collect_interval = 3200 * 1024 * sizeof(void*); -static const size_t max_collect_interval = 500000000UL; // Work really hard to stay within 2GB // Alternative is to risk running out of address space // on 32 bit architectures. -static memsize_t max_total_memory = (memsize_t) 2 * 1024 * 1024 * 1024; +#define MAX32HEAP 1536 * 1024 * 1024 +static memsize_t max_total_memory = (memsize_t) MAX32HEAP; #endif +// heuristic stuff for https://dl.acm.org/doi/10.1145/3563323 +static uint64_t old_pause_time = 0; +static uint64_t old_mut_time = 0; +static uint64_t old_heap_size = 0; +static uint64_t old_alloc_diff = 0; +static uint64_t old_freed_diff = 0; +static uint64_t gc_end_time = 0; + // global variables for GC stats @@ -730,6 +749,7 @@ static int mark_reset_age = 0; static int64_t scanned_bytes; // young bytes scanned while marking static int64_t perm_scanned_bytes; // old bytes scanned while marking int prev_sweep_full = 1; +int under_pressure = 0; // Full collection heuristics static int64_t live_bytes = 0; @@ -916,7 +936,7 @@ void gc_setmark_buf(jl_ptls_t ptls, void *o, uint8_t mark_mode, size_t minsz) JL STATIC_INLINE void maybe_collect(jl_ptls_t ptls) { - if (jl_atomic_load_relaxed(&ptls->gc_num.allocd) >= 0 || jl_gc_debug_check_other()) { + if (jl_atomic_load_relaxed(&gc_heap_stats.heap_size) >= jl_atomic_load_relaxed(&gc_heap_stats.heap_target) || jl_gc_debug_check_other()) { jl_gc_collect(JL_GC_AUTO); } else { @@ -1005,6 +1025,13 @@ STATIC_INLINE jl_value_t *jl_gc_big_alloc_inner(jl_ptls_t ptls, size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + allocsz); jl_atomic_store_relaxed(&ptls->gc_num.bigalloc, jl_atomic_load_relaxed(&ptls->gc_num.bigalloc) + 1); + uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); + if (alloc_acc + allocsz < 16*1024) + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + allocsz); + else { + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + allocsz); + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); + } #ifdef MEMDEBUG memset(v, 0xee, allocsz); #endif @@ -1050,6 +1077,8 @@ static bigval_t **sweep_big_list(int sweep_full, bigval_t **pv) JL_NOTSAFEPOINT if (nxt) nxt->prev = pv; gc_num.freed += v->sz&~3; + jl_atomic_store_relaxed(&gc_heap_stats.heap_size, + jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - (v->sz&~3)); #ifdef MEMDEBUG memset(v, 0xbb, v->sz&~3); #endif @@ -1109,6 +1138,13 @@ void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT jl_ptls_t ptls = jl_current_task->ptls; jl_atomic_store_relaxed(&ptls->gc_num.allocd, jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz); + uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); + if (alloc_acc + sz < 16*1024) + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + sz); + else { + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + sz); + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); + } } static void combine_thread_gc_counts(jl_gc_num_t *dest) JL_NOTSAFEPOINT @@ -1121,12 +1157,13 @@ static void combine_thread_gc_counts(jl_gc_num_t *dest) JL_NOTSAFEPOINT jl_ptls_t ptls = gc_all_tls_states[i]; if (ptls) { dest->allocd += (jl_atomic_load_relaxed(&ptls->gc_num.allocd) + gc_num.interval); - dest->freed += jl_atomic_load_relaxed(&ptls->gc_num.freed); dest->malloc += jl_atomic_load_relaxed(&ptls->gc_num.malloc); dest->realloc += jl_atomic_load_relaxed(&ptls->gc_num.realloc); dest->poolalloc += jl_atomic_load_relaxed(&ptls->gc_num.poolalloc); dest->bigalloc += jl_atomic_load_relaxed(&ptls->gc_num.bigalloc); - dest->freecall += jl_atomic_load_relaxed(&ptls->gc_num.freecall); + uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); + uint64_t free_acc = jl_atomic_load_relaxed(&ptls->gc_num.free_acc); + jl_atomic_store_relaxed(&gc_heap_stats.heap_size, alloc_acc - free_acc + jl_atomic_load_relaxed(&gc_heap_stats.heap_size)); } } } @@ -1183,6 +1220,8 @@ static void jl_gc_free_array(jl_array_t *a) JL_NOTSAFEPOINT jl_free_aligned(d); else free(d); + jl_atomic_store_relaxed(&gc_heap_stats.heap_size, + jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - jl_array_nbytes(a)); gc_num.freed += jl_array_nbytes(a); gc_num.freecall++; } @@ -1257,6 +1296,7 @@ static NOINLINE jl_taggedvalue_t *gc_add_page(jl_gc_pool_t *p) JL_NOTSAFEPOINT set_page_metadata(pg); push_page_metadata_back(&ptls->page_metadata_allocd, pg); jl_taggedvalue_t *fl = gc_reset_page(ptls, p, pg); + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, GC_PAGE_SZ); p->newpages = fl; return fl; } @@ -1451,8 +1491,10 @@ static jl_taggedvalue_t **gc_sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t **allo else if (freed_lazily) { gc_alloc_map_set(pg->data, GC_PAGE_LAZILY_FREED); push_page_metadata_back(lazily_freed, pg); + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, -GC_PAGE_SZ); } else { + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, -GC_PAGE_SZ); #ifdef _P64 // only enable concurrent sweeping on 64bit if (jl_n_sweepthreads == 0) { jl_gc_free_page(pg); @@ -3067,6 +3109,11 @@ JL_DLLEXPORT int64_t jl_gc_live_bytes(void) return live_bytes; } +double jl_gc_smooth(uint64_t old_val, uint64_t new_val, double factor) +{ + return factor * old_val + (1.0-factor) * new_val; +} + size_t jl_maxrss(void); // Only one thread should be running in this function @@ -3081,6 +3128,8 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) jl_gc_markqueue_t *mq = &ptls->mark_queue; uint64_t gc_start_time = jl_hrtime(); + uint64_t mutator_time = gc_start_time - gc_end_time; + uint64_t before_free_heap_size = jl_atomic_load_relaxed(&gc_heap_stats.heap_size); int64_t last_perm_scanned_bytes = perm_scanned_bytes; uint64_t start_mark_time = jl_hrtime(); JL_PROBE_GC_MARK_BEGIN(); @@ -3171,19 +3220,14 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) uint64_t mark_time = end_mark_time - start_mark_time; gc_num.mark_time = mark_time; gc_num.total_mark_time += mark_time; - int64_t allocd = gc_num.allocd; gc_settime_postmark_end(); // marking is over // Flush everything in mark cache gc_sync_all_caches_nolock(ptls); - int64_t live_sz_ub = live_bytes + allocd; - int64_t live_sz_est = scanned_bytes + perm_scanned_bytes; - int64_t estimate_freed = live_sz_ub - live_sz_est; gc_verify(ptls); - gc_stats_all_pool(); gc_stats_big_obj(); objprofile_printall(); @@ -3192,42 +3236,17 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) if (!prev_sweep_full) promoted_bytes += perm_scanned_bytes - last_perm_scanned_bytes; // 5. next collection decision - int not_freed_enough = (collection == JL_GC_AUTO) && estimate_freed < (7*(allocd/10)); - int nptr = 0; + int remset_nptr = 0; + int sweep_full = next_sweep_full; + int recollect = 0; assert(gc_n_threads); for (int i = 0; i < gc_n_threads; i++) { jl_ptls_t ptls2 = gc_all_tls_states[i]; if (ptls2 != NULL) - nptr += ptls2->heap.remset_nptr; + remset_nptr += ptls2->heap.remset_nptr; } + (void)remset_nptr; //Use this information for something? - // many pointers in the intergen frontier => "quick" mark is not quick - int large_frontier = nptr*sizeof(void*) >= default_collect_interval; - int sweep_full = 0; - int recollect = 0; - - // update heuristics only if this GC was automatically triggered - if (collection == JL_GC_AUTO) { - if (large_frontier) { - sweep_full = 1; - gc_num.interval = last_long_collect_interval; - } - if (not_freed_enough || large_frontier) { - gc_num.interval = gc_num.interval * 2; - } - - size_t maxmem = 0; -#ifdef _P64 - // on a big memory machine, increase max_collect_interval to totalmem / nthreads / 2 - maxmem = total_mem / (gc_n_threads - jl_n_gcthreads) / 2; -#endif - if (maxmem < max_collect_interval) - maxmem = max_collect_interval; - if (gc_num.interval > maxmem) { - sweep_full = 1; - gc_num.interval = maxmem; - } - } // If the live data outgrows the suggested max_total_memory // we keep going with minimum intervals and full gcs until @@ -3247,7 +3266,6 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) // on the first collection after sweep_full, and the current scan perm_scanned_bytes = 0; promoted_bytes = 0; - last_long_collect_interval = gc_num.interval; } scanned_bytes = 0; // 6. start sweeping @@ -3272,9 +3290,10 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) if (sweep_full) gc_sweep_perm_alloc(); } + JL_PROBE_GC_SWEEP_END(); - uint64_t gc_end_time = jl_hrtime(); + gc_end_time = jl_hrtime(); uint64_t pause = gc_end_time - gc_start_time; uint64_t sweep_time = gc_end_time - start_sweep_time; gc_num.total_sweep_time += sweep_time; @@ -3283,6 +3302,49 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) gc_num.last_full_sweep = gc_end_time; } + int thrashing = 0; // maybe we should report this to the user or error out? + size_t heap_size = jl_atomic_load_relaxed(&gc_heap_stats.heap_size); + double target_allocs = 0.0; + double min_interval = default_collect_interval; + if (collection == JL_GC_AUTO) { + uint64_t alloc_diff = before_free_heap_size - old_heap_size; + uint64_t freed_diff = before_free_heap_size - heap_size; + double alloc_smooth_factor = 0.95; + double collect_smooth_factor = 0.5; + double tuning_factor = 0.03; + double alloc_mem = jl_gc_smooth(old_alloc_diff, alloc_diff, alloc_smooth_factor); + double alloc_time = jl_gc_smooth(old_mut_time, mutator_time, alloc_smooth_factor); + double gc_mem = jl_gc_smooth(old_freed_diff, freed_diff, collect_smooth_factor); + double gc_time = jl_gc_smooth(old_pause_time, pause, collect_smooth_factor); + old_alloc_diff = alloc_diff; + old_mut_time = mutator_time; + old_freed_diff = freed_diff; + old_pause_time = pause; + old_heap_size = heap_size; + thrashing = gc_time > mutator_time * 98 ? 1 : 0; + if (alloc_mem != 0 && alloc_time != 0 && gc_mem != 0 && gc_time != 0 ) { + double alloc_rate = alloc_mem/alloc_time; + double gc_rate = gc_mem/gc_time; + target_allocs = sqrt(((double)heap_size/min_interval * alloc_rate)/(gc_rate * tuning_factor)); // work on multiples of min interval + } + } + if (target_allocs == 0.0 || thrashing) // If we are thrashing go back to default + target_allocs = 2*sqrt((double)heap_size/min_interval); + + uint64_t target_heap = (uint64_t)target_allocs*min_interval + heap_size; + if (target_heap > max_total_memory && !thrashing) // Allow it to go over if we are thrashing if we die we die + target_heap = max_total_memory; + else if (target_heap < default_collect_interval) + target_heap = default_collect_interval; + jl_atomic_store_relaxed(&gc_heap_stats.heap_target, target_heap); + + double old_ratio = (double)promoted_bytes/(double)heap_size; + if (heap_size > max_total_memory * 0.8 || old_ratio > 0.15) + next_sweep_full = 1; + else + next_sweep_full = 0; + if (heap_size > max_total_memory * 0.8 || thrashing) + under_pressure = 1; // sweeping is over // 7. if it is a quick sweep, put back the remembered objects in queued state // so that we don't trigger the barrier again on them. @@ -3314,55 +3376,19 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) } #endif - _report_gc_finished(pause, gc_num.freed, sweep_full, recollect); - - gc_final_pause_end(gc_start_time, gc_end_time); - gc_time_sweep_pause(gc_end_time, allocd, live_bytes, - estimate_freed, sweep_full); - gc_num.full_sweep += sweep_full; + _report_gc_finished(pause, gc_num.freed, sweep_full, recollect, live_bytes); uint64_t max_memory = last_live_bytes + gc_num.allocd; if (max_memory > gc_num.max_memory) { gc_num.max_memory = max_memory; } - + gc_final_pause_end(gc_start_time, gc_end_time); + gc_time_sweep_pause(gc_end_time, allocd, live_bytes, + estimate_freed, sweep_full); + gc_num.full_sweep += sweep_full; last_live_bytes = live_bytes; - // Can't call inc_live_bytes here because we already added allocd - // to the graph earlier live_bytes += -gc_num.freed + gc_num.allocd; jl_timing_counter_dec(JL_TIMING_COUNTER_HeapSize, gc_num.freed); - if (collection == JL_GC_AUTO) { - //If we aren't freeing enough or are seeing lots and lots of pointers let it increase faster - if (!not_freed_enough || large_frontier) { - int64_t tot = 2 * (live_bytes + gc_num.allocd) / 3; - if (gc_num.interval > tot) { - gc_num.interval = tot; - last_long_collect_interval = tot; - } - // If the current interval is larger than half the live data decrease the interval - } - else { - int64_t half = (live_bytes / 2); - if (gc_num.interval > half) - gc_num.interval = half; - } - - // But never go below default - if (gc_num.interval < default_collect_interval) gc_num.interval = default_collect_interval; - } - - if (gc_num.interval + live_bytes > max_total_memory) { - if (live_bytes < max_total_memory) { - gc_num.interval = max_total_memory - live_bytes; - last_long_collect_interval = max_total_memory - live_bytes; - } - else { - // We can't stay under our goal so let's go back to - // the minimum interval and hope things get better - gc_num.interval = default_collect_interval; - } - } - gc_time_summary(sweep_full, t_start, gc_end_time, gc_num.freed, live_bytes, gc_num.interval, pause, gc_num.time_to_safepoint, @@ -3468,6 +3494,10 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) gc_invoke_callbacks(jl_gc_cb_post_gc_t, gc_cblist_post_gc, (collection)); + if (under_pressure) + gc_invoke_callbacks(jl_gc_cb_notify_gc_pressure_t, + gc_cblist_notify_gc_pressure, ()); + under_pressure = 0; #ifdef _OS_WINDOWS_ SetLastError(last_error); #endif @@ -3553,7 +3583,7 @@ void jl_gc_init(void) arraylist_new(&finalizer_list_marked, 0); arraylist_new(&to_finalize, 0); - + jl_atomic_store_relaxed(&gc_heap_stats.heap_target, default_collect_interval); gc_num.interval = default_collect_interval; last_long_collect_interval = default_collect_interval; gc_num.allocd = 0; @@ -3565,12 +3595,6 @@ void jl_gc_init(void) uint64_t constrained_mem = uv_get_constrained_memory(); if (constrained_mem > 0 && constrained_mem < total_mem) total_mem = constrained_mem; - double percent; - if (total_mem < 128e9) - percent = total_mem * 2.34375e-12 + 0.6; // 60% at 0 gigs and 90% at 128 to not - else // overcommit too much on memory contrained devices - percent = 0.9; - max_total_memory = total_mem * percent; #endif if (jl_options.heap_size_hint) jl_gc_set_max_memory(jl_options.heap_size_hint); @@ -3582,7 +3606,11 @@ JL_DLLEXPORT void jl_gc_set_max_memory(uint64_t max_mem) { if (max_mem > 0 && max_mem < (uint64_t)1 << (sizeof(memsize_t) * 8 - 1)) { + #ifdef _P64 max_total_memory = max_mem; + #else + max_total_memory = max_mem < MAX32HEAP ? max_mem : MAX32HEAP; + #endif } } @@ -3610,6 +3638,13 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz); jl_atomic_store_relaxed(&ptls->gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); + uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); + if (alloc_acc + sz < 16*1024) + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + sz); + else { + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + sz); + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); + } } return malloc(sz); } @@ -3625,6 +3660,13 @@ JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + nm*sz); jl_atomic_store_relaxed(&ptls->gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); + uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); + if (alloc_acc + sz < 16*1024) + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + sz * nm); + else { + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + sz * nm); + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); + } } return calloc(nm, sz); } @@ -3636,10 +3678,13 @@ JL_DLLEXPORT void jl_gc_counted_free_with_size(void *p, size_t sz) free(p); if (pgcstack != NULL && ct->world_age) { jl_ptls_t ptls = ct->ptls; - jl_atomic_store_relaxed(&ptls->gc_num.freed, - jl_atomic_load_relaxed(&ptls->gc_num.freed) + sz); - jl_atomic_store_relaxed(&ptls->gc_num.freecall, - jl_atomic_load_relaxed(&ptls->gc_num.freecall) + 1); + uint64_t free_acc = jl_atomic_load_relaxed(&ptls->gc_num.free_acc); + if (free_acc + sz < 16*1024) + jl_atomic_store_relaxed(&ptls->gc_num.free_acc, free_acc + sz); + else { + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, -(free_acc + sz)); + jl_atomic_store_relaxed(&ptls->gc_num.free_acc, 0); + } } } @@ -3650,14 +3695,12 @@ JL_DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size if (pgcstack != NULL && ct->world_age) { jl_ptls_t ptls = ct->ptls; maybe_collect(ptls); - if (sz < old) - jl_atomic_store_relaxed(&ptls->gc_num.freed, - jl_atomic_load_relaxed(&ptls->gc_num.freed) + (old - sz)); - else + if (!(sz < old)) jl_atomic_store_relaxed(&ptls->gc_num.allocd, jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (sz - old)); jl_atomic_store_relaxed(&ptls->gc_num.realloc, jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1); + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, sz-old); } return realloc(p, sz); } @@ -3732,6 +3775,13 @@ JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + allocsz); jl_atomic_store_relaxed(&ptls->gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); + uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); + if (alloc_acc + allocsz < 16*1024) + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + allocsz); + else { + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + allocsz); + jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); + } int last_errno = errno; #ifdef _OS_WINDOWS_ DWORD last_error = GetLastError(); @@ -3762,15 +3812,12 @@ static void *gc_managed_realloc_(jl_ptls_t ptls, void *d, size_t sz, size_t olds ptls->gc_cache.perm_scanned_bytes += allocsz - oldsz; inc_live_bytes(allocsz - oldsz); } - else if (allocsz < oldsz) - jl_atomic_store_relaxed(&ptls->gc_num.freed, - jl_atomic_load_relaxed(&ptls->gc_num.freed) + (oldsz - allocsz)); - else + else if (!(allocsz < oldsz)) jl_atomic_store_relaxed(&ptls->gc_num.allocd, jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (allocsz - oldsz)); jl_atomic_store_relaxed(&ptls->gc_num.realloc, jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1); - + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, allocsz-oldsz); int last_errno = errno; #ifdef _OS_WINDOWS_ DWORD last_error = GetLastError(); @@ -3859,6 +3906,7 @@ static void *gc_perm_alloc_large(size_t sz, int zero, unsigned align, unsigned o #ifdef _OS_WINDOWS_ SetLastError(last_error); #endif + jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size,sz); errno = last_errno; jl_may_leak(base); assert(align > 0); diff --git a/src/gc.h b/src/gc.h index b1eee5a1d5bda..4bfe5dc328e9d 100644 --- a/src/gc.h +++ b/src/gc.h @@ -10,6 +10,7 @@ #define JL_GC_H #include +#include #include #include #include @@ -257,6 +258,13 @@ typedef struct { pagetable1_t *meta1[REGION2_PG_COUNT]; } pagetable_t; +typedef struct { + _Atomic(size_t) bytes_mapped; + _Atomic(size_t) bytes_resident; + _Atomic(size_t) heap_size; + _Atomic(size_t) heap_target; +} gc_heapstatus_t; + #define GC_PAGE_UNMAPPED 0 #define GC_PAGE_ALLOCATED 1 #define GC_PAGE_LAZILY_FREED 2 @@ -374,6 +382,7 @@ extern int64_t lazy_freed_pages; extern int gc_first_tid; extern int gc_n_threads; extern jl_ptls_t* gc_all_tls_states; +extern gc_heapstatus_t gc_heap_stats; STATIC_INLINE bigval_t *bigval_header(jl_taggedvalue_t *o) JL_NOTSAFEPOINT { @@ -637,7 +646,7 @@ void gc_count_pool(void); size_t jl_array_nbytes(jl_array_t *a) JL_NOTSAFEPOINT; JL_DLLEXPORT void jl_enable_gc_logging(int enable); -void _report_gc_finished(uint64_t pause, uint64_t freed, int full, int recollect) JL_NOTSAFEPOINT; +void _report_gc_finished(uint64_t pause, uint64_t freed, int full, int recollect, int64_t live_bytes) JL_NOTSAFEPOINT; #ifdef __cplusplus } diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index c2b2a1578fd76..a7ffedd5cba10 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -191,6 +191,7 @@ XX(jl_gc_schedule_foreign_sweepfunc) \ XX(jl_gc_set_cb_notify_external_alloc) \ XX(jl_gc_set_cb_notify_external_free) \ + XX(jl_gc_set_cb_notify_gc_pressure) \ XX(jl_gc_set_cb_post_gc) \ XX(jl_gc_set_cb_pre_gc) \ XX(jl_gc_set_cb_root_scanner) \ diff --git a/src/julia_threads.h b/src/julia_threads.h index f4c235243e684..d4cbb88e619ba 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -130,12 +130,12 @@ typedef struct { typedef struct { _Atomic(int64_t) allocd; - _Atomic(int64_t) freed; _Atomic(uint64_t) malloc; _Atomic(uint64_t) realloc; _Atomic(uint64_t) poolalloc; _Atomic(uint64_t) bigalloc; - _Atomic(uint64_t) freecall; + _Atomic(int64_t) free_acc; + _Atomic(uint64_t) alloc_acc; } jl_thread_gc_num_t; typedef struct { diff --git a/test/testenv.jl b/test/testenv.jl index 41706dd24e75e..21f3ee3b31dd1 100644 --- a/test/testenv.jl +++ b/test/testenv.jl @@ -37,6 +37,15 @@ if !@isdefined(testenv_defined) function addprocs_with_testenv(X; rr_allowed=true, kwargs...) exename = rr_allowed ? `$rr_exename $test_exename` : test_exename + if X isa Integer + if Sys.iswindows() + heap_size=round(Int,(Sys.free_memory()/(1024^2)/(X+1))) + heap_size -= 300 # I don't know anymore + else + heap_size=round(Int,(Sys.total_memory()/(1024^2)/(X+1))) + end + push!(test_exeflags.exec, "--heap-size-hint=$(heap_size)M") + end addprocs(X; exename=exename, exeflags=test_exeflags, kwargs...) end