From 01cc143da5e6c86ebf4b58009cb50f291d484590 Mon Sep 17 00:00:00 2001 From: Allan Bazinet Date: Wed, 31 Dec 2025 06:33:32 -0800 Subject: [PATCH] Statistics fixes and improvements A number of fixes and improvements to statistics. Closes #1147 and #1151. While this does not include a fix for #1146, there's an existing PR (#1186) for that issue. Note that while the documentation states that PRs should be landed against dev, issue #1147 necessitates landing this against dev3. --- include/mimalloc-stats.h | 16 ++++++++++---- include/mimalloc/internal.h | 6 +++++ src/init.c | 9 +++++--- src/os.c | 4 ++-- src/page.c | 2 +- src/stats.c | 44 ++++++++++++++++++++++++++----------- 6 files changed, 58 insertions(+), 23 deletions(-) diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index d05ebf8f..3de1626a 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #include -#define MI_STAT_VERSION 2 // increased on every backward incompatible change +#define MI_STAT_VERSION 3 // increased on every backward incompatible change // count allocation over time typedef struct mi_stat_count_s { @@ -25,12 +25,18 @@ typedef struct mi_stat_counter_s { int64_t total; // total count } mi_stat_counter_t; +// average over time +typedef struct mi_stat_average_s { + int64_t total; + int64_t count; +} mi_stat_average_t; + #define MI_STAT_FIELDS() \ MI_STAT_COUNT(pages) /* count of mimalloc pages */ \ MI_STAT_COUNT(reserved) /* reserved memory bytes */ \ MI_STAT_COUNT(committed) /* committed bytes */ \ - MI_STAT_COUNT(reset) /* reset bytes */ \ - MI_STAT_COUNT(purged) /* purged bytes */ \ + MI_STAT_COUNTER(reset) /* reset bytes */ \ + MI_STAT_COUNTER(purged) /* purged bytes */ \ MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \ MI_STAT_COUNT(pages_abandoned) /* abandonded pages count */ \ MI_STAT_COUNT(threads) /* number of threads */ \ @@ -52,7 +58,7 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNTER(arena_purges) \ MI_STAT_COUNTER(pages_extended) /* number of page extensions */ \ MI_STAT_COUNTER(pages_retire) /* number of pages that are retired */ \ - MI_STAT_COUNTER(page_searches) /* searches for a fresh page */ \ + MI_STAT_AVERAGE(page_searches) /* searches for a fresh page */ \ /* only on v1 and v2 */ \ MI_STAT_COUNT(segments) \ MI_STAT_COUNT(segments_abandoned) \ @@ -81,6 +87,7 @@ typedef enum mi_chunkbin_e { #define MI_BIN_HUGE (73U) // see types.h #define MI_STAT_COUNT(stat) mi_stat_count_t stat; #define MI_STAT_COUNTER(stat) mi_stat_counter_t stat; +#define MI_STAT_AVERAGE(stat) mi_stat_average_t stat; typedef struct mi_stats_s { @@ -100,6 +107,7 @@ typedef struct mi_stats_s #undef MI_STAT_COUNT #undef MI_STAT_COUNTER +#undef MI_STAT_AVERAGE // Exported definitions diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index f5aabbcc..b4b74016 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -348,6 +348,10 @@ void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount); void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); +// averages can just be increased +void __mi_stat_average_increase(mi_stat_average_t* stat, size_t amount); +void __mi_stat_average_increase_mt(mi_stat_average_t* stat, size_t amount); + #define mi_subproc_stat_counter_increase(subproc,stat,amount) __mi_stat_counter_increase_mt( &(subproc)->stats.stat, amount) #define mi_subproc_stat_increase(subproc,stat,amount) __mi_stat_increase_mt( &(subproc)->stats.stat, amount) #define mi_subproc_stat_decrease(subproc,stat,amount) __mi_stat_decrease_mt( &(subproc)->stats.stat, amount) @@ -355,6 +359,7 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); #define mi_subproc_stat_adjust_decrease(subproc,stat,amnt) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amnt) #define mi_tld_stat_counter_increase(tld,stat,amount) __mi_stat_counter_increase( &(tld)->stats.stat, amount) +#define mi_tld_stat_average_increase(tld,stat,amount) __mi_stat_average_increase( &(tld)->stats.stat, amount) #define mi_tld_stat_increase(tld,stat,amount) __mi_stat_increase( &(tld)->stats.stat, amount) #define mi_tld_stat_decrease(tld,stat,amount) __mi_stat_decrease( &(tld)->stats.stat, amount) #define mi_tld_stat_adjust_increase(tld,stat,amnt) __mi_stat_adjust_increase( &(tld)->stats.stat, amnt) @@ -365,6 +370,7 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); #define mi_os_stat_decrease(stat,amount) mi_subproc_stat_decrease(_mi_subproc(),stat,amount) #define mi_heap_stat_counter_increase(heap,stat,amount) mi_tld_stat_counter_increase(heap->tld, stat, amount) +#define mi_heap_stat_average_increase(heap,stat,amount) mi_tld_stat_average_increase(heap->tld, stat, amount) #define mi_heap_stat_increase(heap,stat,amount) mi_tld_stat_increase( heap->tld, stat, amount) #define mi_heap_stat_decrease(heap,stat,amount) mi_tld_stat_decrease( heap->tld, stat, amount) #define mi_heap_stat_adjust_decrease(heap,stat,amount) mi_tld_stat_adjust_decrease( heap->tld, stat, amount) diff --git a/src/init.c b/src/init.c index e3a1cb5c..c888780c 100644 --- a/src/init.c +++ b/src/init.c @@ -65,16 +65,19 @@ const mi_page_t _mi_page_empty = { QNULL(MI_LARGE_MAX_OBJ_WSIZE + 2) /* Full queue */ } #define MI_STAT_COUNT_NULL() {0,0,0} +#define MI_STAT_AVERAGE_NULL() {0,0} // Empty statistics #define MI_STATS_NULL \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + { 0 }, { 0 }, \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ { 0 }, { 0 }, { 0 }, { 0 }, \ { 0 }, { 0 }, { 0 }, { 0 }, \ \ - { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, \ + { 0 }, { 0 }, { 0 }, { 0 }, \ + MI_STAT_AVERAGE_NULL(), \ MI_INIT4(MI_STAT_COUNT_NULL), \ { 0 }, { 0 }, { 0 }, { 0 }, \ \ diff --git a/src/os.c b/src/os.c index 47ed7e5c..d58ba4df 100644 --- a/src/os.c +++ b/src/os.c @@ -536,7 +536,7 @@ bool _mi_os_reset(void* addr, size_t size) { size_t csize; void* start = mi_os_page_align_area_conservative(addr, size, &csize); if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr) - mi_os_stat_increase(reset, csize); + mi_os_stat_counter_increase(reset, csize); mi_os_stat_counter_increase(reset_calls, 1); #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN @@ -568,7 +568,7 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, m { if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed? mi_os_stat_counter_increase(purge_calls, 1); - mi_os_stat_increase(purged, size); + mi_os_stat_counter_increase(purged, size); if (commit_fun != NULL) { bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg); diff --git a/src/page.c b/src/page.c index 1eaae6ec..d5117621 100644 --- a/src/page.c +++ b/src/page.c @@ -789,7 +789,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, m page = next; } // for each page - mi_heap_stat_counter_increase(heap, page_searches, count); + mi_heap_stat_average_increase(heap, page_searches, count); // set the page to the best candidate if (page_candidate != NULL) { diff --git a/src/stats.c b/src/stats.c index 63a9c996..0c354167 100644 --- a/src/stats.c +++ b/src/stats.c @@ -46,6 +46,16 @@ void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { stat->total += amount; } +void __mi_stat_average_increase_mt(mi_stat_average_t* stat, size_t amount) { + mi_atomic_addi64_relaxed( &stat->count, (int64_t)1 ); + mi_atomic_addi64_relaxed( &stat->total, (int64_t)amount ); +} + +void __mi_stat_average_increase(mi_stat_average_t* stat, size_t amount) { + stat->count++; + stat->total += amount; +} + void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) { mi_stat_update_mt(stat, (int64_t)amount); } @@ -111,8 +121,15 @@ static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counte mi_atomic_void_addi64_relaxed(&stat->total, &src->total); } +static void mi_stat_average_add_mt(mi_stat_average_t* stat, const mi_stat_average_t* src) { + if (stat==src) return; + mi_atomic_void_addi64_relaxed(&stat->count, &src->count); + mi_atomic_void_addi64_relaxed(&stat->total, &src->total); +} + #define MI_STAT_COUNT(stat) mi_stat_count_add_mt(&stats->stat, &src->stat); #define MI_STAT_COUNTER(stat) mi_stat_counter_add_mt(&stats->stat, &src->stat); +#define MI_STAT_AVERAGE(stat) mi_stat_average_add_mt(&stats->stat, &src->stat); // must be thread safe as it is called from stats_merge static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { @@ -133,6 +150,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { #undef MI_STAT_COUNT #undef MI_STAT_COUNTER +#undef MI_STAT_AVERAGE /* ----------------------------------------------------------- Display statistics @@ -225,12 +243,6 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t mi_stat_print_ex(stat, msg, unit, out, arg, NULL); } -static void mi_stat_peak_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { - _mi_fprintf(out, arg, "%10s:", msg); - mi_print_amount(stat->peak, unit, out, arg); - _mi_fprintf(out, arg, "\n"); -} - #if MI_STAT>1 static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { _mi_fprintf(out, arg, "%10s:", msg); @@ -246,15 +258,13 @@ static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg _mi_fprintf(out, arg, "\n"); } - -static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { - const int64_t avg_tens = (stat->total == 0 ? 0 : (stat->total*10 / stat->total)); +static void mi_stat_average_print(const mi_stat_average_t* stat, const char* msg, mi_output_fun* out, void* arg) { + const int64_t avg_tens = (stat->count == 0 ? 0 : (stat->total*10 / stat->count)); const long avg_whole = (long)(avg_tens/10); const long avg_frac1 = (long)(avg_tens%10); _mi_fprintf(out, arg, "%10s: %5ld.%ld avg\n", msg, avg_whole, avg_frac1); } - static void mi_print_header(mi_output_fun* out, void* arg ) { _mi_fprintf(out, arg, "%10s: %11s %11s %11s %11s %11s\n", "heap stats", "peak ", "total ", "current ", "block ", "total# "); } @@ -341,8 +351,8 @@ void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr #endif mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, ""); mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, ""); - mi_stat_peak_print(&stats->reset, "reset", 1, out, arg ); - mi_stat_peak_print(&stats->purged, "purged", 1, out, arg ); + mi_stat_counter_print(&stats->reset, "reset", out, arg ); + mi_stat_counter_print(&stats->purged, "purged", out, arg ); mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, ""); // mi_stat_print(&stats->segments, "segments", -1, out, arg); // mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg); @@ -365,7 +375,7 @@ void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg); - mi_stat_counter_print_avg(&stats->page_searches, "searches", out, arg); + mi_stat_average_print(&stats->page_searches, "searches", out, arg); _mi_fprintf(out, arg, "%10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); size_t elapsed; @@ -619,8 +629,16 @@ static void mi_heap_buf_print_counter_value(mi_heap_buf_t* hbuf, const char* nam mi_heap_buf_print_value(hbuf, name, stat->total); } +static void mi_heap_buf_print_average_value(mi_heap_buf_t* hbuf, const char* name, mi_stat_average_t* stat) { + char buf[128]; + _mi_snprintf(buf, 128, " \"%s\": { \"count\": %lld, \"total\": %lld },\n", name, stat->count, stat->total); + buf[127] = 0; + mi_heap_buf_print(hbuf, buf); +} + #define MI_STAT_COUNT(stat) mi_heap_buf_print_count_value(&hbuf, #stat, &stats->stat); #define MI_STAT_COUNTER(stat) mi_heap_buf_print_counter_value(&hbuf, #stat, &stats->stat); +#define MI_STAT_AVERAGE(stat) mi_heap_buf_print_average_value(&hbuf, #stat, &stats->stat); char* mi_stats_get_json(size_t output_size, char* output_buf) mi_attr_noexcept { mi_stats_merge();