From 19bbefe136cf8684e126cdb80f7ef2aba88e55dc Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Tue, 31 Mar 2026 18:25:50 -0700 Subject: [PATCH 01/22] Remove dead code: extent_commit_wrapper, large_salloc, tcache_gc_dalloc event waits These functions had zero callers anywhere in the codebase: - extent_commit_wrapper: wrapper never called, _impl used directly - large_salloc: trivial wrapper never called - tcache_gc_dalloc_new_event_wait: no header declaration, no callers - tcache_gc_dalloc_postponed_event_wait: no header declaration, no callers --- include/jemalloc/internal/extent.h | 2 -- include/jemalloc/internal/large_externs.h | 1 - src/extent.c | 7 ------- src/large.c | 5 ----- src/tcache.c | 10 ---------- 5 files changed, 25 deletions(-) diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index e81dff2cd3..a9f81cb78f 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -57,8 +57,6 @@ void extent_dalloc_wrapper_purged( tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata); void extent_destroy_wrapper( tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata); -bool extent_commit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - size_t offset, size_t length); bool extent_purge_lazy_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, size_t offset, size_t length); bool extent_purge_forced_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, diff --git a/include/jemalloc/internal/large_externs.h b/include/jemalloc/internal/large_externs.h index 7cee67520d..84c6c5d62a 100644 --- a/include/jemalloc/internal/large_externs.h +++ b/include/jemalloc/internal/large_externs.h @@ -17,7 +17,6 @@ void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, void large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata); void large_dalloc_finish(tsdn_t *tsdn, edata_t *edata); void large_dalloc(tsdn_t *tsdn, edata_t *edata); -size_t large_salloc(tsdn_t *tsdn, const edata_t *edata); void large_prof_info_get( tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, bool reset_recent); void large_prof_tctx_reset(edata_t *edata); diff --git a/src/extent.c b/src/extent.c index 0a23bbd91d..4b927191ae 100644 --- a/src/extent.c +++ b/src/extent.c @@ -1239,13 +1239,6 @@ extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, return err; } -bool -extent_commit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, - size_t offset, size_t length) { - return extent_commit_impl(tsdn, ehooks, edata, offset, length, - /* growing_retained */ false); -} - static bool extent_decommit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, size_t offset, size_t length) { diff --git a/src/large.c b/src/large.c index 087df99de4..56fa16ab91 100644 --- a/src/large.c +++ b/src/large.c @@ -276,11 +276,6 @@ large_dalloc(tsdn_t *tsdn, edata_t *edata) { arena_decay_tick(tsdn, arena); } -size_t -large_salloc(tsdn_t *tsdn, const edata_t *edata) { - return edata_usize_get(edata); -} - void large_prof_info_get( tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, bool reset_recent) { diff --git a/src/tcache.c b/src/tcache.c index 10fa7c2130..fe210d27ba 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -111,16 +111,6 @@ tcache_gc_postponed_event_wait(tsd_t *tsd) { return TE_MIN_START_WAIT; } -uint64_t -tcache_gc_dalloc_new_event_wait(tsd_t *tsd) { - return opt_tcache_gc_incr_bytes; -} - -uint64_t -tcache_gc_dalloc_postponed_event_wait(tsd_t *tsd) { - return TE_MIN_START_WAIT; -} - static inline void tcache_bin_fill_ctl_init(tcache_slow_t *tcache_slow, szind_t szind) { assert(szind < SC_NBINS); From 176ea0a801338cae1b938c47f0d7dba7ffef0d25 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Tue, 31 Mar 2026 18:12:23 -0700 Subject: [PATCH 02/22] Remove experimental.thread.activity_callback --- include/jemalloc/internal/activity_callback.h | 26 ------- include/jemalloc/internal/tsd_internals.h | 6 +- src/ctl.c | 33 +-------- src/peak_event.c | 13 ---- test/unit/mallctl.c | 73 +------------------ 5 files changed, 3 insertions(+), 148 deletions(-) delete mode 100644 include/jemalloc/internal/activity_callback.h diff --git a/include/jemalloc/internal/activity_callback.h b/include/jemalloc/internal/activity_callback.h deleted file mode 100644 index 6745f1a275..0000000000 --- a/include/jemalloc/internal/activity_callback.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H -#define JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H - -#include "jemalloc/internal/jemalloc_preamble.h" - -/* - * The callback to be executed "periodically", in response to some amount of - * allocator activity. - * - * This callback need not be computing any sort of peak (although that's the - * intended first use case), but we drive it from the peak counter, so it's - * keeps things tidy to keep it here. - * - * The calls to this thunk get driven by the peak_event module. - */ -#define ACTIVITY_CALLBACK_THUNK_INITIALIZER \ - { NULL, NULL } -typedef void (*activity_callback_t)( - void *uctx, uint64_t allocated, uint64_t deallocated); -typedef struct activity_callback_thunk_s activity_callback_thunk_t; -struct activity_callback_thunk_s { - activity_callback_t callback; - void *uctx; -}; - -#endif /* JEMALLOC_INTERNAL_ACTIVITY_CALLBACK_H */ diff --git a/include/jemalloc/internal/tsd_internals.h b/include/jemalloc/internal/tsd_internals.h index f675587d73..53b58d0c97 100644 --- a/include/jemalloc/internal/tsd_internals.h +++ b/include/jemalloc/internal/tsd_internals.h @@ -4,7 +4,6 @@ #define JEMALLOC_INTERNAL_TSD_INTERNALS_H #include "jemalloc/internal/jemalloc_preamble.h" -#include "jemalloc/internal/activity_callback.h" #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/assert.h" #include "jemalloc/internal/bin_types.h" @@ -84,8 +83,6 @@ typedef ql_elm(tsd_t) tsd_link_t; O(tsd_link, tsd_link_t, tsd_link_t) \ O(in_hook, bool, bool) \ O(peak, peak_t, peak_t) \ - O(activity_callback_thunk, activity_callback_thunk_t, \ - activity_callback_thunk_t) \ O(tcache_slow, tcache_slow_t, tcache_slow_t) \ O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) @@ -105,8 +102,7 @@ typedef ql_elm(tsd_t) tsd_link_t; /* sec_shard */ (uint8_t) - 1, \ /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \ /* tsd_link */ {NULL}, /* in_hook */ false, \ - /* peak */ PEAK_INITIALIZER, /* activity_callback_thunk */ \ - ACTIVITY_CALLBACK_THUNK_INITIALIZER, \ + /* peak */ PEAK_INITIALIZER, \ /* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER, \ /* rtree_ctx */ RTREE_CTX_INITIALIZER, diff --git a/src/ctl.c b/src/ctl.c index 4cac5608a4..89824d6a9d 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -365,7 +365,6 @@ CTL_PROTO(experimental_hooks_prof_sample) CTL_PROTO(experimental_hooks_prof_sample_free) CTL_PROTO(experimental_hooks_thread_event) CTL_PROTO(experimental_hooks_safety_check_abort) -CTL_PROTO(experimental_thread_activity_callback) CTL_PROTO(experimental_utilization_query) CTL_PROTO(experimental_utilization_batch_query) CTL_PROTO(experimental_arenas_i_pactivep) @@ -890,9 +889,6 @@ static const ctl_named_node_t experimental_hooks_node[] = { {NAME("thread_event"), CTL(experimental_hooks_thread_event)}, }; -static const ctl_named_node_t experimental_thread_node[] = { - {NAME("activity_callback"), CTL(experimental_thread_activity_callback)}}; - static const ctl_named_node_t experimental_utilization_node[] = { {NAME("query"), CTL(experimental_utilization_query)}, {NAME("batch_query"), CTL(experimental_utilization_batch_query)}}; @@ -916,8 +912,7 @@ static const ctl_named_node_t experimental_node[] = { {NAME("arenas"), CHILD(indexed, experimental_arenas)}, {NAME("arenas_create_ext"), CTL(experimental_arenas_create_ext)}, {NAME("prof_recent"), CHILD(named, experimental_prof_recent)}, - {NAME("batch_alloc"), CTL(experimental_batch_alloc)}, - {NAME("thread"), CHILD(named, experimental_thread)}}; + {NAME("batch_alloc"), CTL(experimental_batch_alloc)}}; static const ctl_named_node_t root_node[] = {{NAME("version"), CTL(version)}, {NAME("epoch"), CTL(epoch)}, @@ -4255,32 +4250,6 @@ experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, return ret; } -static int -experimental_thread_activity_callback_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - - if (!config_stats) { - return ENOENT; - } - - activity_callback_thunk_t t_old = tsd_activity_callback_thunk_get(tsd); - READ(t_old, activity_callback_thunk_t); - - if (newp != NULL) { - /* - * This initialization is unnecessary. If it's omitted, though, - * clang gets confused and warns on the subsequent use of t_new. - */ - activity_callback_thunk_t t_new = {NULL, NULL}; - WRITE(t_new, activity_callback_thunk_t); - tsd_activity_callback_thunk_set(tsd, t_new); - } - ret = 0; -label_return: - return ret; -} - /* * Output six memory utilization entries for an input pointer, the first one of * type (void *) and the remaining five of type size_t, describing the following diff --git a/src/peak_event.c b/src/peak_event.c index e7f54dba87..39f90b70be 100644 --- a/src/peak_event.c +++ b/src/peak_event.c @@ -3,7 +3,6 @@ #include "jemalloc/internal/peak_event.h" -#include "jemalloc/internal/activity_callback.h" #include "jemalloc/internal/peak.h" #include "jemalloc/internal/thread_event_registry.h" @@ -16,17 +15,6 @@ peak_event_update(tsd_t *tsd) { peak_update(peak, alloc, dalloc); } -static void -peak_event_activity_callback(tsd_t *tsd) { - activity_callback_thunk_t *thunk = tsd_activity_callback_thunkp_get( - tsd); - uint64_t alloc = tsd_thread_allocated_get(tsd); - uint64_t dalloc = tsd_thread_deallocated_get(tsd); - if (thunk->callback != NULL) { - thunk->callback(thunk->uctx, alloc, dalloc); - } -} - /* Set current state to zero. */ void peak_event_zero(tsd_t *tsd) { @@ -55,7 +43,6 @@ peak_event_postponed_event_wait(tsd_t *tsd) { static void peak_event_handler(tsd_t *tsd) { peak_event_update(tsd); - peak_event_activity_callback(tsd); } static te_enabled_t diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 4cd0225bed..8f1d2a66ae 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -1332,77 +1332,6 @@ TEST_BEGIN(test_thread_peak) { } TEST_END -typedef struct activity_test_data_s activity_test_data_t; -struct activity_test_data_s { - uint64_t obtained_alloc; - uint64_t obtained_dalloc; -}; - -static void -activity_test_callback(void *uctx, uint64_t alloc, uint64_t dalloc) { - activity_test_data_t *test_data = (activity_test_data_t *)uctx; - test_data->obtained_alloc = alloc; - test_data->obtained_dalloc = dalloc; -} - -TEST_BEGIN(test_thread_activity_callback) { - test_skip_if(!config_stats); - - const size_t big_size = 10 * 1024 * 1024; - void *ptr; - int err; - size_t sz; - - uint64_t *allocatedp; - uint64_t *deallocatedp; - sz = sizeof(allocatedp); - err = mallctl("thread.allocatedp", &allocatedp, &sz, NULL, 0); - assert_d_eq(0, err, ""); - err = mallctl("thread.deallocatedp", &deallocatedp, &sz, NULL, 0); - assert_d_eq(0, err, ""); - - activity_callback_thunk_t old_thunk = { - (activity_callback_t)111, (void *)222}; - - activity_test_data_t test_data = {333, 444}; - activity_callback_thunk_t new_thunk = { - &activity_test_callback, &test_data}; - - sz = sizeof(old_thunk); - err = mallctl("experimental.thread.activity_callback", &old_thunk, &sz, - &new_thunk, sizeof(new_thunk)); - assert_d_eq(0, err, ""); - - expect_true(old_thunk.callback == NULL, "Callback already installed"); - expect_true(old_thunk.uctx == NULL, "Callback data already installed"); - - ptr = mallocx(big_size, 0); - expect_u64_eq(test_data.obtained_alloc, *allocatedp, ""); - expect_u64_eq(test_data.obtained_dalloc, *deallocatedp, ""); - - free(ptr); - expect_u64_eq(test_data.obtained_alloc, *allocatedp, ""); - expect_u64_eq(test_data.obtained_dalloc, *deallocatedp, ""); - - sz = sizeof(old_thunk); - new_thunk = (activity_callback_thunk_t){NULL, NULL}; - err = mallctl("experimental.thread.activity_callback", &old_thunk, &sz, - &new_thunk, sizeof(new_thunk)); - assert_d_eq(0, err, ""); - - expect_true(old_thunk.callback == &activity_test_callback, ""); - expect_true(old_thunk.uctx == &test_data, ""); - - /* Inserting NULL should have turned off tracking. */ - test_data.obtained_alloc = 333; - test_data.obtained_dalloc = 444; - ptr = mallocx(big_size, 0); - free(ptr); - expect_u64_eq(333, test_data.obtained_alloc, ""); - expect_u64_eq(444, test_data.obtained_dalloc, ""); -} -TEST_END - static unsigned nuser_thread_event_cb_calls; static void user_thread_event_cb(bool is_alloc, uint64_t tallocated, uint64_t tdallocated) { @@ -1455,5 +1384,5 @@ main(void) { test_stats_arenas_hpa_shard_counters, test_stats_arenas_hpa_shard_slabs, test_hooks, test_hooks_exhaustion, test_thread_idle, test_thread_peak, - test_thread_activity_callback, test_thread_event_hook); + test_thread_event_hook); } From 513778bcb18f7e98073775d2b358674b14f7433f Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 08:42:54 -0700 Subject: [PATCH 03/22] Fix off-by-one in arenas_bin_i_index and arenas_lextent_i_index bounds checks The index validation used > instead of >=, allowing access at index SC_NBINS (for bins) and SC_NSIZES-SC_NBINS (for lextents), which are one past the valid range. This caused out-of-bounds reads in bin_infos[] and sz_index2size_unsafe(). Add unit tests that verify the boundary indices return ENOENT. --- src/ctl.c | 4 ++-- test/unit/mallctl.c | 47 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/ctl.c b/src/ctl.c index 89824d6a9d..bfc1246969 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -3250,7 +3250,7 @@ CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t) static const ctl_named_node_t * arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { - if (i > SC_NBINS) { + if (i >= SC_NBINS) { return NULL; } return super_arenas_bin_i_node; @@ -3262,7 +3262,7 @@ CTL_RO_NL_GEN(arenas_lextent_i_size, static const ctl_named_node_t * arenas_lextent_i_index( tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { - if (i > SC_NSIZES - SC_NBINS) { + if (i >= SC_NSIZES - SC_NBINS) { return NULL; } return super_arenas_lextent_i_node; diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 8f1d2a66ae..6ab443afca 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -956,6 +956,52 @@ TEST_BEGIN(test_arenas_bin_constants) { } TEST_END +TEST_BEGIN(test_arenas_bin_oob) { + size_t sz; + size_t result; + char buf[128]; + + /* + * Querying the bin at index SC_NBINS should fail because valid + * indices are [0, SC_NBINS). + */ + sz = sizeof(result); + malloc_snprintf( + buf, sizeof(buf), "arenas.bin.%u.size", (unsigned)SC_NBINS); + expect_d_eq(mallctl(buf, (void *)&result, &sz, NULL, 0), ENOENT, + "mallctl() should fail for out-of-bounds bin index SC_NBINS"); + + /* One below the boundary should succeed. */ + malloc_snprintf( + buf, sizeof(buf), "arenas.bin.%u.size", (unsigned)(SC_NBINS - 1)); + expect_d_eq(mallctl(buf, (void *)&result, &sz, NULL, 0), 0, + "mallctl() should succeed for valid bin index SC_NBINS-1"); +} +TEST_END + +TEST_BEGIN(test_arenas_lextent_oob) { + size_t sz; + size_t result; + char buf[128]; + unsigned nlextents = SC_NSIZES - SC_NBINS; + + /* + * Querying the lextent at index nlextents should fail because valid + * indices are [0, nlextents). + */ + sz = sizeof(result); + malloc_snprintf(buf, sizeof(buf), "arenas.lextent.%u.size", nlextents); + expect_d_eq(mallctl(buf, (void *)&result, &sz, NULL, 0), ENOENT, + "mallctl() should fail for out-of-bounds lextent index"); + + /* Querying the last element (nlextents - 1) should succeed. */ + malloc_snprintf( + buf, sizeof(buf), "arenas.lextent.%u.size", nlextents - 1); + expect_d_eq(mallctl(buf, (void *)&result, &sz, NULL, 0), 0, + "mallctl() should succeed for valid lextent index"); +} +TEST_END + TEST_BEGIN(test_arenas_lextent_constants) { #define TEST_ARENAS_LEXTENT_CONSTANT(t, name, expected) \ do { \ @@ -1379,6 +1425,7 @@ main(void) { test_arena_i_dss, test_arena_i_name, test_arena_i_retain_grow_limit, test_arenas_dirty_decay_ms, test_arenas_muzzy_decay_ms, test_arenas_constants, test_arenas_bin_constants, + test_arenas_bin_oob, test_arenas_lextent_oob, test_arenas_lextent_constants, test_arenas_create, test_arenas_lookup, test_prof_active, test_stats_arenas, test_stats_arenas_hpa_shard_counters, From 87f9938de51be77946b02f0ed54cbd32a5ff055b Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 09:02:32 -0700 Subject: [PATCH 04/22] Fix duplicate "nactive_huge" JSON key in HPA shard stats output In both the full_slabs and empty_slabs JSON sections of HPA shard stats, "nactive_huge" was emitted twice instead of emitting "ndirty_huge" as the second entry. This caused ndirty_huge to be missing from the JSON output entirely. Add a unit test that verifies both sections contain "ndirty_huge". --- src/stats.c | 4 +-- test/unit/json_stats.c | 73 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/src/stats.c b/src/stats.c index 22b412bdaa..d906ade6d0 100644 --- a/src/stats.c +++ b/src/stats.c @@ -981,7 +981,7 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter_json_kv( emitter, "nactive_huge", emitter_type_size, &nactive_huge); emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); + emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, &npageslabs_nonhuge); emitter_json_kv( @@ -1022,7 +1022,7 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter_json_kv( emitter, "nactive_huge", emitter_type_size, &nactive_huge); emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); + emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, &npageslabs_nonhuge); emitter_json_kv( diff --git a/test/unit/json_stats.c b/test/unit/json_stats.c index ea8a170b43..eee717c7b6 100644 --- a/test/unit/json_stats.c +++ b/test/unit/json_stats.c @@ -237,7 +237,78 @@ TEST_BEGIN(test_json_stats_mutexes) { } TEST_END +/* + * Verify that hpa_shard JSON stats contain "ndirty_huge" key in both + * full_slabs and empty_slabs sections. A previous bug emitted duplicate + * "nactive_huge" instead of "ndirty_huge". + */ +TEST_BEGIN(test_hpa_shard_json_ndirty_huge) { + test_skip_if(!config_stats); + test_skip_if(!hpa_supported()); + + /* Do some allocation to create HPA state. */ + void *p = mallocx(PAGE, MALLOCX_TCACHE_NONE); + expect_ptr_not_null(p, "Unexpected mallocx failure"); + + uint64_t epoch = 1; + size_t sz = sizeof(epoch); + expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sz), 0, + "Unexpected mallctl() failure"); + + stats_buf_t sbuf; + stats_buf_init(&sbuf); + /* "J" for JSON, include per-arena HPA stats. */ + malloc_stats_print(stats_buf_write_cb, &sbuf, "J"); + + /* + * Find "full_slabs" and check it contains "ndirty_huge". + */ + const char *full_slabs = strstr(sbuf.buf, "\"full_slabs\""); + if (full_slabs != NULL) { + const char *empty_slabs = strstr(full_slabs, "\"empty_slabs\""); + const char *search_end = empty_slabs != NULL + ? empty_slabs + : sbuf.buf + sbuf.len; + /* + * Search for "ndirty_huge" between full_slabs and + * empty_slabs. + */ + const char *ndirty = full_slabs; + bool found = false; + while (ndirty < search_end) { + ndirty = strstr(ndirty, "\"ndirty_huge\""); + if (ndirty != NULL && ndirty < search_end) { + found = true; + break; + } + break; + } + expect_true( + found, "full_slabs section should contain ndirty_huge key"); + } + + /* + * Find "empty_slabs" and check it contains "ndirty_huge". + */ + const char *empty_slabs = strstr(sbuf.buf, "\"empty_slabs\""); + if (empty_slabs != NULL) { + /* Find the end of the empty_slabs object. */ + const char *nonfull = strstr(empty_slabs, "\"nonfull_slabs\""); + const char *search_end = nonfull != NULL ? nonfull + : sbuf.buf + sbuf.len; + const char *ndirty = strstr(empty_slabs, "\"ndirty_huge\""); + bool found = (ndirty != NULL && ndirty < search_end); + expect_true(found, + "empty_slabs section should contain ndirty_huge key"); + } + + stats_buf_fini(&sbuf); + dallocx(p, MALLOCX_TCACHE_NONE); +} +TEST_END + int main(void) { - return test(test_json_stats_mutexes); + return test_no_reentrancy(test_json_stats_mutexes, + test_hpa_shard_json_ndirty_huge); } From a0f2bdf91ddd4e5662790c7cd877052c9009441d Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 09:57:28 -0700 Subject: [PATCH 05/22] Fix missing negation in large_ralloc_no_move usize_min fallback The second expansion attempt in large_ralloc_no_move omitted the ! before large_ralloc_no_move_expand(), inverting the return value. On expansion failure, the function falsely reported success, making callers believe the allocation was expanded in-place when it was not. On expansion success, the function falsely reported failure, causing callers to unnecessarily allocate, copy, and free. Add unit test that verifies the return value matches actual size change. --- Makefile.in | 1 + include/jemalloc/internal/util.h | 3 ++ src/large.c | 2 +- test/unit/large_ralloc.c | 76 ++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 test/unit/large_ralloc.c diff --git a/Makefile.in b/Makefile.in index 459f98fb04..435fc34d8f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -248,6 +248,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/junk_alloc.c \ $(srcroot)test/unit/junk_free.c \ $(srcroot)test/unit/json_stats.c \ + $(srcroot)test/unit/large_ralloc.c \ $(srcroot)test/unit/log.c \ $(srcroot)test/unit/mallctl.c \ $(srcroot)test/unit/malloc_conf_2.c \ diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index bf246c9569..ecfa76b82d 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -20,6 +20,9 @@ */ #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ +/* Number of elements in a fixed-size array. */ +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + /* cpp macro definition stringification. */ #define STRINGIFY_HELPER(x) #x #define STRINGIFY(x) STRINGIFY_HELPER(x) diff --git a/src/large.c b/src/large.c index 56fa16ab91..6ccf49d766 100644 --- a/src/large.c +++ b/src/large.c @@ -147,7 +147,7 @@ large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, } /* Try again, this time with usize_min. */ if (usize_min < usize_max && usize_min > oldusize - && large_ralloc_no_move_expand( + && !large_ralloc_no_move_expand( tsdn, edata, usize_min, zero)) { arena_decay_tick(tsdn, arena_get_from_edata(edata)); return false; diff --git a/test/unit/large_ralloc.c b/test/unit/large_ralloc.c new file mode 100644 index 0000000000..1f08d12595 --- /dev/null +++ b/test/unit/large_ralloc.c @@ -0,0 +1,76 @@ +#include "test/jemalloc_test.h" + +/* + * Test that large_ralloc_no_move causes a failure (returns true) when + * in-place extent expansion cannot succeed for either usize_max or + * usize_min. + * + * A previous bug omitted the ! negation on the second extent expansion + * attempt (usize_min fallback), causing false success (return false) when + * the expansion actually failed. + */ +TEST_BEGIN(test_large_ralloc_no_move_expand_fail) { + /* + * Allocate two adjacent large objects in the same arena to block + * in-place expansion of the first one. + */ + unsigned arena_ind; + size_t sz = sizeof(arena_ind); + expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), + 0, "Unexpected mallctl() failure"); + + int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; + + size_t large_sz = SC_LARGE_MINCLASS; + /* Allocate several blocks to prevent expansion of the first. */ + void *blocks[8]; + for (size_t i = 0; i < ARRAY_SIZE(blocks); i++) { + blocks[i] = mallocx(large_sz, flags); + expect_ptr_not_null(blocks[i], "Unexpected mallocx() failure"); + } + + /* + * Try to expand blocks[0] in place. Use usize_min < usize_max to + * exercise the fallback path. + */ + tsd_t *tsd = tsd_fetch(); + edata_t *edata = emap_edata_lookup( + tsd_tsdn(tsd), &arena_emap_global, blocks[0]); + expect_ptr_not_null(edata, "Unexpected edata lookup failure"); + + size_t oldusize = edata_usize_get(edata); + size_t usize_min = sz_s2u(oldusize + 1); + size_t usize_max = sz_s2u(oldusize * 2); + + /* Ensure min and max are in different size classes. */ + if (usize_min == usize_max) { + usize_max = sz_s2u(usize_min + 1); + } + + bool ret = large_ralloc_no_move( + tsd_tsdn(tsd), edata, usize_min, usize_max, false); + + /* + * With adjacent allocations blocking expansion, this should fail. + * The bug caused ret == false (success) even when expansion failed. + */ + if (!ret) { + /* + * Expansion might actually succeed if adjacent memory + * is free. Verify the size actually changed. + */ + size_t newusize = edata_usize_get(edata); + expect_zu_ge(newusize, usize_min, + "Expansion reported success but size didn't change"); + } + + for (size_t i = 0; i < ARRAY_SIZE(blocks); i++) { + dallocx(blocks[i], flags); + } +} +TEST_END + +int +main(void) { + return test_no_reentrancy(test_large_ralloc_no_move_expand_fail); +} From eab2b29736a3f499f7be1236950ed9aab57c4267 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 09:57:34 -0700 Subject: [PATCH 06/22] Fix off-by-one in stats_arenas_i_bins_j and stats_arenas_i_lextents_j bounds checks Same pattern as arenas_bin_i_index: used > instead of >= allowing access one past the end of bstats[] and lstats[] arrays. Add unit tests that verify boundary indices return ENOENT. --- src/ctl.c | 4 ++-- test/unit/mallctl.c | 54 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/ctl.c b/src/ctl.c index bfc1246969..0b72086cf0 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -3998,7 +3998,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nonfull_slabs, static const ctl_named_node_t * stats_arenas_i_bins_j_index( tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) { - if (j > SC_NBINS) { + if (j >= SC_NBINS) { return NULL; } return super_stats_arenas_i_bins_j_node; @@ -4022,7 +4022,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, static const ctl_named_node_t * stats_arenas_i_lextents_j_index( tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) { - if (j > SC_NSIZES - SC_NBINS) { + if (j >= SC_NSIZES - SC_NBINS) { return NULL; } return super_stats_arenas_i_lextents_j_node; diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 6ab443afca..11710c2787 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -1002,6 +1002,59 @@ TEST_BEGIN(test_arenas_lextent_oob) { } TEST_END +TEST_BEGIN(test_stats_arenas_bins_oob) { + test_skip_if(!config_stats); + size_t sz; + uint64_t result; + char buf[128]; + + uint64_t epoch = 1; + sz = sizeof(epoch); + expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sz), 0, + "Unexpected mallctl() failure"); + + /* SC_NBINS is one past the valid range. */ + sz = sizeof(result); + malloc_snprintf(buf, sizeof(buf), "stats.arenas.0.bins.%u.nmalloc", + (unsigned)SC_NBINS); + expect_d_eq(mallctl(buf, (void *)&result, &sz, NULL, 0), ENOENT, + "mallctl() should fail for out-of-bounds stats bin index"); + + /* SC_NBINS - 1 is valid. */ + malloc_snprintf(buf, sizeof(buf), "stats.arenas.0.bins.%u.nmalloc", + (unsigned)(SC_NBINS - 1)); + expect_d_eq(mallctl(buf, (void *)&result, &sz, NULL, 0), 0, + "mallctl() should succeed for valid stats bin index"); +} +TEST_END + +TEST_BEGIN(test_stats_arenas_lextents_oob) { + test_skip_if(!config_stats); + size_t sz; + uint64_t result; + char buf[128]; + unsigned nlextents = SC_NSIZES - SC_NBINS; + + uint64_t epoch = 1; + sz = sizeof(epoch); + expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sz), 0, + "Unexpected mallctl() failure"); + + /* nlextents is one past the valid range. */ + sz = sizeof(result); + malloc_snprintf( + buf, sizeof(buf), "stats.arenas.0.lextents.%u.nmalloc", nlextents); + expect_d_eq(mallctl(buf, (void *)&result, &sz, NULL, 0), ENOENT, + "mallctl() should fail for out-of-bounds stats lextent index"); + + /* nlextents - 1 is valid. */ + malloc_snprintf(buf, sizeof(buf), "stats.arenas.0.lextents.%u.nmalloc", + nlextents - 1); + expect_d_eq(mallctl(buf, (void *)&result, &sz, NULL, 0), 0, + "mallctl() should succeed for valid stats lextent index"); +} +TEST_END + TEST_BEGIN(test_arenas_lextent_constants) { #define TEST_ARENAS_LEXTENT_CONSTANT(t, name, expected) \ do { \ @@ -1426,6 +1479,7 @@ main(void) { test_arenas_dirty_decay_ms, test_arenas_muzzy_decay_ms, test_arenas_constants, test_arenas_bin_constants, test_arenas_bin_oob, test_arenas_lextent_oob, + test_stats_arenas_bins_oob, test_stats_arenas_lextents_oob, test_arenas_lextent_constants, test_arenas_create, test_arenas_lookup, test_prof_active, test_stats_arenas, test_stats_arenas_hpa_shard_counters, From c2d57040f0d281449febb9bb80287e63bfb271fe Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 10:02:59 -0700 Subject: [PATCH 07/22] Fix out-of-bounds write in malloc_vsnprintf when size is 0 When called with size==0, the else branch wrote to str[size-1] which is str[(size_t)-1], a massive out-of-bounds write. Standard vsnprintf allows size==0 to mean "compute length only, write nothing". Add unit test for the size==0 case. --- src/malloc_io.c | 2 +- test/unit/malloc_io.c | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/malloc_io.c b/src/malloc_io.c index 779cdc05d3..9716c6682d 100644 --- a/src/malloc_io.c +++ b/src/malloc_io.c @@ -692,7 +692,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { label_out: if (i < size) { str[i] = '\0'; - } else { + } else if (size != 0) { str[size - 1] = '\0'; } diff --git a/test/unit/malloc_io.c b/test/unit/malloc_io.c index f78959451e..ee744a7816 100644 --- a/test/unit/malloc_io.c +++ b/test/unit/malloc_io.c @@ -252,8 +252,26 @@ TEST_BEGIN(test_malloc_snprintf) { } TEST_END +TEST_BEGIN(test_malloc_snprintf_zero_size) { + char buf[8]; + size_t result; + + /* + * malloc_snprintf with size==0 should not write anything but should + * return the length that would have been written. A previous bug + * caused an out-of-bounds write via str[size - 1] when size was 0. + */ + memset(buf, 'X', sizeof(buf)); + result = malloc_snprintf(buf, 0, "%s", "hello"); + expect_zu_eq(result, 5, "Expected length 5 for \"hello\""); + /* buf should be untouched. */ + expect_c_eq(buf[0], 'X', "Buffer should not have been modified"); +} +TEST_END + int main(void) { return test(test_malloc_strtoumax_no_endptr, test_malloc_strtoumax, - test_malloc_snprintf_truncated, test_malloc_snprintf); + test_malloc_snprintf_truncated, test_malloc_snprintf, + test_malloc_snprintf_zero_size); } From 3a8bee81f18bd241ba571a6a77c940c8f8cfcfb1 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 10:03:06 -0700 Subject: [PATCH 08/22] Fix pac_mapped stats inflation on allocation failure newly_mapped_size was set unconditionally in the ecache_alloc_grow fallback path, even when the allocation returned NULL. This inflated pac_mapped stats without a corresponding deallocation to correct them. Guard the assignment with an edata != NULL check, matching the pattern used in the batched allocation path above it. --- src/pac.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pac.c b/src/pac.c index 8600113962..ed0f77c2a3 100644 --- a/src/pac.c +++ b/src/pac.c @@ -198,7 +198,9 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, edata = ecache_alloc_grow(tsdn, pac, ehooks, &pac->ecache_retained, NULL, size, alignment, zero, guarded); - newly_mapped_size = size; + if (edata != NULL) { + newly_mapped_size = size; + } } if (config_stats && newly_mapped_size != 0) { From dd30c91eaaf02e5f347e37a49f99eae670b94c88 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 10:04:28 -0700 Subject: [PATCH 09/22] Fix wrong fallback value in os_page_detect when sysconf fails Returned LG_PAGE (log2 of page size, e.g. 12) instead of PAGE (actual page size, e.g. 4096) when sysconf(_SC_PAGESIZE) failed. This would cause os_page to be set to an absurdly small value, breaking all page-aligned operations. --- src/pages.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages.c b/src/pages.c index 2a4f00935c..5c12ae426e 100644 --- a/src/pages.c +++ b/src/pages.c @@ -718,7 +718,7 @@ os_page_detect(void) { #else long result = sysconf(_SC_PAGESIZE); if (result == -1) { - return LG_PAGE; + return PAGE; } return (size_t)result; #endif From 3f6e63e86a193e8a4d685480165812cac6d2350f Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 10:04:36 -0700 Subject: [PATCH 10/22] Fix wrong type for malloc_read_fd return value in prof_stack_range Used size_t (unsigned) instead of ssize_t for the return value of malloc_read_fd, which returns -1 on error. With size_t, -1 becomes a huge positive value, bypassing the error check and corrupting the remaining byte count. --- src/prof_stack_range.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/prof_stack_range.c b/src/prof_stack_range.c index b167b1323a..8ebcab8eef 100644 --- a/src/prof_stack_range.c +++ b/src/prof_stack_range.c @@ -73,17 +73,21 @@ prof_mapping_containing_addr(uintptr_t addr, const char *maps_path, } remaining = malloc_read_fd(fd, buf, sizeof(buf)); - if (remaining <= 0) { + if (remaining < 0) { ret = errno; break; + } else if (remaining == 0) { + break; } line = buf; } else if (line == NULL) { /* case 1: no newline found in buf */ remaining = malloc_read_fd(fd, buf, sizeof(buf)); - if (remaining <= 0) { + if (remaining < 0) { ret = errno; break; + } else if (remaining == 0) { + break; } line = memchr(buf, '\n', remaining); if (line != NULL) { @@ -99,11 +103,13 @@ prof_mapping_containing_addr(uintptr_t addr, const char *maps_path, remaining); /* copy remaining characters to start of buf */ line = buf; - size_t count = malloc_read_fd( + ssize_t count = malloc_read_fd( fd, buf + remaining, sizeof(buf) - remaining); - if (count <= 0) { + if (count < 0) { ret = errno; break; + } else if (count == 0) { + break; } remaining += From 675ab079e7e6f08a74727ec53569ec2db578d515 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 10:04:43 -0700 Subject: [PATCH 11/22] Fix missing release of acquired neighbor edata in extent_try_coalesce_impl When emap_try_acquire_edata_neighbor returned a non-NULL neighbor but the size check failed, the neighbor was never released from extent_state_merging, making it permanently invisible to future allocation and coalescing operations. Release the neighbor when it doesn't meet the size requirement, matching the pattern used in extent_recycle_extract. --- src/extent.c | 44 +++++++++++++++++++++++++++----------------- test/unit/pa.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/src/extent.c b/src/extent.c index 4b927191ae..118c8785fb 100644 --- a/src/extent.c +++ b/src/extent.c @@ -916,15 +916,20 @@ extent_try_coalesce_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t max_next_neighbor = max_size > edata_size_get(edata) ? max_size - edata_size_get(edata) : 0; - if (next != NULL && edata_size_get(next) <= max_next_neighbor) { - if (!extent_coalesce( - tsdn, pac, ehooks, ecache, edata, next, true)) { - if (ecache->delay_coalesce) { - /* Do minimal coalescing. */ - *coalesced = true; - return edata; + if (next != NULL) { + if (edata_size_get(next) > max_next_neighbor) { + emap_release_edata( + tsdn, pac->emap, next, ecache->state); + } else { + if (!extent_coalesce(tsdn, pac, ehooks, ecache, + edata, next, true)) { + if (ecache->delay_coalesce) { + /* Do minimal coalescing. */ + *coalesced = true; + return edata; + } + again = true; } - again = true; } } @@ -934,16 +939,21 @@ extent_try_coalesce_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t max_prev_neighbor = max_size > edata_size_get(edata) ? max_size - edata_size_get(edata) : 0; - if (prev != NULL && edata_size_get(prev) <= max_prev_neighbor) { - if (!extent_coalesce(tsdn, pac, ehooks, ecache, edata, - prev, false)) { - edata = prev; - if (ecache->delay_coalesce) { - /* Do minimal coalescing. */ - *coalesced = true; - return edata; + if (prev != NULL) { + if (edata_size_get(prev) > max_prev_neighbor) { + emap_release_edata( + tsdn, pac->emap, prev, ecache->state); + } else { + if (!extent_coalesce(tsdn, pac, ehooks, ecache, + edata, prev, false)) { + edata = prev; + if (ecache->delay_coalesce) { + /* Do minimal coalescing. */ + *coalesced = true; + return edata; + } + again = true; } - again = true; } } } while (again); diff --git a/test/unit/pa.c b/test/unit/pa.c index 8552225f61..c1562d7be3 100644 --- a/test/unit/pa.c +++ b/test/unit/pa.c @@ -121,7 +121,52 @@ TEST_BEGIN(test_alloc_free_purge_thds) { } TEST_END +TEST_BEGIN(test_failed_coalesce_releases_neighbor) { + test_skip_if(!maps_coalesce); + + test_data_t *test_data = init_test_data(-1, -1); + size_t old_lg_extent_max_active_fit = opt_lg_extent_max_active_fit; + opt_lg_extent_max_active_fit = 0; + + bool deferred_work_generated = false; + size_t unit = SC_LARGE_MINCLASS; + size_t alloc_size = 4 * unit; + edata_t *edata = pa_alloc(TSDN_NULL, &test_data->shard, alloc_size, + PAGE, + /* slab */ false, sz_size2index(alloc_size), /* zero */ false, + /* guarded */ false, &deferred_work_generated); + expect_ptr_not_null(edata, "Unexpected pa_alloc() failure"); + + void *tail_addr = (void *)((uintptr_t)edata_base_get(edata) + unit); + expect_false(pa_shrink(TSDN_NULL, &test_data->shard, edata, alloc_size, + unit, sz_size2index(unit), &deferred_work_generated), + "Unexpected pa_shrink() failure"); + + edata_t *tail = emap_edata_lookup( + TSDN_NULL, &test_data->emap, tail_addr); + expect_ptr_not_null(tail, "Expected dirty tail extent after shrink"); + expect_ptr_eq( + edata_base_get(tail), tail_addr, "Unexpected tail extent address"); + expect_zu_eq( + edata_size_get(tail), 3 * unit, "Unexpected tail extent size"); + expect_d_eq(edata_state_get(tail), extent_state_dirty, + "Expected tail extent to start dirty"); + + pa_dalloc( + TSDN_NULL, &test_data->shard, edata, &deferred_work_generated); + + tail = emap_edata_lookup(TSDN_NULL, &test_data->emap, tail_addr); + expect_ptr_not_null( + tail, "Expected oversized dirty neighbor to remain discoverable"); + expect_d_eq(edata_state_get(tail), extent_state_dirty, + "Failed coalesce must release oversized dirty neighbor"); + + opt_lg_extent_max_active_fit = old_lg_extent_max_active_fit; +} +TEST_END + int main(void) { - return test(test_alloc_free_purge_thds); + return test( + test_alloc_free_purge_thds, test_failed_coalesce_releases_neighbor); } From 234404d324458d4404ef382742741cb4ffbcf921 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 10:06:15 -0700 Subject: [PATCH 12/22] Fix wrong loop variable for array index in sz_boot_pind2sz_tab The sentinel fill loop used sz_pind2sz_tab[pind] (constant) instead of sz_pind2sz_tab[i] (loop variable), writing only to the first entry repeatedly and leaving subsequent entries uninitialized. --- src/sz.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sz.c b/src/sz.c index 4a4c057d51..da92f2b41e 100644 --- a/src/sz.c +++ b/src/sz.c @@ -65,7 +65,7 @@ sz_boot_pind2sz_tab(const sc_data_t *sc_data) { } } for (int i = pind; i <= (int)SC_NPSIZES; i++) { - sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE; + sz_pind2sz_tab[i] = sc_data->large_maxclass + PAGE; } } @@ -93,7 +93,7 @@ sz_boot_size2index_tab(const sc_data_t *sc_data) { size_t dst_max = (SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1; size_t dst_ind = 0; for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max; - sc_ind++) { + sc_ind++) { const sc_t *sc = &sc_data->sc[sc_ind]; size_t sz = (ZU(1) << sc->lg_base) + (ZU(sc->ndelta) << sc->lg_delta); From 2fceece256c0a01a28743652ce3e5cc67723e453 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 10:06:22 -0700 Subject: [PATCH 13/22] Fix extra size argument in edata_init call in extent_alloc_dss An extra 'size' argument was passed where 'slab' (false) should be, shifting all subsequent arguments: slab got size (nonzero=true), szind got false (0), and sn got SC_NSIZES instead of a proper serial number from extent_sn_next(). Match the correct pattern used by the gap edata_init call above. --- src/extent_dss.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/extent_dss.c b/src/extent_dss.c index 3f7a15d0d7..c7c34207eb 100644 --- a/src/extent_dss.c +++ b/src/extent_dss.c @@ -153,11 +153,14 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, - (uintptr_t)gap_addr_page; if (gap_size_page != 0) { edata_init(gap, arena_ind_get(arena), - gap_addr_page, gap_size_page, false, - SC_NSIZES, + gap_addr_page, gap_size_page, + /* slab */ false, + /* szind */ SC_NSIZES, extent_sn_next(&arena->pa_shard.pac), - extent_state_active, false, true, - EXTENT_PAI_PAC, head_state); + extent_state_active, + /* zeroed */ false, + /* committed */ true, + /* pai */ EXTENT_PAI_PAC, head_state); } /* * Compute the address just past the end of the desired @@ -203,9 +206,16 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, arena); edata_init(&edata, arena_ind_get(arena), - ret, size, size, false, SC_NSIZES, - extent_state_active, false, true, - EXTENT_PAI_PAC, head_state); + ret, size, + /* slab */ false, + /* szind */ SC_NSIZES, + extent_sn_next( + &arena->pa_shard.pac), + extent_state_active, + /* zeroed */ false, + /* committed */ true, + /* pai */ EXTENT_PAI_PAC, + head_state); if (extent_purge_forced_wrapper(tsdn, ehooks, &edata, 0, size)) { memset(ret, 0, size); From 5904a421878b31d6a5ec674027b35db63e64537d Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 10:07:05 -0700 Subject: [PATCH 14/22] Fix memory leak of old curr_reg on san_bump_grow_locked failure When san_bump_grow_locked fails, it sets sba->curr_reg to NULL. The old curr_reg (saved in to_destroy) was never freed or restored, leaking the virtual memory extent. Restore sba->curr_reg from to_destroy on failure so the old region remains usable. --- src/san_bump.c | 1 + test/unit/san_bump.c | 89 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/san_bump.c b/src/san_bump.c index 09ed18cabd..110312902c 100644 --- a/src/san_bump.c +++ b/src/san_bump.c @@ -31,6 +31,7 @@ san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t *sba, pac_t *pac, bool err = san_bump_grow_locked( tsdn, sba, pac, ehooks, guarded_size); if (err) { + sba->curr_reg = to_destroy; goto label_err; } } else { diff --git a/test/unit/san_bump.c b/test/unit/san_bump.c index 9aa0210e9e..54d8583d63 100644 --- a/test/unit/san_bump.c +++ b/test/unit/san_bump.c @@ -4,6 +4,50 @@ #include "jemalloc/internal/arena_structs.h" #include "jemalloc/internal/san_bump.h" +static extent_hooks_t *san_bump_default_hooks; +static extent_hooks_t san_bump_hooks; +static bool fail_retained_alloc; +static unsigned retained_alloc_fail_calls; + +static void * +san_bump_fail_alloc_hook(extent_hooks_t *UNUSED extent_hooks, void *new_addr, + size_t size, size_t alignment, bool *zero, bool *commit, + unsigned arena_ind) { + if (fail_retained_alloc && new_addr == NULL + && size >= SBA_RETAINED_ALLOC_SIZE) { + retained_alloc_fail_calls++; + return NULL; + } + return san_bump_default_hooks->alloc(san_bump_default_hooks, new_addr, + size, alignment, zero, commit, arena_ind); +} + +static void +install_san_bump_fail_alloc_hooks(unsigned arena_ind) { + size_t hooks_mib[3]; + size_t hooks_miblen = sizeof(hooks_mib) / sizeof(size_t); + size_t old_size = sizeof(extent_hooks_t *); + size_t new_size = sizeof(extent_hooks_t *); + extent_hooks_t *new_hooks; + extent_hooks_t *old_hooks; + + expect_d_eq( + mallctlnametomib("arena.0.extent_hooks", hooks_mib, &hooks_miblen), + 0, "Unexpected mallctlnametomib() failure"); + hooks_mib[1] = (size_t)arena_ind; + expect_d_eq(mallctlbymib(hooks_mib, hooks_miblen, (void *)&old_hooks, + &old_size, NULL, 0), + 0, "Unexpected extent_hooks error"); + + san_bump_default_hooks = old_hooks; + san_bump_hooks = *old_hooks; + san_bump_hooks.alloc = san_bump_fail_alloc_hook; + new_hooks = &san_bump_hooks; + expect_d_eq(mallctlbymib(hooks_mib, hooks_miblen, NULL, NULL, + (void *)&new_hooks, new_size), + 0, "Unexpected extent_hooks install failure"); +} + TEST_BEGIN(test_san_bump_alloc) { test_skip_if(!maps_coalesce || !opt_retain); @@ -69,6 +113,48 @@ TEST_BEGIN(test_san_bump_alloc) { } TEST_END +TEST_BEGIN(test_failed_grow_preserves_curr_reg) { + test_skip_if(!maps_coalesce || !opt_retain); + + tsdn_t *tsdn = tsdn_fetch(); + + san_bump_alloc_t sba; + san_bump_alloc_init(&sba); + + unsigned arena_ind = do_arena_create(0, 0); + assert_u_ne(arena_ind, UINT_MAX, "Failed to create an arena"); + install_san_bump_fail_alloc_hooks(arena_ind); + + arena_t *arena = arena_get(tsdn, arena_ind, false); + pac_t *pac = &arena->pa_shard.pac; + + size_t small_alloc_size = PAGE * 16; + edata_t *edata = san_bump_alloc(tsdn, &sba, pac, pac_ehooks_get(pac), + small_alloc_size, /* zero */ false); + expect_ptr_not_null(edata, "Initial san_bump allocation failed"); + expect_ptr_not_null(sba.curr_reg, + "Expected retained region remainder after initial allocation"); + + fail_retained_alloc = true; + retained_alloc_fail_calls = 0; + + edata_t *failed = san_bump_alloc(tsdn, &sba, pac, pac_ehooks_get(pac), + SBA_RETAINED_ALLOC_SIZE, /* zero */ false); + expect_ptr_null(failed, "Expected retained grow allocation failure"); + expect_u_eq(retained_alloc_fail_calls, 1, + "Expected exactly one failed retained allocation attempt"); + + edata_t *reused = san_bump_alloc(tsdn, &sba, pac, pac_ehooks_get(pac), + small_alloc_size, /* zero */ false); + expect_ptr_not_null( + reused, "Expected allocator to reuse preexisting current region"); + expect_u_eq(retained_alloc_fail_calls, 1, + "Reuse path should not attempt another retained grow allocation"); + + fail_retained_alloc = false; +} +TEST_END + TEST_BEGIN(test_large_alloc_size) { test_skip_if(!maps_coalesce || !opt_retain); @@ -105,5 +191,6 @@ TEST_END int main(void) { - return test(test_san_bump_alloc, test_large_alloc_size); + return test(test_san_bump_alloc, test_failed_grow_preserves_curr_reg, + test_large_alloc_size); } From 3ac9f96158f3b095496e260259a3c32857eafd28 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Tue, 31 Mar 2026 20:33:49 -0700 Subject: [PATCH 15/22] Run clang-format on test/unit/conf_parse.c --- test/unit/conf_parse.c | 58 ++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/test/unit/conf_parse.c b/test/unit/conf_parse.c index b3fedb4092..eb1078658c 100644 --- a/test/unit/conf_parse.c +++ b/test/unit/conf_parse.c @@ -27,8 +27,8 @@ TEST_END TEST_BEGIN(test_conf_handle_unsigned_in_range) { uintmax_t result = 0; - bool err = conf_handle_unsigned("100", sizeof("100") - 1, - 1, 2048, true, true, true, &result); + bool err = conf_handle_unsigned( + "100", sizeof("100") - 1, 1, 2048, true, true, true, &result); expect_false(err, "Should succeed for in-range value"); expect_u64_eq((uint64_t)result, 100, "result should be 100"); } @@ -36,44 +36,43 @@ TEST_END TEST_BEGIN(test_conf_handle_unsigned_clip_max) { uintmax_t result = 0; - bool err = conf_handle_unsigned("9999", sizeof("9999") - 1, - 1, 2048, true, true, true, &result); + bool err = conf_handle_unsigned( + "9999", sizeof("9999") - 1, 1, 2048, true, true, true, &result); expect_false(err, "Should succeed with clipping"); - expect_u64_eq((uint64_t)result, 2048, - "result should be clipped to max 2048"); + expect_u64_eq( + (uint64_t)result, 2048, "result should be clipped to max 2048"); } TEST_END TEST_BEGIN(test_conf_handle_unsigned_clip_min) { uintmax_t result = 0; - bool err = conf_handle_unsigned("0", sizeof("0") - 1, - 1, 2048, true, true, true, &result); + bool err = conf_handle_unsigned( + "0", sizeof("0") - 1, 1, 2048, true, true, true, &result); expect_false(err, "Should succeed with clipping"); - expect_u64_eq((uint64_t)result, 1, - "result should be clipped to min 1"); + expect_u64_eq((uint64_t)result, 1, "result should be clipped to min 1"); } TEST_END TEST_BEGIN(test_conf_handle_unsigned_no_clip_reject) { uintmax_t result = 0; - bool err = conf_handle_unsigned("9999", sizeof("9999") - 1, - 1, 2048, true, true, false, &result); + bool err = conf_handle_unsigned( + "9999", sizeof("9999") - 1, 1, 2048, true, true, false, &result); expect_true(err, "Should fail for out-of-range value without clip"); } TEST_END TEST_BEGIN(test_conf_handle_unsigned_invalid) { uintmax_t result = 0; - bool err = conf_handle_unsigned("abc", sizeof("abc") - 1, - 1, 2048, true, true, true, &result); + bool err = conf_handle_unsigned( + "abc", sizeof("abc") - 1, 1, 2048, true, true, true, &result); expect_true(err, "Should fail for non-numeric input"); } TEST_END TEST_BEGIN(test_conf_handle_signed_valid) { intmax_t result = 0; - bool err = conf_handle_signed("5000", sizeof("5000") - 1, - -1, INTMAX_MAX, true, false, false, &result); + bool err = conf_handle_signed("5000", sizeof("5000") - 1, -1, + INTMAX_MAX, true, false, false, &result); expect_false(err, "Should succeed for valid value"); expect_d64_eq((int64_t)result, 5000, "result should be 5000"); } @@ -81,8 +80,8 @@ TEST_END TEST_BEGIN(test_conf_handle_signed_negative) { intmax_t result = 0; - bool err = conf_handle_signed("-1", sizeof("-1") - 1, - -1, INTMAX_MAX, true, false, false, &result); + bool err = conf_handle_signed("-1", sizeof("-1") - 1, -1, INTMAX_MAX, + true, false, false, &result); expect_false(err, "Should succeed for -1"); expect_d64_eq((int64_t)result, -1, "result should be -1"); } @@ -90,8 +89,8 @@ TEST_END TEST_BEGIN(test_conf_handle_signed_out_of_range) { intmax_t result = 0; - bool err = conf_handle_signed("5000", sizeof("5000") - 1, - -1, 4999, true, true, false, &result); + bool err = conf_handle_signed( + "5000", sizeof("5000") - 1, -1, 4999, true, true, false, &result); expect_true(err, "Should fail for out-of-range value"); } TEST_END @@ -101,13 +100,14 @@ TEST_BEGIN(test_conf_handle_char_p) { bool err; /* Normal copy. */ - err = conf_handle_char_p("hello", sizeof("hello") - 1, buf, sizeof(buf)); + err = conf_handle_char_p( + "hello", sizeof("hello") - 1, buf, sizeof(buf)); expect_false(err, "Should succeed"); expect_str_eq(buf, "hello", "Should copy string"); /* Truncation. */ - err = conf_handle_char_p("longstring", sizeof("longstring") - 1, - buf, sizeof(buf)); + err = conf_handle_char_p( + "longstring", sizeof("longstring") - 1, buf, sizeof(buf)); expect_false(err, "Should succeed even when truncating"); expect_str_eq(buf, "longstr", "Should truncate to dest_sz - 1"); } @@ -115,16 +115,12 @@ TEST_END int main(void) { - return test(test_conf_handle_bool_true, - test_conf_handle_bool_false, - test_conf_handle_bool_invalid, - test_conf_handle_unsigned_in_range, + return test(test_conf_handle_bool_true, test_conf_handle_bool_false, + test_conf_handle_bool_invalid, test_conf_handle_unsigned_in_range, test_conf_handle_unsigned_clip_max, test_conf_handle_unsigned_clip_min, test_conf_handle_unsigned_no_clip_reject, - test_conf_handle_unsigned_invalid, - test_conf_handle_signed_valid, + test_conf_handle_unsigned_invalid, test_conf_handle_signed_valid, test_conf_handle_signed_negative, - test_conf_handle_signed_out_of_range, - test_conf_handle_char_p); + test_conf_handle_signed_out_of_range, test_conf_handle_char_p); } From b507644cb084d095917aea6e2573c702caff3e5a Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 12:27:00 -0700 Subject: [PATCH 16/22] Fix conf_handle_char_p zero-sized dest and remove unused conf_handle_unsigned --- include/jemalloc/internal/conf.h | 16 +++----- src/conf.c | 41 ++++----------------- test/unit/conf_parse.c | 63 +++++++------------------------- 3 files changed, 27 insertions(+), 93 deletions(-) diff --git a/include/jemalloc/internal/conf.h b/include/jemalloc/internal/conf.h index 216619555b..26983ee9f0 100644 --- a/include/jemalloc/internal/conf.h +++ b/include/jemalloc/internal/conf.h @@ -9,19 +9,15 @@ void malloc_abort_invalid_conf(void); #ifdef JEMALLOC_JET extern bool had_conf_error; + bool conf_next(char const **opts_p, char const **k_p, size_t *klen_p, char const **v_p, size_t *vlen_p); -void conf_error(const char *msg, const char *k, size_t klen, - const char *v, size_t vlen); +void conf_error( + const char *msg, const char *k, size_t klen, const char *v, size_t vlen); bool conf_handle_bool(const char *v, size_t vlen, bool *result); -bool conf_handle_unsigned(const char *v, size_t vlen, - uintmax_t min, uintmax_t max, bool check_min, bool check_max, - bool clip, uintmax_t *result); -bool conf_handle_signed(const char *v, size_t vlen, - intmax_t min, intmax_t max, bool check_min, bool check_max, - bool clip, intmax_t *result); -bool conf_handle_char_p(const char *v, size_t vlen, - char *dest, size_t dest_sz); +bool conf_handle_signed(const char *v, size_t vlen, intmax_t min, intmax_t max, + bool check_min, bool check_max, bool clip, intmax_t *result); +bool conf_handle_char_p(const char *v, size_t vlen, char *dest, size_t dest_sz); #endif #endif /* JEMALLOC_INTERNAL_CONF_H */ diff --git a/src/conf.c b/src/conf.c index 8a23bda62a..65abcd2597 100644 --- a/src/conf.c +++ b/src/conf.c @@ -254,36 +254,8 @@ JEMALLOC_DIAGNOSTIC_PUSH JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-function") JET_EXTERN bool -conf_handle_unsigned(const char *v, size_t vlen, - uintmax_t min, uintmax_t max, bool check_min, bool check_max, - bool clip, uintmax_t *result) { - char *end; - set_errno(0); - uintmax_t mv = (uintmax_t)malloc_strtoumax(v, &end, 0); - if (get_errno() != 0 || (uintptr_t)end - (uintptr_t)v != vlen) { - return true; - } - if (clip) { - if (check_min && mv < min) { - *result = min; - } else if (check_max && mv > max) { - *result = max; - } else { - *result = mv; - } - } else { - if ((check_min && mv < min) || (check_max && mv > max)) { - return true; - } - *result = mv; - } - return false; -} - -JET_EXTERN bool -conf_handle_signed(const char *v, size_t vlen, - intmax_t min, intmax_t max, bool check_min, bool check_max, - bool clip, intmax_t *result) { +conf_handle_signed(const char *v, size_t vlen, intmax_t min, intmax_t max, + bool check_min, bool check_max, bool clip, intmax_t *result) { char *end; set_errno(0); intmax_t mv = (intmax_t)malloc_strtoumax(v, &end, 0); @@ -309,6 +281,9 @@ conf_handle_signed(const char *v, size_t vlen, JET_EXTERN bool conf_handle_char_p(const char *v, size_t vlen, char *dest, size_t dest_sz) { + if (dest_sz == 0) { + return false; + } size_t cpylen = (vlen <= dest_sz - 1) ? vlen : dest_sz - 1; strncpy(dest, v, cpylen); dest[cpylen] = '\0'; @@ -473,11 +448,11 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], continue; } - while (*opts != '\0' - && !conf_next(&opts, &k, &klen, &v, &vlen)) { + while ( + *opts != '\0' && !conf_next(&opts, &k, &klen, &v, &vlen)) { #define CONF_ERROR(msg, k, klen, v, vlen) \ if (!initial_call) { \ - conf_error(msg, k, klen, v, vlen); \ + conf_error(msg, k, klen, v, vlen); \ cur_opt_valid = false; \ } #define CONF_CONTINUE \ diff --git a/test/unit/conf_parse.c b/test/unit/conf_parse.c index eb1078658c..448cc84a92 100644 --- a/test/unit/conf_parse.c +++ b/test/unit/conf_parse.c @@ -25,50 +25,6 @@ TEST_BEGIN(test_conf_handle_bool_invalid) { } TEST_END -TEST_BEGIN(test_conf_handle_unsigned_in_range) { - uintmax_t result = 0; - bool err = conf_handle_unsigned( - "100", sizeof("100") - 1, 1, 2048, true, true, true, &result); - expect_false(err, "Should succeed for in-range value"); - expect_u64_eq((uint64_t)result, 100, "result should be 100"); -} -TEST_END - -TEST_BEGIN(test_conf_handle_unsigned_clip_max) { - uintmax_t result = 0; - bool err = conf_handle_unsigned( - "9999", sizeof("9999") - 1, 1, 2048, true, true, true, &result); - expect_false(err, "Should succeed with clipping"); - expect_u64_eq( - (uint64_t)result, 2048, "result should be clipped to max 2048"); -} -TEST_END - -TEST_BEGIN(test_conf_handle_unsigned_clip_min) { - uintmax_t result = 0; - bool err = conf_handle_unsigned( - "0", sizeof("0") - 1, 1, 2048, true, true, true, &result); - expect_false(err, "Should succeed with clipping"); - expect_u64_eq((uint64_t)result, 1, "result should be clipped to min 1"); -} -TEST_END - -TEST_BEGIN(test_conf_handle_unsigned_no_clip_reject) { - uintmax_t result = 0; - bool err = conf_handle_unsigned( - "9999", sizeof("9999") - 1, 1, 2048, true, true, false, &result); - expect_true(err, "Should fail for out-of-range value without clip"); -} -TEST_END - -TEST_BEGIN(test_conf_handle_unsigned_invalid) { - uintmax_t result = 0; - bool err = conf_handle_unsigned( - "abc", sizeof("abc") - 1, 1, 2048, true, true, true, &result); - expect_true(err, "Should fail for non-numeric input"); -} -TEST_END - TEST_BEGIN(test_conf_handle_signed_valid) { intmax_t result = 0; bool err = conf_handle_signed("5000", sizeof("5000") - 1, -1, @@ -113,14 +69,21 @@ TEST_BEGIN(test_conf_handle_char_p) { } TEST_END +TEST_BEGIN(test_conf_handle_char_p_zero_dest_sz) { + char buf[4] = {'X', 'Y', 'Z', '\0'}; + bool err; + + err = conf_handle_char_p("abc", sizeof("abc") - 1, buf, 0); + expect_false(err, "Should succeed for zero-sized destination"); + expect_c_eq(buf[0], 'X', "Zero-sized destination must not be modified"); +} +TEST_END + int main(void) { return test(test_conf_handle_bool_true, test_conf_handle_bool_false, - test_conf_handle_bool_invalid, test_conf_handle_unsigned_in_range, - test_conf_handle_unsigned_clip_max, - test_conf_handle_unsigned_clip_min, - test_conf_handle_unsigned_no_clip_reject, - test_conf_handle_unsigned_invalid, test_conf_handle_signed_valid, + test_conf_handle_bool_invalid, test_conf_handle_signed_valid, test_conf_handle_signed_negative, - test_conf_handle_signed_out_of_range, test_conf_handle_char_p); + test_conf_handle_signed_out_of_range, test_conf_handle_char_p, + test_conf_handle_char_p_zero_dest_sz); } From a47fa33b5a7d91ab0218436a75b652a2b65588c9 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Tue, 31 Mar 2026 20:37:23 -0700 Subject: [PATCH 17/22] Run clang-format on test/unit/tcache_max.c --- test/unit/tcache_max.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/unit/tcache_max.c b/test/unit/tcache_max.c index d57b2d3b79..653563caf8 100644 --- a/test/unit/tcache_max.c +++ b/test/unit/tcache_max.c @@ -195,9 +195,9 @@ TEST_BEGIN(test_tcache_max) { global_test = true; for (alloc_option = alloc_option_start; alloc_option < alloc_option_end; - alloc_option++) { + alloc_option++) { for (dalloc_option = dalloc_option_start; - dalloc_option < dalloc_option_end; dalloc_option++) { + dalloc_option < dalloc_option_end; dalloc_option++) { /* opt.tcache_max set to 1024 in tcache_max.sh. */ test_tcache_max_impl(1024, alloc_option, dalloc_option); } @@ -318,9 +318,9 @@ tcache_check(void *arg) { expect_zu_eq(tcache_nbins, tcache_max2nbins(new_tcache_max), "Unexpected value for tcache_nbins"); for (unsigned alloc_option = alloc_option_start; - alloc_option < alloc_option_end; alloc_option++) { + alloc_option < alloc_option_end; alloc_option++) { for (unsigned dalloc_option = dalloc_option_start; - dalloc_option < dalloc_option_end; dalloc_option++) { + dalloc_option < dalloc_option_end; dalloc_option++) { test_tcache_max_impl( new_tcache_max, alloc_option, dalloc_option); } From 3cc56d325c15cdb7d6047ed513ab908121c66698 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 13:30:58 -0700 Subject: [PATCH 18/22] Fix large alloc nrequests under-counting on cache misses --- include/jemalloc/internal/tcache_inlines.h | 6 +-- test/unit/tcache_max.c | 47 +++++++++++++++++++++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index 6bd1b339ea..5f8ed317fa 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -163,10 +163,10 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, assert(usize <= tcache_max_get(tcache->tcache_slow)); memset(ret, 0, usize); } + } - if (config_stats) { - bin->tstats.nrequests++; - } + if (config_stats) { + bin->tstats.nrequests++; } return ret; diff --git a/test/unit/tcache_max.c b/test/unit/tcache_max.c index 653563caf8..ab54da39c4 100644 --- a/test/unit/tcache_max.c +++ b/test/unit/tcache_max.c @@ -206,6 +206,50 @@ TEST_BEGIN(test_tcache_max) { } TEST_END +TEST_BEGIN(test_large_tcache_nrequests_on_miss) { + test_skip_if(!config_stats); + test_skip_if(!opt_tcache); + test_skip_if(opt_prof); + test_skip_if(san_uaf_detection_enabled()); + + size_t large; + size_t sz = sizeof(large); + expect_d_eq( + mallctl("arenas.lextent.0.size", (void *)&large, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + expect_d_eq(mallctl("thread.tcache.max", NULL, NULL, (void *)&large, + sizeof(large)), + 0, "Unexpected mallctl() failure"); + expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), 0, + "Unexpected tcache flush failure"); + + tsd_t *tsd = tsd_fetch(); + expect_ptr_not_null(tsd, "Unexpected tsd_fetch() failure"); + tcache_t *tcache = tcache_get(tsd); + expect_ptr_not_null(tcache, "Expected auto tcache"); + + szind_t binind = sz_size2index(large); + expect_true(binind >= SC_NBINS, "Expected large size class"); + cache_bin_t *bin = &tcache->bins[binind]; + bin->tstats.nrequests = 0; + + void *p = mallocx(large, 0); + expect_ptr_not_null(p, "Unexpected mallocx() failure"); + expect_u64_eq(bin->tstats.nrequests, 1, + "Large tcache miss should count as one request"); + + dallocx(p, 0); + p = mallocx(large, 0); + expect_ptr_not_null(p, "Unexpected mallocx() failure"); + expect_u64_eq(bin->tstats.nrequests, 2, + "Large tcache hit should increment request count again"); + + dallocx(p, 0); + expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), 0, + "Unexpected tcache flush failure"); +} +TEST_END + static size_t tcache_max2nbins(size_t tcache_max) { return sz_size2index(tcache_max) + 1; @@ -358,5 +402,6 @@ TEST_END int main(void) { - return test(test_tcache_max, test_thread_tcache_max); + return test(test_tcache_max, test_large_tcache_nrequests_on_miss, + test_thread_tcache_max); } From 6281482c395fdbf721ff1f09f531315744446b35 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 13:50:11 -0700 Subject: [PATCH 19/22] Nest HPA SEC stats inside hpa_shard JSON --- src/stats.c | 3 +- test/unit/json_stats.c | 77 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/stats.c b/src/stats.c index d906ade6d0..fa018ea0d1 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1113,9 +1113,8 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { static void stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { - stats_arena_hpa_shard_sec_print(emitter, i); - emitter_json_object_kv_begin(emitter, "hpa_shard"); + stats_arena_hpa_shard_sec_print(emitter, i); stats_arena_hpa_shard_counters_print(emitter, i, uptime); stats_arena_hpa_shard_slabs_print(emitter, i); emitter_json_object_end(emitter); /* End "hpa_shard" */ diff --git a/test/unit/json_stats.c b/test/unit/json_stats.c index eee717c7b6..d6cbc50c49 100644 --- a/test/unit/json_stats.c +++ b/test/unit/json_stats.c @@ -185,6 +185,41 @@ static const char *arena_mutex_names[] = {"large", "extent_avail", static const size_t num_arena_mutexes = sizeof(arena_mutex_names) / sizeof(arena_mutex_names[0]); +static const char * +json_find_object_end(const char *object_begin) { + int depth = 0; + for (const char *cur = object_begin; *cur != '\0'; cur++) { + if (*cur == '{') { + depth++; + } else if (*cur == '}') { + depth--; + if (depth == 0) { + return cur; + } + } + } + return NULL; +} + +static const char * +json_find_previous_hpa_shard_object( + const char *json, const char *pos, const char **object_end) { + *object_end = NULL; + const char *found = NULL; + const char *cur = json; + const char *next; + + while ((next = strstr(cur, "\"hpa_shard\":{")) != NULL && next < pos) { + found = strchr(next, '{'); + cur = next + 1; + } + if (found == NULL) { + return NULL; + } + *object_end = json_find_object_end(found); + return found; +} + TEST_BEGIN(test_json_stats_mutexes) { test_skip_if(!config_stats); @@ -307,8 +342,48 @@ TEST_BEGIN(test_hpa_shard_json_ndirty_huge) { } TEST_END +TEST_BEGIN(test_hpa_shard_json_contains_sec_stats) { + test_skip_if(!config_stats); + test_skip_if(!hpa_supported()); + + void *p = mallocx(PAGE, MALLOCX_TCACHE_NONE); + expect_ptr_not_null(p, "Unexpected mallocx failure"); + + uint64_t epoch = 1; + size_t sz = sizeof(epoch); + expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sz), 0, + "Unexpected mallctl() failure"); + + stats_buf_t sbuf; + stats_buf_init(&sbuf); + malloc_stats_print(stats_buf_write_cb, &sbuf, "J"); + + const char *sec_bytes = strstr(sbuf.buf, "\"sec_bytes\""); + expect_ptr_not_null(sec_bytes, "JSON output should contain sec_bytes"); + const char *hpa_shard_end = NULL; + const char *hpa_shard = json_find_previous_hpa_shard_object( + sbuf.buf, sec_bytes, &hpa_shard_end); + expect_ptr_not_null(hpa_shard, + "sec_bytes should be associated with an hpa_shard JSON object"); + expect_ptr_not_null(hpa_shard_end, + "Could not find end of enclosing hpa_shard JSON object"); + expect_true(sec_bytes != NULL && sec_bytes < hpa_shard_end, + "sec_bytes should be nested inside hpa_shard JSON object"); + const char *sec_hits = strstr(hpa_shard, "\"sec_hits\""); + expect_true(sec_hits != NULL && sec_hits < hpa_shard_end, + "sec_hits should be nested inside hpa_shard JSON object"); + const char *sec_misses = strstr(hpa_shard, "\"sec_misses\""); + expect_true(sec_misses != NULL && sec_misses < hpa_shard_end, + "sec_misses should be nested inside hpa_shard JSON object"); + + stats_buf_fini(&sbuf); + dallocx(p, MALLOCX_TCACHE_NONE); +} +TEST_END + int main(void) { return test_no_reentrancy(test_json_stats_mutexes, - test_hpa_shard_json_ndirty_huge); + test_hpa_shard_json_ndirty_huge, + test_hpa_shard_json_contains_sec_stats); } From db7d99703d41e58ba2932e98a6e12dd377028231 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 13:54:15 -0700 Subject: [PATCH 20/22] Add TODO to benchmark possibly better policy --- src/sec.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sec.c b/src/sec.c index 5f65362f57..a32545370c 100644 --- a/src/sec.c +++ b/src/sec.c @@ -130,13 +130,17 @@ sec_multishard_trylock_alloc( cur_shard = 0; } } - /* No bin had alloc or had the extent */ + /* + * TODO: Benchmark whether it is worth blocking on all shards here before + * declaring a miss. That could recover more remote-shard hits under + * contention, but it also changes the allocation latency policy. + */ assert(cur_shard == sec_shard_pick(tsdn, sec)); bin = sec_bin_pick(sec, cur_shard, pszind); malloc_mutex_lock(tsdn, &bin->mtx); edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size); if (edata == NULL) { - /* Only now we know it is a miss */ + /* Only now we know it is a miss. */ bin->stats.nmisses++; } malloc_mutex_unlock(tsdn, &bin->mtx); From f265645d02f0bde59833c46977b66acd94dec42e Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 27 Mar 2026 13:58:27 -0700 Subject: [PATCH 21/22] Emit retained HPA slab stats in JSON --- src/stats.c | 6 ++ test/unit/json_stats.c | 122 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/stats.c b/src/stats.c index fa018ea0d1..82458fec45 100644 --- a/src/stats.c +++ b/src/stats.c @@ -988,6 +988,8 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); emitter_json_kv( emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); + emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, + &nretained_nonhuge); emitter_json_object_end(emitter); /* End "full_slabs" */ /* Next, empty slab stats. */ @@ -1029,6 +1031,8 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); emitter_json_kv( emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); + emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, + &nretained_nonhuge); emitter_json_object_end(emitter); /* End "empty_slabs" */ /* Last, nonfull slab stats. */ @@ -1103,6 +1107,8 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { &nactive_nonhuge); emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); + emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, + &nretained_nonhuge); emitter_json_object_end(emitter); } emitter_json_array_end(emitter); /* End "nonfull_slabs" */ diff --git a/test/unit/json_stats.c b/test/unit/json_stats.c index d6cbc50c49..c206974b8a 100644 --- a/test/unit/json_stats.c +++ b/test/unit/json_stats.c @@ -196,6 +196,28 @@ json_find_object_end(const char *object_begin) { if (depth == 0) { return cur; } + if (depth < 0) { + return NULL; + } + } + } + return NULL; +} + +static const char * +json_find_array_end(const char *array_begin) { + int depth = 0; + for (const char *cur = array_begin; *cur != '\0'; cur++) { + if (*cur == '[') { + depth++; + } else if (*cur == ']') { + depth--; + if (depth == 0) { + return cur; + } + if (depth < 0) { + return NULL; + } } } return NULL; @@ -220,6 +242,52 @@ json_find_previous_hpa_shard_object( return found; } +static const char * +json_find_named_object( + const char *json, const char *key, const char **object_end) { + *object_end = NULL; + char search_key[128]; + size_t written = malloc_snprintf( + search_key, sizeof(search_key), "\"%s\":{", key); + if (written >= sizeof(search_key)) { + return NULL; + } + + const char *object_begin = strstr(json, search_key); + if (object_begin == NULL) { + return NULL; + } + object_begin = strchr(object_begin, '{'); + if (object_begin == NULL) { + return NULL; + } + *object_end = json_find_object_end(object_begin); + return object_begin; +} + +static const char * +json_find_named_array( + const char *json, const char *key, const char **array_end) { + *array_end = NULL; + char search_key[128]; + size_t written = malloc_snprintf( + search_key, sizeof(search_key), "\"%s\":[", key); + if (written >= sizeof(search_key)) { + return NULL; + } + + const char *array_begin = strstr(json, search_key); + if (array_begin == NULL) { + return NULL; + } + array_begin = strchr(array_begin, '['); + if (array_begin == NULL) { + return NULL; + } + *array_end = json_find_array_end(array_begin); + return array_begin; +} + TEST_BEGIN(test_json_stats_mutexes) { test_skip_if(!config_stats); @@ -381,9 +449,61 @@ TEST_BEGIN(test_hpa_shard_json_contains_sec_stats) { } TEST_END +TEST_BEGIN(test_hpa_shard_json_contains_retained_stats) { + test_skip_if(!config_stats); + test_skip_if(!hpa_supported()); + + void *p = mallocx(PAGE, MALLOCX_TCACHE_NONE); + expect_ptr_not_null(p, "Unexpected mallocx failure"); + + uint64_t epoch = 1; + size_t sz = sizeof(epoch); + expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sz), 0, + "Unexpected mallctl() failure"); + + stats_buf_t sbuf; + stats_buf_init(&sbuf); + malloc_stats_print(stats_buf_write_cb, &sbuf, "J"); + + const char *full_slabs_end = NULL; + const char *full_slabs = json_find_named_object( + sbuf.buf, "full_slabs", &full_slabs_end); + expect_ptr_not_null( + full_slabs, "JSON output should contain full_slabs"); + const char *full_retained = strstr(full_slabs, "\"nretained_nonhuge\""); + expect_true(full_retained != NULL && full_retained < full_slabs_end, + "full_slabs should contain nretained_nonhuge"); + + const char *empty_slabs_end = NULL; + const char *empty_slabs = json_find_named_object( + sbuf.buf, "empty_slabs", &empty_slabs_end); + expect_ptr_not_null( + empty_slabs, "JSON output should contain empty_slabs"); + const char *empty_retained = strstr( + empty_slabs, "\"nretained_nonhuge\""); + expect_true(empty_retained != NULL && empty_retained < empty_slabs_end, + "empty_slabs should contain nretained_nonhuge"); + + const char *nonfull_slabs_end = NULL; + const char *nonfull_slabs = json_find_named_array( + sbuf.buf, "nonfull_slabs", &nonfull_slabs_end); + expect_ptr_not_null( + nonfull_slabs, "JSON output should contain nonfull_slabs"); + const char *nonfull_retained = strstr( + nonfull_slabs, "\"nretained_nonhuge\""); + expect_true( + nonfull_retained != NULL && nonfull_retained < nonfull_slabs_end, + "nonfull_slabs should contain nretained_nonhuge"); + + stats_buf_fini(&sbuf); + dallocx(p, MALLOCX_TCACHE_NONE); +} +TEST_END + int main(void) { return test_no_reentrancy(test_json_stats_mutexes, test_hpa_shard_json_ndirty_huge, - test_hpa_shard_json_contains_sec_stats); + test_hpa_shard_json_contains_sec_stats, + test_hpa_shard_json_contains_retained_stats); } From 6515df8cec7fe50f6b45069f82bdf685171f9ee7 Mon Sep 17 00:00:00 2001 From: Guangli Dai Date: Tue, 7 Apr 2026 10:41:44 -0700 Subject: [PATCH 22/22] Documentation updates (#2869) * Document new mallctl interfaces added since 5.3.0 Add documentation for the following new mallctl entries: - opt.debug_double_free_max_scan: double-free detection scan limit - opt.prof_bt_max: max profiling backtrace depth - opt.disable_large_size_classes: page-aligned large allocations - opt.process_madvise_max_batch: batched process_madvise purging - thread.tcache.max: per-thread tcache_max control - thread.tcache.ncached_max.read_sizeclass: query ncached_max - thread.tcache.ncached_max.write: set ncached_max per size range - arena..name: get/set arena names - arenas.hugepage: hugepage size - approximate_stats.active: lightweight active bytes estimate Remove config.prof_frameptr since it still needs more development and is still experimental. Co-authored-by: lexprfuncall --- doc/jemalloc.xml.in | 157 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 147 insertions(+), 10 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 2a8573b87f..692658f832 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -897,16 +897,6 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", during build configuration. - - - config.prof_frameptr - (bool) - r- - - was specified - during build configuration. - - config.stats @@ -1419,6 +1409,17 @@ malloc_conf = "xmalloc:true";]]> extent hooks. + + + opt.prof_bt_max + (unsigned) + r- + [] + + Maximum number of stack frames to record in profiling + backtraces. The default is 128. + + opt.prof @@ -1666,6 +1667,53 @@ malloc_conf = "xmalloc:true";]]> testing this behavior. + + + opt.debug_double_free_max_scan + (unsigned) + r- + [] + + Maximum number of cached pointers to scan in the + thread cache when checking for double-free errors on deallocation. + When debug is enabled, each deallocation into the tcache scans up to + this many recently cached pointers to detect whether the same pointer + is being freed twice. Setting this to 0 disables the check. This + option is set to 0 and has no effect when debug is not enabled. The + default is 32. + + + + + opt.disable_large_size_classes + (bool) + r- + + When enabled (the default), large allocations + (i.e. allocations of size >= SC_LARGE_MINCLASS) + are rounded up to the nearest page boundary rather than the nearest + large size class. This minimizes memory overhead, especially when + using hugepages, at the cost of disabling the standard large size + class hierarchy. + + + + + opt.process_madvise_max_batch + (size_t) + r- + + Maximum number of memory regions to include in each + process_madvise + 2 batch call. When set to 0 + (the default), process_madvise is not used, and the standard + madvise + 2 is used instead. Setting this + to a positive value enables batched purging via process_madvise, which + can reduce the number of system calls needed for + purging. + + thread.arena @@ -1802,6 +1850,47 @@ malloc_conf = "xmalloc:true";]]> the developer may find manual flushing useful. + + + thread.tcache.max + (size_t) + rw + + Get or set the maximum cached size class + (tcache_max) for the calling thread's tcache. The + value is clamped to the maximum allowed limit and rounded up to the + nearest size class boundary. Changing this value will resize the + thread cache accordingly. + + + + + thread.tcache.ncached_max.read_sizeclass + (size_t) + rw + + Query the maximum number of cached objects + (ncached_max) for a given size class in the calling + thread's tcache. The size class is passed in via + newp, and the corresponding + ncached_max is returned via + oldp. + + + + + thread.tcache.ncached_max.write + (char *) + -w + + Set the maximum number of cached objects + (ncached_max) for size classes in the calling + thread's tcache. The input is a string of pipe-separated settings, + where each setting specifies a size range and a count, in the same + format as the opt.tcache_ncached_max runtime + option. + + thread.prof.name @@ -1985,6 +2074,24 @@ malloc_conf = "xmalloc:true";]]> linkend="thread.arena">thread.arena. + + + arena.<i>.name + (char *) + rw + + Get or set a descriptive name for arena <i>. + Arena names can be up to 32 characters long (including the null + terminator); longer names are truncated. When reading, the caller + passes a pointer to a pre-allocated buffer (of at least 32 bytes) via + oldp, and + *oldlenp must be + sizeof(char *). + Arena names are also included in the output of malloc_stats_print(). + + + arena.<i>.dss @@ -2342,6 +2449,18 @@ struct extent_hooks_s { Page size. + + + arenas.hugepage + (size_t) + r- + + Hugepage size. This value is also reported in the + output of malloc_stats_print(). + + + arenas.tcache_max @@ -2561,6 +2680,24 @@ struct extent_hooks_s { option for additional information. + + + approximate_stats.active + (size_t) + r- + + Return the total number of bytes in active pages + collected in an unsynchronized manner, without requiring an + epoch update. + As a result, this value should NOT be compared with other + stats. For example, the relative ordering between + approximate_stats.active and stats.active or stats.resident is + not guaranteed. This interface is intended for lightweight monitoring + where an approximate value is sufficient. + + stats.allocated