From 6f5e8dfcd1e590ae0553b77bb31400c142813ac7 Mon Sep 17 00:00:00 2001 From: TCMalloc Team Date: Thu, 9 Jul 2026 11:25:51 -0700 Subject: [PATCH] Turn down in-process fragmentation profile generation. A follow-up change will eliminate the proxy objects that add overhead, but this removes the sole non-test call to `Span::Fragmentation` allowing `allocated_` to be de-std::atomic'd in another change. PiperOrigin-RevId: 945224342 --- tcmalloc/allocation_sampling.cc | 35 ++++++++++++++++ tcmalloc/allocation_sampling.h | 11 +++++ tcmalloc/tcmalloc.cc | 3 +- tcmalloc/testing/profile_test.cc | 69 ++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 2 deletions(-) diff --git a/tcmalloc/allocation_sampling.cc b/tcmalloc/allocation_sampling.cc index 65115749d..eedc410d3 100644 --- a/tcmalloc/allocation_sampling.cc +++ b/tcmalloc/allocation_sampling.cc @@ -48,6 +48,41 @@ GOOGLE_MALLOC_SECTION_BEGIN namespace tcmalloc::tcmalloc_internal { +std::unique_ptr DumpFragmentationProfile(Static& state) { + auto profile = std::make_unique(ProfileType::kFragmentation); + state.sampled_allocation_recorder().Iterate( + [&state, &profile](const SampledAllocation& sampled_allocation) { + // Compute fragmentation to charge to this sample: + const StackTrace& t = sampled_allocation.sampled_stack; + if (t.proxy == nullptr) { + // There is just one object per-span, and neighboring spans + // can be released back to the system, so we charge no + // fragmentation to this sampled object. + return; + } + + // Fetch the span on which the proxy lives so we can examine its + // co-residents. + const PageId p = PageIdContaining(t.proxy); + Span* span = state.pagemap().GetDescriptor(p); + if (span == nullptr || span == &state.invalid_span()) { + // Avoid crashes in production mode code, but report in tests. + TC_ASSERT_NE(span, nullptr); + TC_ASSERT_NE(span, &state.invalid_span()); + return; + } + + const double frag = span->Fragmentation(t.allocated_size); + if (frag > 0) { + // Associate the memory warmth with the actual object, not the proxy. + // The residency information (t.span_start_address) is likely not very + // useful, but we might as well pass it along. + profile->AddTrace(frag, t); + } + }); + return profile; +} + std::unique_ptr DumpHeapProfile(Static& state) { auto profile = std::make_unique(ProfileType::kHeap); profile->SetStartTime(absl::Now()); diff --git a/tcmalloc/allocation_sampling.h b/tcmalloc/allocation_sampling.h index 9753aa7b7..7013d5a0c 100644 --- a/tcmalloc/allocation_sampling.h +++ b/tcmalloc/allocation_sampling.h @@ -39,6 +39,17 @@ namespace tcmalloc::tcmalloc_internal { class Static; +// This function computes a profile that maps a live stack trace to +// the number of bytes of central-cache memory pinned by an allocation +// at that stack trace. +// In the case when span is hosting >= 1 number of small objects (t.proxy != +// nullptr), we call span::Fragmentation() and read `span->allocated_`. It is +// safe to do so since we hold the per-sample lock while iterating over sampled +// allocations. It prevents the sampled allocation that has the proxy object to +// complete deallocation, thus `proxy` can not be returned to the span yet. It +// thus prevents the central free list to return the span to the page heap. +std::unique_ptr DumpFragmentationProfile(Static& state); + std::unique_ptr DumpHeapProfile(Static& state); #if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ // For RSEQ enabled builds, we declare the sampler in percpu.h so that we can diff --git a/tcmalloc/tcmalloc.cc b/tcmalloc/tcmalloc.cc index 7b2754439..f3158e992 100644 --- a/tcmalloc/tcmalloc.cc +++ b/tcmalloc/tcmalloc.cc @@ -278,8 +278,7 @@ extern "C" const ProfileBase* MallocExtension_Internal_SnapshotCurrent( case ProfileType::kHeap: return DumpHeapProfile(tc_globals).release(); case ProfileType::kFragmentation: - // This profile is no longer collected. - return nullptr; + return DumpFragmentationProfile(tc_globals).release(); case ProfileType::kPeakHeap: return tc_globals.peak_heap_tracker().DumpSample().release(); default: diff --git a/tcmalloc/testing/profile_test.cc b/tcmalloc/testing/profile_test.cc index 124a61b25..73e18880a 100644 --- a/tcmalloc/testing/profile_test.cc +++ b/tcmalloc/testing/profile_test.cc @@ -310,6 +310,75 @@ TEST(AllocationSampleTest, SampleAccuracy) { } } +TEST(FragmentationzTest, Accuracy) { + // Increase sampling rate to decrease flakiness. + ScopedProfileSamplingInterval ps(512 * 1024); + // Disable GWP-ASan, since it allocates different sizes than normal samples. + ScopedGuardedSamplingInterval gs(-1); + + // a fairly odd allocation size - will be rounded to 128. This lets + // us find our record in the table. + static const size_t kItemSize = 115; + // allocate about 3.5 GiB: + static const size_t kNumItems = 32 * 1024 * 1024; + + std::vector> keep; + std::vector> drop; + // hint expected sizes: + drop.reserve(kNumItems * 8 / 10); + keep.reserve(kNumItems * 2 / 10); + + // We allocate many items, then free 80% of them "randomly". (To + // decrease noise and speed up, we just keep every 5th one exactly.) + for (int i = 0; i < kNumItems; ++i) { + // Ideally we should use a malloc() here, for consistency; but unique_ptr + // doesn't come with a have a "free()" deleter; use ::operator new instead. + (i % 5 == 0 ? keep : drop) + .push_back(std::unique_ptr( + static_cast(::operator new[](kItemSize)))); + } + drop.resize(0); + + // there are at least 64 items per span here. (8/10)^64 = 6.2e-7 ~= 0 + // probability we actually managed to free a page; every page is fragmented. + // We still have 20% or so of it allocated, so we should see 80% of it + // charged to these allocations as fragmentations. + auto profile = MallocExtension::SnapshotCurrent(ProfileType::kFragmentation); + + // Pull out the fragmentationz entry corresponding to this + size_t requested_size = 0; + size_t allocated_size = 0; + size_t sum = 0; + size_t count = 0; + profile.Iterate([&](const Profile::Sample& e) { + if (e.requested_size != kItemSize) return; + + if (requested_size == 0) { + allocated_size = e.allocated_size; + requested_size = e.requested_size; + } else { + // we will usually have single entry in + // profile, but in builds without optimization + // our fast-path code causes same call-site to + // have two different stack traces. Thus we + // expect and deal with second entry for same + // allocation. + EXPECT_EQ(requested_size, e.requested_size); + EXPECT_EQ(allocated_size, e.allocated_size); + } + sum += e.sum; + count += e.count; + }); + + double frag_bytes = sum; + double real_frag_bytes = + static_cast(allocated_size * kNumItems) * 0.8; + // We should be pretty close with this much data. + EXPECT_NEAR(real_frag_bytes, frag_bytes, real_frag_bytes * 0.15) + << " sum = " << sum << " allocated = " << allocated_size + << " requested = " << requested_size << " count = " << count; +} + extern "C" { void* __alloc_token_0__ZnwmRKSt9nothrow_t(size_t size, std::nothrow_t); void* __alloc_token_1__ZnwmRKSt9nothrow_t(size_t size, std::nothrow_t);