Turn down in-process fragmentation profile generation.

tcmalloc/allocation_sampling.cc

@@ -48,6 +48,41 @@
 GOOGLE_MALLOC_SECTION_BEGIN
 namespace tcmalloc::tcmalloc_internal {
 
+std::unique_ptr<const ProfileBase> DumpFragmentationProfile(Static& state) {
+  auto profile = std::make_unique<StackTraceTable>(ProfileType::kFragmentation);
+  state.sampled_allocation_recorder().Iterate(
+      [&state, &profile](const SampledAllocation& sampled_allocation) {
+        // Compute fragmentation to charge to this sample:
+        const StackTrace& t = sampled_allocation.sampled_stack;
+        if (t.proxy == nullptr) {
+          // There is just one object per-span, and neighboring spans
+          // can be released back to the system, so we charge no
+          // fragmentation to this sampled object.
+          return;
+        }
+
+        // Fetch the span on which the proxy lives so we can examine its
+        // co-residents.
+        const PageId p = PageIdContaining(t.proxy);
+        Span* span = state.pagemap().GetDescriptor(p);
+        if (span == nullptr || span == &state.invalid_span()) {
+          // Avoid crashes in production mode code, but report in tests.
+          TC_ASSERT_NE(span, nullptr);
+          TC_ASSERT_NE(span, &state.invalid_span());
+          return;
+        }
+
+        const double frag = span->Fragmentation(t.allocated_size);
+        if (frag > 0) {
+          // Associate the memory warmth with the actual object, not the proxy.
+          // The residency information (t.span_start_address) is likely not very
+          // useful, but we might as well pass it along.
+          profile->AddTrace(frag, t);
+        }
+      });
+  return profile;
+}
+
 std::unique_ptr<const ProfileBase> DumpHeapProfile(Static& state) {
   auto profile = std::make_unique<StackTraceTable>(ProfileType::kHeap);
   profile->SetStartTime(absl::Now());

tcmalloc/allocation_sampling.h

@@ -39,6 +39,17 @@ namespace tcmalloc::tcmalloc_internal {
 
 class Static;
 
+// This function computes a profile that maps a live stack trace to
+// the number of bytes of central-cache memory pinned by an allocation
+// at that stack trace.
+// In the case when span is hosting >= 1 number of small objects (t.proxy !=
+// nullptr), we call span::Fragmentation() and read `span->allocated_`. It is
+// safe to do so since we hold the per-sample lock while iterating over sampled
+// allocations. It prevents the sampled allocation that has the proxy object to
+// complete deallocation, thus `proxy` can not be returned to the span yet. It
+// thus prevents the central free list to return the span to the page heap.
+std::unique_ptr<const ProfileBase> DumpFragmentationProfile(Static& state);
+
 std::unique_ptr<const ProfileBase> DumpHeapProfile(Static& state);
 #if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ
 // For RSEQ enabled builds, we declare the sampler in percpu.h so that we can

tcmalloc/tcmalloc.cc

@@ -278,8 +278,7 @@ extern "C" const ProfileBase* MallocExtension_Internal_SnapshotCurrent(
     case ProfileType::kHeap:
       return DumpHeapProfile(tc_globals).release();
     case ProfileType::kFragmentation:
-      // This profile is no longer collected.
-      return nullptr;
+      return DumpFragmentationProfile(tc_globals).release();
     case ProfileType::kPeakHeap:
       return tc_globals.peak_heap_tracker().DumpSample().release();
     default:

tcmalloc/testing/profile_test.cc

@@ -310,6 +310,75 @@ TEST(AllocationSampleTest, SampleAccuracy) {
   }
 }
 
+TEST(FragmentationzTest, Accuracy) {
+  // Increase sampling rate to decrease flakiness.
+  ScopedProfileSamplingInterval ps(512 * 1024);
+  // Disable GWP-ASan, since it allocates different sizes than normal samples.
+  ScopedGuardedSamplingInterval gs(-1);
+
+  // a fairly odd allocation size - will be rounded to 128.  This lets
+  // us find our record in the table.
+  static const size_t kItemSize = 115;
+  // allocate about 3.5 GiB:
+  static const size_t kNumItems = 32 * 1024 * 1024;
+
+  std::vector<std::unique_ptr<char[]>> keep;
+  std::vector<std::unique_ptr<char[]>> drop;
+  // hint expected sizes:
+  drop.reserve(kNumItems * 8 / 10);
+  keep.reserve(kNumItems * 2 / 10);
+
+  // We allocate many items, then free 80% of them "randomly". (To
+  // decrease noise and speed up, we just keep every 5th one exactly.)
+  for (int i = 0; i < kNumItems; ++i) {
+    // Ideally we should use a malloc() here, for consistency; but unique_ptr
+    // doesn't come with a have a "free()" deleter; use ::operator new instead.
+    (i % 5 == 0 ? keep : drop)
+        .push_back(std::unique_ptr<char[]>(
+            static_cast<char*>(::operator new[](kItemSize))));
+  }
+  drop.resize(0);
+
+  // there are at least 64 items per span here. (8/10)^64 = 6.2e-7 ~= 0
+  // probability we actually managed to free a page; every page is fragmented.
+  // We still have 20% or so of it allocated, so we should see 80% of it
+  // charged to these allocations as fragmentations.
+  auto profile = MallocExtension::SnapshotCurrent(ProfileType::kFragmentation);
+
+  // Pull out the fragmentationz entry corresponding to this
+  size_t requested_size = 0;
+  size_t allocated_size = 0;
+  size_t sum = 0;
+  size_t count = 0;
+  profile.Iterate([&](const Profile::Sample& e) {
+    if (e.requested_size != kItemSize) return;
+
+    if (requested_size == 0) {
+      allocated_size = e.allocated_size;
+      requested_size = e.requested_size;
+    } else {
+      // we will usually have single entry in
+      // profile, but in builds without optimization
+      // our fast-path code causes same call-site to
+      // have two different stack traces. Thus we
+      // expect and deal with second entry for same
+      // allocation.
+      EXPECT_EQ(requested_size, e.requested_size);
+      EXPECT_EQ(allocated_size, e.allocated_size);
+    }
+    sum += e.sum;
+    count += e.count;
+  });
+
+  double frag_bytes = sum;
+  double real_frag_bytes =
+      static_cast<double>(allocated_size * kNumItems) * 0.8;
+  // We should be pretty close with this much data.
+  EXPECT_NEAR(real_frag_bytes, frag_bytes, real_frag_bytes * 0.15)
+      << " sum = " << sum << " allocated = " << allocated_size
+      << " requested = " << requested_size << " count = " << count;
+}
+
 extern "C" {
 void* __alloc_token_0__ZnwmRKSt9nothrow_t(size_t size, std::nothrow_t);
 void* __alloc_token_1__ZnwmRKSt9nothrow_t(size_t size, std::nothrow_t);