From 27d37345d3eaf69aa0a8e57c7c50d55e500ef202 Mon Sep 17 00:00:00 2001 From: Ben Reisner Date: Wed, 8 Jul 2026 14:50:25 -0700 Subject: [PATCH] Banach fixed-point theorem PiperOrigin-RevId: 944713328 --- tcmalloc/common.h | 2 + tcmalloc/huge_page_filler.h | 89 +++++++---------- tcmalloc/huge_page_filler_fuzz.cc | 10 +- tcmalloc/huge_page_filler_test.cc | 57 +++++------ tcmalloc/internal/pageflags.cc | 6 +- tcmalloc/internal/pageflags.h | 20 +++- tcmalloc/internal/pageflags_test.cc | 76 +++++++++++++-- tcmalloc/internal/profile_builder_test.cc | 4 +- tcmalloc/internal/range_tracker.h | 26 +++++ tcmalloc/internal/range_tracker_test.cc | 68 +++++++++++++ tcmalloc/internal/residency.cc | 19 ++-- tcmalloc/internal/residency.h | 22 ++++- tcmalloc/internal/residency_test.cc | 111 +++++++++++++++++++--- 13 files changed, 378 insertions(+), 132 deletions(-) diff --git a/tcmalloc/common.h b/tcmalloc/common.h index 4e7165c73..f7fae9bf5 100644 --- a/tcmalloc/common.h +++ b/tcmalloc/common.h @@ -35,6 +35,7 @@ #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/memory_tag.h" #include "tcmalloc/internal/optimization.h" +#include "tcmalloc/internal/range_tracker.h" #include "tcmalloc/malloc_extension.h" GOOGLE_MALLOC_SECTION_BEGIN @@ -208,6 +209,7 @@ inline constexpr size_t kMinObjectsToMove = 2; inline constexpr size_t kMaxObjectsToMove = 128; inline constexpr size_t kPageSize = 1 << kPageShift; +using PageBitmap = Bitmap; inline constexpr std::align_val_t kAlignment{8}; // log2 (kAlignment) diff --git a/tcmalloc/huge_page_filler.h b/tcmalloc/huge_page_filler.h index ffab64510..44b24b9c8 100644 --- a/tcmalloc/huge_page_filler.h +++ b/tcmalloc/huge_page_filler.h @@ -218,7 +218,7 @@ class PageTracker : public TList::Elem { Length ReleaseFree(MemoryModifyFunction& unback) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); - Length MarkSubreleased(Bitmap unbacked, + Length MarkSubreleased(tcmalloc_internal::PageBitmap unbacked, int native_pages_per_tcmalloc_page) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); @@ -249,8 +249,8 @@ class PageTracker : public TList::Elem { // used to avoid subreleasing the pages that are being collapsed. bool being_collapsed = false; // Records swap and unbacked bitmaps for this hugepage. - Residency::SinglePageBitmaps bitmaps; - ResidencyBitmap stale; + Residency::SinglePageBitmaps bitmaps; + tcmalloc_internal::PageBitmap stale; // Records whether collapse was skipped due to threshold constraints. bool collapse_skipped = false; // Records whether collapse was skipped due to backoff. @@ -327,7 +327,8 @@ class PageTracker : public TList::Elem { }; NativePageResidencyInfo CountInfoInHugePage( - Residency::SinglePageBitmaps bitmaps, ResidencyBitmap stale) const; + Residency::SinglePageBitmaps bitmaps, + tcmalloc_internal::PageBitmap stale) const; private: HugePage location_; @@ -1310,7 +1311,7 @@ class HugePageFiller { // Utility function to handle a non-hugepage backed `page_tracker` and // mark its unmapped pages appropriately. Length HandleUnbackedHugePage(PageTracker* page_tracker, - Bitmap unbacked) + tcmalloc_internal::PageBitmap unbacked) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); private: @@ -1509,7 +1510,8 @@ inline void PageTracker::SetAnonVmaName(MemoryTagFunction& set_anon_vma_name, } inline PageTracker::NativePageResidencyInfo PageTracker::CountInfoInHugePage( - Residency::SinglePageBitmaps bitmaps, ResidencyBitmap stale) const { + Residency::SinglePageBitmaps bitmaps, + tcmalloc_internal::PageBitmap stale) const { // TODO(b/424551232): Add support for the scenario when native page size is // larger than TCMalloc page size. const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize(); @@ -1518,23 +1520,16 @@ inline PageTracker::NativePageResidencyInfo PageTracker::CountInfoInHugePage( } TC_ASSERT_LE(kNativePagesInHugePage, kMaxResidencyBits); - Bitmap unbacked = bitmaps.unbacked; - Bitmap swapped = bitmaps.swapped; - Bitmap free = free_.bits(); - - TC_ASSERT_EQ(kNativePagesInHugePage % kPagesPerHugePage.raw_num(), 0); - const int shift = kNativePagesInHugePage / kPagesPerHugePage.raw_num(); - const int shift_bits = absl::bit_width(shift - 1); - TC_ASSERT_LT((kNativePagesInHugePage - 1) >> shift_bits, - kPagesPerHugePage.raw_num()); + tcmalloc_internal::PageBitmap unbacked = bitmaps.unbacked; + tcmalloc_internal::PageBitmap swapped = bitmaps.swapped; + tcmalloc_internal::PageBitmap free = free_.bits(); size_t n_unbacked[2] = {0, 0}; size_t n_swapped[2] = {0, 0}; size_t n_stale[2] = {0, 0}; - for (size_t i = 0; i < kNativePagesInHugePage; ++i) { - const int scaled_idx = i >> shift_bits; - bool is_free = !free.GetBit(scaled_idx); + for (size_t i = 0; i < unbacked.size(); ++i) { + bool is_free = !free.GetBit(i); n_swapped[is_free] += swapped.GetBit(i); n_unbacked[is_free] += unbacked.GetBit(i); @@ -1607,8 +1602,9 @@ inline Length PageTracker::ReleaseFree(MemoryModifyFunction& unback) { return Length(count); } -inline Length PageTracker::MarkSubreleased(Bitmap unbacked, - int native_pages_per_tcmalloc_page) { +inline Length PageTracker::MarkSubreleased( + tcmalloc_internal::PageBitmap unbacked, + int native_pages_per_tcmalloc_page) { // TODO(b/525422238): pass the bitmap in TCMalloc pages directly to avoid this // conversion. @@ -1620,47 +1616,25 @@ inline Length PageTracker::MarkSubreleased(Bitmap unbacked, return Length(0); } TC_ASSERT_GT(native_pages_per_tcmalloc_page, 0); - Bitmap free = free_.bits(); - - size_t count = 0; - for (size_t i = 0; i < kPagesPerHugePage.raw_num(); ++i) { - if (!free.GetBit(i) && !released_by_page_.GetBit(i)) { - // Check if all native pages corresponding to this TCMalloc page are - // unbacked. - bool is_unbacked = true; - for (int j = 0; j < native_pages_per_tcmalloc_page; ++j) { - const int bitmap_index = i * native_pages_per_tcmalloc_page + j; - TC_ASSERT_LT(bitmap_index, kMaxResidencyBits); - if (!unbacked.GetBit(bitmap_index)) { - is_unbacked = false; - break; - } - } - // If all native pages corresponding to this TCMalloc page are unbacked, - // and as the TCMalloc page was freed but not released, mark this page - // as released now. - // - // TODO(b/525422238): The residency bitmap was captured outside of the - // lock. So, in a rare case, it's possible that the page was allocated, - // backed and then freed. So, the free page here is actually backed. - // While we currently ignore this case (resulting in underestimating - // RSS), we can potentially fix this by re-investigating the bitmaps - // and marking the pages back to backed to eventually fix this. - if (is_unbacked) { - released_by_page_.SetBit(i); - ++count; - } - } - } - - released_count_ += count; + tcmalloc_internal::PageBitmap free = free_.bits(); + + // TODO(b/525422238): The residency bitmap was captured outside of the + // lock. So, in a rare case, it's possible that the page was allocated, + // backed and then freed. So, the free page here is actually backed. + // While we currently ignore this case (resulting in underestimating + // RSS), we can potentially fix this by re-investigating the bitmaps + // and marking the pages back to backed to eventually fix this. + auto to_release = (~free) & (~released_by_page_) & unbacked; + released_by_page_ = released_by_page_ | to_release; + + released_count_ += to_release.CountBits(); // Mark this is unbroken regardless of whether it had any unbacked free // TCMalloc pages. Marking this will move this tracker to one of the // released lists. unbroken_ = false; TC_ASSERT_LE(Length(released_count_), kPagesPerHugePage); TC_ASSERT_EQ(released_by_page_.CountBits(), released_count_); - return Length(count); + return Length(to_release.CountBits()); } inline MemoryModifyStatus PageTracker::Collapse( @@ -2712,7 +2686,8 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { state.maybe_hugepage_backed = is_hugepage; if (!is_hugepage) { state.bitmaps = - res->GetUnbackedAndSwappedBitmaps(tracker->location().start_addr()); + res->GetUnbackedAndSwappedBitmaps( + tracker->location().start_addr()); if (pf) { pf->GetSinglePageBitmaps(tracker->location().start_addr(), state.stale); @@ -2989,7 +2964,7 @@ inline void HugePageFiller::OnCollapseSuccess(TrackerType* pt) { template inline Length HugePageFiller::HandleUnbackedHugePage( - PageTracker* tracker, Bitmap unbacked) { + PageTracker* tracker, tcmalloc_internal::PageBitmap unbacked) { RemoveFromFillerList(tracker); Length unmapped_length = tracker->MarkSubreleased(unbacked, native_pages_per_tcmalloc_page_); diff --git a/tcmalloc/huge_page_filler_fuzz.cc b/tcmalloc/huge_page_filler_fuzz.cc index f23dac5dd..1c819eaa7 100644 --- a/tcmalloc/huge_page_filler_fuzz.cc +++ b/tcmalloc/huge_page_filler_fuzz.cc @@ -124,8 +124,8 @@ class FakePageFlags : public PageFlagsBase { return PageStats{}; } - absl::StatusCode GetSinglePageBitmaps(const void* addr, - ResidencyBitmap& stale) override { + absl::StatusCode GetSinglePageBitmapsInternal( + const void* addr, ResidencyBitmap& stale) override { return absl::StatusCode::kUnimplemented; } @@ -141,7 +141,8 @@ class FakeResidency : public Residency { return std::nullopt; }; - SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) override { + SinglePageBitmapsInternal GetUnbackedAndSwappedBitmapsInternal( + const void* addr) override { return {unbacked_bitmap, swapped_bitmap, absl::StatusCode::kOk}; }; @@ -151,7 +152,8 @@ class FakeResidency : public Residency { }; private: - absl::flat_hash_map residency_bitmaps_; + absl::flat_hash_map + residency_bitmaps_; }; class MockCollapse final : public MemoryModifyFunction { diff --git a/tcmalloc/huge_page_filler_test.cc b/tcmalloc/huge_page_filler_test.cc index cb6cfbec3..868a43876 100644 --- a/tcmalloc/huge_page_filler_test.cc +++ b/tcmalloc/huge_page_filler_test.cc @@ -289,9 +289,10 @@ class FakePageFlags : public PageFlagsBase { return PageStats{}; } - absl::StatusCode GetSinglePageBitmaps(const void* addr, - ResidencyBitmap& stale) override { + absl::StatusCode GetSinglePageBitmapsInternal( + const void* addr, ResidencyBitmap& stale) override { stale.SetBit(0); + stale.SetBit(1); return absl::StatusCode::kOk; } @@ -328,7 +329,8 @@ class FakeResidency : public Residency { return std::nullopt; }; - SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) override { + SinglePageBitmapsInternal GetUnbackedAndSwappedBitmapsInternal( + const void* addr) override { PageId p = PageIdContaining(addr); HugePage hp = HugePageContaining(p); EXPECT_TRUE(residency_bitmaps_.contains(hp.start_addr())); @@ -344,13 +346,14 @@ class FakeResidency : public Residency { absl::StatusCode::kOk}; } - const size_t kNativePagesInHugePage = kHugePageSize / kPageSize; + const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize(); size_t GetNativePagesInHugePage() const override { return kNativePagesInHugePage; }; private: - absl::flat_hash_map residency_bitmaps_; + absl::flat_hash_map> + residency_bitmaps_; }; TEST_F(PageTrackerTest, AllocSane) { @@ -1428,7 +1431,7 @@ TEST_F(FillerTest, ReleaseFreePagesWhenAnyPageIsSwapped) { EXPECT_FALSE(pageflags.IsHugepageBacked(pa.p.start_addr()).value()); Bitmap unbacked, swapped; - swapped.SetRange(/*index=*/1, /*n=*/1); + swapped.SetRange(/*index=*/0, /*n=*/2); residency.SetUnbackedAndSwappedBitmaps(pa.p.start_addr(), unbacked, swapped); } @@ -2462,7 +2465,7 @@ TEST_F(FillerTestWithSubreleaseUnbacked, SubreleaseUnbackedPartial) { // free TCMalloc page are unbacked, the TCMalloc page is NOT marked as // subreleased. TEST_F(FillerTestWithSubreleaseUnbacked, SubreleaseUnbackedPartialNativePages) { - const size_t kNativePagesInHugePage = kHugePageSize / kPageSize; + const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize(); const int shift = kNativePagesInHugePage / kPagesPerHugePage.raw_num(); if (shift <= 1) { return; @@ -5218,8 +5221,8 @@ TEST_F(FillerTest, ResidencyTelemetry) { EXPECT_THAT(buffer_text, testing::HasSubstr(R"( HugePageFiller: # of sparsely-accessed released hps with a <= # of swapped < b HugePageFiller: < 0<= 0 < 1<= 0 < 2<= 0 < 3<= 0 < 4<= 0 < 5<= 0 -HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 0 -HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 1 <264<= 0 <296<= 0 +HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 1 +HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 0 <264<= 0 <296<= 0 HugePageFiller: <328<= 0 <360<= 0 <392<= 0 <424<= 0 <456<= 0 <488<= 0 HugePageFiller: <504<= 0 <505<= 0 <506<= 0 <507<= 0 <508<= 0 <509<= 0 HugePageFiller: <510<= 0 <511<= 0 @@ -5228,8 +5231,8 @@ HugePageFiller: <510<= 0 <511<= 0 EXPECT_THAT(buffer_text, testing::HasSubstr(R"( HugePageFiller: # of sparsely-accessed released hps with a <= # of unbacked < b HugePageFiller: < 0<= 0 < 1<= 0 < 2<= 0 < 3<= 0 < 4<= 0 < 5<= 0 -HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 0 -HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 1 <264<= 0 <296<= 0 +HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 1 +HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 0 <264<= 0 <296<= 0 HugePageFiller: <328<= 0 <360<= 0 <392<= 0 <424<= 0 <456<= 0 <488<= 0 HugePageFiller: <504<= 0 <505<= 0 <506<= 0 <507<= 0 <508<= 0 <509<= 0 HugePageFiller: <510<= 0 <511<= 0 @@ -5238,7 +5241,7 @@ HugePageFiller: <510<= 0 <511<= 0 EXPECT_THAT(buffer_text, testing::HasSubstr(R"( HugePageFiller: # of sparsely-accessed released hps with a <= # of free AND unbacked < b HugePageFiller: < 0<= 0 < 1<= 0 < 2<= 0 < 3<= 0 < 4<= 0 < 5<= 0 -HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 1 +HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 1 < 72<= 0 <104<= 0 HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 0 <264<= 0 <296<= 0 HugePageFiller: <328<= 0 <360<= 0 <392<= 0 <424<= 0 <456<= 0 <488<= 0 HugePageFiller: <504<= 0 <505<= 0 <506<= 0 <507<= 0 <508<= 0 <509<= 0 @@ -5248,7 +5251,7 @@ HugePageFiller: <510<= 0 <511<= 0 EXPECT_THAT(buffer_text, testing::HasSubstr(R"( HugePageFiller: # of sparsely-accessed released hps with a <= # of free AND swapped < b HugePageFiller: < 0<= 0 < 1<= 0 < 2<= 0 < 3<= 0 < 4<= 0 < 5<= 0 -HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 1 +HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 1 < 72<= 0 <104<= 0 HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 0 <264<= 0 <296<= 0 HugePageFiller: <328<= 0 <360<= 0 <392<= 0 <424<= 0 <456<= 0 <488<= 0 HugePageFiller: <504<= 0 <505<= 0 <506<= 0 <507<= 0 <508<= 0 <509<= 0 @@ -5278,17 +5281,17 @@ HugePageFiller: Of the hugepage backed pages of type sparsely-accessed regular, )")); EXPECT_THAT(buffer_text, testing::HasSubstr(R"( -HugePageFiller: 126 of sparsely-accessed released free native pages are swapped. +HugePageFiller: 63 of sparsely-accessed released free native pages are swapped. )")); EXPECT_THAT(buffer_text, testing::HasSubstr(R"( -HugePageFiller: 130 of sparsely-accessed released used native pages are swapped. +HugePageFiller: 65 of sparsely-accessed released used native pages are swapped. )")); EXPECT_THAT(buffer_text, testing::HasSubstr(R"( -HugePageFiller: 126 of sparsely-accessed released free native pages are unbacked. +HugePageFiller: 63 of sparsely-accessed released free native pages are unbacked. )")); EXPECT_THAT(buffer_text, testing::HasSubstr(R"( -HugePageFiller: 130 of sparsely-accessed released used native pages are unbacked. +HugePageFiller: 65 of sparsely-accessed released used native pages are unbacked. )")); std::string buffer_pbtxt = @@ -5300,17 +5303,17 @@ HugePageFiller: 130 of sparsely-accessed released used native pages are unbacked EXPECT_THAT( buffer_pbtxt, testing::HasSubstr( - "unbacked_histogram { lower_bound: 232 upper_bound: 263 value: 1}")); + "unbacked_histogram { lower_bound: 104 upper_bound: 135 value: 1}")); EXPECT_THAT( buffer_pbtxt, testing::HasSubstr( - "swapped_histogram { lower_bound: 232 upper_bound: 263 value: 1}")); + "swapped_histogram { lower_bound: 104 upper_bound: 135 value: 1}")); EXPECT_THAT(buffer_pbtxt, - testing::HasSubstr("free_unbacked_histogram { lower_bound: 104 " - "upper_bound: 135 value: 1}")); + testing::HasSubstr("free_unbacked_histogram { lower_bound: 40 " + "upper_bound: 71 value: 1}")); EXPECT_THAT(buffer_pbtxt, - testing::HasSubstr("free_swapped_histogram { lower_bound: 104 " - "upper_bound: 135 value: 1}")); + testing::HasSubstr("free_swapped_histogram { lower_bound: 40 " + "upper_bound: 71 value: 1}")); EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_free_pages_non_hugepage_backed: 0")); EXPECT_THAT(buffer_pbtxt, @@ -5319,10 +5322,10 @@ HugePageFiller: 130 of sparsely-accessed released used native pages are unbacked testing::HasSubstr("num_free_pages_hugepage_backed: 126")); EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_used_pages_hugepage_backed: 130")); - EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_swapped: 126")); - EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_swapped: 130")); - EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_unbacked: 126")); - EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_unbacked: 130")); + EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_swapped: 63")); + EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_swapped: 65")); + EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_unbacked: 63")); + EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_unbacked: 65")); EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_stale: 0")); EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_stale: 1")); EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_treated: 1")); diff --git a/tcmalloc/internal/pageflags.cc b/tcmalloc/internal/pageflags.cc index d95dd1e36..0bda56b1a 100644 --- a/tcmalloc/internal/pageflags.cc +++ b/tcmalloc/internal/pageflags.cc @@ -31,6 +31,7 @@ #include "absl/strings/numbers.h" #include "tcmalloc/internal/config.h" #include "tcmalloc/internal/logging.h" +#include "tcmalloc/internal/range_tracker.h" #include "tcmalloc/internal/util.h" GOOGLE_MALLOC_SECTION_BEGIN @@ -335,8 +336,8 @@ std::optional PageFlags::Get(const void* const addr, } return ret; } -absl::StatusCode PageFlags::GetSinglePageBitmaps(const void* addr, - ResidencyBitmap& stale) { +absl::StatusCode PageFlags::GetSinglePageBitmapsInternal( + const void* addr, ResidencyBitmap& stale) { uintptr_t currPage = reinterpret_cast(addr); if ((currPage & (kHugePageSize - 1)) != 0) { TC_LOG("Address is not hugepage aligned"); @@ -394,7 +395,6 @@ absl::StatusCode PageFlags::GetSinglePageBitmaps(const void* addr, if (stale_start != -1) { stale.SetRange(stale_start, kNativePagesInHugePage - stale_start); } - return absl::StatusCode::kOk; } diff --git a/tcmalloc/internal/pageflags.h b/tcmalloc/internal/pageflags.h index c4a62aba1..fe7366145 100644 --- a/tcmalloc/internal/pageflags.h +++ b/tcmalloc/internal/pageflags.h @@ -68,8 +68,20 @@ class PageFlagsBase { PageFlagsBase& operator=(PageFlagsBase&&) = delete; virtual std::optional IsHugepageBacked(const void* addr) = 0; virtual std::optional Get(const void* addr, size_t size) = 0; - virtual absl::StatusCode GetSinglePageBitmaps(const void* addr, - ResidencyBitmap& stale) = 0; + template + absl::StatusCode GetSinglePageBitmaps(const void* addr, Bitmap& stale) { + ResidencyBitmap temp; + auto res = GetSinglePageBitmapsInternal(addr, temp); + if (res != absl::StatusCode::kOk) { + return res; + } + stale = temp.Contract(kHugePageSize / GetPageSize()); + return res; + } + + protected: + virtual absl::StatusCode GetSinglePageBitmapsInternal( + const void* addr, ResidencyBitmap& stale) = 0; }; // PageFlags offers a look at kernel page flags to identify pieces of memory as @@ -101,8 +113,8 @@ class PageFlags final : public PageFlagsBase { // dynamically allocate memory when needed. Using std::optional allows us to // use the function in places where memory allocation is prohibited. std::optional Get(const void* addr, size_t size) override; - absl::StatusCode GetSinglePageBitmaps(const void* addr, - ResidencyBitmap& stale) override; + absl::StatusCode GetSinglePageBitmapsInternal( + const void* addr, ResidencyBitmap& stale) override; std::optional IsHugepageBacked(const void* addr) override; private: diff --git a/tcmalloc/internal/pageflags_test.cc b/tcmalloc/internal/pageflags_test.cc index e6e67a75c..c1ddb165c 100644 --- a/tcmalloc/internal/pageflags_test.cc +++ b/tcmalloc/internal/pageflags_test.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -81,8 +82,13 @@ class PageFlagsFriend { return r_.IsHugepageBacked(addr); } - decltype(auto) GetSinglePageBitmaps(const void* addr, - ResidencyBitmap& stale) { + decltype(auto) GetSinglePageBitmapsInternal(const void* addr, + ResidencyBitmap& stale) { + return r_.GetSinglePageBitmapsInternal(addr, stale); + } + + template + absl::StatusCode GetSinglePageBitmaps(const void* addr, Bitmap& stale) { return r_.GetSinglePageBitmaps(addr, stale); } @@ -604,13 +610,14 @@ TEST(PageFlagsTest, GetSinglePageBitmapsErrorCases) { { PageFlagsFriend s; ResidencyBitmap stale; - EXPECT_EQ(s.GetSinglePageBitmaps(reinterpret_cast(1), stale), - absl::StatusCode::kFailedPrecondition); + EXPECT_EQ( + s.GetSinglePageBitmapsInternal(reinterpret_cast(1), stale), + absl::StatusCode::kFailedPrecondition); } { PageFlagsFriend s("/dev/null/impossible"); ResidencyBitmap stale; - EXPECT_EQ(s.GetSinglePageBitmaps(nullptr, stale), + EXPECT_EQ(s.GetSinglePageBitmapsInternal(nullptr, stale), absl::StatusCode::kUnavailable); } { @@ -619,7 +626,7 @@ TEST(PageFlagsTest, GetSinglePageBitmapsErrorCases) { SetContents(fake_pageflags, "x"); PageFlagsFriend s(fake_pageflags); ResidencyBitmap stale; - EXPECT_EQ(s.GetSinglePageBitmaps(nullptr, stale), + EXPECT_EQ(s.GetSinglePageBitmapsInternal(nullptr, stale), absl::StatusCode::kUnavailable); } { @@ -635,7 +642,7 @@ TEST(PageFlagsTest, GetSinglePageBitmapsErrorCases) { PageFlagsFriend s(fake_pageflags); ResidencyBitmap stale; - EXPECT_EQ(s.GetSinglePageBitmaps(nullptr, stale), + EXPECT_EQ(s.GetSinglePageBitmapsInternal(nullptr, stale), absl::StatusCode::kFailedPrecondition); } } @@ -659,10 +666,63 @@ TEST(PageFlagsTest, GetSinglePageBitmapsSuccess) { ResidencyBitmap stale; stale.Clear(); - EXPECT_EQ(s.GetSinglePageBitmaps(nullptr, stale), absl::StatusCode::kOk); + EXPECT_EQ(s.GetSinglePageBitmapsInternal(nullptr, stale), + absl::StatusCode::kOk); EXPECT_EQ(stale.CountBits(), kMaxResidencyBits / 2); } +template +class GetSinglePageBitmapsWrapperTest : public ::testing::Test {}; + +using WrapperTestSizes = + ::testing::Types, + std::integral_constant, + std::integral_constant>; + +TYPED_TEST_SUITE(GetSinglePageBitmapsWrapperTest, WrapperTestSizes); + +TYPED_TEST(GetSinglePageBitmapsWrapperTest, AlternatingStale) { + constexpr size_t N = TypeParam::value; + + std::string fake_pageflags = + absl::StrCat(testing::TempDir(), "/fake_pageflags_wrapper_", N); + const size_t kPageSize = getpagesize(); + const size_t kNativePagesInHugePage = kHugePageSize / kPageSize; + std::vector data(kNativePagesInHugePage, 0); + + data[0] |= kPageStale; + data[1] |= kPageStale; + data[2] |= kPageStale; + + std::string content(reinterpret_cast(data.data()), + data.size() * sizeof(uint64_t)); + SetContents(fake_pageflags, content); + + PageFlagsFriend s(fake_pageflags); + Bitmap stale; + stale.Clear(); + + EXPECT_EQ(s.GetSinglePageBitmaps(nullptr, stale), absl::StatusCode::kOk); + + if constexpr (N == kMaxResidencyBits) { + EXPECT_EQ(stale.CountBits(), 3); + EXPECT_TRUE(stale.GetBit(0)); + EXPECT_TRUE(stale.GetBit(1)); + EXPECT_TRUE(stale.GetBit(2)); + EXPECT_FALSE(stale.GetBit(3)); + } else if constexpr (N == kMaxResidencyBits / 2) { + EXPECT_EQ(stale.CountBits(), 1); + EXPECT_TRUE(stale.GetBit(0)); + EXPECT_FALSE(stale.GetBit(1)); + } else if constexpr (N == kMaxResidencyBits * 2) { + EXPECT_EQ(stale.CountBits(), 6); + for (size_t i = 0; i < 6; ++i) { + EXPECT_TRUE(stale.GetBit(i)) << i; + } + EXPECT_FALSE(stale.GetBit(6)); + } +} + } // namespace } // namespace tcmalloc_internal } // namespace tcmalloc diff --git a/tcmalloc/internal/profile_builder_test.cc b/tcmalloc/internal/profile_builder_test.cc index 46785da91..e21bd87c7 100644 --- a/tcmalloc/internal/profile_builder_test.cc +++ b/tcmalloc/internal/profile_builder_test.cc @@ -71,8 +71,8 @@ class StubPageFlags final : public PageFlagsBase { public: StubPageFlags() = default; ~StubPageFlags() override = default; - absl::StatusCode GetSinglePageBitmaps(const void* addr, - ResidencyBitmap& stale) override { + absl::StatusCode GetSinglePageBitmapsInternal( + const void* addr, ResidencyBitmap& stale) override { return absl::StatusCode::kUnimplemented; } std::optional Get(const void* addr, size_t size) override { diff --git a/tcmalloc/internal/range_tracker.h b/tcmalloc/internal/range_tracker.h index e7958af23..2352d05b9 100644 --- a/tcmalloc/internal/range_tracker.h +++ b/tcmalloc/internal/range_tracker.h @@ -61,6 +61,32 @@ class Bitmap { // Clears the lowest set bit. Special case is faster than more flexible code. void ClearLowestBit(); + template + Bitmap Contract(size_t src_len) const { + TC_ASSERT_LE(src_len, N); + + TC_ASSERT(src_len % M == 0 || M % src_len == 0); + + Bitmap res; + const size_t src_per_dst = std::max(1, src_len / M); + const size_t dst_per_src = std::max(1, M / src_len); + + size_t dst_idx = 0; + for (size_t src_idx = 0; src_idx < src_len; src_idx += src_per_dst) { + size_t count = 0; + for (size_t j = 0; j < src_per_dst; ++j) { + if (GetBit(src_idx + j)) { + count++; + } + } + if (count == src_per_dst) { + res.SetRange(dst_idx, dst_per_src); + } + dst_idx += dst_per_src; + } + return res; + } + template size_t PopBatch(Visitor visitor, size_t limit) { size_t count = 0; diff --git a/tcmalloc/internal/range_tracker_test.cc b/tcmalloc/internal/range_tracker_test.cc index 521e1d9c9..91a4d7b94 100644 --- a/tcmalloc/internal/range_tracker_test.cc +++ b/tcmalloc/internal/range_tracker_test.cc @@ -347,6 +347,74 @@ TEST_F(BitmapTest, PopBatch) { EXPECT_TRUE(map.IsZero()); } +#ifndef NDEBUG +TEST_F(BitmapTest, ContractAssertionFailures) { + Bitmap<64> map1; + // 1. src_len > N (64) + EXPECT_DEATH(map1.Contract<64>(128), ""); + // 2. src_len (64) doesn't divide M (10) and vice versa + EXPECT_DEATH(map1.Contract<10>(64), ""); +} +#endif + +TEST_F(BitmapTest, Contract) { + // Test case 1: N == M (equal) + { + Bitmap<64> map1; + map1.SetBit(3); + map1.SetBit(17); + map1.SetBit(63); + + auto map2 = map1.Contract<64>(64); + for (size_t i = 0; i < 64; ++i) { + EXPECT_EQ(map1.GetBit(i), map2.GetBit(i)); + } + } + + // Test case 2: N > M (Contracting) + { + Bitmap<64> map1; + // We group by 2 bits. + // Group 0: bits 0, 1 -> both must be set + map1.SetBit(0); + map1.SetBit(1); + + // Group 1: bits 2, 3 -> only one set -> should not be set + map1.SetBit(2); + + // Group 7: bits 14, 15 -> both set + map1.SetBit(14); + map1.SetBit(15); + + auto map2 = map1.Contract<8>(16); + EXPECT_TRUE(map2.GetBit(0)); + EXPECT_FALSE(map2.GetBit(1)); + EXPECT_FALSE(map2.GetBit(2)); + EXPECT_FALSE(map2.GetBit(3)); + EXPECT_FALSE(map2.GetBit(4)); + EXPECT_FALSE(map2.GetBit(5)); + EXPECT_FALSE(map2.GetBit(6)); + EXPECT_TRUE(map2.GetBit(7)); + } + + // Test case 3: N < M (expanding) + { + Bitmap<64> map1; + // We expand by 2 bits. + map1.SetBit(0); + map1.SetBit(7); + + auto map2 = map1.Contract<16>(8); + EXPECT_TRUE(map2.GetBit(0)); + EXPECT_TRUE(map2.GetBit(1)); + EXPECT_FALSE(map2.GetBit(2)); + EXPECT_FALSE(map2.GetBit(3)); + + EXPECT_TRUE(map2.GetBit(14)); + EXPECT_TRUE(map2.GetBit(15)); + } +} + class RangeTrackerTest : public ::testing::Test { protected: std::vector> FreeRanges() { diff --git a/tcmalloc/internal/residency.cc b/tcmalloc/internal/residency.cc index 7b92740e9..d8d896625 100644 --- a/tcmalloc/internal/residency.cc +++ b/tcmalloc/internal/residency.cc @@ -186,20 +186,20 @@ std::optional ResidencyPageMap::Get(const void* const addr, return info; } -Residency::SinglePageBitmaps ResidencyPageMap::GetUnbackedAndSwappedBitmaps( - const void* const addr) { +Residency::SinglePageBitmapsInternal +ResidencyPageMap::GetUnbackedAndSwappedBitmapsInternal(const void* const addr) { Bitmap page_unbacked; Bitmap page_swapped; uintptr_t currPage = reinterpret_cast(addr); if ((currPage & kHugePageMask) != currPage) { TC_LOG("Address is not hugepage aligned"); - return SinglePageBitmaps{page_unbacked, page_swapped, - absl::StatusCode::kFailedPrecondition}; + return SinglePageBitmapsInternal{page_unbacked, page_swapped, + absl::StatusCode::kFailedPrecondition}; } auto res = Seek(currPage); if (res != absl::StatusCode::kOk) { - return SinglePageBitmaps{page_unbacked, page_swapped, - absl::StatusCode::kUnavailable}; + return SinglePageBitmapsInternal{page_unbacked, page_swapped, + absl::StatusCode::kUnavailable}; } auto status = signal_safe_read(fd_, reinterpret_cast(buf_), kSizeOfHugepageInPagemap, nullptr); @@ -208,8 +208,8 @@ Residency::SinglePageBitmaps ResidencyPageMap::GetUnbackedAndSwappedBitmaps( "Could not read from pagemap file due to unexpected number of bytes " "read. Expected %d bytes, got %d bytes", kSizeOfHugepageInPagemap, status); - return SinglePageBitmaps{page_unbacked, page_swapped, - absl::StatusCode::kUnavailable}; + return SinglePageBitmapsInternal{page_unbacked, page_swapped, + absl::StatusCode::kUnavailable}; } for (int native_page_idx = 0; native_page_idx < kNativePagesInHugePage; @@ -223,7 +223,8 @@ Residency::SinglePageBitmaps ResidencyPageMap::GetUnbackedAndSwappedBitmaps( } } } - return SinglePageBitmaps{page_unbacked, page_swapped, absl::StatusCode::kOk}; + return SinglePageBitmapsInternal{page_unbacked, page_swapped, + absl::StatusCode::kOk}; } } // namespace tcmalloc_internal diff --git a/tcmalloc/internal/residency.h b/tcmalloc/internal/residency.h index 440008405..c7791de14 100644 --- a/tcmalloc/internal/residency.h +++ b/tcmalloc/internal/residency.h @@ -47,18 +47,31 @@ class Residency { virtual inline size_t GetNativePagesInHugePage() const = 0; // Struct is ordered with bitmaps first to optimize cacheline usage. + template struct SinglePageBitmaps { - Bitmap unbacked; - Bitmap swapped; + Bitmap unbacked; + Bitmap swapped; absl::StatusCode status; }; + using SinglePageBitmapsInternal = SinglePageBitmaps; + // Using a hugepage-aligned address, parse through /proc/self/pagemap // to output two bitmaps - one for pages that are unbacked and one for pages // that are swapped. Hugepage-sized regions are assumed to be 2MiB in size. A // SinglePageBitmaps struct is returned with the status, the page_unbacked // bitmap, and the page_swapped bitmap. - virtual SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) = 0; + virtual SinglePageBitmapsInternal GetUnbackedAndSwappedBitmapsInternal( + const void* addr) = 0; + + template + SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) { + SinglePageBitmapsInternal res = GetUnbackedAndSwappedBitmapsInternal(addr); + return SinglePageBitmaps{ + .unbacked = res.unbacked.Contract(GetNativePagesInHugePage()), + .swapped = res.swapped.Contract(GetNativePagesInHugePage()), + .status = res.status}; + } }; // Residency offers information about memory residency: whether or not specific @@ -95,7 +108,8 @@ class ResidencyPageMap : public Residency { // that are swapped. Hugepage-sized regions are assumed to be 2MiB in size. A // SinglePageBitmaps struct is returned with the status, the page_unbacked // bitmap, and the page_swapped bitmap. - SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) override; + SinglePageBitmapsInternal GetUnbackedAndSwappedBitmapsInternal( + const void* addr) override; private: // This helper seeks the internal file to the correct location for the given diff --git a/tcmalloc/internal/residency_test.cc b/tcmalloc/internal/residency_test.cc index c93c2d906..624be4e39 100644 --- a/tcmalloc/internal/residency_test.cc +++ b/tcmalloc/internal/residency_test.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -55,8 +56,13 @@ class ResidencySpouse { return r_.Get(std::forward(args)...); } - decltype(auto) GetUnbackedAndSwappedBitmaps(const void* const addr) { - return r_.GetUnbackedAndSwappedBitmaps(addr); + decltype(auto) GetUnbackedAndSwappedBitmapsInternal(const void* const addr) { + return r_.GetUnbackedAndSwappedBitmapsInternal(addr); + } + + template + decltype(auto) GetUnbackedAndSwappedBitmaps(const void* addr) { + return r_.GetUnbackedAndSwappedBitmaps(addr); } private: @@ -208,7 +214,8 @@ void GenerateHolesInSinglePage(absl::string_view filename, int case_num, CHECK_EQ(close(write_fd), 0) << errno; } -Residency::SinglePageBitmaps GenerateExpectedSinglePageBitmaps(int case_num) { +Residency::SinglePageBitmaps<512> GenerateExpectedSinglePageBitmaps( + int case_num) { Bitmap<512> expected_unbacked; Bitmap<512> expected_swapped; switch (case_num) { @@ -270,7 +277,8 @@ bool BitmapsAreEqual(const Bitmap<512>& bitmap1, const Bitmap<512>& bitmap2) { TEST(PageMapTest, GetUnbackedAndSwappedBitmaps) { constexpr int kNumCases = 6; - std::array expected; + std::array, kNumCases> + expected; for (int i = 0; i < kNumCases; ++i) { expected[i] = GenerateExpectedSinglePageBitmaps(i); } @@ -284,8 +292,8 @@ TEST(PageMapTest, GetUnbackedAndSwappedBitmaps) { g.emplace(); ResidencySpouse s(file_path); - Residency::SinglePageBitmaps res = - s.GetUnbackedAndSwappedBitmaps(reinterpret_cast(0)); + auto res = s.GetUnbackedAndSwappedBitmaps( + reinterpret_cast(0)); g.reset(); EXPECT_THAT(res.status, expected[i].status); EXPECT_TRUE(BitmapsAreEqual(res.unbacked, expected[i].unbacked)); @@ -302,8 +310,8 @@ TEST(PageMapTest, CountHolesWithAddressBeyondFirstPage) { Residency::SinglePageBitmaps expected = GenerateExpectedSinglePageBitmaps(5); g.emplace(); ResidencySpouse s(file_path); - Residency::SinglePageBitmaps res = - s.GetUnbackedAndSwappedBitmaps(reinterpret_cast(2 << 21)); + auto res = s.GetUnbackedAndSwappedBitmaps( + reinterpret_cast(2 << 21)); g.reset(); EXPECT_THAT(res.status, expected.status); EXPECT_TRUE(BitmapsAreEqual(res.unbacked, expected.unbacked)); @@ -317,8 +325,8 @@ TEST(PageMapTest, VerifyAddressAlignmentCheckPasses) { /*num_pages=*/512); g.emplace(); ResidencySpouse s(file_path); - Residency::SinglePageBitmaps non_align_addr_res = - s.GetUnbackedAndSwappedBitmaps(reinterpret_cast(0x00001)); + auto non_align_addr_res = s.GetUnbackedAndSwappedBitmaps( + reinterpret_cast(0x00001)); g.reset(); EXPECT_EQ(non_align_addr_res.status, absl::StatusCode::kFailedPrecondition); } @@ -327,8 +335,8 @@ TEST(PageMapTest, VerifyAddressAlignmentBeyondFirstPageFails) { std::optional g; g.emplace(); ResidencySpouse s; - Residency::SinglePageBitmaps res = - s.GetUnbackedAndSwappedBitmaps(reinterpret_cast((2 << 21) + 1)); + auto res = s.GetUnbackedAndSwappedBitmaps( + reinterpret_cast((2 << 21) + 1)); g.reset(); EXPECT_EQ(res.status, absl::StatusCode::kFailedPrecondition); } @@ -346,7 +354,8 @@ TEST(PageMapIntegrationTest, WorksOnActualData) { } g.emplace(); ResidencyPageMap r; - Residency::SinglePageBitmaps res = r.GetUnbackedAndSwappedBitmaps(addr); + Residency::SinglePageBitmapsInternal res = + r.GetUnbackedAndSwappedBitmapsInternal(addr); g.reset(); ASSERT_EQ(res.status, absl::StatusCode::kOk); EXPECT_TRUE(res.unbacked.IsZero()); @@ -357,7 +366,7 @@ TEST(PageMapIntegrationTest, WorksOnActualData) { << errno; g.emplace(); - res = r.GetUnbackedAndSwappedBitmaps(addr); + res = r.GetUnbackedAndSwappedBitmapsInternal(addr); g.reset(); ASSERT_EQ(res.status, absl::StatusCode::kOk); EXPECT_FALSE(res.unbacked.IsZero()); @@ -369,6 +378,80 @@ TEST(PageMapIntegrationTest, WorksOnActualData) { EXPECT_TRUE(res.swapped.IsZero()); } +template +class PageMapIntegrationTest : public ::testing::Test {}; + +using WorksOnActualDataTestSizes = + ::testing::Types, + std::integral_constant, + std::integral_constant>; + +TYPED_TEST_SUITE(PageMapIntegrationTest, WorksOnActualDataTestSizes); + +TYPED_TEST(PageMapIntegrationTest, WorksOnActualDataWrapper) { + constexpr size_t N = TypeParam::value; + std::optional g; + void* addr = mmap(nullptr, 4 << 20, PROT_WRITE, + MAP_ANONYMOUS | MAP_POPULATE | MAP_PRIVATE, -1, 0); + ASSERT_NE(addr, MAP_FAILED) << errno; + auto position = reinterpret_cast(addr); + if ((position & (kHugePageSize - 1)) != 0) { + position |= kHugePageSize - 1; + position++; + addr = reinterpret_cast(position); + } + g.emplace(); + ResidencyPageMap r; + auto res = r.GetUnbackedAndSwappedBitmaps(addr); + g.reset(); + ASSERT_EQ(res.status, absl::StatusCode::kOk); + EXPECT_TRUE(res.unbacked.IsZero()); + EXPECT_TRUE(res.swapped.IsZero()); + // Unmap consecutive pages starting at page 2, and page 16 + ASSERT_EQ(munmap(reinterpret_cast(addr) + 2 * 4096, 2 * 4096), 0) + << errno; + ASSERT_EQ(munmap(reinterpret_cast(addr) + 16 * 4096, 2 * 4096), 0) + << errno; + + g.emplace(); + res = r.GetUnbackedAndSwappedBitmaps(addr); + g.reset(); + ASSERT_EQ(res.status, absl::StatusCode::kOk); + EXPECT_FALSE(res.unbacked.IsZero()); + + if constexpr (N == kMaxResidencyBits) { + ASSERT_TRUE(res.unbacked.GetBit(2)); + ASSERT_TRUE(res.unbacked.GetBit(3)); + ASSERT_TRUE(res.unbacked.GetBit(16)); + ASSERT_TRUE(res.unbacked.GetBit(17)); + res.unbacked.ClearBit(2); + res.unbacked.ClearBit(3); + res.unbacked.ClearBit(16); + res.unbacked.ClearBit(17); + } else if constexpr (N == kMaxResidencyBits / 2) { + ASSERT_TRUE(res.unbacked.GetBit(1)); + ASSERT_TRUE(res.unbacked.GetBit(8)); + res.unbacked.ClearBit(1); + res.unbacked.ClearBit(8); + } else if constexpr (N == kMaxResidencyBits * 2) { + ASSERT_TRUE(res.unbacked.GetBit(4)); + ASSERT_TRUE(res.unbacked.GetBit(5)); + ASSERT_TRUE(res.unbacked.GetBit(32)); + ASSERT_TRUE(res.unbacked.GetBit(33)); + res.unbacked.ClearBit(4); + res.unbacked.ClearBit(5); + res.unbacked.ClearBit(6); + res.unbacked.ClearBit(7); + res.unbacked.ClearBit(32); + res.unbacked.ClearBit(33); + res.unbacked.ClearBit(34); + res.unbacked.ClearBit(35); + } + + EXPECT_TRUE(res.unbacked.IsZero()); + EXPECT_TRUE(res.swapped.IsZero()); +} + } // namespace } // namespace tcmalloc_internal } // namespace tcmalloc