Skip to content

Banach fixed-point theorem #341

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
copybara-service wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Banach fixed-point theorem #341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions tcmalloc/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include "tcmalloc/internal/logging.h"
#include "tcmalloc/internal/memory_tag.h"
#include "tcmalloc/internal/optimization.h"
#include "tcmalloc/internal/range_tracker.h"
#include "tcmalloc/malloc_extension.h"

GOOGLE_MALLOC_SECTION_BEGIN
Expand Down Expand Up @@ -208,6 +209,7 @@ inline constexpr size_t kMinObjectsToMove = 2;
inline constexpr size_t kMaxObjectsToMove = 128;

inline constexpr size_t kPageSize = 1 << kPageShift;
using PageBitmap = Bitmap<kHugePageSize / kPageSize>;

inline constexpr std::align_val_t kAlignment{8};
// log2 (kAlignment)
Expand Down
89 changes: 32 additions & 57 deletions tcmalloc/huge_page_filler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ class PageTracker : public TList<PageTracker>::Elem {
Length ReleaseFree(MemoryModifyFunction& unback)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

Length MarkSubreleased(Bitmap<kMaxResidencyBits> unbacked,
Length MarkSubreleased(tcmalloc_internal::PageBitmap unbacked,
int native_pages_per_tcmalloc_page)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

Expand Down Expand Up @@ -249,8 +249,8 @@ class PageTracker : public TList<PageTracker>::Elem {
// used to avoid subreleasing the pages that are being collapsed.
bool being_collapsed = false;
// Records swap and unbacked bitmaps for this hugepage.
Residency::SinglePageBitmaps bitmaps;
ResidencyBitmap stale;
Residency::SinglePageBitmaps<kPagesPerHugePage.raw_num()> bitmaps;
tcmalloc_internal::PageBitmap stale;
// Records whether collapse was skipped due to threshold constraints.
bool collapse_skipped = false;
// Records whether collapse was skipped due to backoff.
Expand Down Expand Up @@ -327,7 +327,8 @@ class PageTracker : public TList<PageTracker>::Elem {
};

NativePageResidencyInfo CountInfoInHugePage(
Residency::SinglePageBitmaps bitmaps, ResidencyBitmap stale) const;
Residency::SinglePageBitmaps<kPagesPerHugePage.raw_num()> bitmaps,
tcmalloc_internal::PageBitmap stale) const;

private:
HugePage location_;
Expand Down Expand Up @@ -1310,7 +1311,7 @@ class HugePageFiller {
// Utility function to handle a non-hugepage backed `page_tracker` and
// mark its unmapped pages appropriately.
Length HandleUnbackedHugePage(PageTracker* page_tracker,
Bitmap<kMaxResidencyBits> unbacked)
tcmalloc_internal::PageBitmap unbacked)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

private:
Expand Down Expand Up @@ -1509,7 +1510,8 @@ inline void PageTracker::SetAnonVmaName(MemoryTagFunction& set_anon_vma_name,
}

inline PageTracker::NativePageResidencyInfo PageTracker::CountInfoInHugePage(
Residency::SinglePageBitmaps bitmaps, ResidencyBitmap stale) const {
Residency::SinglePageBitmaps<kPagesPerHugePage.raw_num()> bitmaps,
tcmalloc_internal::PageBitmap stale) const {
// TODO(b/424551232): Add support for the scenario when native page size is
// larger than TCMalloc page size.
const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize();
Expand All @@ -1518,23 +1520,16 @@ inline PageTracker::NativePageResidencyInfo PageTracker::CountInfoInHugePage(
}
TC_ASSERT_LE(kNativePagesInHugePage, kMaxResidencyBits);

Bitmap<kMaxResidencyBits> unbacked = bitmaps.unbacked;
Bitmap<kMaxResidencyBits> swapped = bitmaps.swapped;
Bitmap<kPagesPerHugePage.raw_num()> free = free_.bits();

TC_ASSERT_EQ(kNativePagesInHugePage % kPagesPerHugePage.raw_num(), 0);
const int shift = kNativePagesInHugePage / kPagesPerHugePage.raw_num();
const int shift_bits = absl::bit_width<uint8_t>(shift - 1);
TC_ASSERT_LT((kNativePagesInHugePage - 1) >> shift_bits,
kPagesPerHugePage.raw_num());
tcmalloc_internal::PageBitmap unbacked = bitmaps.unbacked;
tcmalloc_internal::PageBitmap swapped = bitmaps.swapped;
tcmalloc_internal::PageBitmap free = free_.bits();

size_t n_unbacked[2] = {0, 0};
size_t n_swapped[2] = {0, 0};
size_t n_stale[2] = {0, 0};

for (size_t i = 0; i < kNativePagesInHugePage; ++i) {
const int scaled_idx = i >> shift_bits;
bool is_free = !free.GetBit(scaled_idx);
for (size_t i = 0; i < unbacked.size(); ++i) {
bool is_free = !free.GetBit(i);

n_swapped[is_free] += swapped.GetBit(i);
n_unbacked[is_free] += unbacked.GetBit(i);
Expand Down Expand Up @@ -1607,8 +1602,9 @@ inline Length PageTracker::ReleaseFree(MemoryModifyFunction& unback) {
return Length(count);
}

inline Length PageTracker::MarkSubreleased(Bitmap<kMaxResidencyBits> unbacked,
int native_pages_per_tcmalloc_page) {
inline Length PageTracker::MarkSubreleased(
tcmalloc_internal::PageBitmap unbacked,
int native_pages_per_tcmalloc_page) {
// TODO(b/525422238): pass the bitmap in TCMalloc pages directly to avoid this
// conversion.

Expand All @@ -1620,47 +1616,25 @@ inline Length PageTracker::MarkSubreleased(Bitmap<kMaxResidencyBits> unbacked,
return Length(0);
}
TC_ASSERT_GT(native_pages_per_tcmalloc_page, 0);
Bitmap<kPagesPerHugePage.raw_num()> free = free_.bits();

size_t count = 0;
for (size_t i = 0; i < kPagesPerHugePage.raw_num(); ++i) {
if (!free.GetBit(i) && !released_by_page_.GetBit(i)) {
// Check if all native pages corresponding to this TCMalloc page are
// unbacked.
bool is_unbacked = true;
for (int j = 0; j < native_pages_per_tcmalloc_page; ++j) {
const int bitmap_index = i * native_pages_per_tcmalloc_page + j;
TC_ASSERT_LT(bitmap_index, kMaxResidencyBits);
if (!unbacked.GetBit(bitmap_index)) {
is_unbacked = false;
break;
}
}
// If all native pages corresponding to this TCMalloc page are unbacked,
// and as the TCMalloc page was freed but not released, mark this page
// as released now.
//
// TODO(b/525422238): The residency bitmap was captured outside of the
// lock. So, in a rare case, it's possible that the page was allocated,
// backed and then freed. So, the free page here is actually backed.
// While we currently ignore this case (resulting in underestimating
// RSS), we can potentially fix this by re-investigating the bitmaps
// and marking the pages back to backed to eventually fix this.
if (is_unbacked) {
released_by_page_.SetBit(i);
++count;
}
}
}

released_count_ += count;
tcmalloc_internal::PageBitmap free = free_.bits();

// TODO(b/525422238): The residency bitmap was captured outside of the
// lock. So, in a rare case, it's possible that the page was allocated,
// backed and then freed. So, the free page here is actually backed.
// While we currently ignore this case (resulting in underestimating
// RSS), we can potentially fix this by re-investigating the bitmaps
// and marking the pages back to backed to eventually fix this.
auto to_release = (~free) & (~released_by_page_) & unbacked;
released_by_page_ = released_by_page_ | to_release;

released_count_ += to_release.CountBits();
// Mark this is unbroken regardless of whether it had any unbacked free
// TCMalloc pages. Marking this will move this tracker to one of the
// released lists.
unbroken_ = false;
TC_ASSERT_LE(Length(released_count_), kPagesPerHugePage);
TC_ASSERT_EQ(released_by_page_.CountBits(), released_count_);
return Length(count);
return Length(to_release.CountBits());
}

inline MemoryModifyStatus PageTracker::Collapse(
Expand Down Expand Up @@ -2712,7 +2686,8 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment {
state.maybe_hugepage_backed = is_hugepage;
if (!is_hugepage) {
state.bitmaps =
res->GetUnbackedAndSwappedBitmaps(tracker->location().start_addr());
res->GetUnbackedAndSwappedBitmaps<kPagesPerHugePage.raw_num()>(
tracker->location().start_addr());
if (pf) {
pf->GetSinglePageBitmaps(tracker->location().start_addr(),
state.stale);
Expand Down Expand Up @@ -2989,7 +2964,7 @@ inline void HugePageFiller<TrackerType>::OnCollapseSuccess(TrackerType* pt) {

template <class TrackerType>
inline Length HugePageFiller<TrackerType>::HandleUnbackedHugePage(
PageTracker* tracker, Bitmap<kMaxResidencyBits> unbacked) {
PageTracker* tracker, tcmalloc_internal::PageBitmap unbacked) {
RemoveFromFillerList(tracker);
Length unmapped_length =
tracker->MarkSubreleased(unbacked, native_pages_per_tcmalloc_page_);
Expand Down
10 changes: 6 additions & 4 deletions tcmalloc/huge_page_filler_fuzz.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ class FakePageFlags : public PageFlagsBase {
return PageStats{};
}

absl::StatusCode GetSinglePageBitmaps(const void* addr,
ResidencyBitmap& stale) override {
absl::StatusCode GetSinglePageBitmapsInternal(
const void* addr, ResidencyBitmap& stale) override {
return absl::StatusCode::kUnimplemented;
}

Expand All @@ -141,7 +141,8 @@ class FakeResidency : public Residency {
return std::nullopt;
};

SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) override {
SinglePageBitmapsInternal GetUnbackedAndSwappedBitmapsInternal(
const void* addr) override {
return {unbacked_bitmap, swapped_bitmap, absl::StatusCode::kOk};
};

Expand All @@ -151,7 +152,8 @@ class FakeResidency : public Residency {
};

private:
absl::flat_hash_map<const void*, SinglePageBitmaps> residency_bitmaps_;
absl::flat_hash_map<const void*, SinglePageBitmapsInternal>
residency_bitmaps_;
};

class MockCollapse final : public MemoryModifyFunction {
Expand Down
57 changes: 30 additions & 27 deletions tcmalloc/huge_page_filler_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,9 +289,10 @@ class FakePageFlags : public PageFlagsBase {
return PageStats{};
}

absl::StatusCode GetSinglePageBitmaps(const void* addr,
ResidencyBitmap& stale) override {
absl::StatusCode GetSinglePageBitmapsInternal(
const void* addr, ResidencyBitmap& stale) override {
stale.SetBit(0);
stale.SetBit(1);
return absl::StatusCode::kOk;
}

Expand Down Expand Up @@ -328,7 +329,8 @@ class FakeResidency : public Residency {
return std::nullopt;
};

SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) override {
SinglePageBitmapsInternal GetUnbackedAndSwappedBitmapsInternal(
const void* addr) override {
PageId p = PageIdContaining(addr);
HugePage hp = HugePageContaining(p);
EXPECT_TRUE(residency_bitmaps_.contains(hp.start_addr()));
Expand All @@ -344,13 +346,14 @@ class FakeResidency : public Residency {
absl::StatusCode::kOk};
}

const size_t kNativePagesInHugePage = kHugePageSize / kPageSize;
const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize();
size_t GetNativePagesInHugePage() const override {
return kNativePagesInHugePage;
};

private:
absl::flat_hash_map<const void*, SinglePageBitmaps> residency_bitmaps_;
absl::flat_hash_map<const void*, SinglePageBitmaps<kMaxResidencyBits>>
residency_bitmaps_;
};

TEST_F(PageTrackerTest, AllocSane) {
Expand Down Expand Up @@ -1428,7 +1431,7 @@ TEST_F(FillerTest, ReleaseFreePagesWhenAnyPageIsSwapped) {
EXPECT_FALSE(pageflags.IsHugepageBacked(pa.p.start_addr()).value());

Bitmap<kMaxResidencyBits> unbacked, swapped;
swapped.SetRange(/*index=*/1, /*n=*/1);
swapped.SetRange(/*index=*/0, /*n=*/2);
residency.SetUnbackedAndSwappedBitmaps(pa.p.start_addr(), unbacked,
swapped);
}
Expand Down Expand Up @@ -2462,7 +2465,7 @@ TEST_F(FillerTestWithSubreleaseUnbacked, SubreleaseUnbackedPartial) {
// free TCMalloc page are unbacked, the TCMalloc page is NOT marked as
// subreleased.
TEST_F(FillerTestWithSubreleaseUnbacked, SubreleaseUnbackedPartialNativePages) {
const size_t kNativePagesInHugePage = kHugePageSize / kPageSize;
const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize();
const int shift = kNativePagesInHugePage / kPagesPerHugePage.raw_num();
if (shift <= 1) {
return;
Expand Down Expand Up @@ -5218,8 +5221,8 @@ TEST_F(FillerTest, ResidencyTelemetry) {
EXPECT_THAT(buffer_text, testing::HasSubstr(R"(
HugePageFiller: # of sparsely-accessed released hps with a <= # of swapped < b
HugePageFiller: < 0<= 0 < 1<= 0 < 2<= 0 < 3<= 0 < 4<= 0 < 5<= 0
HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 0
HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 1 <264<= 0 <296<= 0
HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 1
HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 0 <264<= 0 <296<= 0
HugePageFiller: <328<= 0 <360<= 0 <392<= 0 <424<= 0 <456<= 0 <488<= 0
HugePageFiller: <504<= 0 <505<= 0 <506<= 0 <507<= 0 <508<= 0 <509<= 0
HugePageFiller: <510<= 0 <511<= 0
Expand All @@ -5228,8 +5231,8 @@ HugePageFiller: <510<= 0 <511<= 0
EXPECT_THAT(buffer_text, testing::HasSubstr(R"(
HugePageFiller: # of sparsely-accessed released hps with a <= # of unbacked < b
HugePageFiller: < 0<= 0 < 1<= 0 < 2<= 0 < 3<= 0 < 4<= 0 < 5<= 0
HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 0
HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 1 <264<= 0 <296<= 0
HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 1
HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 0 <264<= 0 <296<= 0
HugePageFiller: <328<= 0 <360<= 0 <392<= 0 <424<= 0 <456<= 0 <488<= 0
HugePageFiller: <504<= 0 <505<= 0 <506<= 0 <507<= 0 <508<= 0 <509<= 0
HugePageFiller: <510<= 0 <511<= 0
Expand All @@ -5238,7 +5241,7 @@ HugePageFiller: <510<= 0 <511<= 0
EXPECT_THAT(buffer_text, testing::HasSubstr(R"(
HugePageFiller: # of sparsely-accessed released hps with a <= # of free AND unbacked < b
HugePageFiller: < 0<= 0 < 1<= 0 < 2<= 0 < 3<= 0 < 4<= 0 < 5<= 0
HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 1
HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 1 < 72<= 0 <104<= 0
HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 0 <264<= 0 <296<= 0
HugePageFiller: <328<= 0 <360<= 0 <392<= 0 <424<= 0 <456<= 0 <488<= 0
HugePageFiller: <504<= 0 <505<= 0 <506<= 0 <507<= 0 <508<= 0 <509<= 0
Expand All @@ -5248,7 +5251,7 @@ HugePageFiller: <510<= 0 <511<= 0
EXPECT_THAT(buffer_text, testing::HasSubstr(R"(
HugePageFiller: # of sparsely-accessed released hps with a <= # of free AND swapped < b
HugePageFiller: < 0<= 0 < 1<= 0 < 2<= 0 < 3<= 0 < 4<= 0 < 5<= 0
HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 0 < 72<= 0 <104<= 1
HugePageFiller: < 6<= 0 < 7<= 0 < 8<= 0 < 40<= 1 < 72<= 0 <104<= 0
HugePageFiller: <136<= 0 <168<= 0 <200<= 0 <232<= 0 <264<= 0 <296<= 0
HugePageFiller: <328<= 0 <360<= 0 <392<= 0 <424<= 0 <456<= 0 <488<= 0
HugePageFiller: <504<= 0 <505<= 0 <506<= 0 <507<= 0 <508<= 0 <509<= 0
Expand Down Expand Up @@ -5278,17 +5281,17 @@ HugePageFiller: Of the hugepage backed pages of type sparsely-accessed regular,
)"));

EXPECT_THAT(buffer_text, testing::HasSubstr(R"(
HugePageFiller: 126 of sparsely-accessed released free native pages are swapped.
HugePageFiller: 63 of sparsely-accessed released free native pages are swapped.
)"));
EXPECT_THAT(buffer_text, testing::HasSubstr(R"(
HugePageFiller: 130 of sparsely-accessed released used native pages are swapped.
HugePageFiller: 65 of sparsely-accessed released used native pages are swapped.
)"));

EXPECT_THAT(buffer_text, testing::HasSubstr(R"(
HugePageFiller: 126 of sparsely-accessed released free native pages are unbacked.
HugePageFiller: 63 of sparsely-accessed released free native pages are unbacked.
)"));
EXPECT_THAT(buffer_text, testing::HasSubstr(R"(
HugePageFiller: 130 of sparsely-accessed released used native pages are unbacked.
HugePageFiller: 65 of sparsely-accessed released used native pages are unbacked.
)"));

std::string buffer_pbtxt =
Expand All @@ -5300,17 +5303,17 @@ HugePageFiller: 130 of sparsely-accessed released used native pages are unbacked
EXPECT_THAT(
buffer_pbtxt,
testing::HasSubstr(
"unbacked_histogram { lower_bound: 232 upper_bound: 263 value: 1}"));
"unbacked_histogram { lower_bound: 104 upper_bound: 135 value: 1}"));
EXPECT_THAT(
buffer_pbtxt,
testing::HasSubstr(
"swapped_histogram { lower_bound: 232 upper_bound: 263 value: 1}"));
"swapped_histogram { lower_bound: 104 upper_bound: 135 value: 1}"));
EXPECT_THAT(buffer_pbtxt,
testing::HasSubstr("free_unbacked_histogram { lower_bound: 104 "
"upper_bound: 135 value: 1}"));
testing::HasSubstr("free_unbacked_histogram { lower_bound: 40 "
"upper_bound: 71 value: 1}"));
EXPECT_THAT(buffer_pbtxt,
testing::HasSubstr("free_swapped_histogram { lower_bound: 104 "
"upper_bound: 135 value: 1}"));
testing::HasSubstr("free_swapped_histogram { lower_bound: 40 "
"upper_bound: 71 value: 1}"));
EXPECT_THAT(buffer_pbtxt,
testing::HasSubstr("num_free_pages_non_hugepage_backed: 0"));
EXPECT_THAT(buffer_pbtxt,
Expand All @@ -5319,10 +5322,10 @@ HugePageFiller: 130 of sparsely-accessed released used native pages are unbacked
testing::HasSubstr("num_free_pages_hugepage_backed: 126"));
EXPECT_THAT(buffer_pbtxt,
testing::HasSubstr("num_used_pages_hugepage_backed: 130"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_swapped: 126"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_swapped: 130"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_unbacked: 126"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_unbacked: 130"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_swapped: 63"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_swapped: 65"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_unbacked: 63"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_unbacked: 65"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_stale: 0"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_stale: 1"));
EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_treated: 1"));
Expand Down
Loading
Loading