Skip to content

Refactor residency to return bitmaps in TCMalloc page granularity. #340

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
copybara-service wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Refactor residency to return bitmaps in TCMalloc page granularity. #340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 33 additions & 53 deletions tcmalloc/huge_page_filler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,8 +218,7 @@ class PageTracker : public TList<PageTracker>::Elem {
Length ReleaseFree(MemoryModifyFunction& unback)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

Length MarkSubreleased(Bitmap<kMaxResidencyBits> unbacked,
int native_pages_per_tcmalloc_page)
Length MarkSubreleased(Bitmap<kMaxResidencyBits> unbacked)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

Bitmap<kPagesPerHugePage.raw_num()> released_by_page() const {
Expand Down Expand Up @@ -702,11 +701,15 @@ class UsageInfo {
} else {
records.num_free_non_hugepage_backed += (free - pt.released_pages());
records.num_used_non_hugepage_backed += pt.used_pages();
const size_t native_pages_per_tcmalloc_page =
kPagesPerHugePage.raw_num() > 0
? (kHugePageSize / GetPageSize() / kPagesPerHugePage.raw_num())
: 0;
Residency::SinglePageBitmaps bitmaps = hugepage_residency_state.bitmaps;
++records.unbacked_histo[NativePageBucketNum(
bitmaps.unbacked.CountBits())];
++records
.swapped_histo[NativePageBucketNum(bitmaps.swapped.CountBits())];
bitmaps.unbacked.CountBits() * native_pages_per_tcmalloc_page)];
++records.swapped_histo[NativePageBucketNum(
bitmaps.swapped.CountBits() * native_pages_per_tcmalloc_page)];
++records.stale_histo[NativePageBucketNum(
hugepage_residency_state.stale.CountBits())];

Expand Down Expand Up @@ -1313,6 +1316,10 @@ class HugePageFiller {
Bitmap<kMaxResidencyBits> unbacked)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

int native_pages_per_tcmalloc_page() const {
return native_pages_per_tcmalloc_page_;
}

private:
// This class wraps an array of N TrackerLists and a Bitmap storing which
// elements are non-empty.
Expand Down Expand Up @@ -1536,8 +1543,8 @@ inline PageTracker::NativePageResidencyInfo PageTracker::CountInfoInHugePage(
const int scaled_idx = i >> shift_bits;
bool is_free = !free.GetBit(scaled_idx);

n_swapped[is_free] += swapped.GetBit(i);
n_unbacked[is_free] += unbacked.GetBit(i);
n_swapped[is_free] += swapped.GetBit(scaled_idx);
n_unbacked[is_free] += unbacked.GetBit(scaled_idx);
n_stale[is_free] += stale.GetBit(i);
}

Expand Down Expand Up @@ -1607,49 +1614,19 @@ inline Length PageTracker::ReleaseFree(MemoryModifyFunction& unback) {
return Length(count);
}

inline Length PageTracker::MarkSubreleased(Bitmap<kMaxResidencyBits> unbacked,
int native_pages_per_tcmalloc_page) {
// TODO(b/525422238): pass the bitmap in TCMalloc pages directly to avoid this
// conversion.

// When TCMalloc page size is smaller than the native page size, mark the page
// as broken. We should additionally mark unbacked pages as subreleased. The
// later case isn't done currently.
if (native_pages_per_tcmalloc_page == 0) {
inline Length PageTracker::MarkSubreleased(Bitmap<kMaxResidencyBits> unbacked) {
const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize();
if (kNativePagesInHugePage < kPagesPerHugePage.raw_num()) {
unbroken_ = false;
return Length(0);
}
TC_ASSERT_GT(native_pages_per_tcmalloc_page, 0);
Bitmap<kPagesPerHugePage.raw_num()> free = free_.bits();

size_t count = 0;
for (size_t i = 0; i < kPagesPerHugePage.raw_num(); ++i) {
if (!free.GetBit(i) && !released_by_page_.GetBit(i)) {
// Check if all native pages corresponding to this TCMalloc page are
// unbacked.
bool is_unbacked = true;
for (int j = 0; j < native_pages_per_tcmalloc_page; ++j) {
const int bitmap_index = i * native_pages_per_tcmalloc_page + j;
TC_ASSERT_LT(bitmap_index, kMaxResidencyBits);
if (!unbacked.GetBit(bitmap_index)) {
is_unbacked = false;
break;
}
}
// If all native pages corresponding to this TCMalloc page are unbacked,
// and as the TCMalloc page was freed but not released, mark this page
// as released now.
//
// TODO(b/525422238): The residency bitmap was captured outside of the
// lock. So, in a rare case, it's possible that the page was allocated,
// backed and then freed. So, the free page here is actually backed.
// While we currently ignore this case (resulting in underestimating
// RSS), we can potentially fix this by re-investigating the bitmaps
// and marking the pages back to backed to eventually fix this.
if (is_unbacked) {
released_by_page_.SetBit(i);
++count;
}
if (!free.GetBit(i) && !released_by_page_.GetBit(i) && unbacked.GetBit(i)) {
released_by_page_.SetBit(i);
++count;
}
}

Expand Down Expand Up @@ -2711,8 +2688,9 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment {
// information.
state.maybe_hugepage_backed = is_hugepage;
if (!is_hugepage) {
state.bitmaps =
res->GetUnbackedAndSwappedBitmaps(tracker->location().start_addr());
state.bitmaps = res->GetUnbackedAndSwappedBitmaps(
tracker->location().start_addr(),
page_filler_.native_pages_per_tcmalloc_page());
if (pf) {
pf->GetSinglePageBitmaps(tracker->location().start_addr(),
state.stale);
Expand All @@ -2721,12 +2699,15 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment {
const bool backoff =
treatment_stats_.collapse_time_max_cycles > max_collapse_cycles;
if (enable_collapse_ == EnableCollapse::kEnabled && !backoff) {
bool should_collapse = enable_unfiltered_collapse_ ==
EnableUnfilteredCollapse::kEnabled ||
(state.bitmaps.swapped.CountBits() <
kMaxSwappedPagesForCollapse &&
state.bitmaps.unbacked.CountBits() <
kMaxUnbackedPagesForCollapse);
bool should_collapse =
enable_unfiltered_collapse_ ==
EnableUnfilteredCollapse::kEnabled ||
(state.bitmaps.swapped.CountBits() *
page_filler_.native_pages_per_tcmalloc_page() <
kMaxSwappedPagesForCollapse &&
state.bitmaps.unbacked.CountBits() *
page_filler_.native_pages_per_tcmalloc_page() <
kMaxUnbackedPagesForCollapse);
if (should_collapse) {
state.maybe_hugepage_backed = TryUserspaceCollapse(tracker);
} else {
Expand Down Expand Up @@ -2991,8 +2972,7 @@ template <class TrackerType>
inline Length HugePageFiller<TrackerType>::HandleUnbackedHugePage(
PageTracker* tracker, Bitmap<kMaxResidencyBits> unbacked) {
RemoveFromFillerList(tracker);
Length unmapped_length =
tracker->MarkSubreleased(unbacked, native_pages_per_tcmalloc_page_);
Length unmapped_length = tracker->MarkSubreleased(unbacked);
subrelease_stats_.total_pages_subreleased += unmapped_length;
unmapped_ += unmapped_length;
unmapping_unaccounted_ += unmapped_length;
Expand Down
34 changes: 31 additions & 3 deletions tcmalloc/huge_page_filler_fuzz.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,11 +141,39 @@ class FakeResidency : public Residency {
return std::nullopt;
};

SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) override {
return {unbacked_bitmap, swapped_bitmap, absl::StatusCode::kOk};
SinglePageBitmaps GetUnbackedAndSwappedBitmaps(
const void* addr, size_t native_pages_per_element) override {
SinglePageBitmaps native_res = {unbacked_bitmap, swapped_bitmap,
absl::StatusCode::kOk};
if (native_pages_per_element <= 1) {
return native_res;
}
SinglePageBitmaps result;
result.status = native_res.status;
size_t num_elements = kNativePagesInHugePage / native_pages_per_element;
for (size_t idx = 0; idx < num_elements; ++idx) {
bool all_unbacked = true;
bool any_swapped = false;
for (size_t j = 0; j < native_pages_per_element; ++j) {
size_t native_idx = idx * native_pages_per_element + j;
if (!native_res.unbacked.GetBit(native_idx)) {
all_unbacked = false;
}
if (native_res.swapped.GetBit(native_idx)) {
any_swapped = true;
}
}
if (all_unbacked) {
result.unbacked.SetBit(idx);
}
if (any_swapped) {
result.swapped.SetBit(idx);
}
}
return result;
};

const size_t kNativePagesInHugePage = kHugePageSize / kPageSize;
const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize();
size_t GetNativePagesInHugePage() const override {
return kNativePagesInHugePage;
};
Expand Down
35 changes: 31 additions & 4 deletions tcmalloc/huge_page_filler_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,11 +328,38 @@ class FakeResidency : public Residency {
return std::nullopt;
};

SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) override {
SinglePageBitmaps GetUnbackedAndSwappedBitmaps(
const void* addr, size_t native_pages_per_element) override {
PageId p = PageIdContaining(addr);
HugePage hp = HugePageContaining(p);
EXPECT_TRUE(residency_bitmaps_.contains(hp.start_addr()));
return residency_bitmaps_[hp.start_addr()];
SinglePageBitmaps native_res = residency_bitmaps_[hp.start_addr()];
if (native_pages_per_element <= 1) {
return native_res;
}
SinglePageBitmaps result;
result.status = native_res.status;
size_t num_elements = kNativePagesInHugePage / native_pages_per_element;
for (size_t idx = 0; idx < num_elements; ++idx) {
bool all_unbacked = true;
bool any_swapped = false;
for (size_t j = 0; j < native_pages_per_element; ++j) {
size_t native_idx = idx * native_pages_per_element + j;
if (!native_res.unbacked.GetBit(native_idx)) {
all_unbacked = false;
}
if (native_res.swapped.GetBit(native_idx)) {
any_swapped = true;
}
}
if (all_unbacked) {
result.unbacked.SetBit(idx);
}
if (any_swapped) {
result.swapped.SetBit(idx);
}
}
return result;
};

void SetUnbackedAndSwappedBitmaps(const void* addr,
Expand All @@ -344,7 +371,7 @@ class FakeResidency : public Residency {
absl::StatusCode::kOk};
}

const size_t kNativePagesInHugePage = kHugePageSize / kPageSize;
const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize();
size_t GetNativePagesInHugePage() const override {
return kNativePagesInHugePage;
};
Expand Down Expand Up @@ -2462,7 +2489,7 @@ TEST_F(FillerTestWithSubreleaseUnbacked, SubreleaseUnbackedPartial) {
// free TCMalloc page are unbacked, the TCMalloc page is NOT marked as
// subreleased.
TEST_F(FillerTestWithSubreleaseUnbacked, SubreleaseUnbackedPartialNativePages) {
const size_t kNativePagesInHugePage = kHugePageSize / kPageSize;
const size_t kNativePagesInHugePage = kHugePageSize / GetPageSize();
const int shift = kNativePagesInHugePage / kPagesPerHugePage.raw_num();
if (shift <= 1) {
return;
Expand Down
1 change: 1 addition & 0 deletions tcmalloc/internal/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,6 +601,7 @@ cc_test(
":range_tracker",
":residency",
":util",
"//tcmalloc:common_8k_pages",
"@com_github_google_benchmark//:benchmark",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
Expand Down
32 changes: 23 additions & 9 deletions tcmalloc/internal/residency.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ std::optional<Residency::Info> ResidencyPageMap::Get(const void* const addr,
}

Residency::SinglePageBitmaps ResidencyPageMap::GetUnbackedAndSwappedBitmaps(
const void* const addr) {
const void* const addr, size_t native_pages_per_element) {
Bitmap<kMaxResidencyBits> page_unbacked;
Bitmap<kMaxResidencyBits> page_swapped;
uintptr_t currPage = reinterpret_cast<uintptr_t>(addr);
Expand All @@ -212,16 +212,30 @@ Residency::SinglePageBitmaps ResidencyPageMap::GetUnbackedAndSwappedBitmaps(
absl::StatusCode::kUnavailable};
}

for (int native_page_idx = 0; native_page_idx < kNativePagesInHugePage;
++native_page_idx) {
uint64_t page_map = buf_[native_page_idx];
if (!PagePresent(page_map)) {
if (PageSwapped(page_map)) {
page_swapped.SetBit(native_page_idx);
} else {
page_unbacked.SetBit(native_page_idx);
if (native_pages_per_element == 0) {
return SinglePageBitmaps{page_unbacked, page_swapped,
absl::StatusCode::kOk};
}
const size_t num_elements = kNativePagesInHugePage / native_pages_per_element;

for (size_t idx = 0; idx < num_elements; ++idx) {
bool all_unbacked = true;
bool any_swapped = false;
for (size_t j = 0; j < native_pages_per_element; ++j) {
uint64_t page_map = buf_[idx * native_pages_per_element + j];
if (PagePresent(page_map)) {
all_unbacked = false;
} else if (PageSwapped(page_map)) {
all_unbacked = false;
any_swapped = true;
}
}
if (all_unbacked) {
page_unbacked.SetBit(idx);
}
if (any_swapped) {
page_swapped.SetBit(idx);
}
}
return SinglePageBitmaps{page_unbacked, page_swapped, absl::StatusCode::kOk};
}
Expand Down
10 changes: 7 additions & 3 deletions tcmalloc/internal/residency.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class Residency {
virtual inline size_t GetNativePagesInHugePage() const = 0;

// Struct is ordered with bitmaps first to optimize cacheline usage.
// The bitmaps are indexed by TCMalloc pages (0 .. kPagesPerHugePage - 1).
struct SinglePageBitmaps {
Bitmap<kMaxResidencyBits> unbacked;
Bitmap<kMaxResidencyBits> swapped;
Expand All @@ -57,8 +58,10 @@ class Residency {
// to output two bitmaps - one for pages that are unbacked and one for pages
// that are swapped. Hugepage-sized regions are assumed to be 2MiB in size. A
// SinglePageBitmaps struct is returned with the status, the page_unbacked
// bitmap, and the page_swapped bitmap.
virtual SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) = 0;
// bitmap, and the page_swapped bitmap. Each bit represents
// native_pages_per_element native OS pages.
virtual SinglePageBitmaps GetUnbackedAndSwappedBitmaps(
const void* addr, size_t native_pages_per_element) = 0;
};

// Residency offers information about memory residency: whether or not specific
Expand Down Expand Up @@ -95,7 +98,8 @@ class ResidencyPageMap : public Residency {
// that are swapped. Hugepage-sized regions are assumed to be 2MiB in size. A
// SinglePageBitmaps struct is returned with the status, the page_unbacked
// bitmap, and the page_swapped bitmap.
SinglePageBitmaps GetUnbackedAndSwappedBitmaps(const void* addr) override;
SinglePageBitmaps GetUnbackedAndSwappedBitmaps(
const void* addr, size_t native_pages_per_element) override;

private:
// This helper seeks the internal file to the correct location for the given
Expand Down
Loading
Loading