From 58818cfb64af329aff7b2401fb628eb7d2491960 Mon Sep 17 00:00:00 2001 From: Qian Ge Date: Sun, 13 Oct 2024 17:07:05 -0700 Subject: [PATCH] Enabling HugeCache to provide an interval for capping its skip-release's demand. PiperOrigin-RevId: 685512884 Change-Id: I970aafe0b4d9200ec75bca519c9c3c5f3f526302 --- tcmalloc/huge_cache.cc | 3 +- tcmalloc/huge_cache.h | 3 + tcmalloc/huge_cache_test.cc | 35 ++++++ tcmalloc/huge_page_subrelease.h | 175 ++++++++++++++------------ tcmalloc/huge_page_subrelease_test.cc | 21 ++++ 5 files changed, 153 insertions(+), 84 deletions(-) diff --git a/tcmalloc/huge_cache.cc b/tcmalloc/huge_cache.cc index 03c99d26c..8b44c60be 100644 --- a/tcmalloc/huge_cache.cc +++ b/tcmalloc/huge_cache.cc @@ -327,7 +327,8 @@ HugeLength HugeCache::GetDesiredReleaseablePages( HLFromPages(cachestats_tracker_.GetRecentPeak(intervals.peak_interval)); } else { required_by_demand = HLFromPages(cachestats_tracker_.GetRecentDemand( - intervals.short_interval, intervals.long_interval)); + intervals.short_interval, intervals.long_interval, + CapDemandInterval())); } HugeLength current = usage() + size(); diff --git a/tcmalloc/huge_cache.h b/tcmalloc/huge_cache.h index 4318971bf..1a42c7511 100644 --- a/tcmalloc/huge_cache.h +++ b/tcmalloc/huge_cache.h @@ -270,6 +270,9 @@ class HugeCache { MemoryModifyFunction& unback_; absl::Duration cache_time_; + // Interval used for capping demand calculated for demand-based release. + absl::Duration CapDemandInterval() const { return absl::Minutes(5); } + using StatsTrackerType = SubreleaseStatsTracker<600>; StatsTrackerType::SubreleaseStats GetSubreleaseStats() const { StatsTrackerType::SubreleaseStats stats; diff --git a/tcmalloc/huge_cache_test.cc b/tcmalloc/huge_cache_test.cc index c8b8aee38..172ef4f14 100644 --- a/tcmalloc/huge_cache_test.cc +++ b/tcmalloc/huge_cache_test.cc @@ -675,6 +675,41 @@ TEST_P(HugeCacheTest, ReleaseByDemandNoHistory) { NHugePages(10)); } +// Tests that the demand is capped by peak within the default interval (5 mins). +TEST_P(HugeCacheTest, ReleaseByDemandCappedByDemandPeak) { + if (!GetDemandBasedRelease()) { + GTEST_SKIP(); + } + EXPECT_CALL(mock_unback_, Unback(testing::_, testing::_)) + .WillRepeatedly(Return(true)); + // Generates a demand pattern that can cause the sum-of-peak issue. + bool released; + // The diff peak: 20 hps - 1 hps = 19 hps. + HugeRange diff_a = cache_.Get(NHugePages(1), &released); + HugeRange diff_b = cache_.Get(NHugePages(20), &released); + Release(diff_a); + Release(diff_b); + Advance(absl::Minutes(5)); + // The long-term demand peak: 15 hps. + HugeRange peak = cache_.Get(NHugePages(15), &released); + Advance(absl::Minutes(1)); + Release(peak); + EXPECT_EQ(cache_.size(), NHugePages(21)); + // Releases partial of the cache as the demand is capped by the 5-mins' peak + // (15 hps). + EXPECT_EQ(cache_.ReleaseCachedPagesByDemand( + NHugePages(100), + SkipSubreleaseIntervals{.short_interval = absl::Minutes(10), + .long_interval = absl::Minutes(10)}, + /*hit_limit=*/false), + NHugePages(6)); + // Releases the rest of the cache. + EXPECT_EQ(cache_.ReleaseCachedPagesByDemand(NHugePages(100), + SkipSubreleaseIntervals{}, + /*hit_limit=*/false), + NHugePages(15)); +} + // Tests demand-based skip release. The test is a modified version of the // FillerTest.SkipSubrelease test by removing parts designed particularly for // subrelease. diff --git a/tcmalloc/huge_page_subrelease.h b/tcmalloc/huge_page_subrelease.h index 84d3a85df..63d384635 100644 --- a/tcmalloc/huge_page_subrelease.h +++ b/tcmalloc/huge_page_subrelease.h @@ -336,94 +336,39 @@ class SubreleaseStatsTracker { // decision. Length GetRecentPeak(absl::Duration peak_interval) { last_skip_subrelease_intervals_.peak_interval = - std::min(peak_interval, epoch_length_ * kEpochs); - Length max_demand_pages; - - int64_t num_epochs = - std::min(peak_interval / epoch_length_, kEpochs); - - tracker_.IterBackwards( - [&](size_t offset, int64_t ts, const SubreleaseStatsEntry& e) { - if (!e.empty()) { - // Identify the maximum number of demand pages we have seen within - // the time interval. - if (e.stats[kStatsAtMaxDemand].num_pages > max_demand_pages) { - max_demand_pages = e.stats[kStatsAtMaxDemand].num_pages; - } - } - }, - num_epochs); - - return max_demand_pages; + std::min(peak_interval, window_); + return CalculateDemandPeak(peak_interval); } - // Calculates demand requirements for skip subrelease: HugePageFiller would - // not subrelease if it has less pages than (or equal to) the required - // amount. We report that the skipping is correct if future demand is going to - // be above the required amount within another realized fragemenation - // interval. The demand requirement is the sum of short-term demand - // fluctuation peak and long-term demand trend. The former is the largest max - // and min demand difference within short_interval, and the latter is the - // largest min demand within long_interval. When both set, short_interval - // should be (significantly) shorter or equal to long_interval to avoid - // realized fragmentation caused by non-recent (short-term) demand spikes. + // Calculates demand requirements for the skip subrelease: we do not + // subrelease if the number of free pages are than (or equal to) the demand + // computed by GetRecentDemand. The demand requirement is the sum of + // short-term demand fluctuation peak with in the last and + // long-term demand trend in the previous . When both are set, + // short_interval should be (significantly) shorter or equal to long_interval + // to avoid realized fragmentation caused by non-recent (short-term) demand + // spikes. The demand is capped to the peak observed in the time series. Length GetRecentDemand(absl::Duration short_interval, absl::Duration long_interval) { - if (short_interval != absl::ZeroDuration() && - long_interval != absl::ZeroDuration()) { - short_interval = std::min(short_interval, long_interval); - } - last_skip_subrelease_intervals_.short_interval = - std::min(short_interval, epoch_length_ * kEpochs); - last_skip_subrelease_intervals_.long_interval = - std::min(long_interval, epoch_length_ * kEpochs); - Length short_term_fluctuation_pages, long_term_trend_pages; - int short_epochs = std::min(short_interval / epoch_length_, kEpochs); - int long_epochs = std::min(long_interval / epoch_length_, kEpochs); - - tracker_.IterBackwards( - [&](size_t offset, int64_t ts, const SubreleaseStatsEntry& e) { - if (!e.empty()) { - Length demand_difference = e.stats[kStatsAtMaxDemand].num_pages - - e.stats[kStatsAtMinDemand].num_pages; - // Identifies the highest demand fluctuation (i.e., difference - // between max_demand and min_demand) that we have seen within the - // time interval. - if (demand_difference > short_term_fluctuation_pages) { - short_term_fluctuation_pages = demand_difference; - } - } - }, - short_epochs); - tracker_.IterBackwards( - [&](size_t offset, int64_t ts, const SubreleaseStatsEntry& e) { - if (!e.empty()) { - // Identifies the long-term demand peak (i.e., largest minimum - // demand) that we have seen within the time interval. - if (e.stats[kStatsAtMinDemand].num_pages > long_term_trend_pages) { - long_term_trend_pages = e.stats[kStatsAtMinDemand].num_pages; - } - } - }, - long_epochs); - - // Since we are taking the sum of peaks, we can end up with a demand peak - // that is larger than the largest peak encountered so far, which could - // lead to OOMs. We adjust the peak in that case, by capping it to the - // largest peak observed in our time series. - Length demand_peak = Length(0); - tracker_.IterBackwards( - [&](size_t offset, int64_t ts, const SubreleaseStatsEntry& e) { - if (!e.empty()) { - if (e.stats[kStatsAtMaxDemand].num_pages > demand_peak) { - demand_peak = e.stats[kStatsAtMaxDemand].num_pages; - } - } - }, - -1); + return GetRecentDemand(short_interval, long_interval, window_); + } - return std::min(demand_peak, - short_term_fluctuation_pages + long_term_trend_pages); + // Calculates demand requirements for the skip subrelease: we do not + // subrelease if the number of free pages are than (or equal to) the demand + // computed by GetRecentDemand. The demand requirement is the sum of + // short-term demand fluctuation peak with in the last and + // long-term demand trend in the previous . When both are set, + // short_interval should be (significantly) shorter or equal to long_interval + // to avoid realized fragmentation caused by non-recent (short-term) demand + // spikes. The demand is capped to the peak observed in the time series over + // the last . + Length GetRecentDemand(absl::Duration short_interval, + absl::Duration long_interval, + absl::Duration peak_interval) { + Length demand_trend = + CalculateCombinedDemandTrend(short_interval, long_interval); + Length demand_peak = CalculateDemandPeak(peak_interval); + return std::min(demand_peak, demand_trend); } // Reports a skipped subrelease, which is evaluated by coming peaks within the @@ -554,6 +499,70 @@ class SubreleaseStatsTracker { bool empty() const { return min_free_pages == kDefaultValue; } }; + // Gets the peak demand recorded in the time series over the last + // . + Length CalculateDemandPeak(absl::Duration peak_interval) { + Length max_demand_pages; + int64_t num_epochs = + std::min(peak_interval / epoch_length_, kEpochs); + tracker_.IterBackwards( + [&](size_t offset, int64_t ts, const SubreleaseStatsEntry& e) { + if (!e.empty()) { + // Identify the maximum number of demand pages we have seen within + // the time interval. + if (e.stats[kStatsAtMaxDemand].num_pages > max_demand_pages) { + max_demand_pages = e.stats[kStatsAtMaxDemand].num_pages; + } + } + }, + num_epochs); + return max_demand_pages; + } + + // Gets the combined demand trend, which is the sum of the maximum demand + // difference in and the maxmin demand in . + Length CalculateCombinedDemandTrend(absl::Duration short_interval, + absl::Duration long_interval) { + if (short_interval != absl::ZeroDuration() && + long_interval != absl::ZeroDuration()) { + short_interval = std::min(short_interval, long_interval); + } + last_skip_subrelease_intervals_.short_interval = + std::min(short_interval, window_); + last_skip_subrelease_intervals_.long_interval = + std::min(long_interval, window_); + Length short_term_fluctuation_pages, long_term_trend_pages; + int short_epochs = std::min(short_interval / epoch_length_, kEpochs); + int long_epochs = std::min(long_interval / epoch_length_, kEpochs); + + tracker_.IterBackwards( + [&](size_t offset, int64_t ts, const SubreleaseStatsEntry& e) { + if (!e.empty()) { + Length demand_difference = e.stats[kStatsAtMaxDemand].num_pages - + e.stats[kStatsAtMinDemand].num_pages; + // Identifies the highest demand fluctuation (i.e., difference + // between max_demand and min_demand) that we have seen within the + // time interval. + if (demand_difference > short_term_fluctuation_pages) { + short_term_fluctuation_pages = demand_difference; + } + } + }, + short_epochs); + tracker_.IterBackwards( + [&](size_t offset, int64_t ts, const SubreleaseStatsEntry& e) { + if (!e.empty()) { + // Identifies the long-term demand peak (i.e., largest minimum + // demand) that we have seen within the time interval. + if (e.stats[kStatsAtMinDemand].num_pages > long_term_trend_pages) { + long_term_trend_pages = e.stats[kStatsAtMinDemand].num_pages; + } + } + }, + long_epochs); + return short_term_fluctuation_pages + long_term_trend_pages; + } + // The tracker reports pages that have been free for at least this interval, // as well as peaks within this interval. The interval is also used for // deciding correctness of skipped subreleases by associating past skipping diff --git a/tcmalloc/huge_page_subrelease_test.cc b/tcmalloc/huge_page_subrelease_test.cc index 8e4aa2ce2..04d0f5e5b 100644 --- a/tcmalloc/huge_page_subrelease_test.cc +++ b/tcmalloc/huge_page_subrelease_test.cc @@ -256,6 +256,27 @@ TEST_F(StatsTrackerTest, ComputeRecentDemand) { tracker_.GetRecentDemand(absl::Minutes(1), absl::Minutes(1))); } +TEST_F(StatsTrackerTest, ComputeRecentDemandAndCappedToPeak) { + // Generates max and min demand in each epoch to create short-term demand + // fluctuations. + GenerateDemandPoint(Length(50), Length(2000)); + GenerateDemandPoint(Length(3000), Length(1000)); + Advance(absl::Minutes(2)); + GenerateDemandPoint(Length(1500), Length(0)); + Advance(absl::Minutes(1)); + GenerateDemandPoint(Length(50), Length(1000)); + GenerateDemandPoint(Length(100), Length(2000)); + // The calculated demand is 2500 (maximum demand diff) + 1500 (max + // min_demand), but capped by the peak observed in the time series. + Length demand_1 = + tracker_.GetRecentDemand(absl::Minutes(5), absl::Minutes(5)); + EXPECT_EQ(demand_1, Length(3000)); + // Capped by the peak observed in 2 mins. + Length demand_2 = tracker_.GetRecentDemand(absl::Minutes(5), absl::Minutes(5), + absl::Minutes(2)); + EXPECT_EQ(demand_2, Length(1500)); +} + TEST_F(StatsTrackerTest, TrackCorrectSubreleaseDecisions) { // First peak (large) GenerateDemandPoint(Length(1000), Length(1000));