|
|
@@ -3386,6 +3386,77 @@ static inline bool is_thp_gfp_mask(gfp_t gfp_mask)
|
|
|
return (gfp_mask & (GFP_TRANSHUGE | __GFP_KSWAPD_RECLAIM)) == GFP_TRANSHUGE;
|
|
|
}
|
|
|
|
|
|
+/*
|
|
|
+ * Maximum number of reclaim retries without any progress before OOM killer
|
|
|
+ * is consider as the only way to move forward.
|
|
|
+ */
|
|
|
+#define MAX_RECLAIM_RETRIES 16
|
|
|
+
|
|
|
+/*
|
|
|
+ * Checks whether it makes sense to retry the reclaim to make a forward progress
|
|
|
+ * for the given allocation request.
|
|
|
+ * The reclaim feedback represented by did_some_progress (any progress during
|
|
|
+ * the last reclaim round), pages_reclaimed (cumulative number of reclaimed
|
|
|
+ * pages) and no_progress_loops (number of reclaim rounds without any progress
|
|
|
+ * in a row) is considered as well as the reclaimable pages on the applicable
|
|
|
+ * zone list (with a backoff mechanism which is a function of no_progress_loops).
|
|
|
+ *
|
|
|
+ * Returns true if a retry is viable or false to enter the oom path.
|
|
|
+ */
|
|
|
+static inline bool
|
|
|
+should_reclaim_retry(gfp_t gfp_mask, unsigned order,
|
|
|
+ struct alloc_context *ac, int alloc_flags,
|
|
|
+ bool did_some_progress, unsigned long pages_reclaimed,
|
|
|
+ int no_progress_loops)
|
|
|
+{
|
|
|
+ struct zone *zone;
|
|
|
+ struct zoneref *z;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Make sure we converge to OOM if we cannot make any progress
|
|
|
+ * several times in the row.
|
|
|
+ */
|
|
|
+ if (no_progress_loops > MAX_RECLAIM_RETRIES)
|
|
|
+ return false;
|
|
|
+
|
|
|
+ if (order > PAGE_ALLOC_COSTLY_ORDER) {
|
|
|
+ if (pages_reclaimed >= (1<<order))
|
|
|
+ return false;
|
|
|
+
|
|
|
+ if (did_some_progress)
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Keep reclaiming pages while there is a chance this will lead somewhere.
|
|
|
+ * If none of the target zones can satisfy our allocation request even
|
|
|
+ * if all reclaimable pages are considered then we are screwed and have
|
|
|
+ * to go OOM.
|
|
|
+ */
|
|
|
+ for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
|
|
|
+ ac->nodemask) {
|
|
|
+ unsigned long available;
|
|
|
+
|
|
|
+ available = zone_reclaimable_pages(zone);
|
|
|
+ available -= DIV_ROUND_UP(no_progress_loops * available,
|
|
|
+ MAX_RECLAIM_RETRIES);
|
|
|
+ available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Would the allocation succeed if we reclaimed the whole
|
|
|
+ * available?
|
|
|
+ */
|
|
|
+ if (__zone_watermark_ok(zone, order, min_wmark_pages(zone),
|
|
|
+ ac->high_zoneidx, alloc_flags, available)) {
|
|
|
+ /* Wait for some write requests to complete then retry */
|
|
|
+ wait_iff_congested(zone, BLK_RW_ASYNC, HZ/50);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return false;
|
|
|
+}
|
|
|
+
|
|
|
static inline struct page *
|
|
|
__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
|
|
|
struct alloc_context *ac)
|
|
|
@@ -3397,6 +3468,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
|
|
|
unsigned long did_some_progress;
|
|
|
enum migrate_mode migration_mode = MIGRATE_ASYNC;
|
|
|
enum compact_result compact_result;
|
|
|
+ int no_progress_loops = 0;
|
|
|
|
|
|
/*
|
|
|
* In the slowpath, we sanity check order to avoid ever trying to
|
|
|
@@ -3525,23 +3597,35 @@ retry:
|
|
|
if (gfp_mask & __GFP_NORETRY)
|
|
|
goto noretry;
|
|
|
|
|
|
- /* Keep reclaiming pages as long as there is reasonable progress */
|
|
|
- pages_reclaimed += did_some_progress;
|
|
|
- if ((did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) ||
|
|
|
- ((gfp_mask & __GFP_REPEAT) && pages_reclaimed < (1 << order))) {
|
|
|
- /* Wait for some write requests to complete then retry */
|
|
|
- wait_iff_congested(ac->preferred_zoneref->zone, BLK_RW_ASYNC, HZ/50);
|
|
|
- goto retry;
|
|
|
+ /*
|
|
|
+ * Do not retry costly high order allocations unless they are
|
|
|
+ * __GFP_REPEAT
|
|
|
+ */
|
|
|
+ if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT))
|
|
|
+ goto noretry;
|
|
|
+
|
|
|
+ if (did_some_progress) {
|
|
|
+ no_progress_loops = 0;
|
|
|
+ pages_reclaimed += did_some_progress;
|
|
|
+ } else {
|
|
|
+ no_progress_loops++;
|
|
|
}
|
|
|
|
|
|
+ if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
|
|
|
+ did_some_progress > 0, pages_reclaimed,
|
|
|
+ no_progress_loops))
|
|
|
+ goto retry;
|
|
|
+
|
|
|
/* Reclaim has failed us, start killing things */
|
|
|
page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
|
|
|
if (page)
|
|
|
goto got_pg;
|
|
|
|
|
|
/* Retry as long as the OOM killer is making progress */
|
|
|
- if (did_some_progress)
|
|
|
+ if (did_some_progress) {
|
|
|
+ no_progress_loops = 0;
|
|
|
goto retry;
|
|
|
+ }
|
|
|
|
|
|
noretry:
|
|
|
/*
|
Michal Hocko