|
@@ -42,6 +42,11 @@ static inline void count_compact_events(enum vm_event_item item, long delta)
|
|
|
#define CREATE_TRACE_POINTS
|
|
|
#include <trace/events/compaction.h>
|
|
|
|
|
|
+#define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order))
|
|
|
+#define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order))
|
|
|
+#define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order)
|
|
|
+#define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order)
|
|
|
+
|
|
|
static unsigned long release_freepages(struct list_head *freelist)
|
|
|
{
|
|
|
struct page *page, *next;
|
|
@@ -161,7 +166,7 @@ static void reset_cached_positions(struct zone *zone)
|
|
|
zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
|
|
|
zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
|
|
|
zone->compact_cached_free_pfn =
|
|
|
- round_down(zone_end_pfn(zone) - 1, pageblock_nr_pages);
|
|
|
+ pageblock_start_pfn(zone_end_pfn(zone) - 1);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -519,10 +524,10 @@ isolate_freepages_range(struct compact_control *cc,
|
|
|
LIST_HEAD(freelist);
|
|
|
|
|
|
pfn = start_pfn;
|
|
|
- block_start_pfn = pfn & ~(pageblock_nr_pages - 1);
|
|
|
+ block_start_pfn = pageblock_start_pfn(pfn);
|
|
|
if (block_start_pfn < cc->zone->zone_start_pfn)
|
|
|
block_start_pfn = cc->zone->zone_start_pfn;
|
|
|
- block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
|
|
|
+ block_end_pfn = pageblock_end_pfn(pfn);
|
|
|
|
|
|
for (; pfn < end_pfn; pfn += isolated,
|
|
|
block_start_pfn = block_end_pfn,
|
|
@@ -538,8 +543,8 @@ isolate_freepages_range(struct compact_control *cc,
|
|
|
* scanning range to right one.
|
|
|
*/
|
|
|
if (pfn >= block_end_pfn) {
|
|
|
- block_start_pfn = pfn & ~(pageblock_nr_pages - 1);
|
|
|
- block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
|
|
|
+ block_start_pfn = pageblock_start_pfn(pfn);
|
|
|
+ block_end_pfn = pageblock_end_pfn(pfn);
|
|
|
block_end_pfn = min(block_end_pfn, end_pfn);
|
|
|
}
|
|
|
|
|
@@ -633,12 +638,13 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
|
|
{
|
|
|
struct zone *zone = cc->zone;
|
|
|
unsigned long nr_scanned = 0, nr_isolated = 0;
|
|
|
- struct list_head *migratelist = &cc->migratepages;
|
|
|
struct lruvec *lruvec;
|
|
|
unsigned long flags = 0;
|
|
|
bool locked = false;
|
|
|
struct page *page = NULL, *valid_page = NULL;
|
|
|
unsigned long start_pfn = low_pfn;
|
|
|
+ bool skip_on_failure = false;
|
|
|
+ unsigned long next_skip_pfn = 0;
|
|
|
|
|
|
/*
|
|
|
* Ensure that there are not too many pages isolated from the LRU
|
|
@@ -659,10 +665,37 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
|
|
if (compact_should_abort(cc))
|
|
|
return 0;
|
|
|
|
|
|
+ if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
|
|
|
+ skip_on_failure = true;
|
|
|
+ next_skip_pfn = block_end_pfn(low_pfn, cc->order);
|
|
|
+ }
|
|
|
+
|
|
|
/* Time to isolate some pages for migration */
|
|
|
for (; low_pfn < end_pfn; low_pfn++) {
|
|
|
bool is_lru;
|
|
|
|
|
|
+ if (skip_on_failure && low_pfn >= next_skip_pfn) {
|
|
|
+ /*
|
|
|
+ * We have isolated all migration candidates in the
|
|
|
+ * previous order-aligned block, and did not skip it due
|
|
|
+ * to failure. We should migrate the pages now and
|
|
|
+ * hopefully succeed compaction.
|
|
|
+ */
|
|
|
+ if (nr_isolated)
|
|
|
+ break;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * We failed to isolate in the previous order-aligned
|
|
|
+ * block. Set the new boundary to the end of the
|
|
|
+ * current block. Note we can't simply increase
|
|
|
+ * next_skip_pfn by 1 << order, as low_pfn might have
|
|
|
+ * been incremented by a higher number due to skipping
|
|
|
+ * a compound or a high-order buddy page in the
|
|
|
+ * previous loop iteration.
|
|
|
+ */
|
|
|
+ next_skip_pfn = block_end_pfn(low_pfn, cc->order);
|
|
|
+ }
|
|
|
+
|
|
|
/*
|
|
|
* Periodically drop the lock (if held) regardless of its
|
|
|
* contention, to give chance to IRQs. Abort async compaction
|
|
@@ -674,7 +707,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
|
|
break;
|
|
|
|
|
|
if (!pfn_valid_within(low_pfn))
|
|
|
- continue;
|
|
|
+ goto isolate_fail;
|
|
|
nr_scanned++;
|
|
|
|
|
|
page = pfn_to_page(low_pfn);
|
|
@@ -729,11 +762,11 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
|
|
if (likely(comp_order < MAX_ORDER))
|
|
|
low_pfn += (1UL << comp_order) - 1;
|
|
|
|
|
|
- continue;
|
|
|
+ goto isolate_fail;
|
|
|
}
|
|
|
|
|
|
if (!is_lru)
|
|
|
- continue;
|
|
|
+ goto isolate_fail;
|
|
|
|
|
|
/*
|
|
|
* Migration will fail if an anonymous page is pinned in memory,
|
|
@@ -742,7 +775,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
|
|
*/
|
|
|
if (!page_mapping(page) &&
|
|
|
page_count(page) > page_mapcount(page))
|
|
|
- continue;
|
|
|
+ goto isolate_fail;
|
|
|
|
|
|
/* If we already hold the lock, we can skip some rechecking */
|
|
|
if (!locked) {
|
|
@@ -753,7 +786,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
|
|
|
|
|
/* Recheck PageLRU and PageCompound under lock */
|
|
|
if (!PageLRU(page))
|
|
|
- continue;
|
|
|
+ goto isolate_fail;
|
|
|
|
|
|
/*
|
|
|
* Page become compound since the non-locked check,
|
|
@@ -762,7 +795,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
|
|
*/
|
|
|
if (unlikely(PageCompound(page))) {
|
|
|
low_pfn += (1UL << compound_order(page)) - 1;
|
|
|
- continue;
|
|
|
+ goto isolate_fail;
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -770,7 +803,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
|
|
|
|
|
/* Try isolate the page */
|
|
|
if (__isolate_lru_page(page, isolate_mode) != 0)
|
|
|
- continue;
|
|
|
+ goto isolate_fail;
|
|
|
|
|
|
VM_BUG_ON_PAGE(PageCompound(page), page);
|
|
|
|
|
@@ -778,15 +811,55 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
|
|
del_page_from_lru_list(page, lruvec, page_lru(page));
|
|
|
|
|
|
isolate_success:
|
|
|
- list_add(&page->lru, migratelist);
|
|
|
+ list_add(&page->lru, &cc->migratepages);
|
|
|
cc->nr_migratepages++;
|
|
|
nr_isolated++;
|
|
|
|
|
|
+ /*
|
|
|
+ * Record where we could have freed pages by migration and not
|
|
|
+ * yet flushed them to buddy allocator.
|
|
|
+ * - this is the lowest page that was isolated and likely be
|
|
|
+ * then freed by migration.
|
|
|
+ */
|
|
|
+ if (!cc->last_migrated_pfn)
|
|
|
+ cc->last_migrated_pfn = low_pfn;
|
|
|
+
|
|
|
/* Avoid isolating too much */
|
|
|
if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) {
|
|
|
++low_pfn;
|
|
|
break;
|
|
|
}
|
|
|
+
|
|
|
+ continue;
|
|
|
+isolate_fail:
|
|
|
+ if (!skip_on_failure)
|
|
|
+ continue;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * We have isolated some pages, but then failed. Release them
|
|
|
+ * instead of migrating, as we cannot form the cc->order buddy
|
|
|
+ * page anyway.
|
|
|
+ */
|
|
|
+ if (nr_isolated) {
|
|
|
+ if (locked) {
|
|
|
+ spin_unlock_irqrestore(&zone->lru_lock, flags);
|
|
|
+ locked = false;
|
|
|
+ }
|
|
|
+ acct_isolated(zone, cc);
|
|
|
+ putback_movable_pages(&cc->migratepages);
|
|
|
+ cc->nr_migratepages = 0;
|
|
|
+ cc->last_migrated_pfn = 0;
|
|
|
+ nr_isolated = 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (low_pfn < next_skip_pfn) {
|
|
|
+ low_pfn = next_skip_pfn - 1;
|
|
|
+ /*
|
|
|
+ * The check near the loop beginning would have updated
|
|
|
+ * next_skip_pfn too, but this is a bit simpler.
|
|
|
+ */
|
|
|
+ next_skip_pfn += 1UL << cc->order;
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -834,10 +907,10 @@ isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
|
|
|
|
|
|
/* Scan block by block. First and last block may be incomplete */
|
|
|
pfn = start_pfn;
|
|
|
- block_start_pfn = pfn & ~(pageblock_nr_pages - 1);
|
|
|
+ block_start_pfn = pageblock_start_pfn(pfn);
|
|
|
if (block_start_pfn < cc->zone->zone_start_pfn)
|
|
|
block_start_pfn = cc->zone->zone_start_pfn;
|
|
|
- block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
|
|
|
+ block_end_pfn = pageblock_end_pfn(pfn);
|
|
|
|
|
|
for (; pfn < end_pfn; pfn = block_end_pfn,
|
|
|
block_start_pfn = block_end_pfn,
|
|
@@ -924,10 +997,10 @@ static void isolate_freepages(struct compact_control *cc)
|
|
|
* is using.
|
|
|
*/
|
|
|
isolate_start_pfn = cc->free_pfn;
|
|
|
- block_start_pfn = cc->free_pfn & ~(pageblock_nr_pages-1);
|
|
|
+ block_start_pfn = pageblock_start_pfn(cc->free_pfn);
|
|
|
block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
|
|
|
zone_end_pfn(zone));
|
|
|
- low_pfn = ALIGN(cc->migrate_pfn + 1, pageblock_nr_pages);
|
|
|
+ low_pfn = pageblock_end_pfn(cc->migrate_pfn);
|
|
|
|
|
|
/*
|
|
|
* Isolate free pages until enough are available to migrate the
|
|
@@ -1070,7 +1143,6 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
|
|
|
unsigned long block_start_pfn;
|
|
|
unsigned long block_end_pfn;
|
|
|
unsigned long low_pfn;
|
|
|
- unsigned long isolate_start_pfn;
|
|
|
struct page *page;
|
|
|
const isolate_mode_t isolate_mode =
|
|
|
(sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
|
|
@@ -1081,12 +1153,12 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
|
|
|
* initialized by compact_zone()
|
|
|
*/
|
|
|
low_pfn = cc->migrate_pfn;
|
|
|
- block_start_pfn = cc->migrate_pfn & ~(pageblock_nr_pages - 1);
|
|
|
+ block_start_pfn = pageblock_start_pfn(low_pfn);
|
|
|
if (block_start_pfn < zone->zone_start_pfn)
|
|
|
block_start_pfn = zone->zone_start_pfn;
|
|
|
|
|
|
/* Only scan within a pageblock boundary */
|
|
|
- block_end_pfn = ALIGN(low_pfn + 1, pageblock_nr_pages);
|
|
|
+ block_end_pfn = pageblock_end_pfn(low_pfn);
|
|
|
|
|
|
/*
|
|
|
* Iterate over whole pageblocks until we find the first suitable.
|
|
@@ -1125,7 +1197,6 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
|
|
|
continue;
|
|
|
|
|
|
/* Perform the isolation */
|
|
|
- isolate_start_pfn = low_pfn;
|
|
|
low_pfn = isolate_migratepages_block(cc, low_pfn,
|
|
|
block_end_pfn, isolate_mode);
|
|
|
|
|
@@ -1134,15 +1205,6 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
|
|
|
return ISOLATE_ABORT;
|
|
|
}
|
|
|
|
|
|
- /*
|
|
|
- * Record where we could have freed pages by migration and not
|
|
|
- * yet flushed them to buddy allocator.
|
|
|
- * - this is the lowest page that could have been isolated and
|
|
|
- * then freed by migration.
|
|
|
- */
|
|
|
- if (cc->nr_migratepages && !cc->last_migrated_pfn)
|
|
|
- cc->last_migrated_pfn = isolate_start_pfn;
|
|
|
-
|
|
|
/*
|
|
|
* Either we isolated something and proceed with migration. Or
|
|
|
* we failed and compact_zone should decide if we should
|
|
@@ -1251,7 +1313,8 @@ static int compact_finished(struct zone *zone, struct compact_control *cc,
|
|
|
* COMPACT_CONTINUE - If compaction should run now
|
|
|
*/
|
|
|
static unsigned long __compaction_suitable(struct zone *zone, int order,
|
|
|
- int alloc_flags, int classzone_idx)
|
|
|
+ unsigned int alloc_flags,
|
|
|
+ int classzone_idx)
|
|
|
{
|
|
|
int fragindex;
|
|
|
unsigned long watermark;
|
|
@@ -1296,7 +1359,8 @@ static unsigned long __compaction_suitable(struct zone *zone, int order,
|
|
|
}
|
|
|
|
|
|
unsigned long compaction_suitable(struct zone *zone, int order,
|
|
|
- int alloc_flags, int classzone_idx)
|
|
|
+ unsigned int alloc_flags,
|
|
|
+ int classzone_idx)
|
|
|
{
|
|
|
unsigned long ret;
|
|
|
|
|
@@ -1343,7 +1407,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
|
|
|
cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync];
|
|
|
cc->free_pfn = zone->compact_cached_free_pfn;
|
|
|
if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
|
|
|
- cc->free_pfn = round_down(end_pfn - 1, pageblock_nr_pages);
|
|
|
+ cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
|
|
|
zone->compact_cached_free_pfn = cc->free_pfn;
|
|
|
}
|
|
|
if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
|
|
@@ -1398,6 +1462,18 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
|
|
|
ret = COMPACT_CONTENDED;
|
|
|
goto out;
|
|
|
}
|
|
|
+ /*
|
|
|
+ * We failed to migrate at least one page in the current
|
|
|
+ * order-aligned block, so skip the rest of it.
|
|
|
+ */
|
|
|
+ if (cc->direct_compaction &&
|
|
|
+ (cc->mode == MIGRATE_ASYNC)) {
|
|
|
+ cc->migrate_pfn = block_end_pfn(
|
|
|
+ cc->migrate_pfn - 1, cc->order);
|
|
|
+ /* Draining pcplists is useless in this case */
|
|
|
+ cc->last_migrated_pfn = 0;
|
|
|
+
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
check_drain:
|
|
@@ -1411,7 +1487,7 @@ check_drain:
|
|
|
if (cc->order > 0 && cc->last_migrated_pfn) {
|
|
|
int cpu;
|
|
|
unsigned long current_block_start =
|
|
|
- cc->migrate_pfn & ~((1UL << cc->order) - 1);
|
|
|
+ block_start_pfn(cc->migrate_pfn, cc->order);
|
|
|
|
|
|
if (cc->last_migrated_pfn < current_block_start) {
|
|
|
cpu = get_cpu();
|
|
@@ -1436,7 +1512,7 @@ out:
|
|
|
cc->nr_freepages = 0;
|
|
|
VM_BUG_ON(free_pfn == 0);
|
|
|
/* The cached pfn is always the first in a pageblock */
|
|
|
- free_pfn &= ~(pageblock_nr_pages-1);
|
|
|
+ free_pfn = pageblock_start_pfn(free_pfn);
|
|
|
/*
|
|
|
* Only go back, not forward. The cached pfn might have been
|
|
|
* already reset to zone end in compact_finished()
|
|
@@ -1456,7 +1532,7 @@ out:
|
|
|
|
|
|
static unsigned long compact_zone_order(struct zone *zone, int order,
|
|
|
gfp_t gfp_mask, enum migrate_mode mode, int *contended,
|
|
|
- int alloc_flags, int classzone_idx)
|
|
|
+ unsigned int alloc_flags, int classzone_idx)
|
|
|
{
|
|
|
unsigned long ret;
|
|
|
struct compact_control cc = {
|
|
@@ -1497,8 +1573,8 @@ int sysctl_extfrag_threshold = 500;
|
|
|
* This is the main entry point for direct page compaction.
|
|
|
*/
|
|
|
unsigned long try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
|
|
|
- int alloc_flags, const struct alloc_context *ac,
|
|
|
- enum migrate_mode mode, int *contended)
|
|
|
+ unsigned int alloc_flags, const struct alloc_context *ac,
|
|
|
+ enum migrate_mode mode, int *contended)
|
|
|
{
|
|
|
int may_enter_fs = gfp_mask & __GFP_FS;
|
|
|
int may_perform_io = gfp_mask & __GFP_IO;
|
|
@@ -1526,7 +1602,7 @@ unsigned long try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
|
|
|
|
|
|
status = compact_zone_order(zone, order, gfp_mask, mode,
|
|
|
&zone_contended, alloc_flags,
|
|
|
- ac->classzone_idx);
|
|
|
+ ac_classzone_idx(ac));
|
|
|
rc = max(status, rc);
|
|
|
/*
|
|
|
* It takes at least one zone that wasn't lock contended
|
|
@@ -1536,7 +1612,7 @@ unsigned long try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
|
|
|
|
|
|
/* If a normal allocation would succeed, stop compacting */
|
|
|
if (zone_watermark_ok(zone, order, low_wmark_pages(zone),
|
|
|
- ac->classzone_idx, alloc_flags)) {
|
|
|
+ ac_classzone_idx(ac), alloc_flags)) {
|
|
|
/*
|
|
|
* We think the allocation will succeed in this zone,
|
|
|
* but it is not certain, hence the false. The caller
|