|
@@ -125,6 +125,24 @@ unsigned long dirty_balance_reserve __read_mostly;
|
|
|
int percpu_pagelist_fraction;
|
|
|
gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
|
|
|
|
|
|
+/*
|
|
|
+ * A cached value of the page's pageblock's migratetype, used when the page is
|
|
|
+ * put on a pcplist. Used to avoid the pageblock migratetype lookup when
|
|
|
+ * freeing from pcplists in most cases, at the cost of possibly becoming stale.
|
|
|
+ * Also the migratetype set in the page does not necessarily match the pcplist
|
|
|
+ * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
|
|
|
+ * other index - this ensures that it will be put on the correct CMA freelist.
|
|
|
+ */
|
|
|
+static inline int get_pcppage_migratetype(struct page *page)
|
|
|
+{
|
|
|
+ return page->index;
|
|
|
+}
|
|
|
+
|
|
|
+static inline void set_pcppage_migratetype(struct page *page, int migratetype)
|
|
|
+{
|
|
|
+ page->index = migratetype;
|
|
|
+}
|
|
|
+
|
|
|
#ifdef CONFIG_PM_SLEEP
|
|
|
/*
|
|
|
* The following functions are used by the suspend/hibernate code to temporarily
|
|
@@ -789,7 +807,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
|
|
|
/* must delete as __free_one_page list manipulates */
|
|
|
list_del(&page->lru);
|
|
|
|
|
|
- mt = get_freepage_migratetype(page);
|
|
|
+ mt = get_pcppage_migratetype(page);
|
|
|
/* MIGRATE_ISOLATE page should not go to pcplists */
|
|
|
VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
|
|
|
/* Pageblock could have been isolated meanwhile */
|
|
@@ -956,7 +974,6 @@ static void __free_pages_ok(struct page *page, unsigned int order)
|
|
|
migratetype = get_pfnblock_migratetype(page, pfn);
|
|
|
local_irq_save(flags);
|
|
|
__count_vm_events(PGFREE, 1 << order);
|
|
|
- set_freepage_migratetype(page, migratetype);
|
|
|
free_one_page(page_zone(page), page, pfn, order, migratetype);
|
|
|
local_irq_restore(flags);
|
|
|
}
|
|
@@ -1384,7 +1401,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
|
|
|
rmv_page_order(page);
|
|
|
area->nr_free--;
|
|
|
expand(zone, page, order, current_order, area, migratetype);
|
|
|
- set_freepage_migratetype(page, migratetype);
|
|
|
+ set_pcppage_migratetype(page, migratetype);
|
|
|
return page;
|
|
|
}
|
|
|
|
|
@@ -1461,7 +1478,6 @@ int move_freepages(struct zone *zone,
|
|
|
order = page_order(page);
|
|
|
list_move(&page->lru,
|
|
|
&zone->free_area[order].free_list[migratetype]);
|
|
|
- set_freepage_migratetype(page, migratetype);
|
|
|
page += 1 << order;
|
|
|
pages_moved += 1 << order;
|
|
|
}
|
|
@@ -1631,14 +1647,13 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
|
|
|
expand(zone, page, order, current_order, area,
|
|
|
start_migratetype);
|
|
|
/*
|
|
|
- * The freepage_migratetype may differ from pageblock's
|
|
|
+ * The pcppage_migratetype may differ from pageblock's
|
|
|
* migratetype depending on the decisions in
|
|
|
- * try_to_steal_freepages(). This is OK as long as it
|
|
|
- * does not differ for MIGRATE_CMA pageblocks. For CMA
|
|
|
- * we need to make sure unallocated pages flushed from
|
|
|
- * pcp lists are returned to the correct freelist.
|
|
|
+ * find_suitable_fallback(). This is OK as long as it does not
|
|
|
+ * differ for MIGRATE_CMA pageblocks. Those can be used as
|
|
|
+ * fallback only via special __rmqueue_cma_fallback() function
|
|
|
*/
|
|
|
- set_freepage_migratetype(page, start_migratetype);
|
|
|
+ set_pcppage_migratetype(page, start_migratetype);
|
|
|
|
|
|
trace_mm_page_alloc_extfrag(page, order, current_order,
|
|
|
start_migratetype, fallback_mt);
|
|
@@ -1714,7 +1729,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
|
|
|
else
|
|
|
list_add_tail(&page->lru, list);
|
|
|
list = &page->lru;
|
|
|
- if (is_migrate_cma(get_freepage_migratetype(page)))
|
|
|
+ if (is_migrate_cma(get_pcppage_migratetype(page)))
|
|
|
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
|
|
|
-(1 << order));
|
|
|
}
|
|
@@ -1911,7 +1926,7 @@ void free_hot_cold_page(struct page *page, bool cold)
|
|
|
return;
|
|
|
|
|
|
migratetype = get_pfnblock_migratetype(page, pfn);
|
|
|
- set_freepage_migratetype(page, migratetype);
|
|
|
+ set_pcppage_migratetype(page, migratetype);
|
|
|
local_irq_save(flags);
|
|
|
__count_vm_event(PGFREE);
|
|
|
|
|
@@ -2116,7 +2131,7 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
|
|
|
if (!page)
|
|
|
goto failed;
|
|
|
__mod_zone_freepage_state(zone, -(1 << order),
|
|
|
- get_freepage_migratetype(page));
|
|
|
+ get_pcppage_migratetype(page));
|
|
|
}
|
|
|
|
|
|
__mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
|