|
@@ -466,29 +466,6 @@ static inline void rmv_page_order(struct page *page)
|
|
|
set_page_private(page, 0);
|
|
|
}
|
|
|
|
|
|
-/*
|
|
|
- * Locate the struct page for both the matching buddy in our
|
|
|
- * pair (buddy1) and the combined O(n+1) page they form (page).
|
|
|
- *
|
|
|
- * 1) Any buddy B1 will have an order O twin B2 which satisfies
|
|
|
- * the following equation:
|
|
|
- * B2 = B1 ^ (1 << O)
|
|
|
- * For example, if the starting buddy (buddy2) is #8 its order
|
|
|
- * 1 buddy is #10:
|
|
|
- * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
|
|
|
- *
|
|
|
- * 2) Any buddy B will have an order O+1 parent P which
|
|
|
- * satisfies the following equation:
|
|
|
- * P = B & ~(1 << O)
|
|
|
- *
|
|
|
- * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
|
|
|
- */
|
|
|
-static inline unsigned long
|
|
|
-__find_buddy_index(unsigned long page_idx, unsigned int order)
|
|
|
-{
|
|
|
- return page_idx ^ (1 << order);
|
|
|
-}
|
|
|
-
|
|
|
/*
|
|
|
* This function checks whether a page is free && is the buddy
|
|
|
* we can do coalesce a page and its buddy if
|
|
@@ -569,6 +546,7 @@ static inline void __free_one_page(struct page *page,
|
|
|
unsigned long combined_idx;
|
|
|
unsigned long uninitialized_var(buddy_idx);
|
|
|
struct page *buddy;
|
|
|
+ int max_order = MAX_ORDER;
|
|
|
|
|
|
VM_BUG_ON(!zone_is_initialized(zone));
|
|
|
|
|
@@ -577,13 +555,24 @@ static inline void __free_one_page(struct page *page,
|
|
|
return;
|
|
|
|
|
|
VM_BUG_ON(migratetype == -1);
|
|
|
+ if (is_migrate_isolate(migratetype)) {
|
|
|
+ /*
|
|
|
+ * We restrict max order of merging to prevent merge
|
|
|
+ * between freepages on isolate pageblock and normal
|
|
|
+ * pageblock. Without this, pageblock isolation
|
|
|
+ * could cause incorrect freepage accounting.
|
|
|
+ */
|
|
|
+ max_order = min(MAX_ORDER, pageblock_order + 1);
|
|
|
+ } else {
|
|
|
+ __mod_zone_freepage_state(zone, 1 << order, migratetype);
|
|
|
+ }
|
|
|
|
|
|
- page_idx = pfn & ((1 << MAX_ORDER) - 1);
|
|
|
+ page_idx = pfn & ((1 << max_order) - 1);
|
|
|
|
|
|
VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
|
|
|
VM_BUG_ON_PAGE(bad_range(zone, page), page);
|
|
|
|
|
|
- while (order < MAX_ORDER-1) {
|
|
|
+ while (order < max_order - 1) {
|
|
|
buddy_idx = __find_buddy_index(page_idx, order);
|
|
|
buddy = page + (buddy_idx - page_idx);
|
|
|
if (!page_is_buddy(page, buddy, order))
|
|
@@ -594,9 +583,11 @@ static inline void __free_one_page(struct page *page,
|
|
|
*/
|
|
|
if (page_is_guard(buddy)) {
|
|
|
clear_page_guard_flag(buddy);
|
|
|
- set_page_private(page, 0);
|
|
|
- __mod_zone_freepage_state(zone, 1 << order,
|
|
|
- migratetype);
|
|
|
+ set_page_private(buddy, 0);
|
|
|
+ if (!is_migrate_isolate(migratetype)) {
|
|
|
+ __mod_zone_freepage_state(zone, 1 << order,
|
|
|
+ migratetype);
|
|
|
+ }
|
|
|
} else {
|
|
|
list_del(&buddy->lru);
|
|
|
zone->free_area[order].nr_free--;
|
|
@@ -715,14 +706,12 @@ static void free_pcppages_bulk(struct zone *zone, int count,
|
|
|
/* must delete as __free_one_page list manipulates */
|
|
|
list_del(&page->lru);
|
|
|
mt = get_freepage_migratetype(page);
|
|
|
+ if (unlikely(has_isolate_pageblock(zone)))
|
|
|
+ mt = get_pageblock_migratetype(page);
|
|
|
+
|
|
|
/* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
|
|
|
__free_one_page(page, page_to_pfn(page), zone, 0, mt);
|
|
|
trace_mm_page_pcpu_drain(page, 0, mt);
|
|
|
- if (likely(!is_migrate_isolate_page(page))) {
|
|
|
- __mod_zone_page_state(zone, NR_FREE_PAGES, 1);
|
|
|
- if (is_migrate_cma(mt))
|
|
|
- __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1);
|
|
|
- }
|
|
|
} while (--to_free && --batch_free && !list_empty(list));
|
|
|
}
|
|
|
spin_unlock(&zone->lock);
|
|
@@ -739,9 +728,11 @@ static void free_one_page(struct zone *zone,
|
|
|
if (nr_scanned)
|
|
|
__mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
|
|
|
|
|
|
+ if (unlikely(has_isolate_pageblock(zone) ||
|
|
|
+ is_migrate_isolate(migratetype))) {
|
|
|
+ migratetype = get_pfnblock_migratetype(page, pfn);
|
|
|
+ }
|
|
|
__free_one_page(page, pfn, zone, order, migratetype);
|
|
|
- if (unlikely(!is_migrate_isolate(migratetype)))
|
|
|
- __mod_zone_freepage_state(zone, 1 << order, migratetype);
|
|
|
spin_unlock(&zone->lock);
|
|
|
}
|
|
|
|
|
@@ -1484,7 +1475,7 @@ void split_page(struct page *page, unsigned int order)
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(split_page);
|
|
|
|
|
|
-static int __isolate_free_page(struct page *page, unsigned int order)
|
|
|
+int __isolate_free_page(struct page *page, unsigned int order)
|
|
|
{
|
|
|
unsigned long watermark;
|
|
|
struct zone *zone;
|
|
@@ -6408,13 +6399,12 @@ int alloc_contig_range(unsigned long start, unsigned long end,
|
|
|
|
|
|
/* Make sure the range is really isolated. */
|
|
|
if (test_pages_isolated(outer_start, end, false)) {
|
|
|
- pr_warn("alloc_contig_range test_pages_isolated(%lx, %lx) failed\n",
|
|
|
- outer_start, end);
|
|
|
+ pr_info("%s: [%lx, %lx) PFNs busy\n",
|
|
|
+ __func__, outer_start, end);
|
|
|
ret = -EBUSY;
|
|
|
goto done;
|
|
|
}
|
|
|
|
|
|
-
|
|
|
/* Grab isolated pages from freelists. */
|
|
|
outer_end = isolate_freepages_range(&cc, outer_start, end);
|
|
|
if (!outer_end) {
|