|
|
@@ -1085,10 +1085,10 @@ static void free_pcppages_bulk(struct zone *zone, int count,
|
|
|
{
|
|
|
int migratetype = 0;
|
|
|
int batch_free = 0;
|
|
|
- unsigned long nr_scanned;
|
|
|
+ unsigned long nr_scanned, flags;
|
|
|
bool isolated_pageblocks;
|
|
|
|
|
|
- spin_lock(&zone->lock);
|
|
|
+ spin_lock_irqsave(&zone->lock, flags);
|
|
|
isolated_pageblocks = has_isolate_pageblock(zone);
|
|
|
nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
|
|
|
if (nr_scanned)
|
|
|
@@ -1137,7 +1137,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
|
|
|
trace_mm_page_pcpu_drain(page, 0, mt);
|
|
|
} while (--count && --batch_free && !list_empty(list));
|
|
|
}
|
|
|
- spin_unlock(&zone->lock);
|
|
|
+ spin_unlock_irqrestore(&zone->lock, flags);
|
|
|
}
|
|
|
|
|
|
static void free_one_page(struct zone *zone,
|
|
|
@@ -1145,8 +1145,9 @@ static void free_one_page(struct zone *zone,
|
|
|
unsigned int order,
|
|
|
int migratetype)
|
|
|
{
|
|
|
- unsigned long nr_scanned;
|
|
|
- spin_lock(&zone->lock);
|
|
|
+ unsigned long nr_scanned, flags;
|
|
|
+ spin_lock_irqsave(&zone->lock, flags);
|
|
|
+ __count_vm_events(PGFREE, 1 << order);
|
|
|
nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
|
|
|
if (nr_scanned)
|
|
|
__mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
|
|
|
@@ -1156,7 +1157,7 @@ static void free_one_page(struct zone *zone,
|
|
|
migratetype = get_pfnblock_migratetype(page, pfn);
|
|
|
}
|
|
|
__free_one_page(page, pfn, zone, order, migratetype);
|
|
|
- spin_unlock(&zone->lock);
|
|
|
+ spin_unlock_irqrestore(&zone->lock, flags);
|
|
|
}
|
|
|
|
|
|
static void __meminit __init_single_page(struct page *page, unsigned long pfn,
|
|
|
@@ -1234,7 +1235,6 @@ void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
|
|
|
|
|
|
static void __free_pages_ok(struct page *page, unsigned int order)
|
|
|
{
|
|
|
- unsigned long flags;
|
|
|
int migratetype;
|
|
|
unsigned long pfn = page_to_pfn(page);
|
|
|
|
|
|
@@ -1242,10 +1242,7 @@ static void __free_pages_ok(struct page *page, unsigned int order)
|
|
|
return;
|
|
|
|
|
|
migratetype = get_pfnblock_migratetype(page, pfn);
|
|
|
- local_irq_save(flags);
|
|
|
- __count_vm_events(PGFREE, 1 << order);
|
|
|
free_one_page(page_zone(page), page, pfn, order, migratetype);
|
|
|
- local_irq_restore(flags);
|
|
|
}
|
|
|
|
|
|
static void __init __free_pages_boot_core(struct page *page, unsigned int order)
|
|
|
@@ -2217,8 +2214,9 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
|
|
|
int migratetype, bool cold)
|
|
|
{
|
|
|
int i, alloced = 0;
|
|
|
+ unsigned long flags;
|
|
|
|
|
|
- spin_lock(&zone->lock);
|
|
|
+ spin_lock_irqsave(&zone->lock, flags);
|
|
|
for (i = 0; i < count; ++i) {
|
|
|
struct page *page = __rmqueue(zone, order, migratetype);
|
|
|
if (unlikely(page == NULL))
|
|
|
@@ -2254,7 +2252,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
|
|
|
* pages added to the pcp list.
|
|
|
*/
|
|
|
__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
|
|
|
- spin_unlock(&zone->lock);
|
|
|
+ spin_unlock_irqrestore(&zone->lock, flags);
|
|
|
return alloced;
|
|
|
}
|
|
|
|
|
|
@@ -2475,17 +2473,20 @@ void free_hot_cold_page(struct page *page, bool cold)
|
|
|
{
|
|
|
struct zone *zone = page_zone(page);
|
|
|
struct per_cpu_pages *pcp;
|
|
|
- unsigned long flags;
|
|
|
unsigned long pfn = page_to_pfn(page);
|
|
|
int migratetype;
|
|
|
|
|
|
+ if (in_interrupt()) {
|
|
|
+ __free_pages_ok(page, 0);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
if (!free_pcp_prepare(page))
|
|
|
return;
|
|
|
|
|
|
migratetype = get_pfnblock_migratetype(page, pfn);
|
|
|
set_pcppage_migratetype(page, migratetype);
|
|
|
- local_irq_save(flags);
|
|
|
- __count_vm_event(PGFREE);
|
|
|
+ preempt_disable();
|
|
|
|
|
|
/*
|
|
|
* We only track unmovable, reclaimable and movable on pcp lists.
|
|
|
@@ -2502,6 +2503,7 @@ void free_hot_cold_page(struct page *page, bool cold)
|
|
|
migratetype = MIGRATE_MOVABLE;
|
|
|
}
|
|
|
|
|
|
+ __count_vm_event(PGFREE);
|
|
|
pcp = &this_cpu_ptr(zone->pageset)->pcp;
|
|
|
if (!cold)
|
|
|
list_add(&page->lru, &pcp->lists[migratetype]);
|
|
|
@@ -2515,7 +2517,7 @@ void free_hot_cold_page(struct page *page, bool cold)
|
|
|
}
|
|
|
|
|
|
out:
|
|
|
- local_irq_restore(flags);
|
|
|
+ preempt_enable();
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
@@ -2640,6 +2642,8 @@ static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
|
|
|
{
|
|
|
struct page *page;
|
|
|
|
|
|
+ VM_BUG_ON(in_interrupt());
|
|
|
+
|
|
|
do {
|
|
|
if (list_empty(list)) {
|
|
|
pcp->count += rmqueue_bulk(zone, 0,
|
|
|
@@ -2670,9 +2674,8 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
|
|
|
struct list_head *list;
|
|
|
bool cold = ((gfp_flags & __GFP_COLD) != 0);
|
|
|
struct page *page;
|
|
|
- unsigned long flags;
|
|
|
|
|
|
- local_irq_save(flags);
|
|
|
+ preempt_disable();
|
|
|
pcp = &this_cpu_ptr(zone->pageset)->pcp;
|
|
|
list = &pcp->lists[migratetype];
|
|
|
page = __rmqueue_pcplist(zone, migratetype, cold, pcp, list);
|
|
|
@@ -2680,7 +2683,7 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
|
|
|
__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
|
|
|
zone_statistics(preferred_zone, zone);
|
|
|
}
|
|
|
- local_irq_restore(flags);
|
|
|
+ preempt_enable();
|
|
|
return page;
|
|
|
}
|
|
|
|
|
|
@@ -2696,7 +2699,7 @@ struct page *rmqueue(struct zone *preferred_zone,
|
|
|
unsigned long flags;
|
|
|
struct page *page;
|
|
|
|
|
|
- if (likely(order == 0)) {
|
|
|
+ if (likely(order == 0) && !in_interrupt()) {
|
|
|
page = rmqueue_pcplist(preferred_zone, zone, order,
|
|
|
gfp_flags, migratetype);
|
|
|
goto out;
|
Mel Gorman