|
@@ -621,38 +621,18 @@ static int move_to_new_page(struct page *newpage, struct page *page,
|
|
|
return rc;
|
|
return rc;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-/*
|
|
|
|
|
- * Obtain the lock on page, remove all ptes and migrate the page
|
|
|
|
|
- * to the newly allocated page in newpage.
|
|
|
|
|
- */
|
|
|
|
|
-static int unmap_and_move(new_page_t get_new_page, unsigned long private,
|
|
|
|
|
- struct page *page, int force, bool offlining, bool sync)
|
|
|
|
|
|
|
+static int __unmap_and_move(struct page *page, struct page *newpage,
|
|
|
|
|
+ int force, bool offlining, bool sync)
|
|
|
{
|
|
{
|
|
|
- int rc = 0;
|
|
|
|
|
- int *result = NULL;
|
|
|
|
|
- struct page *newpage = get_new_page(page, private, &result);
|
|
|
|
|
|
|
+ int rc = -EAGAIN;
|
|
|
int remap_swapcache = 1;
|
|
int remap_swapcache = 1;
|
|
|
int charge = 0;
|
|
int charge = 0;
|
|
|
struct mem_cgroup *mem;
|
|
struct mem_cgroup *mem;
|
|
|
struct anon_vma *anon_vma = NULL;
|
|
struct anon_vma *anon_vma = NULL;
|
|
|
|
|
|
|
|
- if (!newpage)
|
|
|
|
|
- return -ENOMEM;
|
|
|
|
|
-
|
|
|
|
|
- if (page_count(page) == 1) {
|
|
|
|
|
- /* page was freed from under us. So we are done. */
|
|
|
|
|
- goto move_newpage;
|
|
|
|
|
- }
|
|
|
|
|
- if (unlikely(PageTransHuge(page)))
|
|
|
|
|
- if (unlikely(split_huge_page(page)))
|
|
|
|
|
- goto move_newpage;
|
|
|
|
|
-
|
|
|
|
|
- /* prepare cgroup just returns 0 or -ENOMEM */
|
|
|
|
|
- rc = -EAGAIN;
|
|
|
|
|
-
|
|
|
|
|
if (!trylock_page(page)) {
|
|
if (!trylock_page(page)) {
|
|
|
if (!force || !sync)
|
|
if (!force || !sync)
|
|
|
- goto move_newpage;
|
|
|
|
|
|
|
+ goto out;
|
|
|
|
|
|
|
|
/*
|
|
/*
|
|
|
* It's not safe for direct compaction to call lock_page.
|
|
* It's not safe for direct compaction to call lock_page.
|
|
@@ -668,7 +648,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
|
|
|
* altogether.
|
|
* altogether.
|
|
|
*/
|
|
*/
|
|
|
if (current->flags & PF_MEMALLOC)
|
|
if (current->flags & PF_MEMALLOC)
|
|
|
- goto move_newpage;
|
|
|
|
|
|
|
+ goto out;
|
|
|
|
|
|
|
|
lock_page(page);
|
|
lock_page(page);
|
|
|
}
|
|
}
|
|
@@ -785,27 +765,52 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
|
|
|
mem_cgroup_end_migration(mem, page, newpage, rc == 0);
|
|
mem_cgroup_end_migration(mem, page, newpage, rc == 0);
|
|
|
unlock:
|
|
unlock:
|
|
|
unlock_page(page);
|
|
unlock_page(page);
|
|
|
|
|
+out:
|
|
|
|
|
+ return rc;
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
-move_newpage:
|
|
|
|
|
|
|
+/*
|
|
|
|
|
+ * Obtain the lock on page, remove all ptes and migrate the page
|
|
|
|
|
+ * to the newly allocated page in newpage.
|
|
|
|
|
+ */
|
|
|
|
|
+static int unmap_and_move(new_page_t get_new_page, unsigned long private,
|
|
|
|
|
+ struct page *page, int force, bool offlining, bool sync)
|
|
|
|
|
+{
|
|
|
|
|
+ int rc = 0;
|
|
|
|
|
+ int *result = NULL;
|
|
|
|
|
+ struct page *newpage = get_new_page(page, private, &result);
|
|
|
|
|
+
|
|
|
|
|
+ if (!newpage)
|
|
|
|
|
+ return -ENOMEM;
|
|
|
|
|
+
|
|
|
|
|
+ if (page_count(page) == 1) {
|
|
|
|
|
+ /* page was freed from under us. So we are done. */
|
|
|
|
|
+ goto out;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if (unlikely(PageTransHuge(page)))
|
|
|
|
|
+ if (unlikely(split_huge_page(page)))
|
|
|
|
|
+ goto out;
|
|
|
|
|
+
|
|
|
|
|
+ rc = __unmap_and_move(page, newpage, force, offlining, sync);
|
|
|
|
|
+out:
|
|
|
if (rc != -EAGAIN) {
|
|
if (rc != -EAGAIN) {
|
|
|
- /*
|
|
|
|
|
- * A page that has been migrated has all references
|
|
|
|
|
- * removed and will be freed. A page that has not been
|
|
|
|
|
- * migrated will have kepts its references and be
|
|
|
|
|
- * restored.
|
|
|
|
|
- */
|
|
|
|
|
- list_del(&page->lru);
|
|
|
|
|
|
|
+ /*
|
|
|
|
|
+ * A page that has been migrated has all references
|
|
|
|
|
+ * removed and will be freed. A page that has not been
|
|
|
|
|
+ * migrated will have kepts its references and be
|
|
|
|
|
+ * restored.
|
|
|
|
|
+ */
|
|
|
|
|
+ list_del(&page->lru);
|
|
|
dec_zone_page_state(page, NR_ISOLATED_ANON +
|
|
dec_zone_page_state(page, NR_ISOLATED_ANON +
|
|
|
page_is_file_cache(page));
|
|
page_is_file_cache(page));
|
|
|
putback_lru_page(page);
|
|
putback_lru_page(page);
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
/*
|
|
/*
|
|
|
* Move the new page to the LRU. If migration was not successful
|
|
* Move the new page to the LRU. If migration was not successful
|
|
|
* then this will free the page.
|
|
* then this will free the page.
|
|
|
*/
|
|
*/
|
|
|
putback_lru_page(newpage);
|
|
putback_lru_page(newpage);
|
|
|
-
|
|
|
|
|
if (result) {
|
|
if (result) {
|
|
|
if (rc)
|
|
if (rc)
|
|
|
*result = rc;
|
|
*result = rc;
|