|
|
@@ -1298,15 +1298,9 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
|
|
|
/*
|
|
|
* We can only reuse the page if nobody else maps the huge page or it's
|
|
|
- * part. We can do it by checking page_mapcount() on each sub-page, but
|
|
|
- * it's expensive.
|
|
|
- * The cheaper way is to check page_count() to be equal 1: every
|
|
|
- * mapcount takes page reference reference, so this way we can
|
|
|
- * guarantee, that the PMD is the only mapping.
|
|
|
- * This can give false negative if somebody pinned the page, but that's
|
|
|
- * fine.
|
|
|
+ * part.
|
|
|
*/
|
|
|
- if (page_mapcount(page) == 1 && page_count(page) == 1) {
|
|
|
+ if (page_trans_huge_mapcount(page, NULL) == 1) {
|
|
|
pmd_t entry;
|
|
|
entry = pmd_mkyoung(orig_pmd);
|
|
|
entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
|
|
|
@@ -2079,7 +2073,8 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
|
|
|
if (pte_write(pteval)) {
|
|
|
writable = true;
|
|
|
} else {
|
|
|
- if (PageSwapCache(page) && !reuse_swap_page(page)) {
|
|
|
+ if (PageSwapCache(page) &&
|
|
|
+ !reuse_swap_page(page, NULL)) {
|
|
|
unlock_page(page);
|
|
|
result = SCAN_SWAP_CACHE_PAGE;
|
|
|
goto out;
|
|
|
@@ -3222,6 +3217,64 @@ int total_mapcount(struct page *page)
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
+/*
|
|
|
+ * This calculates accurately how many mappings a transparent hugepage
|
|
|
+ * has (unlike page_mapcount() which isn't fully accurate). This full
|
|
|
+ * accuracy is primarily needed to know if copy-on-write faults can
|
|
|
+ * reuse the page and change the mapping to read-write instead of
|
|
|
+ * copying them. At the same time this returns the total_mapcount too.
|
|
|
+ *
|
|
|
+ * The function returns the highest mapcount any one of the subpages
|
|
|
+ * has. If the return value is one, even if different processes are
|
|
|
+ * mapping different subpages of the transparent hugepage, they can
|
|
|
+ * all reuse it, because each process is reusing a different subpage.
|
|
|
+ *
|
|
|
+ * The total_mapcount is instead counting all virtual mappings of the
|
|
|
+ * subpages. If the total_mapcount is equal to "one", it tells the
|
|
|
+ * caller all mappings belong to the same "mm" and in turn the
|
|
|
+ * anon_vma of the transparent hugepage can become the vma->anon_vma
|
|
|
+ * local one as no other process may be mapping any of the subpages.
|
|
|
+ *
|
|
|
+ * It would be more accurate to replace page_mapcount() with
|
|
|
+ * page_trans_huge_mapcount(), however we only use
|
|
|
+ * page_trans_huge_mapcount() in the copy-on-write faults where we
|
|
|
+ * need full accuracy to avoid breaking page pinning, because
|
|
|
+ * page_trans_huge_mapcount() is slower than page_mapcount().
|
|
|
+ */
|
|
|
+int page_trans_huge_mapcount(struct page *page, int *total_mapcount)
|
|
|
+{
|
|
|
+ int i, ret, _total_mapcount, mapcount;
|
|
|
+
|
|
|
+ /* hugetlbfs shouldn't call it */
|
|
|
+ VM_BUG_ON_PAGE(PageHuge(page), page);
|
|
|
+
|
|
|
+ if (likely(!PageTransCompound(page))) {
|
|
|
+ mapcount = atomic_read(&page->_mapcount) + 1;
|
|
|
+ if (total_mapcount)
|
|
|
+ *total_mapcount = mapcount;
|
|
|
+ return mapcount;
|
|
|
+ }
|
|
|
+
|
|
|
+ page = compound_head(page);
|
|
|
+
|
|
|
+ _total_mapcount = ret = 0;
|
|
|
+ for (i = 0; i < HPAGE_PMD_NR; i++) {
|
|
|
+ mapcount = atomic_read(&page[i]._mapcount) + 1;
|
|
|
+ ret = max(ret, mapcount);
|
|
|
+ _total_mapcount += mapcount;
|
|
|
+ }
|
|
|
+ if (PageDoubleMap(page)) {
|
|
|
+ ret -= 1;
|
|
|
+ _total_mapcount -= HPAGE_PMD_NR;
|
|
|
+ }
|
|
|
+ mapcount = compound_mapcount(page);
|
|
|
+ ret += mapcount;
|
|
|
+ _total_mapcount += mapcount;
|
|
|
+ if (total_mapcount)
|
|
|
+ *total_mapcount = _total_mapcount;
|
|
|
+ return ret;
|
|
|
+}
|
|
|
+
|
|
|
/*
|
|
|
* This function splits huge page into normal pages. @page can point to any
|
|
|
* subpage of huge page to split. Split doesn't change the position of @page.
|
Andrea Arcangeli