|
@@ -116,6 +116,19 @@ retry:
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ if (flags & FOLL_SPLIT && PageTransCompound(page)) {
|
|
|
+ int ret;
|
|
|
+ get_page(page);
|
|
|
+ pte_unmap_unlock(ptep, ptl);
|
|
|
+ lock_page(page);
|
|
|
+ ret = split_huge_page(page);
|
|
|
+ unlock_page(page);
|
|
|
+ put_page(page);
|
|
|
+ if (ret)
|
|
|
+ return ERR_PTR(ret);
|
|
|
+ goto retry;
|
|
|
+ }
|
|
|
+
|
|
|
if (flags & FOLL_GET)
|
|
|
get_page_foll(page);
|
|
|
if (flags & FOLL_TOUCH) {
|
|
@@ -220,27 +233,45 @@ struct page *follow_page_mask(struct vm_area_struct *vma,
|
|
|
}
|
|
|
if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
|
|
|
return no_page_table(vma, flags);
|
|
|
- if (pmd_trans_huge(*pmd)) {
|
|
|
- if (flags & FOLL_SPLIT) {
|
|
|
+ if (likely(!pmd_trans_huge(*pmd)))
|
|
|
+ return follow_page_pte(vma, address, pmd, flags);
|
|
|
+
|
|
|
+ ptl = pmd_lock(mm, pmd);
|
|
|
+ if (unlikely(!pmd_trans_huge(*pmd))) {
|
|
|
+ spin_unlock(ptl);
|
|
|
+ return follow_page_pte(vma, address, pmd, flags);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (unlikely(pmd_trans_splitting(*pmd))) {
|
|
|
+ spin_unlock(ptl);
|
|
|
+ wait_split_huge_page(vma->anon_vma, pmd);
|
|
|
+ return follow_page_pte(vma, address, pmd, flags);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (flags & FOLL_SPLIT) {
|
|
|
+ int ret;
|
|
|
+ page = pmd_page(*pmd);
|
|
|
+ if (is_huge_zero_page(page)) {
|
|
|
+ spin_unlock(ptl);
|
|
|
+ ret = 0;
|
|
|
split_huge_page_pmd(vma, address, pmd);
|
|
|
- return follow_page_pte(vma, address, pmd, flags);
|
|
|
- }
|
|
|
- ptl = pmd_lock(mm, pmd);
|
|
|
- if (likely(pmd_trans_huge(*pmd))) {
|
|
|
- if (unlikely(pmd_trans_splitting(*pmd))) {
|
|
|
- spin_unlock(ptl);
|
|
|
- wait_split_huge_page(vma->anon_vma, pmd);
|
|
|
- } else {
|
|
|
- page = follow_trans_huge_pmd(vma, address,
|
|
|
- pmd, flags);
|
|
|
- spin_unlock(ptl);
|
|
|
- *page_mask = HPAGE_PMD_NR - 1;
|
|
|
- return page;
|
|
|
- }
|
|
|
- } else
|
|
|
+ } else {
|
|
|
+ get_page(page);
|
|
|
spin_unlock(ptl);
|
|
|
+ lock_page(page);
|
|
|
+ ret = split_huge_page(page);
|
|
|
+ unlock_page(page);
|
|
|
+ put_page(page);
|
|
|
+ }
|
|
|
+
|
|
|
+ return ret ? ERR_PTR(ret) :
|
|
|
+ follow_page_pte(vma, address, pmd, flags);
|
|
|
}
|
|
|
- return follow_page_pte(vma, address, pmd, flags);
|
|
|
+
|
|
|
+ page = follow_trans_huge_pmd(vma, address, pmd, flags);
|
|
|
+ spin_unlock(ptl);
|
|
|
+ *page_mask = HPAGE_PMD_NR - 1;
|
|
|
+ return page;
|
|
|
}
|
|
|
|
|
|
static int get_gate_page(struct mm_struct *mm, unsigned long address,
|