|
|
@@ -55,9 +55,6 @@
|
|
|
|
|
|
#include "mmu_decl.h"
|
|
|
|
|
|
-#define CREATE_TRACE_POINTS
|
|
|
-#include <trace/events/thp.h>
|
|
|
-
|
|
|
#ifdef CONFIG_PPC_STD_MMU_64
|
|
|
#if TASK_SIZE_USER64 > (1UL << (ESID_BITS + SID_SHIFT))
|
|
|
#error TASK_SIZE_USER64 exceeds user VSID range
|
|
|
@@ -435,359 +432,3 @@ void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
|
|
|
}
|
|
|
}
|
|
|
#endif
|
|
|
-
|
|
|
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
|
-
|
|
|
-/*
|
|
|
- * This is called when relaxing access to a hugepage. It's also called in the page
|
|
|
- * fault path when we don't hit any of the major fault cases, ie, a minor
|
|
|
- * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have
|
|
|
- * handled those two for us, we additionally deal with missing execute
|
|
|
- * permission here on some processors
|
|
|
- */
|
|
|
-int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
|
|
|
- pmd_t *pmdp, pmd_t entry, int dirty)
|
|
|
-{
|
|
|
- int changed;
|
|
|
-#ifdef CONFIG_DEBUG_VM
|
|
|
- WARN_ON(!pmd_trans_huge(*pmdp));
|
|
|
- assert_spin_locked(&vma->vm_mm->page_table_lock);
|
|
|
-#endif
|
|
|
- changed = !pmd_same(*(pmdp), entry);
|
|
|
- if (changed) {
|
|
|
- __ptep_set_access_flags(pmdp_ptep(pmdp), pmd_pte(entry));
|
|
|
- /*
|
|
|
- * Since we are not supporting SW TLB systems, we don't
|
|
|
- * have any thing similar to flush_tlb_page_nohash()
|
|
|
- */
|
|
|
- }
|
|
|
- return changed;
|
|
|
-}
|
|
|
-
|
|
|
-unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
|
|
|
- pmd_t *pmdp, unsigned long clr,
|
|
|
- unsigned long set)
|
|
|
-{
|
|
|
-
|
|
|
- __be64 old_be, tmp;
|
|
|
- unsigned long old;
|
|
|
-
|
|
|
-#ifdef CONFIG_DEBUG_VM
|
|
|
- WARN_ON(!pmd_trans_huge(*pmdp));
|
|
|
- assert_spin_locked(&mm->page_table_lock);
|
|
|
-#endif
|
|
|
-
|
|
|
- __asm__ __volatile__(
|
|
|
- "1: ldarx %0,0,%3\n\
|
|
|
- and. %1,%0,%6\n\
|
|
|
- bne- 1b \n\
|
|
|
- andc %1,%0,%4 \n\
|
|
|
- or %1,%1,%7\n\
|
|
|
- stdcx. %1,0,%3 \n\
|
|
|
- bne- 1b"
|
|
|
- : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp)
|
|
|
- : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp),
|
|
|
- "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))
|
|
|
- : "cc" );
|
|
|
-
|
|
|
- old = be64_to_cpu(old_be);
|
|
|
-
|
|
|
- trace_hugepage_update(addr, old, clr, set);
|
|
|
- if (old & H_PAGE_HASHPTE)
|
|
|
- hpte_do_hugepage_flush(mm, addr, pmdp, old);
|
|
|
- return old;
|
|
|
-}
|
|
|
-
|
|
|
-pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
|
|
|
- pmd_t *pmdp)
|
|
|
-{
|
|
|
- pmd_t pmd;
|
|
|
-
|
|
|
- VM_BUG_ON(address & ~HPAGE_PMD_MASK);
|
|
|
- VM_BUG_ON(pmd_trans_huge(*pmdp));
|
|
|
-
|
|
|
- pmd = *pmdp;
|
|
|
- pmd_clear(pmdp);
|
|
|
- /*
|
|
|
- * Wait for all pending hash_page to finish. This is needed
|
|
|
- * in case of subpage collapse. When we collapse normal pages
|
|
|
- * to hugepage, we first clear the pmd, then invalidate all
|
|
|
- * the PTE entries. The assumption here is that any low level
|
|
|
- * page fault will see a none pmd and take the slow path that
|
|
|
- * will wait on mmap_sem. But we could very well be in a
|
|
|
- * hash_page with local ptep pointer value. Such a hash page
|
|
|
- * can result in adding new HPTE entries for normal subpages.
|
|
|
- * That means we could be modifying the page content as we
|
|
|
- * copy them to a huge page. So wait for parallel hash_page
|
|
|
- * to finish before invalidating HPTE entries. We can do this
|
|
|
- * by sending an IPI to all the cpus and executing a dummy
|
|
|
- * function there.
|
|
|
- */
|
|
|
- kick_all_cpus_sync();
|
|
|
- /*
|
|
|
- * Now invalidate the hpte entries in the range
|
|
|
- * covered by pmd. This make sure we take a
|
|
|
- * fault and will find the pmd as none, which will
|
|
|
- * result in a major fault which takes mmap_sem and
|
|
|
- * hence wait for collapse to complete. Without this
|
|
|
- * the __collapse_huge_page_copy can result in copying
|
|
|
- * the old content.
|
|
|
- */
|
|
|
- flush_tlb_pmd_range(vma->vm_mm, &pmd, address);
|
|
|
- return pmd;
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * We currently remove entries from the hashtable regardless of whether
|
|
|
- * the entry was young or dirty.
|
|
|
- *
|
|
|
- * We should be more intelligent about this but for the moment we override
|
|
|
- * these functions and force a tlb flush unconditionally
|
|
|
- */
|
|
|
-int pmdp_test_and_clear_young(struct vm_area_struct *vma,
|
|
|
- unsigned long address, pmd_t *pmdp)
|
|
|
-{
|
|
|
- return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp);
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * We want to put the pgtable in pmd and use pgtable for tracking
|
|
|
- * the base page size hptes
|
|
|
- */
|
|
|
-void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
|
|
|
- pgtable_t pgtable)
|
|
|
-{
|
|
|
- pgtable_t *pgtable_slot;
|
|
|
- assert_spin_locked(&mm->page_table_lock);
|
|
|
- /*
|
|
|
- * we store the pgtable in the second half of PMD
|
|
|
- */
|
|
|
- pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
|
|
|
- *pgtable_slot = pgtable;
|
|
|
- /*
|
|
|
- * expose the deposited pgtable to other cpus.
|
|
|
- * before we set the hugepage PTE at pmd level
|
|
|
- * hash fault code looks at the deposted pgtable
|
|
|
- * to store hash index values.
|
|
|
- */
|
|
|
- smp_wmb();
|
|
|
-}
|
|
|
-
|
|
|
-pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
|
|
|
-{
|
|
|
- pgtable_t pgtable;
|
|
|
- pgtable_t *pgtable_slot;
|
|
|
-
|
|
|
- assert_spin_locked(&mm->page_table_lock);
|
|
|
- pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
|
|
|
- pgtable = *pgtable_slot;
|
|
|
- /*
|
|
|
- * Once we withdraw, mark the entry NULL.
|
|
|
- */
|
|
|
- *pgtable_slot = NULL;
|
|
|
- /*
|
|
|
- * We store HPTE information in the deposited PTE fragment.
|
|
|
- * zero out the content on withdraw.
|
|
|
- */
|
|
|
- memset(pgtable, 0, PTE_FRAG_SIZE);
|
|
|
- return pgtable;
|
|
|
-}
|
|
|
-
|
|
|
-void pmdp_huge_split_prepare(struct vm_area_struct *vma,
|
|
|
- unsigned long address, pmd_t *pmdp)
|
|
|
-{
|
|
|
- VM_BUG_ON(address & ~HPAGE_PMD_MASK);
|
|
|
- VM_BUG_ON(REGION_ID(address) != USER_REGION_ID);
|
|
|
-
|
|
|
- /*
|
|
|
- * We can't mark the pmd none here, because that will cause a race
|
|
|
- * against exit_mmap. We need to continue mark pmd TRANS HUGE, while
|
|
|
- * we spilt, but at the same time we wan't rest of the ppc64 code
|
|
|
- * not to insert hash pte on this, because we will be modifying
|
|
|
- * the deposited pgtable in the caller of this function. Hence
|
|
|
- * clear the _PAGE_USER so that we move the fault handling to
|
|
|
- * higher level function and that will serialize against ptl.
|
|
|
- * We need to flush existing hash pte entries here even though,
|
|
|
- * the translation is still valid, because we will withdraw
|
|
|
- * pgtable_t after this.
|
|
|
- */
|
|
|
- pmd_hugepage_update(vma->vm_mm, address, pmdp, 0, _PAGE_PRIVILEGED);
|
|
|
-}
|
|
|
-
|
|
|
-
|
|
|
-/*
|
|
|
- * set a new huge pmd. We should not be called for updating
|
|
|
- * an existing pmd entry. That should go via pmd_hugepage_update.
|
|
|
- */
|
|
|
-void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
|
|
- pmd_t *pmdp, pmd_t pmd)
|
|
|
-{
|
|
|
-#ifdef CONFIG_DEBUG_VM
|
|
|
- WARN_ON(pte_present(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp)));
|
|
|
- assert_spin_locked(&mm->page_table_lock);
|
|
|
- WARN_ON(!pmd_trans_huge(pmd));
|
|
|
-#endif
|
|
|
- trace_hugepage_set_pmd(addr, pmd_val(pmd));
|
|
|
- return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd));
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * We use this to invalidate a pmdp entry before switching from a
|
|
|
- * hugepte to regular pmd entry.
|
|
|
- */
|
|
|
-void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
|
|
|
- pmd_t *pmdp)
|
|
|
-{
|
|
|
- pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
|
|
|
-
|
|
|
- /*
|
|
|
- * This ensures that generic code that rely on IRQ disabling
|
|
|
- * to prevent a parallel THP split work as expected.
|
|
|
- */
|
|
|
- kick_all_cpus_sync();
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * A linux hugepage PMD was changed and the corresponding hash table entries
|
|
|
- * neesd to be flushed.
|
|
|
- */
|
|
|
-void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
|
|
|
- pmd_t *pmdp, unsigned long old_pmd)
|
|
|
-{
|
|
|
- int ssize;
|
|
|
- unsigned int psize;
|
|
|
- unsigned long vsid;
|
|
|
- unsigned long flags = 0;
|
|
|
- const struct cpumask *tmp;
|
|
|
-
|
|
|
- /* get the base page size,vsid and segment size */
|
|
|
-#ifdef CONFIG_DEBUG_VM
|
|
|
- psize = get_slice_psize(mm, addr);
|
|
|
- BUG_ON(psize == MMU_PAGE_16M);
|
|
|
-#endif
|
|
|
- if (old_pmd & H_PAGE_COMBO)
|
|
|
- psize = MMU_PAGE_4K;
|
|
|
- else
|
|
|
- psize = MMU_PAGE_64K;
|
|
|
-
|
|
|
- if (!is_kernel_addr(addr)) {
|
|
|
- ssize = user_segment_size(addr);
|
|
|
- vsid = get_vsid(mm->context.id, addr, ssize);
|
|
|
- WARN_ON(vsid == 0);
|
|
|
- } else {
|
|
|
- vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
|
|
|
- ssize = mmu_kernel_ssize;
|
|
|
- }
|
|
|
-
|
|
|
- tmp = cpumask_of(smp_processor_id());
|
|
|
- if (cpumask_equal(mm_cpumask(mm), tmp))
|
|
|
- flags |= HPTE_LOCAL_UPDATE;
|
|
|
-
|
|
|
- return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, flags);
|
|
|
-}
|
|
|
-
|
|
|
-static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)
|
|
|
-{
|
|
|
- return __pmd(pmd_val(pmd) | pgprot_val(pgprot));
|
|
|
-}
|
|
|
-
|
|
|
-pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot)
|
|
|
-{
|
|
|
- unsigned long pmdv;
|
|
|
-
|
|
|
- pmdv = (pfn << PAGE_SHIFT) & PTE_RPN_MASK;
|
|
|
- return pmd_set_protbits(__pmd(pmdv), pgprot);
|
|
|
-}
|
|
|
-
|
|
|
-pmd_t mk_pmd(struct page *page, pgprot_t pgprot)
|
|
|
-{
|
|
|
- return pfn_pmd(page_to_pfn(page), pgprot);
|
|
|
-}
|
|
|
-
|
|
|
-pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
|
|
-{
|
|
|
- unsigned long pmdv;
|
|
|
-
|
|
|
- pmdv = pmd_val(pmd);
|
|
|
- pmdv &= _HPAGE_CHG_MASK;
|
|
|
- return pmd_set_protbits(__pmd(pmdv), newprot);
|
|
|
-}
|
|
|
-
|
|
|
-/*
|
|
|
- * This is called at the end of handling a user page fault, when the
|
|
|
- * fault has been handled by updating a HUGE PMD entry in the linux page tables.
|
|
|
- * We use it to preload an HPTE into the hash table corresponding to
|
|
|
- * the updated linux HUGE PMD entry.
|
|
|
- */
|
|
|
-void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
|
|
|
- pmd_t *pmd)
|
|
|
-{
|
|
|
- return;
|
|
|
-}
|
|
|
-
|
|
|
-pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
|
|
|
- unsigned long addr, pmd_t *pmdp)
|
|
|
-{
|
|
|
- pmd_t old_pmd;
|
|
|
- pgtable_t pgtable;
|
|
|
- unsigned long old;
|
|
|
- pgtable_t *pgtable_slot;
|
|
|
-
|
|
|
- old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
|
|
|
- old_pmd = __pmd(old);
|
|
|
- /*
|
|
|
- * We have pmd == none and we are holding page_table_lock.
|
|
|
- * So we can safely go and clear the pgtable hash
|
|
|
- * index info.
|
|
|
- */
|
|
|
- pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
|
|
|
- pgtable = *pgtable_slot;
|
|
|
- /*
|
|
|
- * Let's zero out old valid and hash index details
|
|
|
- * hash fault look at them.
|
|
|
- */
|
|
|
- memset(pgtable, 0, PTE_FRAG_SIZE);
|
|
|
- /*
|
|
|
- * Serialize against find_linux_pte_or_hugepte which does lock-less
|
|
|
- * lookup in page tables with local interrupts disabled. For huge pages
|
|
|
- * it casts pmd_t to pte_t. Since format of pte_t is different from
|
|
|
- * pmd_t we want to prevent transit from pmd pointing to page table
|
|
|
- * to pmd pointing to huge page (and back) while interrupts are disabled.
|
|
|
- * We clear pmd to possibly replace it with page table pointer in
|
|
|
- * different code paths. So make sure we wait for the parallel
|
|
|
- * find_linux_pte_or_hugepage to finish.
|
|
|
- */
|
|
|
- kick_all_cpus_sync();
|
|
|
- return old_pmd;
|
|
|
-}
|
|
|
-
|
|
|
-int has_transparent_hugepage(void)
|
|
|
-{
|
|
|
-
|
|
|
- if (!mmu_has_feature(MMU_FTR_16M_PAGE))
|
|
|
- return 0;
|
|
|
- /*
|
|
|
- * We support THP only if PMD_SIZE is 16MB.
|
|
|
- */
|
|
|
- if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT)
|
|
|
- return 0;
|
|
|
- /*
|
|
|
- * We need to make sure that we support 16MB hugepage in a segement
|
|
|
- * with base page size 64K or 4K. We only enable THP with a PAGE_SIZE
|
|
|
- * of 64K.
|
|
|
- */
|
|
|
- /*
|
|
|
- * If we have 64K HPTE, we will be using that by default
|
|
|
- */
|
|
|
- if (mmu_psize_defs[MMU_PAGE_64K].shift &&
|
|
|
- (mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1))
|
|
|
- return 0;
|
|
|
- /*
|
|
|
- * Ok we only have 4K HPTE
|
|
|
- */
|
|
|
- if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1)
|
|
|
- return 0;
|
|
|
-
|
|
|
- return 1;
|
|
|
-}
|
|
|
-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
Aneesh Kumar K.V