pgtable.h 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750
  1. #ifndef _ASM_GENERIC_PGTABLE_H
  2. #define _ASM_GENERIC_PGTABLE_H
  3. #ifndef __ASSEMBLY__
  4. #ifdef CONFIG_MMU
  5. #include <linux/mm_types.h>
  6. #include <linux/bug.h>
  7. #include <linux/errno.h>
  8. #if 4 - defined(__PAGETABLE_PUD_FOLDED) - defined(__PAGETABLE_PMD_FOLDED) != \
  9. CONFIG_PGTABLE_LEVELS
  10. #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{PUD,PMD}_FOLDED
  11. #endif
  12. /*
  13. * On almost all architectures and configurations, 0 can be used as the
  14. * upper ceiling to free_pgtables(): on many architectures it has the same
  15. * effect as using TASK_SIZE. However, there is one configuration which
  16. * must impose a more careful limit, to avoid freeing kernel pgtables.
  17. */
  18. #ifndef USER_PGTABLES_CEILING
  19. #define USER_PGTABLES_CEILING 0UL
  20. #endif
  21. #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
  22. extern int ptep_set_access_flags(struct vm_area_struct *vma,
  23. unsigned long address, pte_t *ptep,
  24. pte_t entry, int dirty);
  25. #endif
  26. #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
  27. extern int pmdp_set_access_flags(struct vm_area_struct *vma,
  28. unsigned long address, pmd_t *pmdp,
  29. pmd_t entry, int dirty);
  30. #endif
  31. #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
  32. static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
  33. unsigned long address,
  34. pte_t *ptep)
  35. {
  36. pte_t pte = *ptep;
  37. int r = 1;
  38. if (!pte_young(pte))
  39. r = 0;
  40. else
  41. set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte));
  42. return r;
  43. }
  44. #endif
  45. #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
  46. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  47. static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
  48. unsigned long address,
  49. pmd_t *pmdp)
  50. {
  51. pmd_t pmd = *pmdp;
  52. int r = 1;
  53. if (!pmd_young(pmd))
  54. r = 0;
  55. else
  56. set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd));
  57. return r;
  58. }
  59. #else /* CONFIG_TRANSPARENT_HUGEPAGE */
  60. static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
  61. unsigned long address,
  62. pmd_t *pmdp)
  63. {
  64. BUG();
  65. return 0;
  66. }
  67. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  68. #endif
  69. #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
  70. int ptep_clear_flush_young(struct vm_area_struct *vma,
  71. unsigned long address, pte_t *ptep);
  72. #endif
  73. #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
  74. int pmdp_clear_flush_young(struct vm_area_struct *vma,
  75. unsigned long address, pmd_t *pmdp);
  76. #endif
  77. #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
  78. static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
  79. unsigned long address,
  80. pte_t *ptep)
  81. {
  82. pte_t pte = *ptep;
  83. pte_clear(mm, address, ptep);
  84. return pte;
  85. }
  86. #endif
  87. #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
  88. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  89. static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
  90. unsigned long address,
  91. pmd_t *pmdp)
  92. {
  93. pmd_t pmd = *pmdp;
  94. pmd_clear(pmdp);
  95. return pmd;
  96. }
  97. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  98. #endif
  99. #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
  100. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  101. static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm,
  102. unsigned long address, pmd_t *pmdp,
  103. int full)
  104. {
  105. return pmdp_huge_get_and_clear(mm, address, pmdp);
  106. }
  107. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  108. #endif
  109. #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
  110. static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
  111. unsigned long address, pte_t *ptep,
  112. int full)
  113. {
  114. pte_t pte;
  115. pte = ptep_get_and_clear(mm, address, ptep);
  116. return pte;
  117. }
  118. #endif
  119. /*
  120. * Some architectures may be able to avoid expensive synchronization
  121. * primitives when modifications are made to PTE's which are already
  122. * not present, or in the process of an address space destruction.
  123. */
  124. #ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
  125. static inline void pte_clear_not_present_full(struct mm_struct *mm,
  126. unsigned long address,
  127. pte_t *ptep,
  128. int full)
  129. {
  130. pte_clear(mm, address, ptep);
  131. }
  132. #endif
  133. #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
  134. extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
  135. unsigned long address,
  136. pte_t *ptep);
  137. #endif
  138. #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
  139. extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
  140. unsigned long address,
  141. pmd_t *pmdp);
  142. #endif
  143. #ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
  144. struct mm_struct;
  145. static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
  146. {
  147. pte_t old_pte = *ptep;
  148. set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));
  149. }
  150. #endif
  151. #ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT
  152. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  153. static inline void pmdp_set_wrprotect(struct mm_struct *mm,
  154. unsigned long address, pmd_t *pmdp)
  155. {
  156. pmd_t old_pmd = *pmdp;
  157. set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd));
  158. }
  159. #else /* CONFIG_TRANSPARENT_HUGEPAGE */
  160. static inline void pmdp_set_wrprotect(struct mm_struct *mm,
  161. unsigned long address, pmd_t *pmdp)
  162. {
  163. BUG();
  164. }
  165. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  166. #endif
  167. #ifndef __HAVE_ARCH_PMDP_SPLITTING_FLUSH
  168. extern void pmdp_splitting_flush(struct vm_area_struct *vma,
  169. unsigned long address, pmd_t *pmdp);
  170. #endif
  171. #ifndef pmdp_collapse_flush
  172. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  173. extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
  174. unsigned long address, pmd_t *pmdp);
  175. #else
  176. static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
  177. unsigned long address,
  178. pmd_t *pmdp)
  179. {
  180. BUILD_BUG();
  181. return *pmdp;
  182. }
  183. #define pmdp_collapse_flush pmdp_collapse_flush
  184. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  185. #endif
  186. #ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
  187. extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
  188. pgtable_t pgtable);
  189. #endif
  190. #ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
  191. extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
  192. #endif
  193. #ifndef __HAVE_ARCH_PMDP_INVALIDATE
  194. extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
  195. pmd_t *pmdp);
  196. #endif
  197. #ifndef __HAVE_ARCH_PTE_SAME
  198. static inline int pte_same(pte_t pte_a, pte_t pte_b)
  199. {
  200. return pte_val(pte_a) == pte_val(pte_b);
  201. }
  202. #endif
  203. #ifndef __HAVE_ARCH_PTE_UNUSED
  204. /*
  205. * Some architectures provide facilities to virtualization guests
  206. * so that they can flag allocated pages as unused. This allows the
  207. * host to transparently reclaim unused pages. This function returns
  208. * whether the pte's page is unused.
  209. */
  210. static inline int pte_unused(pte_t pte)
  211. {
  212. return 0;
  213. }
  214. #endif
  215. #ifndef __HAVE_ARCH_PMD_SAME
  216. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  217. static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
  218. {
  219. return pmd_val(pmd_a) == pmd_val(pmd_b);
  220. }
  221. #else /* CONFIG_TRANSPARENT_HUGEPAGE */
  222. static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
  223. {
  224. BUG();
  225. return 0;
  226. }
  227. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  228. #endif
  229. #ifndef __HAVE_ARCH_PGD_OFFSET_GATE
  230. #define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
  231. #endif
  232. #ifndef __HAVE_ARCH_MOVE_PTE
  233. #define move_pte(pte, prot, old_addr, new_addr) (pte)
  234. #endif
  235. #ifndef pte_accessible
  236. # define pte_accessible(mm, pte) ((void)(pte), 1)
  237. #endif
  238. #ifndef flush_tlb_fix_spurious_fault
  239. #define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address)
  240. #endif
  241. #ifndef pgprot_noncached
  242. #define pgprot_noncached(prot) (prot)
  243. #endif
  244. #ifndef pgprot_writecombine
  245. #define pgprot_writecombine pgprot_noncached
  246. #endif
  247. #ifndef pgprot_writethrough
  248. #define pgprot_writethrough pgprot_noncached
  249. #endif
  250. #ifndef pgprot_device
  251. #define pgprot_device pgprot_noncached
  252. #endif
  253. #ifndef pgprot_modify
  254. #define pgprot_modify pgprot_modify
  255. static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
  256. {
  257. if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot)))
  258. newprot = pgprot_noncached(newprot);
  259. if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot)))
  260. newprot = pgprot_writecombine(newprot);
  261. if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot)))
  262. newprot = pgprot_device(newprot);
  263. return newprot;
  264. }
  265. #endif
  266. /*
  267. * When walking page tables, get the address of the next boundary,
  268. * or the end address of the range if that comes earlier. Although no
  269. * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout.
  270. */
  271. #define pgd_addr_end(addr, end) \
  272. ({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
  273. (__boundary - 1 < (end) - 1)? __boundary: (end); \
  274. })
  275. #ifndef pud_addr_end
  276. #define pud_addr_end(addr, end) \
  277. ({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \
  278. (__boundary - 1 < (end) - 1)? __boundary: (end); \
  279. })
  280. #endif
  281. #ifndef pmd_addr_end
  282. #define pmd_addr_end(addr, end) \
  283. ({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \
  284. (__boundary - 1 < (end) - 1)? __boundary: (end); \
  285. })
  286. #endif
  287. /*
  288. * When walking page tables, we usually want to skip any p?d_none entries;
  289. * and any p?d_bad entries - reporting the error before resetting to none.
  290. * Do the tests inline, but report and clear the bad entry in mm/memory.c.
  291. */
  292. void pgd_clear_bad(pgd_t *);
  293. void pud_clear_bad(pud_t *);
  294. void pmd_clear_bad(pmd_t *);
  295. static inline int pgd_none_or_clear_bad(pgd_t *pgd)
  296. {
  297. if (pgd_none(*pgd))
  298. return 1;
  299. if (unlikely(pgd_bad(*pgd))) {
  300. pgd_clear_bad(pgd);
  301. return 1;
  302. }
  303. return 0;
  304. }
  305. static inline int pud_none_or_clear_bad(pud_t *pud)
  306. {
  307. if (pud_none(*pud))
  308. return 1;
  309. if (unlikely(pud_bad(*pud))) {
  310. pud_clear_bad(pud);
  311. return 1;
  312. }
  313. return 0;
  314. }
  315. static inline int pmd_none_or_clear_bad(pmd_t *pmd)
  316. {
  317. if (pmd_none(*pmd))
  318. return 1;
  319. if (unlikely(pmd_bad(*pmd))) {
  320. pmd_clear_bad(pmd);
  321. return 1;
  322. }
  323. return 0;
  324. }
  325. static inline pte_t __ptep_modify_prot_start(struct mm_struct *mm,
  326. unsigned long addr,
  327. pte_t *ptep)
  328. {
  329. /*
  330. * Get the current pte state, but zero it out to make it
  331. * non-present, preventing the hardware from asynchronously
  332. * updating it.
  333. */
  334. return ptep_get_and_clear(mm, addr, ptep);
  335. }
  336. static inline void __ptep_modify_prot_commit(struct mm_struct *mm,
  337. unsigned long addr,
  338. pte_t *ptep, pte_t pte)
  339. {
  340. /*
  341. * The pte is non-present, so there's no hardware state to
  342. * preserve.
  343. */
  344. set_pte_at(mm, addr, ptep, pte);
  345. }
  346. #ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
  347. /*
  348. * Start a pte protection read-modify-write transaction, which
  349. * protects against asynchronous hardware modifications to the pte.
  350. * The intention is not to prevent the hardware from making pte
  351. * updates, but to prevent any updates it may make from being lost.
  352. *
  353. * This does not protect against other software modifications of the
  354. * pte; the appropriate pte lock must be held over the transation.
  355. *
  356. * Note that this interface is intended to be batchable, meaning that
  357. * ptep_modify_prot_commit may not actually update the pte, but merely
  358. * queue the update to be done at some later time. The update must be
  359. * actually committed before the pte lock is released, however.
  360. */
  361. static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
  362. unsigned long addr,
  363. pte_t *ptep)
  364. {
  365. return __ptep_modify_prot_start(mm, addr, ptep);
  366. }
  367. /*
  368. * Commit an update to a pte, leaving any hardware-controlled bits in
  369. * the PTE unmodified.
  370. */
  371. static inline void ptep_modify_prot_commit(struct mm_struct *mm,
  372. unsigned long addr,
  373. pte_t *ptep, pte_t pte)
  374. {
  375. __ptep_modify_prot_commit(mm, addr, ptep, pte);
  376. }
  377. #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
  378. #endif /* CONFIG_MMU */
  379. /*
  380. * A facility to provide lazy MMU batching. This allows PTE updates and
  381. * page invalidations to be delayed until a call to leave lazy MMU mode
  382. * is issued. Some architectures may benefit from doing this, and it is
  383. * beneficial for both shadow and direct mode hypervisors, which may batch
  384. * the PTE updates which happen during this window. Note that using this
  385. * interface requires that read hazards be removed from the code. A read
  386. * hazard could result in the direct mode hypervisor case, since the actual
  387. * write to the page tables may not yet have taken place, so reads though
  388. * a raw PTE pointer after it has been modified are not guaranteed to be
  389. * up to date. This mode can only be entered and left under the protection of
  390. * the page table locks for all page tables which may be modified. In the UP
  391. * case, this is required so that preemption is disabled, and in the SMP case,
  392. * it must synchronize the delayed page table writes properly on other CPUs.
  393. */
  394. #ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE
  395. #define arch_enter_lazy_mmu_mode() do {} while (0)
  396. #define arch_leave_lazy_mmu_mode() do {} while (0)
  397. #define arch_flush_lazy_mmu_mode() do {} while (0)
  398. #endif
  399. /*
  400. * A facility to provide batching of the reload of page tables and
  401. * other process state with the actual context switch code for
  402. * paravirtualized guests. By convention, only one of the batched
  403. * update (lazy) modes (CPU, MMU) should be active at any given time,
  404. * entry should never be nested, and entry and exits should always be
  405. * paired. This is for sanity of maintaining and reasoning about the
  406. * kernel code. In this case, the exit (end of the context switch) is
  407. * in architecture-specific code, and so doesn't need a generic
  408. * definition.
  409. */
  410. #ifndef __HAVE_ARCH_START_CONTEXT_SWITCH
  411. #define arch_start_context_switch(prev) do {} while (0)
  412. #endif
  413. #ifndef CONFIG_HAVE_ARCH_SOFT_DIRTY
  414. static inline int pte_soft_dirty(pte_t pte)
  415. {
  416. return 0;
  417. }
  418. static inline int pmd_soft_dirty(pmd_t pmd)
  419. {
  420. return 0;
  421. }
  422. static inline pte_t pte_mksoft_dirty(pte_t pte)
  423. {
  424. return pte;
  425. }
  426. static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
  427. {
  428. return pmd;
  429. }
  430. static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
  431. {
  432. return pte;
  433. }
  434. static inline int pte_swp_soft_dirty(pte_t pte)
  435. {
  436. return 0;
  437. }
  438. static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
  439. {
  440. return pte;
  441. }
  442. #endif
  443. #ifndef __HAVE_PFNMAP_TRACKING
  444. /*
  445. * Interfaces that can be used by architecture code to keep track of
  446. * memory type of pfn mappings specified by the remap_pfn_range,
  447. * vm_insert_pfn.
  448. */
  449. /*
  450. * track_pfn_remap is called when a _new_ pfn mapping is being established
  451. * by remap_pfn_range() for physical range indicated by pfn and size.
  452. */
  453. static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
  454. unsigned long pfn, unsigned long addr,
  455. unsigned long size)
  456. {
  457. return 0;
  458. }
  459. /*
  460. * track_pfn_insert is called when a _new_ single pfn is established
  461. * by vm_insert_pfn().
  462. */
  463. static inline int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
  464. unsigned long pfn)
  465. {
  466. return 0;
  467. }
  468. /*
  469. * track_pfn_copy is called when vma that is covering the pfnmap gets
  470. * copied through copy_page_range().
  471. */
  472. static inline int track_pfn_copy(struct vm_area_struct *vma)
  473. {
  474. return 0;
  475. }
  476. /*
  477. * untrack_pfn_vma is called while unmapping a pfnmap for a region.
  478. * untrack can be called for a specific region indicated by pfn and size or
  479. * can be for the entire vma (in which case pfn, size are zero).
  480. */
  481. static inline void untrack_pfn(struct vm_area_struct *vma,
  482. unsigned long pfn, unsigned long size)
  483. {
  484. }
  485. #else
  486. extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
  487. unsigned long pfn, unsigned long addr,
  488. unsigned long size);
  489. extern int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
  490. unsigned long pfn);
  491. extern int track_pfn_copy(struct vm_area_struct *vma);
  492. extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
  493. unsigned long size);
  494. #endif
  495. #ifdef __HAVE_COLOR_ZERO_PAGE
  496. static inline int is_zero_pfn(unsigned long pfn)
  497. {
  498. extern unsigned long zero_pfn;
  499. unsigned long offset_from_zero_pfn = pfn - zero_pfn;
  500. return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
  501. }
  502. #define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr))
  503. #else
  504. static inline int is_zero_pfn(unsigned long pfn)
  505. {
  506. extern unsigned long zero_pfn;
  507. return pfn == zero_pfn;
  508. }
  509. static inline unsigned long my_zero_pfn(unsigned long addr)
  510. {
  511. extern unsigned long zero_pfn;
  512. return zero_pfn;
  513. }
  514. #endif
  515. #ifdef CONFIG_MMU
  516. #ifndef CONFIG_TRANSPARENT_HUGEPAGE
  517. static inline int pmd_trans_huge(pmd_t pmd)
  518. {
  519. return 0;
  520. }
  521. static inline int pmd_trans_splitting(pmd_t pmd)
  522. {
  523. return 0;
  524. }
  525. #ifndef __HAVE_ARCH_PMD_WRITE
  526. static inline int pmd_write(pmd_t pmd)
  527. {
  528. BUG();
  529. return 0;
  530. }
  531. #endif /* __HAVE_ARCH_PMD_WRITE */
  532. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  533. #ifndef pmd_read_atomic
  534. static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
  535. {
  536. /*
  537. * Depend on compiler for an atomic pmd read. NOTE: this is
  538. * only going to work, if the pmdval_t isn't larger than
  539. * an unsigned long.
  540. */
  541. return *pmdp;
  542. }
  543. #endif
  544. #ifndef pmd_move_must_withdraw
  545. static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl,
  546. spinlock_t *old_pmd_ptl)
  547. {
  548. /*
  549. * With split pmd lock we also need to move preallocated
  550. * PTE page table if new_pmd is on different PMD page table.
  551. */
  552. return new_pmd_ptl != old_pmd_ptl;
  553. }
  554. #endif
  555. /*
  556. * This function is meant to be used by sites walking pagetables with
  557. * the mmap_sem hold in read mode to protect against MADV_DONTNEED and
  558. * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd
  559. * into a null pmd and the transhuge page fault can convert a null pmd
  560. * into an hugepmd or into a regular pmd (if the hugepage allocation
  561. * fails). While holding the mmap_sem in read mode the pmd becomes
  562. * stable and stops changing under us only if it's not null and not a
  563. * transhuge pmd. When those races occurs and this function makes a
  564. * difference vs the standard pmd_none_or_clear_bad, the result is
  565. * undefined so behaving like if the pmd was none is safe (because it
  566. * can return none anyway). The compiler level barrier() is critically
  567. * important to compute the two checks atomically on the same pmdval.
  568. *
  569. * For 32bit kernels with a 64bit large pmd_t this automatically takes
  570. * care of reading the pmd atomically to avoid SMP race conditions
  571. * against pmd_populate() when the mmap_sem is hold for reading by the
  572. * caller (a special atomic read not done by "gcc" as in the generic
  573. * version above, is also needed when THP is disabled because the page
  574. * fault can populate the pmd from under us).
  575. */
  576. static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd)
  577. {
  578. pmd_t pmdval = pmd_read_atomic(pmd);
  579. /*
  580. * The barrier will stabilize the pmdval in a register or on
  581. * the stack so that it will stop changing under the code.
  582. *
  583. * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE,
  584. * pmd_read_atomic is allowed to return a not atomic pmdval
  585. * (for example pointing to an hugepage that has never been
  586. * mapped in the pmd). The below checks will only care about
  587. * the low part of the pmd with 32bit PAE x86 anyway, with the
  588. * exception of pmd_none(). So the important thing is that if
  589. * the low part of the pmd is found null, the high part will
  590. * be also null or the pmd_none() check below would be
  591. * confused.
  592. */
  593. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  594. barrier();
  595. #endif
  596. if (pmd_none(pmdval) || pmd_trans_huge(pmdval))
  597. return 1;
  598. if (unlikely(pmd_bad(pmdval))) {
  599. pmd_clear_bad(pmd);
  600. return 1;
  601. }
  602. return 0;
  603. }
  604. /*
  605. * This is a noop if Transparent Hugepage Support is not built into
  606. * the kernel. Otherwise it is equivalent to
  607. * pmd_none_or_trans_huge_or_clear_bad(), and shall only be called in
  608. * places that already verified the pmd is not none and they want to
  609. * walk ptes while holding the mmap sem in read mode (write mode don't
  610. * need this). If THP is not enabled, the pmd can't go away under the
  611. * code even if MADV_DONTNEED runs, but if THP is enabled we need to
  612. * run a pmd_trans_unstable before walking the ptes after
  613. * split_huge_page_pmd returns (because it may have run when the pmd
  614. * become null, but then a page fault can map in a THP and not a
  615. * regular page).
  616. */
  617. static inline int pmd_trans_unstable(pmd_t *pmd)
  618. {
  619. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  620. return pmd_none_or_trans_huge_or_clear_bad(pmd);
  621. #else
  622. return 0;
  623. #endif
  624. }
  625. #ifndef CONFIG_NUMA_BALANCING
  626. /*
  627. * Technically a PTE can be PROTNONE even when not doing NUMA balancing but
  628. * the only case the kernel cares is for NUMA balancing and is only ever set
  629. * when the VMA is accessible. For PROT_NONE VMAs, the PTEs are not marked
  630. * _PAGE_PROTNONE so by by default, implement the helper as "always no". It
  631. * is the responsibility of the caller to distinguish between PROT_NONE
  632. * protections and NUMA hinting fault protections.
  633. */
  634. static inline int pte_protnone(pte_t pte)
  635. {
  636. return 0;
  637. }
  638. static inline int pmd_protnone(pmd_t pmd)
  639. {
  640. return 0;
  641. }
  642. #endif /* CONFIG_NUMA_BALANCING */
  643. #endif /* CONFIG_MMU */
  644. #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
  645. int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);
  646. int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);
  647. int pud_clear_huge(pud_t *pud);
  648. int pmd_clear_huge(pmd_t *pmd);
  649. #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
  650. static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
  651. {
  652. return 0;
  653. }
  654. static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
  655. {
  656. return 0;
  657. }
  658. static inline int pud_clear_huge(pud_t *pud)
  659. {
  660. return 0;
  661. }
  662. static inline int pmd_clear_huge(pmd_t *pmd)
  663. {
  664. return 0;
  665. }
  666. #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
  667. #endif /* !__ASSEMBLY__ */
  668. #ifndef io_remap_pfn_range
  669. #define io_remap_pfn_range remap_pfn_range
  670. #endif
  671. #endif /* _ASM_GENERIC_PGTABLE_H */