page-flags.h 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693
  1. /*
  2. * Macros for manipulating and testing page->flags
  3. */
  4. #ifndef PAGE_FLAGS_H
  5. #define PAGE_FLAGS_H
  6. #include <linux/types.h>
  7. #include <linux/bug.h>
  8. #include <linux/mmdebug.h>
  9. #ifndef __GENERATING_BOUNDS_H
  10. #include <linux/mm_types.h>
  11. #include <generated/bounds.h>
  12. #endif /* !__GENERATING_BOUNDS_H */
  13. /*
  14. * Various page->flags bits:
  15. *
  16. * PG_reserved is set for special pages, which can never be swapped out. Some
  17. * of them might not even exist (eg empty_bad_page)...
  18. *
  19. * The PG_private bitflag is set on pagecache pages if they contain filesystem
  20. * specific data (which is normally at page->private). It can be used by
  21. * private allocations for its own usage.
  22. *
  23. * During initiation of disk I/O, PG_locked is set. This bit is set before I/O
  24. * and cleared when writeback _starts_ or when read _completes_. PG_writeback
  25. * is set before writeback starts and cleared when it finishes.
  26. *
  27. * PG_locked also pins a page in pagecache, and blocks truncation of the file
  28. * while it is held.
  29. *
  30. * page_waitqueue(page) is a wait queue of all tasks waiting for the page
  31. * to become unlocked.
  32. *
  33. * PG_uptodate tells whether the page's contents is valid. When a read
  34. * completes, the page becomes uptodate, unless a disk I/O error happened.
  35. *
  36. * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
  37. * file-backed pagecache (see mm/vmscan.c).
  38. *
  39. * PG_error is set to indicate that an I/O error occurred on this page.
  40. *
  41. * PG_arch_1 is an architecture specific page state bit. The generic code
  42. * guarantees that this bit is cleared for a page when it first is entered into
  43. * the page cache.
  44. *
  45. * PG_highmem pages are not permanently mapped into the kernel virtual address
  46. * space, they need to be kmapped separately for doing IO on the pages. The
  47. * struct page (these bits with information) are always mapped into kernel
  48. * address space...
  49. *
  50. * PG_hwpoison indicates that a page got corrupted in hardware and contains
  51. * data with incorrect ECC bits that triggered a machine check. Accessing is
  52. * not safe since it may cause another machine check. Don't touch!
  53. */
  54. /*
  55. * Don't use the *_dontuse flags. Use the macros. Otherwise you'll break
  56. * locked- and dirty-page accounting.
  57. *
  58. * The page flags field is split into two parts, the main flags area
  59. * which extends from the low bits upwards, and the fields area which
  60. * extends from the high bits downwards.
  61. *
  62. * | FIELD | ... | FLAGS |
  63. * N-1 ^ 0
  64. * (NR_PAGEFLAGS)
  65. *
  66. * The fields area is reserved for fields mapping zone, node (for NUMA) and
  67. * SPARSEMEM section (for variants of SPARSEMEM that require section ids like
  68. * SPARSEMEM_EXTREME with !SPARSEMEM_VMEMMAP).
  69. */
  70. enum pageflags {
  71. PG_locked, /* Page is locked. Don't touch. */
  72. PG_error,
  73. PG_referenced,
  74. PG_uptodate,
  75. PG_dirty,
  76. PG_lru,
  77. PG_active,
  78. PG_slab,
  79. PG_owner_priv_1, /* Owner use. If pagecache, fs may use*/
  80. PG_arch_1,
  81. PG_reserved,
  82. PG_private, /* If pagecache, has fs-private data */
  83. PG_private_2, /* If pagecache, has fs aux data */
  84. PG_writeback, /* Page is under writeback */
  85. PG_head, /* A head page */
  86. PG_swapcache, /* Swap page: swp_entry_t in private */
  87. PG_mappedtodisk, /* Has blocks allocated on-disk */
  88. PG_reclaim, /* To be reclaimed asap */
  89. PG_swapbacked, /* Page is backed by RAM/swap */
  90. PG_unevictable, /* Page is "unevictable" */
  91. #ifdef CONFIG_MMU
  92. PG_mlocked, /* Page is vma mlocked */
  93. #endif
  94. #ifdef CONFIG_ARCH_USES_PG_UNCACHED
  95. PG_uncached, /* Page has been mapped as uncached */
  96. #endif
  97. #ifdef CONFIG_MEMORY_FAILURE
  98. PG_hwpoison, /* hardware poisoned page. Don't touch */
  99. #endif
  100. #if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
  101. PG_young,
  102. PG_idle,
  103. #endif
  104. __NR_PAGEFLAGS,
  105. /* Filesystems */
  106. PG_checked = PG_owner_priv_1,
  107. /* Two page bits are conscripted by FS-Cache to maintain local caching
  108. * state. These bits are set on pages belonging to the netfs's inodes
  109. * when those inodes are being locally cached.
  110. */
  111. PG_fscache = PG_private_2, /* page backed by cache */
  112. /* XEN */
  113. /* Pinned in Xen as a read-only pagetable page. */
  114. PG_pinned = PG_owner_priv_1,
  115. /* Pinned as part of domain save (see xen_mm_pin_all()). */
  116. PG_savepinned = PG_dirty,
  117. /* Has a grant mapping of another (foreign) domain's page. */
  118. PG_foreign = PG_owner_priv_1,
  119. /* SLOB */
  120. PG_slob_free = PG_private,
  121. /* Compound pages. Stored in first tail page's flags */
  122. PG_double_map = PG_private_2,
  123. };
  124. #ifndef __GENERATING_BOUNDS_H
  125. struct page; /* forward declaration */
  126. static inline struct page *compound_head(struct page *page)
  127. {
  128. unsigned long head = READ_ONCE(page->compound_head);
  129. if (unlikely(head & 1))
  130. return (struct page *) (head - 1);
  131. return page;
  132. }
  133. static inline int PageTail(struct page *page)
  134. {
  135. return READ_ONCE(page->compound_head) & 1;
  136. }
  137. static inline int PageCompound(struct page *page)
  138. {
  139. return test_bit(PG_head, &page->flags) || PageTail(page);
  140. }
  141. /*
  142. * Page flags policies wrt compound pages
  143. *
  144. * PF_ANY:
  145. * the page flag is relevant for small, head and tail pages.
  146. *
  147. * PF_HEAD:
  148. * for compound page all operations related to the page flag applied to
  149. * head page.
  150. *
  151. * PF_NO_TAIL:
  152. * modifications of the page flag must be done on small or head pages,
  153. * checks can be done on tail pages too.
  154. *
  155. * PF_NO_COMPOUND:
  156. * the page flag is not relevant for compound pages.
  157. */
  158. #define PF_ANY(page, enforce) page
  159. #define PF_HEAD(page, enforce) compound_head(page)
  160. #define PF_NO_TAIL(page, enforce) ({ \
  161. VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
  162. compound_head(page);})
  163. #define PF_NO_COMPOUND(page, enforce) ({ \
  164. VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
  165. page;})
  166. /*
  167. * Macros to create function definitions for page flags
  168. */
  169. #define TESTPAGEFLAG(uname, lname, policy) \
  170. static inline int Page##uname(struct page *page) \
  171. { return test_bit(PG_##lname, &policy(page, 0)->flags); }
  172. #define SETPAGEFLAG(uname, lname, policy) \
  173. static inline void SetPage##uname(struct page *page) \
  174. { set_bit(PG_##lname, &policy(page, 1)->flags); }
  175. #define CLEARPAGEFLAG(uname, lname, policy) \
  176. static inline void ClearPage##uname(struct page *page) \
  177. { clear_bit(PG_##lname, &policy(page, 1)->flags); }
  178. #define __SETPAGEFLAG(uname, lname, policy) \
  179. static inline void __SetPage##uname(struct page *page) \
  180. { __set_bit(PG_##lname, &policy(page, 1)->flags); }
  181. #define __CLEARPAGEFLAG(uname, lname, policy) \
  182. static inline void __ClearPage##uname(struct page *page) \
  183. { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
  184. #define TESTSETFLAG(uname, lname, policy) \
  185. static inline int TestSetPage##uname(struct page *page) \
  186. { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
  187. #define TESTCLEARFLAG(uname, lname, policy) \
  188. static inline int TestClearPage##uname(struct page *page) \
  189. { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
  190. #define PAGEFLAG(uname, lname, policy) \
  191. TESTPAGEFLAG(uname, lname, policy) \
  192. SETPAGEFLAG(uname, lname, policy) \
  193. CLEARPAGEFLAG(uname, lname, policy)
  194. #define __PAGEFLAG(uname, lname, policy) \
  195. TESTPAGEFLAG(uname, lname, policy) \
  196. __SETPAGEFLAG(uname, lname, policy) \
  197. __CLEARPAGEFLAG(uname, lname, policy)
  198. #define TESTSCFLAG(uname, lname, policy) \
  199. TESTSETFLAG(uname, lname, policy) \
  200. TESTCLEARFLAG(uname, lname, policy)
  201. #define TESTPAGEFLAG_FALSE(uname) \
  202. static inline int Page##uname(const struct page *page) { return 0; }
  203. #define SETPAGEFLAG_NOOP(uname) \
  204. static inline void SetPage##uname(struct page *page) { }
  205. #define CLEARPAGEFLAG_NOOP(uname) \
  206. static inline void ClearPage##uname(struct page *page) { }
  207. #define __CLEARPAGEFLAG_NOOP(uname) \
  208. static inline void __ClearPage##uname(struct page *page) { }
  209. #define TESTSETFLAG_FALSE(uname) \
  210. static inline int TestSetPage##uname(struct page *page) { return 0; }
  211. #define TESTCLEARFLAG_FALSE(uname) \
  212. static inline int TestClearPage##uname(struct page *page) { return 0; }
  213. #define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
  214. SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
  215. #define TESTSCFLAG_FALSE(uname) \
  216. TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
  217. __PAGEFLAG(Locked, locked, PF_NO_TAIL)
  218. PAGEFLAG(Error, error, PF_NO_COMPOUND) TESTCLEARFLAG(Error, error, PF_NO_COMPOUND)
  219. PAGEFLAG(Referenced, referenced, PF_HEAD)
  220. TESTCLEARFLAG(Referenced, referenced, PF_HEAD)
  221. __SETPAGEFLAG(Referenced, referenced, PF_HEAD)
  222. PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
  223. __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
  224. PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
  225. PAGEFLAG(Active, active, PF_HEAD) __CLEARPAGEFLAG(Active, active, PF_HEAD)
  226. TESTCLEARFLAG(Active, active, PF_HEAD)
  227. __PAGEFLAG(Slab, slab, PF_NO_TAIL)
  228. __PAGEFLAG(SlobFree, slob_free, PF_NO_TAIL)
  229. PAGEFLAG(Checked, checked, PF_NO_COMPOUND) /* Used by some filesystems */
  230. /* Xen */
  231. PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
  232. TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
  233. PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
  234. PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
  235. PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
  236. __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
  237. PAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
  238. __CLEARPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
  239. __SETPAGEFLAG(SwapBacked, swapbacked, PF_NO_TAIL)
  240. /*
  241. * Private page markings that may be used by the filesystem that owns the page
  242. * for its own purposes.
  243. * - PG_private and PG_private_2 cause releasepage() and co to be invoked
  244. */
  245. PAGEFLAG(Private, private, PF_ANY) __SETPAGEFLAG(Private, private, PF_ANY)
  246. __CLEARPAGEFLAG(Private, private, PF_ANY)
  247. PAGEFLAG(Private2, private_2, PF_ANY) TESTSCFLAG(Private2, private_2, PF_ANY)
  248. PAGEFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
  249. TESTCLEARFLAG(OwnerPriv1, owner_priv_1, PF_ANY)
  250. /*
  251. * Only test-and-set exist for PG_writeback. The unconditional operators are
  252. * risky: they bypass page accounting.
  253. */
  254. TESTPAGEFLAG(Writeback, writeback, PF_NO_COMPOUND)
  255. TESTSCFLAG(Writeback, writeback, PF_NO_COMPOUND)
  256. PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_COMPOUND)
  257. /* PG_readahead is only used for reads; PG_reclaim is only for writes */
  258. PAGEFLAG(Reclaim, reclaim, PF_NO_COMPOUND)
  259. TESTCLEARFLAG(Reclaim, reclaim, PF_NO_COMPOUND)
  260. PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
  261. TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
  262. #ifdef CONFIG_HIGHMEM
  263. /*
  264. * Must use a macro here due to header dependency issues. page_zone() is not
  265. * available at this point.
  266. */
  267. #define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
  268. #else
  269. PAGEFLAG_FALSE(HighMem)
  270. #endif
  271. #ifdef CONFIG_SWAP
  272. PAGEFLAG(SwapCache, swapcache, PF_NO_COMPOUND)
  273. #else
  274. PAGEFLAG_FALSE(SwapCache)
  275. #endif
  276. PAGEFLAG(Unevictable, unevictable, PF_HEAD)
  277. __CLEARPAGEFLAG(Unevictable, unevictable, PF_HEAD)
  278. TESTCLEARFLAG(Unevictable, unevictable, PF_HEAD)
  279. #ifdef CONFIG_MMU
  280. PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
  281. __CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
  282. TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
  283. #else
  284. PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
  285. TESTSCFLAG_FALSE(Mlocked)
  286. #endif
  287. #ifdef CONFIG_ARCH_USES_PG_UNCACHED
  288. PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
  289. #else
  290. PAGEFLAG_FALSE(Uncached)
  291. #endif
  292. #ifdef CONFIG_MEMORY_FAILURE
  293. PAGEFLAG(HWPoison, hwpoison, PF_ANY)
  294. TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
  295. #define __PG_HWPOISON (1UL << PG_hwpoison)
  296. #else
  297. PAGEFLAG_FALSE(HWPoison)
  298. #define __PG_HWPOISON 0
  299. #endif
  300. #if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
  301. TESTPAGEFLAG(Young, young, PF_ANY)
  302. SETPAGEFLAG(Young, young, PF_ANY)
  303. TESTCLEARFLAG(Young, young, PF_ANY)
  304. PAGEFLAG(Idle, idle, PF_ANY)
  305. #endif
  306. /*
  307. * On an anonymous page mapped into a user virtual memory area,
  308. * page->mapping points to its anon_vma, not to a struct address_space;
  309. * with the PAGE_MAPPING_ANON bit set to distinguish it. See rmap.h.
  310. *
  311. * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
  312. * the PAGE_MAPPING_KSM bit may be set along with the PAGE_MAPPING_ANON bit;
  313. * and then page->mapping points, not to an anon_vma, but to a private
  314. * structure which KSM associates with that merged page. See ksm.h.
  315. *
  316. * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.
  317. *
  318. * Please note that, confusingly, "page_mapping" refers to the inode
  319. * address_space which maps the page from disk; whereas "page_mapped"
  320. * refers to user virtual address space into which the page is mapped.
  321. */
  322. #define PAGE_MAPPING_ANON 1
  323. #define PAGE_MAPPING_KSM 2
  324. #define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
  325. static inline int PageAnon(struct page *page)
  326. {
  327. page = compound_head(page);
  328. return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
  329. }
  330. #ifdef CONFIG_KSM
  331. /*
  332. * A KSM page is one of those write-protected "shared pages" or "merged pages"
  333. * which KSM maps into multiple mms, wherever identical anonymous page content
  334. * is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
  335. * anon_vma, but to that page's node of the stable tree.
  336. */
  337. static inline int PageKsm(struct page *page)
  338. {
  339. page = compound_head(page);
  340. return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
  341. (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
  342. }
  343. #else
  344. TESTPAGEFLAG_FALSE(Ksm)
  345. #endif
  346. u64 stable_page_flags(struct page *page);
  347. static inline int PageUptodate(struct page *page)
  348. {
  349. int ret;
  350. page = compound_head(page);
  351. ret = test_bit(PG_uptodate, &(page)->flags);
  352. /*
  353. * Must ensure that the data we read out of the page is loaded
  354. * _after_ we've loaded page->flags to check for PageUptodate.
  355. * We can skip the barrier if the page is not uptodate, because
  356. * we wouldn't be reading anything from it.
  357. *
  358. * See SetPageUptodate() for the other side of the story.
  359. */
  360. if (ret)
  361. smp_rmb();
  362. return ret;
  363. }
  364. static inline void __SetPageUptodate(struct page *page)
  365. {
  366. VM_BUG_ON_PAGE(PageTail(page), page);
  367. smp_wmb();
  368. __set_bit(PG_uptodate, &page->flags);
  369. }
  370. static inline void SetPageUptodate(struct page *page)
  371. {
  372. VM_BUG_ON_PAGE(PageTail(page), page);
  373. /*
  374. * Memory barrier must be issued before setting the PG_uptodate bit,
  375. * so that all previous stores issued in order to bring the page
  376. * uptodate are actually visible before PageUptodate becomes true.
  377. */
  378. smp_wmb();
  379. set_bit(PG_uptodate, &page->flags);
  380. }
  381. CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
  382. int test_clear_page_writeback(struct page *page);
  383. int __test_set_page_writeback(struct page *page, bool keep_write);
  384. #define test_set_page_writeback(page) \
  385. __test_set_page_writeback(page, false)
  386. #define test_set_page_writeback_keepwrite(page) \
  387. __test_set_page_writeback(page, true)
  388. static inline void set_page_writeback(struct page *page)
  389. {
  390. test_set_page_writeback(page);
  391. }
  392. static inline void set_page_writeback_keepwrite(struct page *page)
  393. {
  394. test_set_page_writeback_keepwrite(page);
  395. }
  396. __PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
  397. static inline void set_compound_head(struct page *page, struct page *head)
  398. {
  399. WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
  400. }
  401. static inline void clear_compound_head(struct page *page)
  402. {
  403. WRITE_ONCE(page->compound_head, 0);
  404. }
  405. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  406. static inline void ClearPageCompound(struct page *page)
  407. {
  408. BUG_ON(!PageHead(page));
  409. ClearPageHead(page);
  410. }
  411. #endif
  412. #define PG_head_mask ((1L << PG_head))
  413. #ifdef CONFIG_HUGETLB_PAGE
  414. int PageHuge(struct page *page);
  415. int PageHeadHuge(struct page *page);
  416. bool page_huge_active(struct page *page);
  417. #else
  418. TESTPAGEFLAG_FALSE(Huge)
  419. TESTPAGEFLAG_FALSE(HeadHuge)
  420. static inline bool page_huge_active(struct page *page)
  421. {
  422. return 0;
  423. }
  424. #endif
  425. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  426. /*
  427. * PageHuge() only returns true for hugetlbfs pages, but not for
  428. * normal or transparent huge pages.
  429. *
  430. * PageTransHuge() returns true for both transparent huge and
  431. * hugetlbfs pages, but not normal pages. PageTransHuge() can only be
  432. * called only in the core VM paths where hugetlbfs pages can't exist.
  433. */
  434. static inline int PageTransHuge(struct page *page)
  435. {
  436. VM_BUG_ON_PAGE(PageTail(page), page);
  437. return PageHead(page);
  438. }
  439. /*
  440. * PageTransCompound returns true for both transparent huge pages
  441. * and hugetlbfs pages, so it should only be called when it's known
  442. * that hugetlbfs pages aren't involved.
  443. */
  444. static inline int PageTransCompound(struct page *page)
  445. {
  446. return PageCompound(page);
  447. }
  448. /*
  449. * PageTransTail returns true for both transparent huge pages
  450. * and hugetlbfs pages, so it should only be called when it's known
  451. * that hugetlbfs pages aren't involved.
  452. */
  453. static inline int PageTransTail(struct page *page)
  454. {
  455. return PageTail(page);
  456. }
  457. /*
  458. * PageDoubleMap indicates that the compound page is mapped with PTEs as well
  459. * as PMDs.
  460. *
  461. * This is required for optimization of rmap operations for THP: we can postpone
  462. * per small page mapcount accounting (and its overhead from atomic operations)
  463. * until the first PMD split.
  464. *
  465. * For the page PageDoubleMap means ->_mapcount in all sub-pages is offset up
  466. * by one. This reference will go away with last compound_mapcount.
  467. *
  468. * See also __split_huge_pmd_locked() and page_remove_anon_compound_rmap().
  469. */
  470. static inline int PageDoubleMap(struct page *page)
  471. {
  472. return PageHead(page) && test_bit(PG_double_map, &page[1].flags);
  473. }
  474. static inline int TestSetPageDoubleMap(struct page *page)
  475. {
  476. VM_BUG_ON_PAGE(!PageHead(page), page);
  477. return test_and_set_bit(PG_double_map, &page[1].flags);
  478. }
  479. static inline int TestClearPageDoubleMap(struct page *page)
  480. {
  481. VM_BUG_ON_PAGE(!PageHead(page), page);
  482. return test_and_clear_bit(PG_double_map, &page[1].flags);
  483. }
  484. #else
  485. TESTPAGEFLAG_FALSE(TransHuge)
  486. TESTPAGEFLAG_FALSE(TransCompound)
  487. TESTPAGEFLAG_FALSE(TransTail)
  488. TESTPAGEFLAG_FALSE(DoubleMap)
  489. TESTSETFLAG_FALSE(DoubleMap)
  490. TESTCLEARFLAG_FALSE(DoubleMap)
  491. #endif
  492. /*
  493. * PageBuddy() indicate that the page is free and in the buddy system
  494. * (see mm/page_alloc.c).
  495. *
  496. * PAGE_BUDDY_MAPCOUNT_VALUE must be <= -2 but better not too close to
  497. * -2 so that an underflow of the page_mapcount() won't be mistaken
  498. * for a genuine PAGE_BUDDY_MAPCOUNT_VALUE. -128 can be created very
  499. * efficiently by most CPU architectures.
  500. */
  501. #define PAGE_BUDDY_MAPCOUNT_VALUE (-128)
  502. static inline int PageBuddy(struct page *page)
  503. {
  504. return atomic_read(&page->_mapcount) == PAGE_BUDDY_MAPCOUNT_VALUE;
  505. }
  506. static inline void __SetPageBuddy(struct page *page)
  507. {
  508. VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page);
  509. atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);
  510. }
  511. static inline void __ClearPageBuddy(struct page *page)
  512. {
  513. VM_BUG_ON_PAGE(!PageBuddy(page), page);
  514. atomic_set(&page->_mapcount, -1);
  515. }
  516. extern bool is_free_buddy_page(struct page *page);
  517. #define PAGE_BALLOON_MAPCOUNT_VALUE (-256)
  518. static inline int PageBalloon(struct page *page)
  519. {
  520. return atomic_read(&page->_mapcount) == PAGE_BALLOON_MAPCOUNT_VALUE;
  521. }
  522. static inline void __SetPageBalloon(struct page *page)
  523. {
  524. VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page);
  525. atomic_set(&page->_mapcount, PAGE_BALLOON_MAPCOUNT_VALUE);
  526. }
  527. static inline void __ClearPageBalloon(struct page *page)
  528. {
  529. VM_BUG_ON_PAGE(!PageBalloon(page), page);
  530. atomic_set(&page->_mapcount, -1);
  531. }
  532. /*
  533. * If network-based swap is enabled, sl*b must keep track of whether pages
  534. * were allocated from pfmemalloc reserves.
  535. */
  536. static inline int PageSlabPfmemalloc(struct page *page)
  537. {
  538. VM_BUG_ON_PAGE(!PageSlab(page), page);
  539. return PageActive(page);
  540. }
  541. static inline void SetPageSlabPfmemalloc(struct page *page)
  542. {
  543. VM_BUG_ON_PAGE(!PageSlab(page), page);
  544. SetPageActive(page);
  545. }
  546. static inline void __ClearPageSlabPfmemalloc(struct page *page)
  547. {
  548. VM_BUG_ON_PAGE(!PageSlab(page), page);
  549. __ClearPageActive(page);
  550. }
  551. static inline void ClearPageSlabPfmemalloc(struct page *page)
  552. {
  553. VM_BUG_ON_PAGE(!PageSlab(page), page);
  554. ClearPageActive(page);
  555. }
  556. #ifdef CONFIG_MMU
  557. #define __PG_MLOCKED (1 << PG_mlocked)
  558. #else
  559. #define __PG_MLOCKED 0
  560. #endif
  561. /*
  562. * Flags checked when a page is freed. Pages being freed should not have
  563. * these flags set. It they are, there is a problem.
  564. */
  565. #define PAGE_FLAGS_CHECK_AT_FREE \
  566. (1 << PG_lru | 1 << PG_locked | \
  567. 1 << PG_private | 1 << PG_private_2 | \
  568. 1 << PG_writeback | 1 << PG_reserved | \
  569. 1 << PG_slab | 1 << PG_swapcache | 1 << PG_active | \
  570. 1 << PG_unevictable | __PG_MLOCKED)
  571. /*
  572. * Flags checked when a page is prepped for return by the page allocator.
  573. * Pages being prepped should not have these flags set. It they are set,
  574. * there has been a kernel bug or struct page corruption.
  575. *
  576. * __PG_HWPOISON is exceptional because it needs to be kept beyond page's
  577. * alloc-free cycle to prevent from reusing the page.
  578. */
  579. #define PAGE_FLAGS_CHECK_AT_PREP \
  580. (((1 << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
  581. #define PAGE_FLAGS_PRIVATE \
  582. (1 << PG_private | 1 << PG_private_2)
  583. /**
  584. * page_has_private - Determine if page has private stuff
  585. * @page: The page to be checked
  586. *
  587. * Determine if a page has private stuff, indicating that release routines
  588. * should be invoked upon it.
  589. */
  590. static inline int page_has_private(struct page *page)
  591. {
  592. return !!(page->flags & PAGE_FLAGS_PRIVATE);
  593. }
  594. #undef PF_ANY
  595. #undef PF_HEAD
  596. #undef PF_NO_TAIL
  597. #undef PF_NO_COMPOUND
  598. #endif /* !__GENERATING_BOUNDS_H */
  599. #endif /* PAGE_FLAGS_H */