page_owner.c 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313
  1. #include <linux/debugfs.h>
  2. #include <linux/mm.h>
  3. #include <linux/slab.h>
  4. #include <linux/uaccess.h>
  5. #include <linux/bootmem.h>
  6. #include <linux/stacktrace.h>
  7. #include <linux/page_owner.h>
  8. #include "internal.h"
  9. static bool page_owner_disabled = true;
  10. bool page_owner_inited __read_mostly;
  11. static void init_early_allocated_pages(void);
  12. static int early_page_owner_param(char *buf)
  13. {
  14. if (!buf)
  15. return -EINVAL;
  16. if (strcmp(buf, "on") == 0)
  17. page_owner_disabled = false;
  18. return 0;
  19. }
  20. early_param("page_owner", early_page_owner_param);
  21. static bool need_page_owner(void)
  22. {
  23. if (page_owner_disabled)
  24. return false;
  25. return true;
  26. }
  27. static void init_page_owner(void)
  28. {
  29. if (page_owner_disabled)
  30. return;
  31. page_owner_inited = true;
  32. init_early_allocated_pages();
  33. }
  34. struct page_ext_operations page_owner_ops = {
  35. .need = need_page_owner,
  36. .init = init_page_owner,
  37. };
  38. void __reset_page_owner(struct page *page, unsigned int order)
  39. {
  40. int i;
  41. struct page_ext *page_ext;
  42. for (i = 0; i < (1 << order); i++) {
  43. page_ext = lookup_page_ext(page + i);
  44. __clear_bit(PAGE_EXT_OWNER, &page_ext->flags);
  45. }
  46. }
  47. void __set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask)
  48. {
  49. struct page_ext *page_ext = lookup_page_ext(page);
  50. struct stack_trace trace = {
  51. .nr_entries = 0,
  52. .max_entries = ARRAY_SIZE(page_ext->trace_entries),
  53. .entries = &page_ext->trace_entries[0],
  54. .skip = 3,
  55. };
  56. save_stack_trace(&trace);
  57. page_ext->order = order;
  58. page_ext->gfp_mask = gfp_mask;
  59. page_ext->nr_entries = trace.nr_entries;
  60. __set_bit(PAGE_EXT_OWNER, &page_ext->flags);
  61. }
  62. static ssize_t
  63. print_page_owner(char __user *buf, size_t count, unsigned long pfn,
  64. struct page *page, struct page_ext *page_ext)
  65. {
  66. int ret;
  67. int pageblock_mt, page_mt;
  68. char *kbuf;
  69. struct stack_trace trace = {
  70. .nr_entries = page_ext->nr_entries,
  71. .entries = &page_ext->trace_entries[0],
  72. };
  73. kbuf = kmalloc(count, GFP_KERNEL);
  74. if (!kbuf)
  75. return -ENOMEM;
  76. ret = snprintf(kbuf, count,
  77. "Page allocated via order %u, mask 0x%x\n",
  78. page_ext->order, page_ext->gfp_mask);
  79. if (ret >= count)
  80. goto err;
  81. /* Print information relevant to grouping pages by mobility */
  82. pageblock_mt = get_pfnblock_migratetype(page, pfn);
  83. page_mt = gfpflags_to_migratetype(page_ext->gfp_mask);
  84. ret += snprintf(kbuf + ret, count - ret,
  85. "PFN %lu Block %lu type %d %s Flags %s%s%s%s%s%s%s%s%s%s%s%s\n",
  86. pfn,
  87. pfn >> pageblock_order,
  88. pageblock_mt,
  89. pageblock_mt != page_mt ? "Fallback" : " ",
  90. PageLocked(page) ? "K" : " ",
  91. PageError(page) ? "E" : " ",
  92. PageReferenced(page) ? "R" : " ",
  93. PageUptodate(page) ? "U" : " ",
  94. PageDirty(page) ? "D" : " ",
  95. PageLRU(page) ? "L" : " ",
  96. PageActive(page) ? "A" : " ",
  97. PageSlab(page) ? "S" : " ",
  98. PageWriteback(page) ? "W" : " ",
  99. PageCompound(page) ? "C" : " ",
  100. PageSwapCache(page) ? "B" : " ",
  101. PageMappedToDisk(page) ? "M" : " ");
  102. if (ret >= count)
  103. goto err;
  104. ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0);
  105. if (ret >= count)
  106. goto err;
  107. ret += snprintf(kbuf + ret, count - ret, "\n");
  108. if (ret >= count)
  109. goto err;
  110. if (copy_to_user(buf, kbuf, ret))
  111. ret = -EFAULT;
  112. kfree(kbuf);
  113. return ret;
  114. err:
  115. kfree(kbuf);
  116. return -ENOMEM;
  117. }
  118. static ssize_t
  119. read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
  120. {
  121. unsigned long pfn;
  122. struct page *page;
  123. struct page_ext *page_ext;
  124. if (!page_owner_inited)
  125. return -EINVAL;
  126. page = NULL;
  127. pfn = min_low_pfn + *ppos;
  128. /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
  129. while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
  130. pfn++;
  131. drain_all_pages(NULL);
  132. /* Find an allocated page */
  133. for (; pfn < max_pfn; pfn++) {
  134. /*
  135. * If the new page is in a new MAX_ORDER_NR_PAGES area,
  136. * validate the area as existing, skip it if not
  137. */
  138. if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
  139. pfn += MAX_ORDER_NR_PAGES - 1;
  140. continue;
  141. }
  142. /* Check for holes within a MAX_ORDER area */
  143. if (!pfn_valid_within(pfn))
  144. continue;
  145. page = pfn_to_page(pfn);
  146. if (PageBuddy(page)) {
  147. unsigned long freepage_order = page_order_unsafe(page);
  148. if (freepage_order < MAX_ORDER)
  149. pfn += (1UL << freepage_order) - 1;
  150. continue;
  151. }
  152. page_ext = lookup_page_ext(page);
  153. /*
  154. * Some pages could be missed by concurrent allocation or free,
  155. * because we don't hold the zone lock.
  156. */
  157. if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
  158. continue;
  159. /* Record the next PFN to read in the file offset */
  160. *ppos = (pfn - min_low_pfn) + 1;
  161. return print_page_owner(buf, count, pfn, page, page_ext);
  162. }
  163. return 0;
  164. }
  165. static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
  166. {
  167. struct page *page;
  168. struct page_ext *page_ext;
  169. unsigned long pfn = zone->zone_start_pfn, block_end_pfn;
  170. unsigned long end_pfn = pfn + zone->spanned_pages;
  171. unsigned long count = 0;
  172. /* Scan block by block. First and last block may be incomplete */
  173. pfn = zone->zone_start_pfn;
  174. /*
  175. * Walk the zone in pageblock_nr_pages steps. If a page block spans
  176. * a zone boundary, it will be double counted between zones. This does
  177. * not matter as the mixed block count will still be correct
  178. */
  179. for (; pfn < end_pfn; ) {
  180. if (!pfn_valid(pfn)) {
  181. pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
  182. continue;
  183. }
  184. block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
  185. block_end_pfn = min(block_end_pfn, end_pfn);
  186. page = pfn_to_page(pfn);
  187. for (; pfn < block_end_pfn; pfn++) {
  188. if (!pfn_valid_within(pfn))
  189. continue;
  190. page = pfn_to_page(pfn);
  191. /*
  192. * We are safe to check buddy flag and order, because
  193. * this is init stage and only single thread runs.
  194. */
  195. if (PageBuddy(page)) {
  196. pfn += (1UL << page_order(page)) - 1;
  197. continue;
  198. }
  199. if (PageReserved(page))
  200. continue;
  201. page_ext = lookup_page_ext(page);
  202. /* Maybe overraping zone */
  203. if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
  204. continue;
  205. /* Found early allocated page */
  206. set_page_owner(page, 0, 0);
  207. count++;
  208. }
  209. }
  210. pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
  211. pgdat->node_id, zone->name, count);
  212. }
  213. static void init_zones_in_node(pg_data_t *pgdat)
  214. {
  215. struct zone *zone;
  216. struct zone *node_zones = pgdat->node_zones;
  217. unsigned long flags;
  218. for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
  219. if (!populated_zone(zone))
  220. continue;
  221. spin_lock_irqsave(&zone->lock, flags);
  222. init_pages_in_zone(pgdat, zone);
  223. spin_unlock_irqrestore(&zone->lock, flags);
  224. }
  225. }
  226. static void init_early_allocated_pages(void)
  227. {
  228. pg_data_t *pgdat;
  229. drain_all_pages(NULL);
  230. for_each_online_pgdat(pgdat)
  231. init_zones_in_node(pgdat);
  232. }
  233. static const struct file_operations proc_page_owner_operations = {
  234. .read = read_page_owner,
  235. };
  236. static int __init pageowner_init(void)
  237. {
  238. struct dentry *dentry;
  239. if (!page_owner_inited) {
  240. pr_info("page_owner is disabled\n");
  241. return 0;
  242. }
  243. dentry = debugfs_create_file("page_owner", S_IRUSR, NULL,
  244. NULL, &proc_page_owner_operations);
  245. if (IS_ERR(dentry))
  246. return PTR_ERR(dentry);
  247. return 0;
  248. }
  249. module_init(pageowner_init)