khugepaged.c 49 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946
  1. // SPDX-License-Identifier: GPL-2.0
  2. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  3. #include <linux/mm.h>
  4. #include <linux/sched.h>
  5. #include <linux/sched/mm.h>
  6. #include <linux/sched/coredump.h>
  7. #include <linux/mmu_notifier.h>
  8. #include <linux/rmap.h>
  9. #include <linux/swap.h>
  10. #include <linux/mm_inline.h>
  11. #include <linux/kthread.h>
  12. #include <linux/khugepaged.h>
  13. #include <linux/freezer.h>
  14. #include <linux/mman.h>
  15. #include <linux/hashtable.h>
  16. #include <linux/userfaultfd_k.h>
  17. #include <linux/page_idle.h>
  18. #include <linux/swapops.h>
  19. #include <linux/shmem_fs.h>
  20. #include <asm/tlb.h>
  21. #include <asm/pgalloc.h>
  22. #include "internal.h"
  23. enum scan_result {
  24. SCAN_FAIL,
  25. SCAN_SUCCEED,
  26. SCAN_PMD_NULL,
  27. SCAN_EXCEED_NONE_PTE,
  28. SCAN_PTE_NON_PRESENT,
  29. SCAN_PAGE_RO,
  30. SCAN_LACK_REFERENCED_PAGE,
  31. SCAN_PAGE_NULL,
  32. SCAN_SCAN_ABORT,
  33. SCAN_PAGE_COUNT,
  34. SCAN_PAGE_LRU,
  35. SCAN_PAGE_LOCK,
  36. SCAN_PAGE_ANON,
  37. SCAN_PAGE_COMPOUND,
  38. SCAN_ANY_PROCESS,
  39. SCAN_VMA_NULL,
  40. SCAN_VMA_CHECK,
  41. SCAN_ADDRESS_RANGE,
  42. SCAN_SWAP_CACHE_PAGE,
  43. SCAN_DEL_PAGE_LRU,
  44. SCAN_ALLOC_HUGE_PAGE_FAIL,
  45. SCAN_CGROUP_CHARGE_FAIL,
  46. SCAN_EXCEED_SWAP_PTE,
  47. SCAN_TRUNCATED,
  48. };
  49. #define CREATE_TRACE_POINTS
  50. #include <trace/events/huge_memory.h>
  51. /* default scan 8*512 pte (or vmas) every 30 second */
  52. static unsigned int khugepaged_pages_to_scan __read_mostly;
  53. static unsigned int khugepaged_pages_collapsed;
  54. static unsigned int khugepaged_full_scans;
  55. static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
  56. /* during fragmentation poll the hugepage allocator once every minute */
  57. static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
  58. static unsigned long khugepaged_sleep_expire;
  59. static DEFINE_SPINLOCK(khugepaged_mm_lock);
  60. static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
  61. /*
  62. * default collapse hugepages if there is at least one pte mapped like
  63. * it would have happened if the vma was large enough during page
  64. * fault.
  65. */
  66. static unsigned int khugepaged_max_ptes_none __read_mostly;
  67. static unsigned int khugepaged_max_ptes_swap __read_mostly;
  68. #define MM_SLOTS_HASH_BITS 10
  69. static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
  70. static struct kmem_cache *mm_slot_cache __read_mostly;
  71. /**
  72. * struct mm_slot - hash lookup from mm to mm_slot
  73. * @hash: hash collision list
  74. * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
  75. * @mm: the mm that this information is valid for
  76. */
  77. struct mm_slot {
  78. struct hlist_node hash;
  79. struct list_head mm_node;
  80. struct mm_struct *mm;
  81. };
  82. /**
  83. * struct khugepaged_scan - cursor for scanning
  84. * @mm_head: the head of the mm list to scan
  85. * @mm_slot: the current mm_slot we are scanning
  86. * @address: the next address inside that to be scanned
  87. *
  88. * There is only the one khugepaged_scan instance of this cursor structure.
  89. */
  90. struct khugepaged_scan {
  91. struct list_head mm_head;
  92. struct mm_slot *mm_slot;
  93. unsigned long address;
  94. };
  95. static struct khugepaged_scan khugepaged_scan = {
  96. .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
  97. };
  98. #ifdef CONFIG_SYSFS
  99. static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
  100. struct kobj_attribute *attr,
  101. char *buf)
  102. {
  103. return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);
  104. }
  105. static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
  106. struct kobj_attribute *attr,
  107. const char *buf, size_t count)
  108. {
  109. unsigned long msecs;
  110. int err;
  111. err = kstrtoul(buf, 10, &msecs);
  112. if (err || msecs > UINT_MAX)
  113. return -EINVAL;
  114. khugepaged_scan_sleep_millisecs = msecs;
  115. khugepaged_sleep_expire = 0;
  116. wake_up_interruptible(&khugepaged_wait);
  117. return count;
  118. }
  119. static struct kobj_attribute scan_sleep_millisecs_attr =
  120. __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,
  121. scan_sleep_millisecs_store);
  122. static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
  123. struct kobj_attribute *attr,
  124. char *buf)
  125. {
  126. return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
  127. }
  128. static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
  129. struct kobj_attribute *attr,
  130. const char *buf, size_t count)
  131. {
  132. unsigned long msecs;
  133. int err;
  134. err = kstrtoul(buf, 10, &msecs);
  135. if (err || msecs > UINT_MAX)
  136. return -EINVAL;
  137. khugepaged_alloc_sleep_millisecs = msecs;
  138. khugepaged_sleep_expire = 0;
  139. wake_up_interruptible(&khugepaged_wait);
  140. return count;
  141. }
  142. static struct kobj_attribute alloc_sleep_millisecs_attr =
  143. __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,
  144. alloc_sleep_millisecs_store);
  145. static ssize_t pages_to_scan_show(struct kobject *kobj,
  146. struct kobj_attribute *attr,
  147. char *buf)
  148. {
  149. return sprintf(buf, "%u\n", khugepaged_pages_to_scan);
  150. }
  151. static ssize_t pages_to_scan_store(struct kobject *kobj,
  152. struct kobj_attribute *attr,
  153. const char *buf, size_t count)
  154. {
  155. int err;
  156. unsigned long pages;
  157. err = kstrtoul(buf, 10, &pages);
  158. if (err || !pages || pages > UINT_MAX)
  159. return -EINVAL;
  160. khugepaged_pages_to_scan = pages;
  161. return count;
  162. }
  163. static struct kobj_attribute pages_to_scan_attr =
  164. __ATTR(pages_to_scan, 0644, pages_to_scan_show,
  165. pages_to_scan_store);
  166. static ssize_t pages_collapsed_show(struct kobject *kobj,
  167. struct kobj_attribute *attr,
  168. char *buf)
  169. {
  170. return sprintf(buf, "%u\n", khugepaged_pages_collapsed);
  171. }
  172. static struct kobj_attribute pages_collapsed_attr =
  173. __ATTR_RO(pages_collapsed);
  174. static ssize_t full_scans_show(struct kobject *kobj,
  175. struct kobj_attribute *attr,
  176. char *buf)
  177. {
  178. return sprintf(buf, "%u\n", khugepaged_full_scans);
  179. }
  180. static struct kobj_attribute full_scans_attr =
  181. __ATTR_RO(full_scans);
  182. static ssize_t khugepaged_defrag_show(struct kobject *kobj,
  183. struct kobj_attribute *attr, char *buf)
  184. {
  185. return single_hugepage_flag_show(kobj, attr, buf,
  186. TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
  187. }
  188. static ssize_t khugepaged_defrag_store(struct kobject *kobj,
  189. struct kobj_attribute *attr,
  190. const char *buf, size_t count)
  191. {
  192. return single_hugepage_flag_store(kobj, attr, buf, count,
  193. TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
  194. }
  195. static struct kobj_attribute khugepaged_defrag_attr =
  196. __ATTR(defrag, 0644, khugepaged_defrag_show,
  197. khugepaged_defrag_store);
  198. /*
  199. * max_ptes_none controls if khugepaged should collapse hugepages over
  200. * any unmapped ptes in turn potentially increasing the memory
  201. * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
  202. * reduce the available free memory in the system as it
  203. * runs. Increasing max_ptes_none will instead potentially reduce the
  204. * free memory in the system during the khugepaged scan.
  205. */
  206. static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,
  207. struct kobj_attribute *attr,
  208. char *buf)
  209. {
  210. return sprintf(buf, "%u\n", khugepaged_max_ptes_none);
  211. }
  212. static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,
  213. struct kobj_attribute *attr,
  214. const char *buf, size_t count)
  215. {
  216. int err;
  217. unsigned long max_ptes_none;
  218. err = kstrtoul(buf, 10, &max_ptes_none);
  219. if (err || max_ptes_none > HPAGE_PMD_NR-1)
  220. return -EINVAL;
  221. khugepaged_max_ptes_none = max_ptes_none;
  222. return count;
  223. }
  224. static struct kobj_attribute khugepaged_max_ptes_none_attr =
  225. __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
  226. khugepaged_max_ptes_none_store);
  227. static ssize_t khugepaged_max_ptes_swap_show(struct kobject *kobj,
  228. struct kobj_attribute *attr,
  229. char *buf)
  230. {
  231. return sprintf(buf, "%u\n", khugepaged_max_ptes_swap);
  232. }
  233. static ssize_t khugepaged_max_ptes_swap_store(struct kobject *kobj,
  234. struct kobj_attribute *attr,
  235. const char *buf, size_t count)
  236. {
  237. int err;
  238. unsigned long max_ptes_swap;
  239. err = kstrtoul(buf, 10, &max_ptes_swap);
  240. if (err || max_ptes_swap > HPAGE_PMD_NR-1)
  241. return -EINVAL;
  242. khugepaged_max_ptes_swap = max_ptes_swap;
  243. return count;
  244. }
  245. static struct kobj_attribute khugepaged_max_ptes_swap_attr =
  246. __ATTR(max_ptes_swap, 0644, khugepaged_max_ptes_swap_show,
  247. khugepaged_max_ptes_swap_store);
  248. static struct attribute *khugepaged_attr[] = {
  249. &khugepaged_defrag_attr.attr,
  250. &khugepaged_max_ptes_none_attr.attr,
  251. &pages_to_scan_attr.attr,
  252. &pages_collapsed_attr.attr,
  253. &full_scans_attr.attr,
  254. &scan_sleep_millisecs_attr.attr,
  255. &alloc_sleep_millisecs_attr.attr,
  256. &khugepaged_max_ptes_swap_attr.attr,
  257. NULL,
  258. };
  259. struct attribute_group khugepaged_attr_group = {
  260. .attrs = khugepaged_attr,
  261. .name = "khugepaged",
  262. };
  263. #endif /* CONFIG_SYSFS */
  264. #define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
  265. int hugepage_madvise(struct vm_area_struct *vma,
  266. unsigned long *vm_flags, int advice)
  267. {
  268. switch (advice) {
  269. case MADV_HUGEPAGE:
  270. #ifdef CONFIG_S390
  271. /*
  272. * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
  273. * can't handle this properly after s390_enable_sie, so we simply
  274. * ignore the madvise to prevent qemu from causing a SIGSEGV.
  275. */
  276. if (mm_has_pgste(vma->vm_mm))
  277. return 0;
  278. #endif
  279. *vm_flags &= ~VM_NOHUGEPAGE;
  280. *vm_flags |= VM_HUGEPAGE;
  281. /*
  282. * If the vma become good for khugepaged to scan,
  283. * register it here without waiting a page fault that
  284. * may not happen any time soon.
  285. */
  286. if (!(*vm_flags & VM_NO_KHUGEPAGED) &&
  287. khugepaged_enter_vma_merge(vma, *vm_flags))
  288. return -ENOMEM;
  289. break;
  290. case MADV_NOHUGEPAGE:
  291. *vm_flags &= ~VM_HUGEPAGE;
  292. *vm_flags |= VM_NOHUGEPAGE;
  293. /*
  294. * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
  295. * this vma even if we leave the mm registered in khugepaged if
  296. * it got registered before VM_NOHUGEPAGE was set.
  297. */
  298. break;
  299. }
  300. return 0;
  301. }
  302. int __init khugepaged_init(void)
  303. {
  304. mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
  305. sizeof(struct mm_slot),
  306. __alignof__(struct mm_slot), 0, NULL);
  307. if (!mm_slot_cache)
  308. return -ENOMEM;
  309. khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;
  310. khugepaged_max_ptes_none = HPAGE_PMD_NR - 1;
  311. khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;
  312. return 0;
  313. }
  314. void __init khugepaged_destroy(void)
  315. {
  316. kmem_cache_destroy(mm_slot_cache);
  317. }
  318. static inline struct mm_slot *alloc_mm_slot(void)
  319. {
  320. if (!mm_slot_cache) /* initialization failed */
  321. return NULL;
  322. return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
  323. }
  324. static inline void free_mm_slot(struct mm_slot *mm_slot)
  325. {
  326. kmem_cache_free(mm_slot_cache, mm_slot);
  327. }
  328. static struct mm_slot *get_mm_slot(struct mm_struct *mm)
  329. {
  330. struct mm_slot *mm_slot;
  331. hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
  332. if (mm == mm_slot->mm)
  333. return mm_slot;
  334. return NULL;
  335. }
  336. static void insert_to_mm_slots_hash(struct mm_struct *mm,
  337. struct mm_slot *mm_slot)
  338. {
  339. mm_slot->mm = mm;
  340. hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
  341. }
  342. static inline int khugepaged_test_exit(struct mm_struct *mm)
  343. {
  344. return atomic_read(&mm->mm_users) == 0;
  345. }
  346. int __khugepaged_enter(struct mm_struct *mm)
  347. {
  348. struct mm_slot *mm_slot;
  349. int wakeup;
  350. mm_slot = alloc_mm_slot();
  351. if (!mm_slot)
  352. return -ENOMEM;
  353. /* __khugepaged_exit() must not run from under us */
  354. VM_BUG_ON_MM(khugepaged_test_exit(mm), mm);
  355. if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
  356. free_mm_slot(mm_slot);
  357. return 0;
  358. }
  359. spin_lock(&khugepaged_mm_lock);
  360. insert_to_mm_slots_hash(mm, mm_slot);
  361. /*
  362. * Insert just behind the scanning cursor, to let the area settle
  363. * down a little.
  364. */
  365. wakeup = list_empty(&khugepaged_scan.mm_head);
  366. list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
  367. spin_unlock(&khugepaged_mm_lock);
  368. mmgrab(mm);
  369. if (wakeup)
  370. wake_up_interruptible(&khugepaged_wait);
  371. return 0;
  372. }
  373. int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
  374. unsigned long vm_flags)
  375. {
  376. unsigned long hstart, hend;
  377. if (!vma->anon_vma)
  378. /*
  379. * Not yet faulted in so we will register later in the
  380. * page fault if needed.
  381. */
  382. return 0;
  383. if (vma->vm_ops || (vm_flags & VM_NO_KHUGEPAGED))
  384. /* khugepaged not yet working on file or special mappings */
  385. return 0;
  386. hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
  387. hend = vma->vm_end & HPAGE_PMD_MASK;
  388. if (hstart < hend)
  389. return khugepaged_enter(vma, vm_flags);
  390. return 0;
  391. }
  392. void __khugepaged_exit(struct mm_struct *mm)
  393. {
  394. struct mm_slot *mm_slot;
  395. int free = 0;
  396. spin_lock(&khugepaged_mm_lock);
  397. mm_slot = get_mm_slot(mm);
  398. if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
  399. hash_del(&mm_slot->hash);
  400. list_del(&mm_slot->mm_node);
  401. free = 1;
  402. }
  403. spin_unlock(&khugepaged_mm_lock);
  404. if (free) {
  405. clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
  406. free_mm_slot(mm_slot);
  407. mmdrop(mm);
  408. } else if (mm_slot) {
  409. /*
  410. * This is required to serialize against
  411. * khugepaged_test_exit() (which is guaranteed to run
  412. * under mmap sem read mode). Stop here (after we
  413. * return all pagetables will be destroyed) until
  414. * khugepaged has finished working on the pagetables
  415. * under the mmap_sem.
  416. */
  417. down_write(&mm->mmap_sem);
  418. up_write(&mm->mmap_sem);
  419. }
  420. }
  421. static void release_pte_page(struct page *page)
  422. {
  423. dec_node_page_state(page, NR_ISOLATED_ANON + page_is_file_cache(page));
  424. unlock_page(page);
  425. putback_lru_page(page);
  426. }
  427. static void release_pte_pages(pte_t *pte, pte_t *_pte)
  428. {
  429. while (--_pte >= pte) {
  430. pte_t pteval = *_pte;
  431. if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)))
  432. release_pte_page(pte_page(pteval));
  433. }
  434. }
  435. static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
  436. unsigned long address,
  437. pte_t *pte)
  438. {
  439. struct page *page = NULL;
  440. pte_t *_pte;
  441. int none_or_zero = 0, result = 0, referenced = 0;
  442. bool writable = false;
  443. for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
  444. _pte++, address += PAGE_SIZE) {
  445. pte_t pteval = *_pte;
  446. if (pte_none(pteval) || (pte_present(pteval) &&
  447. is_zero_pfn(pte_pfn(pteval)))) {
  448. if (!userfaultfd_armed(vma) &&
  449. ++none_or_zero <= khugepaged_max_ptes_none) {
  450. continue;
  451. } else {
  452. result = SCAN_EXCEED_NONE_PTE;
  453. goto out;
  454. }
  455. }
  456. if (!pte_present(pteval)) {
  457. result = SCAN_PTE_NON_PRESENT;
  458. goto out;
  459. }
  460. page = vm_normal_page(vma, address, pteval);
  461. if (unlikely(!page)) {
  462. result = SCAN_PAGE_NULL;
  463. goto out;
  464. }
  465. /* TODO: teach khugepaged to collapse THP mapped with pte */
  466. if (PageCompound(page)) {
  467. result = SCAN_PAGE_COMPOUND;
  468. goto out;
  469. }
  470. VM_BUG_ON_PAGE(!PageAnon(page), page);
  471. /*
  472. * We can do it before isolate_lru_page because the
  473. * page can't be freed from under us. NOTE: PG_lock
  474. * is needed to serialize against split_huge_page
  475. * when invoked from the VM.
  476. */
  477. if (!trylock_page(page)) {
  478. result = SCAN_PAGE_LOCK;
  479. goto out;
  480. }
  481. /*
  482. * cannot use mapcount: can't collapse if there's a gup pin.
  483. * The page must only be referenced by the scanned process
  484. * and page swap cache.
  485. */
  486. if (page_count(page) != 1 + PageSwapCache(page)) {
  487. unlock_page(page);
  488. result = SCAN_PAGE_COUNT;
  489. goto out;
  490. }
  491. if (pte_write(pteval)) {
  492. writable = true;
  493. } else {
  494. if (PageSwapCache(page) &&
  495. !reuse_swap_page(page, NULL)) {
  496. unlock_page(page);
  497. result = SCAN_SWAP_CACHE_PAGE;
  498. goto out;
  499. }
  500. /*
  501. * Page is not in the swap cache. It can be collapsed
  502. * into a THP.
  503. */
  504. }
  505. /*
  506. * Isolate the page to avoid collapsing an hugepage
  507. * currently in use by the VM.
  508. */
  509. if (isolate_lru_page(page)) {
  510. unlock_page(page);
  511. result = SCAN_DEL_PAGE_LRU;
  512. goto out;
  513. }
  514. inc_node_page_state(page,
  515. NR_ISOLATED_ANON + page_is_file_cache(page));
  516. VM_BUG_ON_PAGE(!PageLocked(page), page);
  517. VM_BUG_ON_PAGE(PageLRU(page), page);
  518. /* There should be enough young pte to collapse the page */
  519. if (pte_young(pteval) ||
  520. page_is_young(page) || PageReferenced(page) ||
  521. mmu_notifier_test_young(vma->vm_mm, address))
  522. referenced++;
  523. }
  524. if (likely(writable)) {
  525. if (likely(referenced)) {
  526. result = SCAN_SUCCEED;
  527. trace_mm_collapse_huge_page_isolate(page, none_or_zero,
  528. referenced, writable, result);
  529. return 1;
  530. }
  531. } else {
  532. result = SCAN_PAGE_RO;
  533. }
  534. out:
  535. release_pte_pages(pte, _pte);
  536. trace_mm_collapse_huge_page_isolate(page, none_or_zero,
  537. referenced, writable, result);
  538. return 0;
  539. }
  540. static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
  541. struct vm_area_struct *vma,
  542. unsigned long address,
  543. spinlock_t *ptl)
  544. {
  545. pte_t *_pte;
  546. for (_pte = pte; _pte < pte + HPAGE_PMD_NR;
  547. _pte++, page++, address += PAGE_SIZE) {
  548. pte_t pteval = *_pte;
  549. struct page *src_page;
  550. if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
  551. clear_user_highpage(page, address);
  552. add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
  553. if (is_zero_pfn(pte_pfn(pteval))) {
  554. /*
  555. * ptl mostly unnecessary.
  556. */
  557. spin_lock(ptl);
  558. /*
  559. * paravirt calls inside pte_clear here are
  560. * superfluous.
  561. */
  562. pte_clear(vma->vm_mm, address, _pte);
  563. spin_unlock(ptl);
  564. }
  565. } else {
  566. src_page = pte_page(pteval);
  567. copy_user_highpage(page, src_page, address, vma);
  568. VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page);
  569. release_pte_page(src_page);
  570. /*
  571. * ptl mostly unnecessary, but preempt has to
  572. * be disabled to update the per-cpu stats
  573. * inside page_remove_rmap().
  574. */
  575. spin_lock(ptl);
  576. /*
  577. * paravirt calls inside pte_clear here are
  578. * superfluous.
  579. */
  580. pte_clear(vma->vm_mm, address, _pte);
  581. page_remove_rmap(src_page, false);
  582. spin_unlock(ptl);
  583. free_page_and_swap_cache(src_page);
  584. }
  585. }
  586. }
  587. static void khugepaged_alloc_sleep(void)
  588. {
  589. DEFINE_WAIT(wait);
  590. add_wait_queue(&khugepaged_wait, &wait);
  591. freezable_schedule_timeout_interruptible(
  592. msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
  593. remove_wait_queue(&khugepaged_wait, &wait);
  594. }
  595. static int khugepaged_node_load[MAX_NUMNODES];
  596. static bool khugepaged_scan_abort(int nid)
  597. {
  598. int i;
  599. /*
  600. * If node_reclaim_mode is disabled, then no extra effort is made to
  601. * allocate memory locally.
  602. */
  603. if (!node_reclaim_mode)
  604. return false;
  605. /* If there is a count for this node already, it must be acceptable */
  606. if (khugepaged_node_load[nid])
  607. return false;
  608. for (i = 0; i < MAX_NUMNODES; i++) {
  609. if (!khugepaged_node_load[i])
  610. continue;
  611. if (node_distance(nid, i) > RECLAIM_DISTANCE)
  612. return true;
  613. }
  614. return false;
  615. }
  616. /* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
  617. static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
  618. {
  619. return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT;
  620. }
  621. #ifdef CONFIG_NUMA
  622. static int khugepaged_find_target_node(void)
  623. {
  624. static int last_khugepaged_target_node = NUMA_NO_NODE;
  625. int nid, target_node = 0, max_value = 0;
  626. /* find first node with max normal pages hit */
  627. for (nid = 0; nid < MAX_NUMNODES; nid++)
  628. if (khugepaged_node_load[nid] > max_value) {
  629. max_value = khugepaged_node_load[nid];
  630. target_node = nid;
  631. }
  632. /* do some balance if several nodes have the same hit record */
  633. if (target_node <= last_khugepaged_target_node)
  634. for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
  635. nid++)
  636. if (max_value == khugepaged_node_load[nid]) {
  637. target_node = nid;
  638. break;
  639. }
  640. last_khugepaged_target_node = target_node;
  641. return target_node;
  642. }
  643. static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
  644. {
  645. if (IS_ERR(*hpage)) {
  646. if (!*wait)
  647. return false;
  648. *wait = false;
  649. *hpage = NULL;
  650. khugepaged_alloc_sleep();
  651. } else if (*hpage) {
  652. put_page(*hpage);
  653. *hpage = NULL;
  654. }
  655. return true;
  656. }
  657. static struct page *
  658. khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
  659. {
  660. VM_BUG_ON_PAGE(*hpage, *hpage);
  661. *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER);
  662. if (unlikely(!*hpage)) {
  663. count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
  664. *hpage = ERR_PTR(-ENOMEM);
  665. return NULL;
  666. }
  667. prep_transhuge_page(*hpage);
  668. count_vm_event(THP_COLLAPSE_ALLOC);
  669. return *hpage;
  670. }
  671. #else
  672. static int khugepaged_find_target_node(void)
  673. {
  674. return 0;
  675. }
  676. static inline struct page *alloc_khugepaged_hugepage(void)
  677. {
  678. struct page *page;
  679. page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(),
  680. HPAGE_PMD_ORDER);
  681. if (page)
  682. prep_transhuge_page(page);
  683. return page;
  684. }
  685. static struct page *khugepaged_alloc_hugepage(bool *wait)
  686. {
  687. struct page *hpage;
  688. do {
  689. hpage = alloc_khugepaged_hugepage();
  690. if (!hpage) {
  691. count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
  692. if (!*wait)
  693. return NULL;
  694. *wait = false;
  695. khugepaged_alloc_sleep();
  696. } else
  697. count_vm_event(THP_COLLAPSE_ALLOC);
  698. } while (unlikely(!hpage) && likely(khugepaged_enabled()));
  699. return hpage;
  700. }
  701. static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
  702. {
  703. if (!*hpage)
  704. *hpage = khugepaged_alloc_hugepage(wait);
  705. if (unlikely(!*hpage))
  706. return false;
  707. return true;
  708. }
  709. static struct page *
  710. khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
  711. {
  712. VM_BUG_ON(!*hpage);
  713. return *hpage;
  714. }
  715. #endif
  716. static bool hugepage_vma_check(struct vm_area_struct *vma)
  717. {
  718. if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
  719. (vma->vm_flags & VM_NOHUGEPAGE) ||
  720. test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
  721. return false;
  722. if (shmem_file(vma->vm_file)) {
  723. if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
  724. return false;
  725. return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
  726. HPAGE_PMD_NR);
  727. }
  728. if (!vma->anon_vma || vma->vm_ops)
  729. return false;
  730. if (is_vma_temporary_stack(vma))
  731. return false;
  732. return !(vma->vm_flags & VM_NO_KHUGEPAGED);
  733. }
  734. /*
  735. * If mmap_sem temporarily dropped, revalidate vma
  736. * before taking mmap_sem.
  737. * Return 0 if succeeds, otherwise return none-zero
  738. * value (scan code).
  739. */
  740. static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
  741. struct vm_area_struct **vmap)
  742. {
  743. struct vm_area_struct *vma;
  744. unsigned long hstart, hend;
  745. if (unlikely(khugepaged_test_exit(mm)))
  746. return SCAN_ANY_PROCESS;
  747. *vmap = vma = find_vma(mm, address);
  748. if (!vma)
  749. return SCAN_VMA_NULL;
  750. hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
  751. hend = vma->vm_end & HPAGE_PMD_MASK;
  752. if (address < hstart || address + HPAGE_PMD_SIZE > hend)
  753. return SCAN_ADDRESS_RANGE;
  754. if (!hugepage_vma_check(vma))
  755. return SCAN_VMA_CHECK;
  756. return 0;
  757. }
  758. /*
  759. * Bring missing pages in from swap, to complete THP collapse.
  760. * Only done if khugepaged_scan_pmd believes it is worthwhile.
  761. *
  762. * Called and returns without pte mapped or spinlocks held,
  763. * but with mmap_sem held to protect against vma changes.
  764. */
  765. static bool __collapse_huge_page_swapin(struct mm_struct *mm,
  766. struct vm_area_struct *vma,
  767. unsigned long address, pmd_t *pmd,
  768. int referenced)
  769. {
  770. int swapped_in = 0, ret = 0;
  771. struct vm_fault vmf = {
  772. .vma = vma,
  773. .address = address,
  774. .flags = FAULT_FLAG_ALLOW_RETRY,
  775. .pmd = pmd,
  776. .pgoff = linear_page_index(vma, address),
  777. };
  778. /* we only decide to swapin, if there is enough young ptes */
  779. if (referenced < HPAGE_PMD_NR/2) {
  780. trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
  781. return false;
  782. }
  783. vmf.pte = pte_offset_map(pmd, address);
  784. for (; vmf.address < address + HPAGE_PMD_NR*PAGE_SIZE;
  785. vmf.pte++, vmf.address += PAGE_SIZE) {
  786. vmf.orig_pte = *vmf.pte;
  787. if (!is_swap_pte(vmf.orig_pte))
  788. continue;
  789. swapped_in++;
  790. ret = do_swap_page(&vmf);
  791. /* do_swap_page returns VM_FAULT_RETRY with released mmap_sem */
  792. if (ret & VM_FAULT_RETRY) {
  793. down_read(&mm->mmap_sem);
  794. if (hugepage_vma_revalidate(mm, address, &vmf.vma)) {
  795. /* vma is no longer available, don't continue to swapin */
  796. trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
  797. return false;
  798. }
  799. /* check if the pmd is still valid */
  800. if (mm_find_pmd(mm, address) != pmd) {
  801. trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
  802. return false;
  803. }
  804. }
  805. if (ret & VM_FAULT_ERROR) {
  806. trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
  807. return false;
  808. }
  809. /* pte is unmapped now, we need to map it */
  810. vmf.pte = pte_offset_map(pmd, vmf.address);
  811. }
  812. vmf.pte--;
  813. pte_unmap(vmf.pte);
  814. trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1);
  815. return true;
  816. }
  817. static void collapse_huge_page(struct mm_struct *mm,
  818. unsigned long address,
  819. struct page **hpage,
  820. int node, int referenced)
  821. {
  822. pmd_t *pmd, _pmd;
  823. pte_t *pte;
  824. pgtable_t pgtable;
  825. struct page *new_page;
  826. spinlock_t *pmd_ptl, *pte_ptl;
  827. int isolated = 0, result = 0;
  828. struct mem_cgroup *memcg;
  829. struct vm_area_struct *vma;
  830. unsigned long mmun_start; /* For mmu_notifiers */
  831. unsigned long mmun_end; /* For mmu_notifiers */
  832. gfp_t gfp;
  833. VM_BUG_ON(address & ~HPAGE_PMD_MASK);
  834. /* Only allocate from the target node */
  835. gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
  836. /*
  837. * Before allocating the hugepage, release the mmap_sem read lock.
  838. * The allocation can take potentially a long time if it involves
  839. * sync compaction, and we do not need to hold the mmap_sem during
  840. * that. We will recheck the vma after taking it again in write mode.
  841. */
  842. up_read(&mm->mmap_sem);
  843. new_page = khugepaged_alloc_page(hpage, gfp, node);
  844. if (!new_page) {
  845. result = SCAN_ALLOC_HUGE_PAGE_FAIL;
  846. goto out_nolock;
  847. }
  848. if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) {
  849. result = SCAN_CGROUP_CHARGE_FAIL;
  850. goto out_nolock;
  851. }
  852. down_read(&mm->mmap_sem);
  853. result = hugepage_vma_revalidate(mm, address, &vma);
  854. if (result) {
  855. mem_cgroup_cancel_charge(new_page, memcg, true);
  856. up_read(&mm->mmap_sem);
  857. goto out_nolock;
  858. }
  859. pmd = mm_find_pmd(mm, address);
  860. if (!pmd) {
  861. result = SCAN_PMD_NULL;
  862. mem_cgroup_cancel_charge(new_page, memcg, true);
  863. up_read(&mm->mmap_sem);
  864. goto out_nolock;
  865. }
  866. /*
  867. * __collapse_huge_page_swapin always returns with mmap_sem locked.
  868. * If it fails, we release mmap_sem and jump out_nolock.
  869. * Continuing to collapse causes inconsistency.
  870. */
  871. if (!__collapse_huge_page_swapin(mm, vma, address, pmd, referenced)) {
  872. mem_cgroup_cancel_charge(new_page, memcg, true);
  873. up_read(&mm->mmap_sem);
  874. goto out_nolock;
  875. }
  876. up_read(&mm->mmap_sem);
  877. /*
  878. * Prevent all access to pagetables with the exception of
  879. * gup_fast later handled by the ptep_clear_flush and the VM
  880. * handled by the anon_vma lock + PG_lock.
  881. */
  882. down_write(&mm->mmap_sem);
  883. result = hugepage_vma_revalidate(mm, address, &vma);
  884. if (result)
  885. goto out;
  886. /* check if the pmd is still valid */
  887. if (mm_find_pmd(mm, address) != pmd)
  888. goto out;
  889. anon_vma_lock_write(vma->anon_vma);
  890. pte = pte_offset_map(pmd, address);
  891. pte_ptl = pte_lockptr(mm, pmd);
  892. mmun_start = address;
  893. mmun_end = address + HPAGE_PMD_SIZE;
  894. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  895. pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
  896. /*
  897. * After this gup_fast can't run anymore. This also removes
  898. * any huge TLB entry from the CPU so we won't allow
  899. * huge and small TLB entries for the same virtual address
  900. * to avoid the risk of CPU bugs in that area.
  901. */
  902. _pmd = pmdp_collapse_flush(vma, address, pmd);
  903. spin_unlock(pmd_ptl);
  904. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  905. spin_lock(pte_ptl);
  906. isolated = __collapse_huge_page_isolate(vma, address, pte);
  907. spin_unlock(pte_ptl);
  908. if (unlikely(!isolated)) {
  909. pte_unmap(pte);
  910. spin_lock(pmd_ptl);
  911. BUG_ON(!pmd_none(*pmd));
  912. /*
  913. * We can only use set_pmd_at when establishing
  914. * hugepmds and never for establishing regular pmds that
  915. * points to regular pagetables. Use pmd_populate for that
  916. */
  917. pmd_populate(mm, pmd, pmd_pgtable(_pmd));
  918. spin_unlock(pmd_ptl);
  919. anon_vma_unlock_write(vma->anon_vma);
  920. result = SCAN_FAIL;
  921. goto out;
  922. }
  923. /*
  924. * All pages are isolated and locked so anon_vma rmap
  925. * can't run anymore.
  926. */
  927. anon_vma_unlock_write(vma->anon_vma);
  928. __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl);
  929. pte_unmap(pte);
  930. __SetPageUptodate(new_page);
  931. pgtable = pmd_pgtable(_pmd);
  932. _pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
  933. _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
  934. /*
  935. * spin_lock() below is not the equivalent of smp_wmb(), so
  936. * this is needed to avoid the copy_huge_page writes to become
  937. * visible after the set_pmd_at() write.
  938. */
  939. smp_wmb();
  940. spin_lock(pmd_ptl);
  941. BUG_ON(!pmd_none(*pmd));
  942. page_add_new_anon_rmap(new_page, vma, address, true);
  943. mem_cgroup_commit_charge(new_page, memcg, false, true);
  944. lru_cache_add_active_or_unevictable(new_page, vma);
  945. pgtable_trans_huge_deposit(mm, pmd, pgtable);
  946. set_pmd_at(mm, address, pmd, _pmd);
  947. update_mmu_cache_pmd(vma, address, pmd);
  948. spin_unlock(pmd_ptl);
  949. *hpage = NULL;
  950. khugepaged_pages_collapsed++;
  951. result = SCAN_SUCCEED;
  952. out_up_write:
  953. up_write(&mm->mmap_sem);
  954. out_nolock:
  955. trace_mm_collapse_huge_page(mm, isolated, result);
  956. return;
  957. out:
  958. mem_cgroup_cancel_charge(new_page, memcg, true);
  959. goto out_up_write;
  960. }
  961. static int khugepaged_scan_pmd(struct mm_struct *mm,
  962. struct vm_area_struct *vma,
  963. unsigned long address,
  964. struct page **hpage)
  965. {
  966. pmd_t *pmd;
  967. pte_t *pte, *_pte;
  968. int ret = 0, none_or_zero = 0, result = 0, referenced = 0;
  969. struct page *page = NULL;
  970. unsigned long _address;
  971. spinlock_t *ptl;
  972. int node = NUMA_NO_NODE, unmapped = 0;
  973. bool writable = false;
  974. VM_BUG_ON(address & ~HPAGE_PMD_MASK);
  975. pmd = mm_find_pmd(mm, address);
  976. if (!pmd) {
  977. result = SCAN_PMD_NULL;
  978. goto out;
  979. }
  980. memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
  981. pte = pte_offset_map_lock(mm, pmd, address, &ptl);
  982. for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
  983. _pte++, _address += PAGE_SIZE) {
  984. pte_t pteval = *_pte;
  985. if (is_swap_pte(pteval)) {
  986. if (++unmapped <= khugepaged_max_ptes_swap) {
  987. continue;
  988. } else {
  989. result = SCAN_EXCEED_SWAP_PTE;
  990. goto out_unmap;
  991. }
  992. }
  993. if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
  994. if (!userfaultfd_armed(vma) &&
  995. ++none_or_zero <= khugepaged_max_ptes_none) {
  996. continue;
  997. } else {
  998. result = SCAN_EXCEED_NONE_PTE;
  999. goto out_unmap;
  1000. }
  1001. }
  1002. if (!pte_present(pteval)) {
  1003. result = SCAN_PTE_NON_PRESENT;
  1004. goto out_unmap;
  1005. }
  1006. if (pte_write(pteval))
  1007. writable = true;
  1008. page = vm_normal_page(vma, _address, pteval);
  1009. if (unlikely(!page)) {
  1010. result = SCAN_PAGE_NULL;
  1011. goto out_unmap;
  1012. }
  1013. /* TODO: teach khugepaged to collapse THP mapped with pte */
  1014. if (PageCompound(page)) {
  1015. result = SCAN_PAGE_COMPOUND;
  1016. goto out_unmap;
  1017. }
  1018. /*
  1019. * Record which node the original page is from and save this
  1020. * information to khugepaged_node_load[].
  1021. * Khupaged will allocate hugepage from the node has the max
  1022. * hit record.
  1023. */
  1024. node = page_to_nid(page);
  1025. if (khugepaged_scan_abort(node)) {
  1026. result = SCAN_SCAN_ABORT;
  1027. goto out_unmap;
  1028. }
  1029. khugepaged_node_load[node]++;
  1030. if (!PageLRU(page)) {
  1031. result = SCAN_PAGE_LRU;
  1032. goto out_unmap;
  1033. }
  1034. if (PageLocked(page)) {
  1035. result = SCAN_PAGE_LOCK;
  1036. goto out_unmap;
  1037. }
  1038. if (!PageAnon(page)) {
  1039. result = SCAN_PAGE_ANON;
  1040. goto out_unmap;
  1041. }
  1042. /*
  1043. * cannot use mapcount: can't collapse if there's a gup pin.
  1044. * The page must only be referenced by the scanned process
  1045. * and page swap cache.
  1046. */
  1047. if (page_count(page) != 1 + PageSwapCache(page)) {
  1048. result = SCAN_PAGE_COUNT;
  1049. goto out_unmap;
  1050. }
  1051. if (pte_young(pteval) ||
  1052. page_is_young(page) || PageReferenced(page) ||
  1053. mmu_notifier_test_young(vma->vm_mm, address))
  1054. referenced++;
  1055. }
  1056. if (writable) {
  1057. if (referenced) {
  1058. result = SCAN_SUCCEED;
  1059. ret = 1;
  1060. } else {
  1061. result = SCAN_LACK_REFERENCED_PAGE;
  1062. }
  1063. } else {
  1064. result = SCAN_PAGE_RO;
  1065. }
  1066. out_unmap:
  1067. pte_unmap_unlock(pte, ptl);
  1068. if (ret) {
  1069. node = khugepaged_find_target_node();
  1070. /* collapse_huge_page will return with the mmap_sem released */
  1071. collapse_huge_page(mm, address, hpage, node, referenced);
  1072. }
  1073. out:
  1074. trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced,
  1075. none_or_zero, result, unmapped);
  1076. return ret;
  1077. }
  1078. static void collect_mm_slot(struct mm_slot *mm_slot)
  1079. {
  1080. struct mm_struct *mm = mm_slot->mm;
  1081. VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
  1082. if (khugepaged_test_exit(mm)) {
  1083. /* free mm_slot */
  1084. hash_del(&mm_slot->hash);
  1085. list_del(&mm_slot->mm_node);
  1086. /*
  1087. * Not strictly needed because the mm exited already.
  1088. *
  1089. * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
  1090. */
  1091. /* khugepaged_mm_lock actually not necessary for the below */
  1092. free_mm_slot(mm_slot);
  1093. mmdrop(mm);
  1094. }
  1095. }
  1096. #if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
  1097. static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
  1098. {
  1099. struct vm_area_struct *vma;
  1100. unsigned long addr;
  1101. pmd_t *pmd, _pmd;
  1102. i_mmap_lock_write(mapping);
  1103. vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
  1104. /* probably overkill */
  1105. if (vma->anon_vma)
  1106. continue;
  1107. addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
  1108. if (addr & ~HPAGE_PMD_MASK)
  1109. continue;
  1110. if (vma->vm_end < addr + HPAGE_PMD_SIZE)
  1111. continue;
  1112. pmd = mm_find_pmd(vma->vm_mm, addr);
  1113. if (!pmd)
  1114. continue;
  1115. /*
  1116. * We need exclusive mmap_sem to retract page table.
  1117. * If trylock fails we would end up with pte-mapped THP after
  1118. * re-fault. Not ideal, but it's more important to not disturb
  1119. * the system too much.
  1120. */
  1121. if (down_write_trylock(&vma->vm_mm->mmap_sem)) {
  1122. spinlock_t *ptl = pmd_lock(vma->vm_mm, pmd);
  1123. /* assume page table is clear */
  1124. _pmd = pmdp_collapse_flush(vma, addr, pmd);
  1125. spin_unlock(ptl);
  1126. up_write(&vma->vm_mm->mmap_sem);
  1127. mm_dec_nr_ptes(vma->vm_mm);
  1128. pte_free(vma->vm_mm, pmd_pgtable(_pmd));
  1129. }
  1130. }
  1131. i_mmap_unlock_write(mapping);
  1132. }
  1133. /**
  1134. * collapse_shmem - collapse small tmpfs/shmem pages into huge one.
  1135. *
  1136. * Basic scheme is simple, details are more complex:
  1137. * - allocate and freeze a new huge page;
  1138. * - scan over radix tree replacing old pages the new one
  1139. * + swap in pages if necessary;
  1140. * + fill in gaps;
  1141. * + keep old pages around in case if rollback is required;
  1142. * - if replacing succeed:
  1143. * + copy data over;
  1144. * + free old pages;
  1145. * + unfreeze huge page;
  1146. * - if replacing failed;
  1147. * + put all pages back and unfreeze them;
  1148. * + restore gaps in the radix-tree;
  1149. * + free huge page;
  1150. */
  1151. static void collapse_shmem(struct mm_struct *mm,
  1152. struct address_space *mapping, pgoff_t start,
  1153. struct page **hpage, int node)
  1154. {
  1155. gfp_t gfp;
  1156. struct page *page, *new_page, *tmp;
  1157. struct mem_cgroup *memcg;
  1158. pgoff_t index, end = start + HPAGE_PMD_NR;
  1159. LIST_HEAD(pagelist);
  1160. struct radix_tree_iter iter;
  1161. void **slot;
  1162. int nr_none = 0, result = SCAN_SUCCEED;
  1163. VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
  1164. /* Only allocate from the target node */
  1165. gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
  1166. new_page = khugepaged_alloc_page(hpage, gfp, node);
  1167. if (!new_page) {
  1168. result = SCAN_ALLOC_HUGE_PAGE_FAIL;
  1169. goto out;
  1170. }
  1171. if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) {
  1172. result = SCAN_CGROUP_CHARGE_FAIL;
  1173. goto out;
  1174. }
  1175. new_page->index = start;
  1176. new_page->mapping = mapping;
  1177. __SetPageSwapBacked(new_page);
  1178. __SetPageLocked(new_page);
  1179. BUG_ON(!page_ref_freeze(new_page, 1));
  1180. /*
  1181. * At this point the new_page is 'frozen' (page_count() is zero), locked
  1182. * and not up-to-date. It's safe to insert it into radix tree, because
  1183. * nobody would be able to map it or use it in other way until we
  1184. * unfreeze it.
  1185. */
  1186. index = start;
  1187. xa_lock_irq(&mapping->i_pages);
  1188. radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
  1189. int n = min(iter.index, end) - index;
  1190. /*
  1191. * Handle holes in the radix tree: charge it from shmem and
  1192. * insert relevant subpage of new_page into the radix-tree.
  1193. */
  1194. if (n && !shmem_charge(mapping->host, n)) {
  1195. result = SCAN_FAIL;
  1196. break;
  1197. }
  1198. nr_none += n;
  1199. for (; index < min(iter.index, end); index++) {
  1200. radix_tree_insert(&mapping->i_pages, index,
  1201. new_page + (index % HPAGE_PMD_NR));
  1202. }
  1203. /* We are done. */
  1204. if (index >= end)
  1205. break;
  1206. page = radix_tree_deref_slot_protected(slot,
  1207. &mapping->i_pages.xa_lock);
  1208. if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) {
  1209. xa_unlock_irq(&mapping->i_pages);
  1210. /* swap in or instantiate fallocated page */
  1211. if (shmem_getpage(mapping->host, index, &page,
  1212. SGP_NOHUGE)) {
  1213. result = SCAN_FAIL;
  1214. goto tree_unlocked;
  1215. }
  1216. xa_lock_irq(&mapping->i_pages);
  1217. } else if (trylock_page(page)) {
  1218. get_page(page);
  1219. } else {
  1220. result = SCAN_PAGE_LOCK;
  1221. break;
  1222. }
  1223. /*
  1224. * The page must be locked, so we can drop the i_pages lock
  1225. * without racing with truncate.
  1226. */
  1227. VM_BUG_ON_PAGE(!PageLocked(page), page);
  1228. VM_BUG_ON_PAGE(!PageUptodate(page), page);
  1229. VM_BUG_ON_PAGE(PageTransCompound(page), page);
  1230. if (page_mapping(page) != mapping) {
  1231. result = SCAN_TRUNCATED;
  1232. goto out_unlock;
  1233. }
  1234. xa_unlock_irq(&mapping->i_pages);
  1235. if (isolate_lru_page(page)) {
  1236. result = SCAN_DEL_PAGE_LRU;
  1237. goto out_isolate_failed;
  1238. }
  1239. if (page_mapped(page))
  1240. unmap_mapping_pages(mapping, index, 1, false);
  1241. xa_lock_irq(&mapping->i_pages);
  1242. slot = radix_tree_lookup_slot(&mapping->i_pages, index);
  1243. VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot,
  1244. &mapping->i_pages.xa_lock), page);
  1245. VM_BUG_ON_PAGE(page_mapped(page), page);
  1246. /*
  1247. * The page is expected to have page_count() == 3:
  1248. * - we hold a pin on it;
  1249. * - one reference from radix tree;
  1250. * - one from isolate_lru_page;
  1251. */
  1252. if (!page_ref_freeze(page, 3)) {
  1253. result = SCAN_PAGE_COUNT;
  1254. goto out_lru;
  1255. }
  1256. /*
  1257. * Add the page to the list to be able to undo the collapse if
  1258. * something go wrong.
  1259. */
  1260. list_add_tail(&page->lru, &pagelist);
  1261. /* Finally, replace with the new page. */
  1262. radix_tree_replace_slot(&mapping->i_pages, slot,
  1263. new_page + (index % HPAGE_PMD_NR));
  1264. slot = radix_tree_iter_resume(slot, &iter);
  1265. index++;
  1266. continue;
  1267. out_lru:
  1268. xa_unlock_irq(&mapping->i_pages);
  1269. putback_lru_page(page);
  1270. out_isolate_failed:
  1271. unlock_page(page);
  1272. put_page(page);
  1273. goto tree_unlocked;
  1274. out_unlock:
  1275. unlock_page(page);
  1276. put_page(page);
  1277. break;
  1278. }
  1279. /*
  1280. * Handle hole in radix tree at the end of the range.
  1281. * This code only triggers if there's nothing in radix tree
  1282. * beyond 'end'.
  1283. */
  1284. if (result == SCAN_SUCCEED && index < end) {
  1285. int n = end - index;
  1286. if (!shmem_charge(mapping->host, n)) {
  1287. result = SCAN_FAIL;
  1288. goto tree_locked;
  1289. }
  1290. for (; index < end; index++) {
  1291. radix_tree_insert(&mapping->i_pages, index,
  1292. new_page + (index % HPAGE_PMD_NR));
  1293. }
  1294. nr_none += n;
  1295. }
  1296. tree_locked:
  1297. xa_unlock_irq(&mapping->i_pages);
  1298. tree_unlocked:
  1299. if (result == SCAN_SUCCEED) {
  1300. unsigned long flags;
  1301. struct zone *zone = page_zone(new_page);
  1302. /*
  1303. * Replacing old pages with new one has succeed, now we need to
  1304. * copy the content and free old pages.
  1305. */
  1306. list_for_each_entry_safe(page, tmp, &pagelist, lru) {
  1307. copy_highpage(new_page + (page->index % HPAGE_PMD_NR),
  1308. page);
  1309. list_del(&page->lru);
  1310. unlock_page(page);
  1311. page_ref_unfreeze(page, 1);
  1312. page->mapping = NULL;
  1313. ClearPageActive(page);
  1314. ClearPageUnevictable(page);
  1315. put_page(page);
  1316. }
  1317. local_irq_save(flags);
  1318. __inc_node_page_state(new_page, NR_SHMEM_THPS);
  1319. if (nr_none) {
  1320. __mod_node_page_state(zone->zone_pgdat, NR_FILE_PAGES, nr_none);
  1321. __mod_node_page_state(zone->zone_pgdat, NR_SHMEM, nr_none);
  1322. }
  1323. local_irq_restore(flags);
  1324. /*
  1325. * Remove pte page tables, so we can re-faulti
  1326. * the page as huge.
  1327. */
  1328. retract_page_tables(mapping, start);
  1329. /* Everything is ready, let's unfreeze the new_page */
  1330. set_page_dirty(new_page);
  1331. SetPageUptodate(new_page);
  1332. page_ref_unfreeze(new_page, HPAGE_PMD_NR);
  1333. mem_cgroup_commit_charge(new_page, memcg, false, true);
  1334. lru_cache_add_anon(new_page);
  1335. unlock_page(new_page);
  1336. *hpage = NULL;
  1337. } else {
  1338. /* Something went wrong: rollback changes to the radix-tree */
  1339. shmem_uncharge(mapping->host, nr_none);
  1340. xa_lock_irq(&mapping->i_pages);
  1341. radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
  1342. if (iter.index >= end)
  1343. break;
  1344. page = list_first_entry_or_null(&pagelist,
  1345. struct page, lru);
  1346. if (!page || iter.index < page->index) {
  1347. if (!nr_none)
  1348. break;
  1349. nr_none--;
  1350. /* Put holes back where they were */
  1351. radix_tree_delete(&mapping->i_pages, iter.index);
  1352. continue;
  1353. }
  1354. VM_BUG_ON_PAGE(page->index != iter.index, page);
  1355. /* Unfreeze the page. */
  1356. list_del(&page->lru);
  1357. page_ref_unfreeze(page, 2);
  1358. radix_tree_replace_slot(&mapping->i_pages, slot, page);
  1359. slot = radix_tree_iter_resume(slot, &iter);
  1360. xa_unlock_irq(&mapping->i_pages);
  1361. putback_lru_page(page);
  1362. unlock_page(page);
  1363. xa_lock_irq(&mapping->i_pages);
  1364. }
  1365. VM_BUG_ON(nr_none);
  1366. xa_unlock_irq(&mapping->i_pages);
  1367. /* Unfreeze new_page, caller would take care about freeing it */
  1368. page_ref_unfreeze(new_page, 1);
  1369. mem_cgroup_cancel_charge(new_page, memcg, true);
  1370. unlock_page(new_page);
  1371. new_page->mapping = NULL;
  1372. }
  1373. out:
  1374. VM_BUG_ON(!list_empty(&pagelist));
  1375. /* TODO: tracepoints */
  1376. }
  1377. static void khugepaged_scan_shmem(struct mm_struct *mm,
  1378. struct address_space *mapping,
  1379. pgoff_t start, struct page **hpage)
  1380. {
  1381. struct page *page = NULL;
  1382. struct radix_tree_iter iter;
  1383. void **slot;
  1384. int present, swap;
  1385. int node = NUMA_NO_NODE;
  1386. int result = SCAN_SUCCEED;
  1387. present = 0;
  1388. swap = 0;
  1389. memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
  1390. rcu_read_lock();
  1391. radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
  1392. if (iter.index >= start + HPAGE_PMD_NR)
  1393. break;
  1394. page = radix_tree_deref_slot(slot);
  1395. if (radix_tree_deref_retry(page)) {
  1396. slot = radix_tree_iter_retry(&iter);
  1397. continue;
  1398. }
  1399. if (radix_tree_exception(page)) {
  1400. if (++swap > khugepaged_max_ptes_swap) {
  1401. result = SCAN_EXCEED_SWAP_PTE;
  1402. break;
  1403. }
  1404. continue;
  1405. }
  1406. if (PageTransCompound(page)) {
  1407. result = SCAN_PAGE_COMPOUND;
  1408. break;
  1409. }
  1410. node = page_to_nid(page);
  1411. if (khugepaged_scan_abort(node)) {
  1412. result = SCAN_SCAN_ABORT;
  1413. break;
  1414. }
  1415. khugepaged_node_load[node]++;
  1416. if (!PageLRU(page)) {
  1417. result = SCAN_PAGE_LRU;
  1418. break;
  1419. }
  1420. if (page_count(page) != 1 + page_mapcount(page)) {
  1421. result = SCAN_PAGE_COUNT;
  1422. break;
  1423. }
  1424. /*
  1425. * We probably should check if the page is referenced here, but
  1426. * nobody would transfer pte_young() to PageReferenced() for us.
  1427. * And rmap walk here is just too costly...
  1428. */
  1429. present++;
  1430. if (need_resched()) {
  1431. slot = radix_tree_iter_resume(slot, &iter);
  1432. cond_resched_rcu();
  1433. }
  1434. }
  1435. rcu_read_unlock();
  1436. if (result == SCAN_SUCCEED) {
  1437. if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) {
  1438. result = SCAN_EXCEED_NONE_PTE;
  1439. } else {
  1440. node = khugepaged_find_target_node();
  1441. collapse_shmem(mm, mapping, start, hpage, node);
  1442. }
  1443. }
  1444. /* TODO: tracepoints */
  1445. }
  1446. #else
  1447. static void khugepaged_scan_shmem(struct mm_struct *mm,
  1448. struct address_space *mapping,
  1449. pgoff_t start, struct page **hpage)
  1450. {
  1451. BUILD_BUG();
  1452. }
  1453. #endif
  1454. static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
  1455. struct page **hpage)
  1456. __releases(&khugepaged_mm_lock)
  1457. __acquires(&khugepaged_mm_lock)
  1458. {
  1459. struct mm_slot *mm_slot;
  1460. struct mm_struct *mm;
  1461. struct vm_area_struct *vma;
  1462. int progress = 0;
  1463. VM_BUG_ON(!pages);
  1464. VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
  1465. if (khugepaged_scan.mm_slot)
  1466. mm_slot = khugepaged_scan.mm_slot;
  1467. else {
  1468. mm_slot = list_entry(khugepaged_scan.mm_head.next,
  1469. struct mm_slot, mm_node);
  1470. khugepaged_scan.address = 0;
  1471. khugepaged_scan.mm_slot = mm_slot;
  1472. }
  1473. spin_unlock(&khugepaged_mm_lock);
  1474. mm = mm_slot->mm;
  1475. /*
  1476. * Don't wait for semaphore (to avoid long wait times). Just move to
  1477. * the next mm on the list.
  1478. */
  1479. vma = NULL;
  1480. if (unlikely(!down_read_trylock(&mm->mmap_sem)))
  1481. goto breakouterloop_mmap_sem;
  1482. if (likely(!khugepaged_test_exit(mm)))
  1483. vma = find_vma(mm, khugepaged_scan.address);
  1484. progress++;
  1485. for (; vma; vma = vma->vm_next) {
  1486. unsigned long hstart, hend;
  1487. cond_resched();
  1488. if (unlikely(khugepaged_test_exit(mm))) {
  1489. progress++;
  1490. break;
  1491. }
  1492. if (!hugepage_vma_check(vma)) {
  1493. skip:
  1494. progress++;
  1495. continue;
  1496. }
  1497. hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
  1498. hend = vma->vm_end & HPAGE_PMD_MASK;
  1499. if (hstart >= hend)
  1500. goto skip;
  1501. if (khugepaged_scan.address > hend)
  1502. goto skip;
  1503. if (khugepaged_scan.address < hstart)
  1504. khugepaged_scan.address = hstart;
  1505. VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
  1506. while (khugepaged_scan.address < hend) {
  1507. int ret;
  1508. cond_resched();
  1509. if (unlikely(khugepaged_test_exit(mm)))
  1510. goto breakouterloop;
  1511. VM_BUG_ON(khugepaged_scan.address < hstart ||
  1512. khugepaged_scan.address + HPAGE_PMD_SIZE >
  1513. hend);
  1514. if (shmem_file(vma->vm_file)) {
  1515. struct file *file;
  1516. pgoff_t pgoff = linear_page_index(vma,
  1517. khugepaged_scan.address);
  1518. if (!shmem_huge_enabled(vma))
  1519. goto skip;
  1520. file = get_file(vma->vm_file);
  1521. up_read(&mm->mmap_sem);
  1522. ret = 1;
  1523. khugepaged_scan_shmem(mm, file->f_mapping,
  1524. pgoff, hpage);
  1525. fput(file);
  1526. } else {
  1527. ret = khugepaged_scan_pmd(mm, vma,
  1528. khugepaged_scan.address,
  1529. hpage);
  1530. }
  1531. /* move to next address */
  1532. khugepaged_scan.address += HPAGE_PMD_SIZE;
  1533. progress += HPAGE_PMD_NR;
  1534. if (ret)
  1535. /* we released mmap_sem so break loop */
  1536. goto breakouterloop_mmap_sem;
  1537. if (progress >= pages)
  1538. goto breakouterloop;
  1539. }
  1540. }
  1541. breakouterloop:
  1542. up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */
  1543. breakouterloop_mmap_sem:
  1544. spin_lock(&khugepaged_mm_lock);
  1545. VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);
  1546. /*
  1547. * Release the current mm_slot if this mm is about to die, or
  1548. * if we scanned all vmas of this mm.
  1549. */
  1550. if (khugepaged_test_exit(mm) || !vma) {
  1551. /*
  1552. * Make sure that if mm_users is reaching zero while
  1553. * khugepaged runs here, khugepaged_exit will find
  1554. * mm_slot not pointing to the exiting mm.
  1555. */
  1556. if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {
  1557. khugepaged_scan.mm_slot = list_entry(
  1558. mm_slot->mm_node.next,
  1559. struct mm_slot, mm_node);
  1560. khugepaged_scan.address = 0;
  1561. } else {
  1562. khugepaged_scan.mm_slot = NULL;
  1563. khugepaged_full_scans++;
  1564. }
  1565. collect_mm_slot(mm_slot);
  1566. }
  1567. return progress;
  1568. }
  1569. static int khugepaged_has_work(void)
  1570. {
  1571. return !list_empty(&khugepaged_scan.mm_head) &&
  1572. khugepaged_enabled();
  1573. }
  1574. static int khugepaged_wait_event(void)
  1575. {
  1576. return !list_empty(&khugepaged_scan.mm_head) ||
  1577. kthread_should_stop();
  1578. }
  1579. static void khugepaged_do_scan(void)
  1580. {
  1581. struct page *hpage = NULL;
  1582. unsigned int progress = 0, pass_through_head = 0;
  1583. unsigned int pages = khugepaged_pages_to_scan;
  1584. bool wait = true;
  1585. barrier(); /* write khugepaged_pages_to_scan to local stack */
  1586. while (progress < pages) {
  1587. if (!khugepaged_prealloc_page(&hpage, &wait))
  1588. break;
  1589. cond_resched();
  1590. if (unlikely(kthread_should_stop() || try_to_freeze()))
  1591. break;
  1592. spin_lock(&khugepaged_mm_lock);
  1593. if (!khugepaged_scan.mm_slot)
  1594. pass_through_head++;
  1595. if (khugepaged_has_work() &&
  1596. pass_through_head < 2)
  1597. progress += khugepaged_scan_mm_slot(pages - progress,
  1598. &hpage);
  1599. else
  1600. progress = pages;
  1601. spin_unlock(&khugepaged_mm_lock);
  1602. }
  1603. if (!IS_ERR_OR_NULL(hpage))
  1604. put_page(hpage);
  1605. }
  1606. static bool khugepaged_should_wakeup(void)
  1607. {
  1608. return kthread_should_stop() ||
  1609. time_after_eq(jiffies, khugepaged_sleep_expire);
  1610. }
  1611. static void khugepaged_wait_work(void)
  1612. {
  1613. if (khugepaged_has_work()) {
  1614. const unsigned long scan_sleep_jiffies =
  1615. msecs_to_jiffies(khugepaged_scan_sleep_millisecs);
  1616. if (!scan_sleep_jiffies)
  1617. return;
  1618. khugepaged_sleep_expire = jiffies + scan_sleep_jiffies;
  1619. wait_event_freezable_timeout(khugepaged_wait,
  1620. khugepaged_should_wakeup(),
  1621. scan_sleep_jiffies);
  1622. return;
  1623. }
  1624. if (khugepaged_enabled())
  1625. wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
  1626. }
  1627. static int khugepaged(void *none)
  1628. {
  1629. struct mm_slot *mm_slot;
  1630. set_freezable();
  1631. set_user_nice(current, MAX_NICE);
  1632. while (!kthread_should_stop()) {
  1633. khugepaged_do_scan();
  1634. khugepaged_wait_work();
  1635. }
  1636. spin_lock(&khugepaged_mm_lock);
  1637. mm_slot = khugepaged_scan.mm_slot;
  1638. khugepaged_scan.mm_slot = NULL;
  1639. if (mm_slot)
  1640. collect_mm_slot(mm_slot);
  1641. spin_unlock(&khugepaged_mm_lock);
  1642. return 0;
  1643. }
  1644. static void set_recommended_min_free_kbytes(void)
  1645. {
  1646. struct zone *zone;
  1647. int nr_zones = 0;
  1648. unsigned long recommended_min;
  1649. for_each_populated_zone(zone) {
  1650. /*
  1651. * We don't need to worry about fragmentation of
  1652. * ZONE_MOVABLE since it only has movable pages.
  1653. */
  1654. if (zone_idx(zone) > gfp_zone(GFP_USER))
  1655. continue;
  1656. nr_zones++;
  1657. }
  1658. /* Ensure 2 pageblocks are free to assist fragmentation avoidance */
  1659. recommended_min = pageblock_nr_pages * nr_zones * 2;
  1660. /*
  1661. * Make sure that on average at least two pageblocks are almost free
  1662. * of another type, one for a migratetype to fall back to and a
  1663. * second to avoid subsequent fallbacks of other types There are 3
  1664. * MIGRATE_TYPES we care about.
  1665. */
  1666. recommended_min += pageblock_nr_pages * nr_zones *
  1667. MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;
  1668. /* don't ever allow to reserve more than 5% of the lowmem */
  1669. recommended_min = min(recommended_min,
  1670. (unsigned long) nr_free_buffer_pages() / 20);
  1671. recommended_min <<= (PAGE_SHIFT-10);
  1672. if (recommended_min > min_free_kbytes) {
  1673. if (user_min_free_kbytes >= 0)
  1674. pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n",
  1675. min_free_kbytes, recommended_min);
  1676. min_free_kbytes = recommended_min;
  1677. }
  1678. setup_per_zone_wmarks();
  1679. }
  1680. int start_stop_khugepaged(void)
  1681. {
  1682. static struct task_struct *khugepaged_thread __read_mostly;
  1683. static DEFINE_MUTEX(khugepaged_mutex);
  1684. int err = 0;
  1685. mutex_lock(&khugepaged_mutex);
  1686. if (khugepaged_enabled()) {
  1687. if (!khugepaged_thread)
  1688. khugepaged_thread = kthread_run(khugepaged, NULL,
  1689. "khugepaged");
  1690. if (IS_ERR(khugepaged_thread)) {
  1691. pr_err("khugepaged: kthread_run(khugepaged) failed\n");
  1692. err = PTR_ERR(khugepaged_thread);
  1693. khugepaged_thread = NULL;
  1694. goto fail;
  1695. }
  1696. if (!list_empty(&khugepaged_scan.mm_head))
  1697. wake_up_interruptible(&khugepaged_wait);
  1698. set_recommended_min_free_kbytes();
  1699. } else if (khugepaged_thread) {
  1700. kthread_stop(khugepaged_thread);
  1701. khugepaged_thread = NULL;
  1702. }
  1703. fail:
  1704. mutex_unlock(&khugepaged_mutex);
  1705. return err;
  1706. }