mremap.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587
  1. /*
  2. * mm/mremap.c
  3. *
  4. * (C) Copyright 1996 Linus Torvalds
  5. *
  6. * Address space accounting code <alan@lxorguk.ukuu.org.uk>
  7. * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
  8. */
  9. #include <linux/mm.h>
  10. #include <linux/hugetlb.h>
  11. #include <linux/shm.h>
  12. #include <linux/ksm.h>
  13. #include <linux/mman.h>
  14. #include <linux/swap.h>
  15. #include <linux/capability.h>
  16. #include <linux/fs.h>
  17. #include <linux/swapops.h>
  18. #include <linux/highmem.h>
  19. #include <linux/security.h>
  20. #include <linux/syscalls.h>
  21. #include <linux/mmu_notifier.h>
  22. #include <linux/sched/sysctl.h>
  23. #include <linux/uaccess.h>
  24. #include <asm/cacheflush.h>
  25. #include <asm/tlbflush.h>
  26. #include "internal.h"
  27. static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
  28. {
  29. pgd_t *pgd;
  30. pud_t *pud;
  31. pmd_t *pmd;
  32. pgd = pgd_offset(mm, addr);
  33. if (pgd_none_or_clear_bad(pgd))
  34. return NULL;
  35. pud = pud_offset(pgd, addr);
  36. if (pud_none_or_clear_bad(pud))
  37. return NULL;
  38. pmd = pmd_offset(pud, addr);
  39. if (pmd_none(*pmd))
  40. return NULL;
  41. return pmd;
  42. }
  43. static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
  44. unsigned long addr)
  45. {
  46. pgd_t *pgd;
  47. pud_t *pud;
  48. pmd_t *pmd;
  49. pgd = pgd_offset(mm, addr);
  50. pud = pud_alloc(mm, pgd, addr);
  51. if (!pud)
  52. return NULL;
  53. pmd = pmd_alloc(mm, pud, addr);
  54. if (!pmd)
  55. return NULL;
  56. VM_BUG_ON(pmd_trans_huge(*pmd));
  57. return pmd;
  58. }
  59. static pte_t move_soft_dirty_pte(pte_t pte)
  60. {
  61. /*
  62. * Set soft dirty bit so we can notice
  63. * in userspace the ptes were moved.
  64. */
  65. #ifdef CONFIG_MEM_SOFT_DIRTY
  66. if (pte_present(pte))
  67. pte = pte_mksoft_dirty(pte);
  68. else if (is_swap_pte(pte))
  69. pte = pte_swp_mksoft_dirty(pte);
  70. #endif
  71. return pte;
  72. }
  73. static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
  74. unsigned long old_addr, unsigned long old_end,
  75. struct vm_area_struct *new_vma, pmd_t *new_pmd,
  76. unsigned long new_addr, bool need_rmap_locks)
  77. {
  78. struct address_space *mapping = NULL;
  79. struct anon_vma *anon_vma = NULL;
  80. struct mm_struct *mm = vma->vm_mm;
  81. pte_t *old_pte, *new_pte, pte;
  82. spinlock_t *old_ptl, *new_ptl;
  83. /*
  84. * When need_rmap_locks is true, we take the i_mmap_rwsem and anon_vma
  85. * locks to ensure that rmap will always observe either the old or the
  86. * new ptes. This is the easiest way to avoid races with
  87. * truncate_pagecache(), page migration, etc...
  88. *
  89. * When need_rmap_locks is false, we use other ways to avoid
  90. * such races:
  91. *
  92. * - During exec() shift_arg_pages(), we use a specially tagged vma
  93. * which rmap call sites look for using is_vma_temporary_stack().
  94. *
  95. * - During mremap(), new_vma is often known to be placed after vma
  96. * in rmap traversal order. This ensures rmap will always observe
  97. * either the old pte, or the new pte, or both (the page table locks
  98. * serialize access to individual ptes, but only rmap traversal
  99. * order guarantees that we won't miss both the old and new ptes).
  100. */
  101. if (need_rmap_locks) {
  102. if (vma->vm_file) {
  103. mapping = vma->vm_file->f_mapping;
  104. i_mmap_lock_write(mapping);
  105. }
  106. if (vma->anon_vma) {
  107. anon_vma = vma->anon_vma;
  108. anon_vma_lock_write(anon_vma);
  109. }
  110. }
  111. /*
  112. * We don't have to worry about the ordering of src and dst
  113. * pte locks because exclusive mmap_sem prevents deadlock.
  114. */
  115. old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
  116. new_pte = pte_offset_map(new_pmd, new_addr);
  117. new_ptl = pte_lockptr(mm, new_pmd);
  118. if (new_ptl != old_ptl)
  119. spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
  120. arch_enter_lazy_mmu_mode();
  121. for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
  122. new_pte++, new_addr += PAGE_SIZE) {
  123. if (pte_none(*old_pte))
  124. continue;
  125. pte = ptep_get_and_clear(mm, old_addr, old_pte);
  126. pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
  127. pte = move_soft_dirty_pte(pte);
  128. set_pte_at(mm, new_addr, new_pte, pte);
  129. }
  130. arch_leave_lazy_mmu_mode();
  131. if (new_ptl != old_ptl)
  132. spin_unlock(new_ptl);
  133. pte_unmap(new_pte - 1);
  134. pte_unmap_unlock(old_pte - 1, old_ptl);
  135. if (anon_vma)
  136. anon_vma_unlock_write(anon_vma);
  137. if (mapping)
  138. i_mmap_unlock_write(mapping);
  139. }
  140. #define LATENCY_LIMIT (64 * PAGE_SIZE)
  141. unsigned long move_page_tables(struct vm_area_struct *vma,
  142. unsigned long old_addr, struct vm_area_struct *new_vma,
  143. unsigned long new_addr, unsigned long len,
  144. bool need_rmap_locks)
  145. {
  146. unsigned long extent, next, old_end;
  147. pmd_t *old_pmd, *new_pmd;
  148. bool need_flush = false;
  149. unsigned long mmun_start; /* For mmu_notifiers */
  150. unsigned long mmun_end; /* For mmu_notifiers */
  151. old_end = old_addr + len;
  152. flush_cache_range(vma, old_addr, old_end);
  153. mmun_start = old_addr;
  154. mmun_end = old_end;
  155. mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
  156. for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
  157. cond_resched();
  158. next = (old_addr + PMD_SIZE) & PMD_MASK;
  159. /* even if next overflowed, extent below will be ok */
  160. extent = next - old_addr;
  161. if (extent > old_end - old_addr)
  162. extent = old_end - old_addr;
  163. old_pmd = get_old_pmd(vma->vm_mm, old_addr);
  164. if (!old_pmd)
  165. continue;
  166. new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);
  167. if (!new_pmd)
  168. break;
  169. if (pmd_trans_huge(*old_pmd)) {
  170. int err = 0;
  171. if (extent == HPAGE_PMD_SIZE) {
  172. VM_BUG_ON_VMA(vma->vm_file || !vma->anon_vma,
  173. vma);
  174. /* See comment in move_ptes() */
  175. if (need_rmap_locks)
  176. anon_vma_lock_write(vma->anon_vma);
  177. err = move_huge_pmd(vma, new_vma, old_addr,
  178. new_addr, old_end,
  179. old_pmd, new_pmd);
  180. if (need_rmap_locks)
  181. anon_vma_unlock_write(vma->anon_vma);
  182. }
  183. if (err > 0) {
  184. need_flush = true;
  185. continue;
  186. } else if (!err) {
  187. split_huge_page_pmd(vma, old_addr, old_pmd);
  188. }
  189. VM_BUG_ON(pmd_trans_huge(*old_pmd));
  190. }
  191. if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma,
  192. new_pmd, new_addr))
  193. break;
  194. next = (new_addr + PMD_SIZE) & PMD_MASK;
  195. if (extent > next - new_addr)
  196. extent = next - new_addr;
  197. if (extent > LATENCY_LIMIT)
  198. extent = LATENCY_LIMIT;
  199. move_ptes(vma, old_pmd, old_addr, old_addr + extent,
  200. new_vma, new_pmd, new_addr, need_rmap_locks);
  201. need_flush = true;
  202. }
  203. if (likely(need_flush))
  204. flush_tlb_range(vma, old_end-len, old_addr);
  205. mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
  206. return len + old_addr - old_end; /* how much done */
  207. }
  208. static unsigned long move_vma(struct vm_area_struct *vma,
  209. unsigned long old_addr, unsigned long old_len,
  210. unsigned long new_len, unsigned long new_addr, bool *locked)
  211. {
  212. struct mm_struct *mm = vma->vm_mm;
  213. struct vm_area_struct *new_vma;
  214. unsigned long vm_flags = vma->vm_flags;
  215. unsigned long new_pgoff;
  216. unsigned long moved_len;
  217. unsigned long excess = 0;
  218. unsigned long hiwater_vm;
  219. int split = 0;
  220. int err;
  221. bool need_rmap_locks;
  222. /*
  223. * We'd prefer to avoid failure later on in do_munmap:
  224. * which may split one vma into three before unmapping.
  225. */
  226. if (mm->map_count >= sysctl_max_map_count - 3)
  227. return -ENOMEM;
  228. /*
  229. * Advise KSM to break any KSM pages in the area to be moved:
  230. * it would be confusing if they were to turn up at the new
  231. * location, where they happen to coincide with different KSM
  232. * pages recently unmapped. But leave vma->vm_flags as it was,
  233. * so KSM can come around to merge on vma and new_vma afterwards.
  234. */
  235. err = ksm_madvise(vma, old_addr, old_addr + old_len,
  236. MADV_UNMERGEABLE, &vm_flags);
  237. if (err)
  238. return err;
  239. new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
  240. new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
  241. &need_rmap_locks);
  242. if (!new_vma)
  243. return -ENOMEM;
  244. moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,
  245. need_rmap_locks);
  246. if (moved_len < old_len) {
  247. /*
  248. * On error, move entries back from new area to old,
  249. * which will succeed since page tables still there,
  250. * and then proceed to unmap new area instead of old.
  251. */
  252. move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,
  253. true);
  254. vma = new_vma;
  255. old_len = new_len;
  256. old_addr = new_addr;
  257. new_addr = -ENOMEM;
  258. } else if (vma->vm_file && vma->vm_file->f_op->mremap)
  259. vma->vm_file->f_op->mremap(vma->vm_file, new_vma);
  260. /* Conceal VM_ACCOUNT so old reservation is not undone */
  261. if (vm_flags & VM_ACCOUNT) {
  262. vma->vm_flags &= ~VM_ACCOUNT;
  263. excess = vma->vm_end - vma->vm_start - old_len;
  264. if (old_addr > vma->vm_start &&
  265. old_addr + old_len < vma->vm_end)
  266. split = 1;
  267. }
  268. /*
  269. * If we failed to move page tables we still do total_vm increment
  270. * since do_munmap() will decrement it by old_len == new_len.
  271. *
  272. * Since total_vm is about to be raised artificially high for a
  273. * moment, we need to restore high watermark afterwards: if stats
  274. * are taken meanwhile, total_vm and hiwater_vm appear too high.
  275. * If this were a serious issue, we'd add a flag to do_munmap().
  276. */
  277. hiwater_vm = mm->hiwater_vm;
  278. vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT);
  279. if (do_munmap(mm, old_addr, old_len) < 0) {
  280. /* OOM: unable to split vma, just get accounts right */
  281. vm_unacct_memory(excess >> PAGE_SHIFT);
  282. excess = 0;
  283. }
  284. mm->hiwater_vm = hiwater_vm;
  285. /* Restore VM_ACCOUNT if one or two pieces of vma left */
  286. if (excess) {
  287. vma->vm_flags |= VM_ACCOUNT;
  288. if (split)
  289. vma->vm_next->vm_flags |= VM_ACCOUNT;
  290. }
  291. if (vm_flags & VM_LOCKED) {
  292. mm->locked_vm += new_len >> PAGE_SHIFT;
  293. *locked = true;
  294. }
  295. return new_addr;
  296. }
  297. static struct vm_area_struct *vma_to_resize(unsigned long addr,
  298. unsigned long old_len, unsigned long new_len, unsigned long *p)
  299. {
  300. struct mm_struct *mm = current->mm;
  301. struct vm_area_struct *vma = find_vma(mm, addr);
  302. if (!vma || vma->vm_start > addr)
  303. goto Efault;
  304. if (is_vm_hugetlb_page(vma))
  305. goto Einval;
  306. /* We can't remap across vm area boundaries */
  307. if (old_len > vma->vm_end - addr)
  308. goto Efault;
  309. /* Need to be careful about a growing mapping */
  310. if (new_len > old_len) {
  311. unsigned long pgoff;
  312. if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))
  313. goto Efault;
  314. pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
  315. pgoff += vma->vm_pgoff;
  316. if (pgoff + (new_len >> PAGE_SHIFT) < pgoff)
  317. goto Einval;
  318. }
  319. if (vma->vm_flags & VM_LOCKED) {
  320. unsigned long locked, lock_limit;
  321. locked = mm->locked_vm << PAGE_SHIFT;
  322. lock_limit = rlimit(RLIMIT_MEMLOCK);
  323. locked += new_len - old_len;
  324. if (locked > lock_limit && !capable(CAP_IPC_LOCK))
  325. goto Eagain;
  326. }
  327. if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT))
  328. goto Enomem;
  329. if (vma->vm_flags & VM_ACCOUNT) {
  330. unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
  331. if (security_vm_enough_memory_mm(mm, charged))
  332. goto Efault;
  333. *p = charged;
  334. }
  335. return vma;
  336. Efault: /* very odd choice for most of the cases, but... */
  337. return ERR_PTR(-EFAULT);
  338. Einval:
  339. return ERR_PTR(-EINVAL);
  340. Enomem:
  341. return ERR_PTR(-ENOMEM);
  342. Eagain:
  343. return ERR_PTR(-EAGAIN);
  344. }
  345. static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
  346. unsigned long new_addr, unsigned long new_len, bool *locked)
  347. {
  348. struct mm_struct *mm = current->mm;
  349. struct vm_area_struct *vma;
  350. unsigned long ret = -EINVAL;
  351. unsigned long charged = 0;
  352. unsigned long map_flags;
  353. if (new_addr & ~PAGE_MASK)
  354. goto out;
  355. if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
  356. goto out;
  357. /* Check if the location we're moving into overlaps the
  358. * old location at all, and fail if it does.
  359. */
  360. if ((new_addr <= addr) && (new_addr+new_len) > addr)
  361. goto out;
  362. if ((addr <= new_addr) && (addr+old_len) > new_addr)
  363. goto out;
  364. ret = do_munmap(mm, new_addr, new_len);
  365. if (ret)
  366. goto out;
  367. if (old_len >= new_len) {
  368. ret = do_munmap(mm, addr+new_len, old_len - new_len);
  369. if (ret && old_len != new_len)
  370. goto out;
  371. old_len = new_len;
  372. }
  373. vma = vma_to_resize(addr, old_len, new_len, &charged);
  374. if (IS_ERR(vma)) {
  375. ret = PTR_ERR(vma);
  376. goto out;
  377. }
  378. map_flags = MAP_FIXED;
  379. if (vma->vm_flags & VM_MAYSHARE)
  380. map_flags |= MAP_SHARED;
  381. ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +
  382. ((addr - vma->vm_start) >> PAGE_SHIFT),
  383. map_flags);
  384. if (ret & ~PAGE_MASK)
  385. goto out1;
  386. ret = move_vma(vma, addr, old_len, new_len, new_addr, locked);
  387. if (!(ret & ~PAGE_MASK))
  388. goto out;
  389. out1:
  390. vm_unacct_memory(charged);
  391. out:
  392. return ret;
  393. }
  394. static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
  395. {
  396. unsigned long end = vma->vm_end + delta;
  397. if (end < vma->vm_end) /* overflow */
  398. return 0;
  399. if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */
  400. return 0;
  401. if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,
  402. 0, MAP_FIXED) & ~PAGE_MASK)
  403. return 0;
  404. return 1;
  405. }
  406. /*
  407. * Expand (or shrink) an existing mapping, potentially moving it at the
  408. * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
  409. *
  410. * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
  411. * This option implies MREMAP_MAYMOVE.
  412. */
  413. SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
  414. unsigned long, new_len, unsigned long, flags,
  415. unsigned long, new_addr)
  416. {
  417. struct mm_struct *mm = current->mm;
  418. struct vm_area_struct *vma;
  419. unsigned long ret = -EINVAL;
  420. unsigned long charged = 0;
  421. bool locked = false;
  422. if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
  423. return ret;
  424. if (flags & MREMAP_FIXED && !(flags & MREMAP_MAYMOVE))
  425. return ret;
  426. if (addr & ~PAGE_MASK)
  427. return ret;
  428. old_len = PAGE_ALIGN(old_len);
  429. new_len = PAGE_ALIGN(new_len);
  430. /*
  431. * We allow a zero old-len as a special case
  432. * for DOS-emu "duplicate shm area" thing. But
  433. * a zero new-len is nonsensical.
  434. */
  435. if (!new_len)
  436. return ret;
  437. down_write(&current->mm->mmap_sem);
  438. if (flags & MREMAP_FIXED) {
  439. ret = mremap_to(addr, old_len, new_addr, new_len,
  440. &locked);
  441. goto out;
  442. }
  443. /*
  444. * Always allow a shrinking remap: that just unmaps
  445. * the unnecessary pages..
  446. * do_munmap does all the needed commit accounting
  447. */
  448. if (old_len >= new_len) {
  449. ret = do_munmap(mm, addr+new_len, old_len - new_len);
  450. if (ret && old_len != new_len)
  451. goto out;
  452. ret = addr;
  453. goto out;
  454. }
  455. /*
  456. * Ok, we need to grow..
  457. */
  458. vma = vma_to_resize(addr, old_len, new_len, &charged);
  459. if (IS_ERR(vma)) {
  460. ret = PTR_ERR(vma);
  461. goto out;
  462. }
  463. /* old_len exactly to the end of the area..
  464. */
  465. if (old_len == vma->vm_end - addr) {
  466. /* can we just expand the current mapping? */
  467. if (vma_expandable(vma, new_len - old_len)) {
  468. int pages = (new_len - old_len) >> PAGE_SHIFT;
  469. if (vma_adjust(vma, vma->vm_start, addr + new_len,
  470. vma->vm_pgoff, NULL)) {
  471. ret = -ENOMEM;
  472. goto out;
  473. }
  474. vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
  475. if (vma->vm_flags & VM_LOCKED) {
  476. mm->locked_vm += pages;
  477. locked = true;
  478. new_addr = addr;
  479. }
  480. ret = addr;
  481. goto out;
  482. }
  483. }
  484. /*
  485. * We weren't able to just expand or shrink the area,
  486. * we need to create a new one and move it..
  487. */
  488. ret = -ENOMEM;
  489. if (flags & MREMAP_MAYMOVE) {
  490. unsigned long map_flags = 0;
  491. if (vma->vm_flags & VM_MAYSHARE)
  492. map_flags |= MAP_SHARED;
  493. new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
  494. vma->vm_pgoff +
  495. ((addr - vma->vm_start) >> PAGE_SHIFT),
  496. map_flags);
  497. if (new_addr & ~PAGE_MASK) {
  498. ret = new_addr;
  499. goto out;
  500. }
  501. ret = move_vma(vma, addr, old_len, new_len, new_addr, &locked);
  502. }
  503. out:
  504. if (ret & ~PAGE_MASK)
  505. vm_unacct_memory(charged);
  506. up_write(&current->mm->mmap_sem);
  507. if (locked && new_len > old_len)
  508. mm_populate(new_addr + old_len, new_len - old_len);
  509. return ret;
  510. }