fault.c 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275
  1. /*
  2. * Copyright (C) 1995 Linus Torvalds
  3. * Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs.
  4. * Copyright (C) 2008-2009, Red Hat Inc., Ingo Molnar
  5. */
  6. #include <linux/magic.h> /* STACK_END_MAGIC */
  7. #include <linux/sched.h> /* test_thread_flag(), ... */
  8. #include <linux/kdebug.h> /* oops_begin/end, ... */
  9. #include <linux/module.h> /* search_exception_table */
  10. #include <linux/bootmem.h> /* max_low_pfn */
  11. #include <linux/kprobes.h> /* __kprobes, ... */
  12. #include <linux/mmiotrace.h> /* kmmio_handler, ... */
  13. #include <linux/perf_event.h> /* perf_sw_event */
  14. #include <linux/hugetlb.h> /* hstate_index_to_shift */
  15. #include <linux/prefetch.h> /* prefetchw */
  16. #include <linux/context_tracking.h> /* exception_enter(), ... */
  17. #include <asm/traps.h> /* dotraplinkage, ... */
  18. #include <asm/pgalloc.h> /* pgd_*(), ... */
  19. #include <asm/kmemcheck.h> /* kmemcheck_*(), ... */
  20. #include <asm/fixmap.h> /* VSYSCALL_START */
  21. #define CREATE_TRACE_POINTS
  22. #include <asm/trace/exceptions.h>
  23. /*
  24. * Page fault error code bits:
  25. *
  26. * bit 0 == 0: no page found 1: protection fault
  27. * bit 1 == 0: read access 1: write access
  28. * bit 2 == 0: kernel-mode access 1: user-mode access
  29. * bit 3 == 1: use of reserved bit detected
  30. * bit 4 == 1: fault was an instruction fetch
  31. */
  32. enum x86_pf_error_code {
  33. PF_PROT = 1 << 0,
  34. PF_WRITE = 1 << 1,
  35. PF_USER = 1 << 2,
  36. PF_RSVD = 1 << 3,
  37. PF_INSTR = 1 << 4,
  38. };
  39. /*
  40. * Returns 0 if mmiotrace is disabled, or if the fault is not
  41. * handled by mmiotrace:
  42. */
  43. static inline int __kprobes
  44. kmmio_fault(struct pt_regs *regs, unsigned long addr)
  45. {
  46. if (unlikely(is_kmmio_active()))
  47. if (kmmio_handler(regs, addr) == 1)
  48. return -1;
  49. return 0;
  50. }
  51. static inline int __kprobes kprobes_fault(struct pt_regs *regs)
  52. {
  53. int ret = 0;
  54. /* kprobe_running() needs smp_processor_id() */
  55. if (kprobes_built_in() && !user_mode_vm(regs)) {
  56. preempt_disable();
  57. if (kprobe_running() && kprobe_fault_handler(regs, 14))
  58. ret = 1;
  59. preempt_enable();
  60. }
  61. return ret;
  62. }
  63. /*
  64. * Prefetch quirks:
  65. *
  66. * 32-bit mode:
  67. *
  68. * Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
  69. * Check that here and ignore it.
  70. *
  71. * 64-bit mode:
  72. *
  73. * Sometimes the CPU reports invalid exceptions on prefetch.
  74. * Check that here and ignore it.
  75. *
  76. * Opcode checker based on code by Richard Brunner.
  77. */
  78. static inline int
  79. check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
  80. unsigned char opcode, int *prefetch)
  81. {
  82. unsigned char instr_hi = opcode & 0xf0;
  83. unsigned char instr_lo = opcode & 0x0f;
  84. switch (instr_hi) {
  85. case 0x20:
  86. case 0x30:
  87. /*
  88. * Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes.
  89. * In X86_64 long mode, the CPU will signal invalid
  90. * opcode if some of these prefixes are present so
  91. * X86_64 will never get here anyway
  92. */
  93. return ((instr_lo & 7) == 0x6);
  94. #ifdef CONFIG_X86_64
  95. case 0x40:
  96. /*
  97. * In AMD64 long mode 0x40..0x4F are valid REX prefixes
  98. * Need to figure out under what instruction mode the
  99. * instruction was issued. Could check the LDT for lm,
  100. * but for now it's good enough to assume that long
  101. * mode only uses well known segments or kernel.
  102. */
  103. return (!user_mode(regs) || user_64bit_mode(regs));
  104. #endif
  105. case 0x60:
  106. /* 0x64 thru 0x67 are valid prefixes in all modes. */
  107. return (instr_lo & 0xC) == 0x4;
  108. case 0xF0:
  109. /* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */
  110. return !instr_lo || (instr_lo>>1) == 1;
  111. case 0x00:
  112. /* Prefetch instruction is 0x0F0D or 0x0F18 */
  113. if (probe_kernel_address(instr, opcode))
  114. return 0;
  115. *prefetch = (instr_lo == 0xF) &&
  116. (opcode == 0x0D || opcode == 0x18);
  117. return 0;
  118. default:
  119. return 0;
  120. }
  121. }
  122. static int
  123. is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
  124. {
  125. unsigned char *max_instr;
  126. unsigned char *instr;
  127. int prefetch = 0;
  128. /*
  129. * If it was a exec (instruction fetch) fault on NX page, then
  130. * do not ignore the fault:
  131. */
  132. if (error_code & PF_INSTR)
  133. return 0;
  134. instr = (void *)convert_ip_to_linear(current, regs);
  135. max_instr = instr + 15;
  136. if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
  137. return 0;
  138. while (instr < max_instr) {
  139. unsigned char opcode;
  140. if (probe_kernel_address(instr, opcode))
  141. break;
  142. instr++;
  143. if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
  144. break;
  145. }
  146. return prefetch;
  147. }
  148. static void
  149. force_sig_info_fault(int si_signo, int si_code, unsigned long address,
  150. struct task_struct *tsk, int fault)
  151. {
  152. unsigned lsb = 0;
  153. siginfo_t info;
  154. info.si_signo = si_signo;
  155. info.si_errno = 0;
  156. info.si_code = si_code;
  157. info.si_addr = (void __user *)address;
  158. if (fault & VM_FAULT_HWPOISON_LARGE)
  159. lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
  160. if (fault & VM_FAULT_HWPOISON)
  161. lsb = PAGE_SHIFT;
  162. info.si_addr_lsb = lsb;
  163. force_sig_info(si_signo, &info, tsk);
  164. }
  165. DEFINE_SPINLOCK(pgd_lock);
  166. LIST_HEAD(pgd_list);
  167. #ifdef CONFIG_X86_32
  168. static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
  169. {
  170. unsigned index = pgd_index(address);
  171. pgd_t *pgd_k;
  172. pud_t *pud, *pud_k;
  173. pmd_t *pmd, *pmd_k;
  174. pgd += index;
  175. pgd_k = init_mm.pgd + index;
  176. if (!pgd_present(*pgd_k))
  177. return NULL;
  178. /*
  179. * set_pgd(pgd, *pgd_k); here would be useless on PAE
  180. * and redundant with the set_pmd() on non-PAE. As would
  181. * set_pud.
  182. */
  183. pud = pud_offset(pgd, address);
  184. pud_k = pud_offset(pgd_k, address);
  185. if (!pud_present(*pud_k))
  186. return NULL;
  187. pmd = pmd_offset(pud, address);
  188. pmd_k = pmd_offset(pud_k, address);
  189. if (!pmd_present(*pmd_k))
  190. return NULL;
  191. if (!pmd_present(*pmd))
  192. set_pmd(pmd, *pmd_k);
  193. else
  194. BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
  195. return pmd_k;
  196. }
  197. void vmalloc_sync_all(void)
  198. {
  199. unsigned long address;
  200. if (SHARED_KERNEL_PMD)
  201. return;
  202. for (address = VMALLOC_START & PMD_MASK;
  203. address >= TASK_SIZE && address < FIXADDR_TOP;
  204. address += PMD_SIZE) {
  205. struct page *page;
  206. spin_lock(&pgd_lock);
  207. list_for_each_entry(page, &pgd_list, lru) {
  208. spinlock_t *pgt_lock;
  209. pmd_t *ret;
  210. /* the pgt_lock only for Xen */
  211. pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
  212. spin_lock(pgt_lock);
  213. ret = vmalloc_sync_one(page_address(page), address);
  214. spin_unlock(pgt_lock);
  215. if (!ret)
  216. break;
  217. }
  218. spin_unlock(&pgd_lock);
  219. }
  220. }
  221. /*
  222. * 32-bit:
  223. *
  224. * Handle a fault on the vmalloc or module mapping area
  225. */
  226. static noinline __kprobes int vmalloc_fault(unsigned long address)
  227. {
  228. unsigned long pgd_paddr;
  229. pmd_t *pmd_k;
  230. pte_t *pte_k;
  231. /* Make sure we are in vmalloc area: */
  232. if (!(address >= VMALLOC_START && address < VMALLOC_END))
  233. return -1;
  234. WARN_ON_ONCE(in_nmi());
  235. /*
  236. * Synchronize this task's top level page-table
  237. * with the 'reference' page table.
  238. *
  239. * Do _not_ use "current" here. We might be inside
  240. * an interrupt in the middle of a task switch..
  241. */
  242. pgd_paddr = read_cr3();
  243. pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
  244. if (!pmd_k)
  245. return -1;
  246. pte_k = pte_offset_kernel(pmd_k, address);
  247. if (!pte_present(*pte_k))
  248. return -1;
  249. return 0;
  250. }
  251. /*
  252. * Did it hit the DOS screen memory VA from vm86 mode?
  253. */
  254. static inline void
  255. check_v8086_mode(struct pt_regs *regs, unsigned long address,
  256. struct task_struct *tsk)
  257. {
  258. unsigned long bit;
  259. if (!v8086_mode(regs))
  260. return;
  261. bit = (address - 0xA0000) >> PAGE_SHIFT;
  262. if (bit < 32)
  263. tsk->thread.screen_bitmap |= 1 << bit;
  264. }
  265. static bool low_pfn(unsigned long pfn)
  266. {
  267. return pfn < max_low_pfn;
  268. }
  269. static void dump_pagetable(unsigned long address)
  270. {
  271. pgd_t *base = __va(read_cr3());
  272. pgd_t *pgd = &base[pgd_index(address)];
  273. pmd_t *pmd;
  274. pte_t *pte;
  275. #ifdef CONFIG_X86_PAE
  276. printk("*pdpt = %016Lx ", pgd_val(*pgd));
  277. if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))
  278. goto out;
  279. #endif
  280. pmd = pmd_offset(pud_offset(pgd, address), address);
  281. printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));
  282. /*
  283. * We must not directly access the pte in the highpte
  284. * case if the page table is located in highmem.
  285. * And let's rather not kmap-atomic the pte, just in case
  286. * it's allocated already:
  287. */
  288. if (!low_pfn(pmd_pfn(*pmd)) || !pmd_present(*pmd) || pmd_large(*pmd))
  289. goto out;
  290. pte = pte_offset_kernel(pmd, address);
  291. printk("*pte = %0*Lx ", sizeof(*pte) * 2, (u64)pte_val(*pte));
  292. out:
  293. printk("\n");
  294. }
  295. #else /* CONFIG_X86_64: */
  296. void vmalloc_sync_all(void)
  297. {
  298. sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END);
  299. }
  300. /*
  301. * 64-bit:
  302. *
  303. * Handle a fault on the vmalloc area
  304. *
  305. * This assumes no large pages in there.
  306. */
  307. static noinline __kprobes int vmalloc_fault(unsigned long address)
  308. {
  309. pgd_t *pgd, *pgd_ref;
  310. pud_t *pud, *pud_ref;
  311. pmd_t *pmd, *pmd_ref;
  312. pte_t *pte, *pte_ref;
  313. /* Make sure we are in vmalloc area: */
  314. if (!(address >= VMALLOC_START && address < VMALLOC_END))
  315. return -1;
  316. WARN_ON_ONCE(in_nmi());
  317. /*
  318. * Copy kernel mappings over when needed. This can also
  319. * happen within a race in page table update. In the later
  320. * case just flush:
  321. */
  322. pgd = pgd_offset(current->active_mm, address);
  323. pgd_ref = pgd_offset_k(address);
  324. if (pgd_none(*pgd_ref))
  325. return -1;
  326. if (pgd_none(*pgd)) {
  327. set_pgd(pgd, *pgd_ref);
  328. arch_flush_lazy_mmu_mode();
  329. } else {
  330. BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
  331. }
  332. /*
  333. * Below here mismatches are bugs because these lower tables
  334. * are shared:
  335. */
  336. pud = pud_offset(pgd, address);
  337. pud_ref = pud_offset(pgd_ref, address);
  338. if (pud_none(*pud_ref))
  339. return -1;
  340. if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
  341. BUG();
  342. pmd = pmd_offset(pud, address);
  343. pmd_ref = pmd_offset(pud_ref, address);
  344. if (pmd_none(*pmd_ref))
  345. return -1;
  346. if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
  347. BUG();
  348. pte_ref = pte_offset_kernel(pmd_ref, address);
  349. if (!pte_present(*pte_ref))
  350. return -1;
  351. pte = pte_offset_kernel(pmd, address);
  352. /*
  353. * Don't use pte_page here, because the mappings can point
  354. * outside mem_map, and the NUMA hash lookup cannot handle
  355. * that:
  356. */
  357. if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
  358. BUG();
  359. return 0;
  360. }
  361. #ifdef CONFIG_CPU_SUP_AMD
  362. static const char errata93_warning[] =
  363. KERN_ERR
  364. "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
  365. "******* Working around it, but it may cause SEGVs or burn power.\n"
  366. "******* Please consider a BIOS update.\n"
  367. "******* Disabling USB legacy in the BIOS may also help.\n";
  368. #endif
  369. /*
  370. * No vm86 mode in 64-bit mode:
  371. */
  372. static inline void
  373. check_v8086_mode(struct pt_regs *regs, unsigned long address,
  374. struct task_struct *tsk)
  375. {
  376. }
  377. static int bad_address(void *p)
  378. {
  379. unsigned long dummy;
  380. return probe_kernel_address((unsigned long *)p, dummy);
  381. }
  382. static void dump_pagetable(unsigned long address)
  383. {
  384. pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);
  385. pgd_t *pgd = base + pgd_index(address);
  386. pud_t *pud;
  387. pmd_t *pmd;
  388. pte_t *pte;
  389. if (bad_address(pgd))
  390. goto bad;
  391. printk("PGD %lx ", pgd_val(*pgd));
  392. if (!pgd_present(*pgd))
  393. goto out;
  394. pud = pud_offset(pgd, address);
  395. if (bad_address(pud))
  396. goto bad;
  397. printk("PUD %lx ", pud_val(*pud));
  398. if (!pud_present(*pud) || pud_large(*pud))
  399. goto out;
  400. pmd = pmd_offset(pud, address);
  401. if (bad_address(pmd))
  402. goto bad;
  403. printk("PMD %lx ", pmd_val(*pmd));
  404. if (!pmd_present(*pmd) || pmd_large(*pmd))
  405. goto out;
  406. pte = pte_offset_kernel(pmd, address);
  407. if (bad_address(pte))
  408. goto bad;
  409. printk("PTE %lx", pte_val(*pte));
  410. out:
  411. printk("\n");
  412. return;
  413. bad:
  414. printk("BAD\n");
  415. }
  416. #endif /* CONFIG_X86_64 */
  417. /*
  418. * Workaround for K8 erratum #93 & buggy BIOS.
  419. *
  420. * BIOS SMM functions are required to use a specific workaround
  421. * to avoid corruption of the 64bit RIP register on C stepping K8.
  422. *
  423. * A lot of BIOS that didn't get tested properly miss this.
  424. *
  425. * The OS sees this as a page fault with the upper 32bits of RIP cleared.
  426. * Try to work around it here.
  427. *
  428. * Note we only handle faults in kernel here.
  429. * Does nothing on 32-bit.
  430. */
  431. static int is_errata93(struct pt_regs *regs, unsigned long address)
  432. {
  433. #if defined(CONFIG_X86_64) && defined(CONFIG_CPU_SUP_AMD)
  434. if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD
  435. || boot_cpu_data.x86 != 0xf)
  436. return 0;
  437. if (address != regs->ip)
  438. return 0;
  439. if ((address >> 32) != 0)
  440. return 0;
  441. address |= 0xffffffffUL << 32;
  442. if ((address >= (u64)_stext && address <= (u64)_etext) ||
  443. (address >= MODULES_VADDR && address <= MODULES_END)) {
  444. printk_once(errata93_warning);
  445. regs->ip = address;
  446. return 1;
  447. }
  448. #endif
  449. return 0;
  450. }
  451. /*
  452. * Work around K8 erratum #100 K8 in compat mode occasionally jumps
  453. * to illegal addresses >4GB.
  454. *
  455. * We catch this in the page fault handler because these addresses
  456. * are not reachable. Just detect this case and return. Any code
  457. * segment in LDT is compatibility mode.
  458. */
  459. static int is_errata100(struct pt_regs *regs, unsigned long address)
  460. {
  461. #ifdef CONFIG_X86_64
  462. if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
  463. return 1;
  464. #endif
  465. return 0;
  466. }
  467. static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
  468. {
  469. #ifdef CONFIG_X86_F00F_BUG
  470. unsigned long nr;
  471. /*
  472. * Pentium F0 0F C7 C8 bug workaround:
  473. */
  474. if (boot_cpu_has_bug(X86_BUG_F00F)) {
  475. nr = (address - idt_descr.address) >> 3;
  476. if (nr == 6) {
  477. do_invalid_op(regs, 0);
  478. return 1;
  479. }
  480. }
  481. #endif
  482. return 0;
  483. }
  484. static const char nx_warning[] = KERN_CRIT
  485. "kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n";
  486. static void
  487. show_fault_oops(struct pt_regs *regs, unsigned long error_code,
  488. unsigned long address)
  489. {
  490. if (!oops_may_print())
  491. return;
  492. if (error_code & PF_INSTR) {
  493. unsigned int level;
  494. pte_t *pte = lookup_address(address, &level);
  495. if (pte && pte_present(*pte) && !pte_exec(*pte))
  496. printk(nx_warning, from_kuid(&init_user_ns, current_uid()));
  497. }
  498. printk(KERN_ALERT "BUG: unable to handle kernel ");
  499. if (address < PAGE_SIZE)
  500. printk(KERN_CONT "NULL pointer dereference");
  501. else
  502. printk(KERN_CONT "paging request");
  503. printk(KERN_CONT " at %p\n", (void *) address);
  504. printk(KERN_ALERT "IP:");
  505. printk_address(regs->ip);
  506. dump_pagetable(address);
  507. }
  508. static noinline void
  509. pgtable_bad(struct pt_regs *regs, unsigned long error_code,
  510. unsigned long address)
  511. {
  512. struct task_struct *tsk;
  513. unsigned long flags;
  514. int sig;
  515. flags = oops_begin();
  516. tsk = current;
  517. sig = SIGKILL;
  518. printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
  519. tsk->comm, address);
  520. dump_pagetable(address);
  521. tsk->thread.cr2 = address;
  522. tsk->thread.trap_nr = X86_TRAP_PF;
  523. tsk->thread.error_code = error_code;
  524. if (__die("Bad pagetable", regs, error_code))
  525. sig = 0;
  526. oops_end(flags, regs, sig);
  527. }
  528. static noinline void
  529. no_context(struct pt_regs *regs, unsigned long error_code,
  530. unsigned long address, int signal, int si_code)
  531. {
  532. struct task_struct *tsk = current;
  533. unsigned long *stackend;
  534. unsigned long flags;
  535. int sig;
  536. /* Are we prepared to handle this kernel fault? */
  537. if (fixup_exception(regs)) {
  538. /*
  539. * Any interrupt that takes a fault gets the fixup. This makes
  540. * the below recursive fault logic only apply to a faults from
  541. * task context.
  542. */
  543. if (in_interrupt())
  544. return;
  545. /*
  546. * Per the above we're !in_interrupt(), aka. task context.
  547. *
  548. * In this case we need to make sure we're not recursively
  549. * faulting through the emulate_vsyscall() logic.
  550. */
  551. if (current_thread_info()->sig_on_uaccess_error && signal) {
  552. tsk->thread.trap_nr = X86_TRAP_PF;
  553. tsk->thread.error_code = error_code | PF_USER;
  554. tsk->thread.cr2 = address;
  555. /* XXX: hwpoison faults will set the wrong code. */
  556. force_sig_info_fault(signal, si_code, address, tsk, 0);
  557. }
  558. /*
  559. * Barring that, we can do the fixup and be happy.
  560. */
  561. return;
  562. }
  563. /*
  564. * 32-bit:
  565. *
  566. * Valid to do another page fault here, because if this fault
  567. * had been triggered by is_prefetch fixup_exception would have
  568. * handled it.
  569. *
  570. * 64-bit:
  571. *
  572. * Hall of shame of CPU/BIOS bugs.
  573. */
  574. if (is_prefetch(regs, error_code, address))
  575. return;
  576. if (is_errata93(regs, address))
  577. return;
  578. /*
  579. * Oops. The kernel tried to access some bad page. We'll have to
  580. * terminate things with extreme prejudice:
  581. */
  582. flags = oops_begin();
  583. show_fault_oops(regs, error_code, address);
  584. stackend = end_of_stack(tsk);
  585. if (tsk != &init_task && *stackend != STACK_END_MAGIC)
  586. printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
  587. tsk->thread.cr2 = address;
  588. tsk->thread.trap_nr = X86_TRAP_PF;
  589. tsk->thread.error_code = error_code;
  590. sig = SIGKILL;
  591. if (__die("Oops", regs, error_code))
  592. sig = 0;
  593. /* Executive summary in case the body of the oops scrolled away */
  594. printk(KERN_DEFAULT "CR2: %016lx\n", address);
  595. oops_end(flags, regs, sig);
  596. }
  597. /*
  598. * Print out info about fatal segfaults, if the show_unhandled_signals
  599. * sysctl is set:
  600. */
  601. static inline void
  602. show_signal_msg(struct pt_regs *regs, unsigned long error_code,
  603. unsigned long address, struct task_struct *tsk)
  604. {
  605. if (!unhandled_signal(tsk, SIGSEGV))
  606. return;
  607. if (!printk_ratelimit())
  608. return;
  609. printk("%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
  610. task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
  611. tsk->comm, task_pid_nr(tsk), address,
  612. (void *)regs->ip, (void *)regs->sp, error_code);
  613. print_vma_addr(KERN_CONT " in ", regs->ip);
  614. printk(KERN_CONT "\n");
  615. }
  616. static void
  617. __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
  618. unsigned long address, int si_code)
  619. {
  620. struct task_struct *tsk = current;
  621. /* User mode accesses just cause a SIGSEGV */
  622. if (error_code & PF_USER) {
  623. /*
  624. * It's possible to have interrupts off here:
  625. */
  626. local_irq_enable();
  627. /*
  628. * Valid to do another page fault here because this one came
  629. * from user space:
  630. */
  631. if (is_prefetch(regs, error_code, address))
  632. return;
  633. if (is_errata100(regs, address))
  634. return;
  635. #ifdef CONFIG_X86_64
  636. /*
  637. * Instruction fetch faults in the vsyscall page might need
  638. * emulation.
  639. */
  640. if (unlikely((error_code & PF_INSTR) &&
  641. ((address & ~0xfff) == VSYSCALL_START))) {
  642. if (emulate_vsyscall(regs, address))
  643. return;
  644. }
  645. #endif
  646. /* Kernel addresses are always protection faults: */
  647. if (address >= TASK_SIZE)
  648. error_code |= PF_PROT;
  649. if (likely(show_unhandled_signals))
  650. show_signal_msg(regs, error_code, address, tsk);
  651. tsk->thread.cr2 = address;
  652. tsk->thread.error_code = error_code;
  653. tsk->thread.trap_nr = X86_TRAP_PF;
  654. force_sig_info_fault(SIGSEGV, si_code, address, tsk, 0);
  655. return;
  656. }
  657. if (is_f00f_bug(regs, address))
  658. return;
  659. no_context(regs, error_code, address, SIGSEGV, si_code);
  660. }
  661. static noinline void
  662. bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
  663. unsigned long address)
  664. {
  665. __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
  666. }
  667. static void
  668. __bad_area(struct pt_regs *regs, unsigned long error_code,
  669. unsigned long address, int si_code)
  670. {
  671. struct mm_struct *mm = current->mm;
  672. /*
  673. * Something tried to access memory that isn't in our memory map..
  674. * Fix it, but check if it's kernel or user first..
  675. */
  676. up_read(&mm->mmap_sem);
  677. __bad_area_nosemaphore(regs, error_code, address, si_code);
  678. }
  679. static noinline void
  680. bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
  681. {
  682. __bad_area(regs, error_code, address, SEGV_MAPERR);
  683. }
  684. static noinline void
  685. bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
  686. unsigned long address)
  687. {
  688. __bad_area(regs, error_code, address, SEGV_ACCERR);
  689. }
  690. static void
  691. do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
  692. unsigned int fault)
  693. {
  694. struct task_struct *tsk = current;
  695. struct mm_struct *mm = tsk->mm;
  696. int code = BUS_ADRERR;
  697. up_read(&mm->mmap_sem);
  698. /* Kernel mode? Handle exceptions or die: */
  699. if (!(error_code & PF_USER)) {
  700. no_context(regs, error_code, address, SIGBUS, BUS_ADRERR);
  701. return;
  702. }
  703. /* User-space => ok to do another page fault: */
  704. if (is_prefetch(regs, error_code, address))
  705. return;
  706. tsk->thread.cr2 = address;
  707. tsk->thread.error_code = error_code;
  708. tsk->thread.trap_nr = X86_TRAP_PF;
  709. #ifdef CONFIG_MEMORY_FAILURE
  710. if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
  711. printk(KERN_ERR
  712. "MCE: Killing %s:%d due to hardware memory corruption fault at %lx\n",
  713. tsk->comm, tsk->pid, address);
  714. code = BUS_MCEERR_AR;
  715. }
  716. #endif
  717. force_sig_info_fault(SIGBUS, code, address, tsk, fault);
  718. }
  719. static noinline void
  720. mm_fault_error(struct pt_regs *regs, unsigned long error_code,
  721. unsigned long address, unsigned int fault)
  722. {
  723. if (fatal_signal_pending(current) && !(error_code & PF_USER)) {
  724. up_read(&current->mm->mmap_sem);
  725. no_context(regs, error_code, address, 0, 0);
  726. return;
  727. }
  728. if (fault & VM_FAULT_OOM) {
  729. /* Kernel mode? Handle exceptions or die: */
  730. if (!(error_code & PF_USER)) {
  731. up_read(&current->mm->mmap_sem);
  732. no_context(regs, error_code, address,
  733. SIGSEGV, SEGV_MAPERR);
  734. return;
  735. }
  736. up_read(&current->mm->mmap_sem);
  737. /*
  738. * We ran out of memory, call the OOM killer, and return the
  739. * userspace (which will retry the fault, or kill us if we got
  740. * oom-killed):
  741. */
  742. pagefault_out_of_memory();
  743. } else {
  744. if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|
  745. VM_FAULT_HWPOISON_LARGE))
  746. do_sigbus(regs, error_code, address, fault);
  747. else
  748. BUG();
  749. }
  750. }
  751. static int spurious_fault_check(unsigned long error_code, pte_t *pte)
  752. {
  753. if ((error_code & PF_WRITE) && !pte_write(*pte))
  754. return 0;
  755. if ((error_code & PF_INSTR) && !pte_exec(*pte))
  756. return 0;
  757. return 1;
  758. }
  759. /*
  760. * Handle a spurious fault caused by a stale TLB entry.
  761. *
  762. * This allows us to lazily refresh the TLB when increasing the
  763. * permissions of a kernel page (RO -> RW or NX -> X). Doing it
  764. * eagerly is very expensive since that implies doing a full
  765. * cross-processor TLB flush, even if no stale TLB entries exist
  766. * on other processors.
  767. *
  768. * There are no security implications to leaving a stale TLB when
  769. * increasing the permissions on a page.
  770. */
  771. static noinline __kprobes int
  772. spurious_fault(unsigned long error_code, unsigned long address)
  773. {
  774. pgd_t *pgd;
  775. pud_t *pud;
  776. pmd_t *pmd;
  777. pte_t *pte;
  778. int ret;
  779. /* Reserved-bit violation or user access to kernel space? */
  780. if (error_code & (PF_USER | PF_RSVD))
  781. return 0;
  782. pgd = init_mm.pgd + pgd_index(address);
  783. if (!pgd_present(*pgd))
  784. return 0;
  785. pud = pud_offset(pgd, address);
  786. if (!pud_present(*pud))
  787. return 0;
  788. if (pud_large(*pud))
  789. return spurious_fault_check(error_code, (pte_t *) pud);
  790. pmd = pmd_offset(pud, address);
  791. if (!pmd_present(*pmd))
  792. return 0;
  793. if (pmd_large(*pmd))
  794. return spurious_fault_check(error_code, (pte_t *) pmd);
  795. pte = pte_offset_kernel(pmd, address);
  796. if (!pte_present(*pte))
  797. return 0;
  798. ret = spurious_fault_check(error_code, pte);
  799. if (!ret)
  800. return 0;
  801. /*
  802. * Make sure we have permissions in PMD.
  803. * If not, then there's a bug in the page tables:
  804. */
  805. ret = spurious_fault_check(error_code, (pte_t *) pmd);
  806. WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
  807. return ret;
  808. }
  809. int show_unhandled_signals = 1;
  810. static inline int
  811. access_error(unsigned long error_code, struct vm_area_struct *vma)
  812. {
  813. if (error_code & PF_WRITE) {
  814. /* write, present and write, not present: */
  815. if (unlikely(!(vma->vm_flags & VM_WRITE)))
  816. return 1;
  817. return 0;
  818. }
  819. /* read, present: */
  820. if (unlikely(error_code & PF_PROT))
  821. return 1;
  822. /* read, not present: */
  823. if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
  824. return 1;
  825. return 0;
  826. }
  827. static int fault_in_kernel_space(unsigned long address)
  828. {
  829. return address >= TASK_SIZE_MAX;
  830. }
  831. static inline bool smap_violation(int error_code, struct pt_regs *regs)
  832. {
  833. if (error_code & PF_USER)
  834. return false;
  835. if (!user_mode_vm(regs) && (regs->flags & X86_EFLAGS_AC))
  836. return false;
  837. return true;
  838. }
  839. /*
  840. * This routine handles page faults. It determines the address,
  841. * and the problem, and then passes it off to one of the appropriate
  842. * routines.
  843. */
  844. static void __kprobes
  845. __do_page_fault(struct pt_regs *regs, unsigned long error_code)
  846. {
  847. struct vm_area_struct *vma;
  848. struct task_struct *tsk;
  849. unsigned long address;
  850. struct mm_struct *mm;
  851. int fault;
  852. unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
  853. tsk = current;
  854. mm = tsk->mm;
  855. /* Get the faulting address: */
  856. address = read_cr2();
  857. /*
  858. * Detect and handle instructions that would cause a page fault for
  859. * both a tracked kernel page and a userspace page.
  860. */
  861. if (kmemcheck_active(regs))
  862. kmemcheck_hide(regs);
  863. prefetchw(&mm->mmap_sem);
  864. if (unlikely(kmmio_fault(regs, address)))
  865. return;
  866. /*
  867. * We fault-in kernel-space virtual memory on-demand. The
  868. * 'reference' page table is init_mm.pgd.
  869. *
  870. * NOTE! We MUST NOT take any locks for this case. We may
  871. * be in an interrupt or a critical region, and should
  872. * only copy the information from the master page table,
  873. * nothing more.
  874. *
  875. * This verifies that the fault happens in kernel space
  876. * (error_code & 4) == 0, and that the fault was not a
  877. * protection error (error_code & 9) == 0.
  878. */
  879. if (unlikely(fault_in_kernel_space(address))) {
  880. if (!(error_code & (PF_RSVD | PF_USER | PF_PROT))) {
  881. if (vmalloc_fault(address) >= 0)
  882. return;
  883. if (kmemcheck_fault(regs, address, error_code))
  884. return;
  885. }
  886. /* Can handle a stale RO->RW TLB: */
  887. if (spurious_fault(error_code, address))
  888. return;
  889. /* kprobes don't want to hook the spurious faults: */
  890. if (kprobes_fault(regs))
  891. return;
  892. /*
  893. * Don't take the mm semaphore here. If we fixup a prefetch
  894. * fault we could otherwise deadlock:
  895. */
  896. bad_area_nosemaphore(regs, error_code, address);
  897. return;
  898. }
  899. /* kprobes don't want to hook the spurious faults: */
  900. if (unlikely(kprobes_fault(regs)))
  901. return;
  902. if (unlikely(error_code & PF_RSVD))
  903. pgtable_bad(regs, error_code, address);
  904. if (static_cpu_has(X86_FEATURE_SMAP)) {
  905. if (unlikely(smap_violation(error_code, regs))) {
  906. bad_area_nosemaphore(regs, error_code, address);
  907. return;
  908. }
  909. }
  910. /*
  911. * If we're in an interrupt, have no user context or are running
  912. * in an atomic region then we must not take the fault:
  913. */
  914. if (unlikely(in_atomic() || !mm)) {
  915. bad_area_nosemaphore(regs, error_code, address);
  916. return;
  917. }
  918. /*
  919. * It's safe to allow irq's after cr2 has been saved and the
  920. * vmalloc fault has been handled.
  921. *
  922. * User-mode registers count as a user access even for any
  923. * potential system fault or CPU buglet:
  924. */
  925. if (user_mode_vm(regs)) {
  926. local_irq_enable();
  927. error_code |= PF_USER;
  928. flags |= FAULT_FLAG_USER;
  929. } else {
  930. if (regs->flags & X86_EFLAGS_IF)
  931. local_irq_enable();
  932. }
  933. perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
  934. if (error_code & PF_WRITE)
  935. flags |= FAULT_FLAG_WRITE;
  936. /*
  937. * When running in the kernel we expect faults to occur only to
  938. * addresses in user space. All other faults represent errors in
  939. * the kernel and should generate an OOPS. Unfortunately, in the
  940. * case of an erroneous fault occurring in a code path which already
  941. * holds mmap_sem we will deadlock attempting to validate the fault
  942. * against the address space. Luckily the kernel only validly
  943. * references user space from well defined areas of code, which are
  944. * listed in the exceptions table.
  945. *
  946. * As the vast majority of faults will be valid we will only perform
  947. * the source reference check when there is a possibility of a
  948. * deadlock. Attempt to lock the address space, if we cannot we then
  949. * validate the source. If this is invalid we can skip the address
  950. * space check, thus avoiding the deadlock:
  951. */
  952. if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
  953. if ((error_code & PF_USER) == 0 &&
  954. !search_exception_tables(regs->ip)) {
  955. bad_area_nosemaphore(regs, error_code, address);
  956. return;
  957. }
  958. retry:
  959. down_read(&mm->mmap_sem);
  960. } else {
  961. /*
  962. * The above down_read_trylock() might have succeeded in
  963. * which case we'll have missed the might_sleep() from
  964. * down_read():
  965. */
  966. might_sleep();
  967. }
  968. vma = find_vma(mm, address);
  969. if (unlikely(!vma)) {
  970. bad_area(regs, error_code, address);
  971. return;
  972. }
  973. if (likely(vma->vm_start <= address))
  974. goto good_area;
  975. if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
  976. bad_area(regs, error_code, address);
  977. return;
  978. }
  979. if (error_code & PF_USER) {
  980. /*
  981. * Accessing the stack below %sp is always a bug.
  982. * The large cushion allows instructions like enter
  983. * and pusha to work. ("enter $65535, $31" pushes
  984. * 32 pointers and then decrements %sp by 65535.)
  985. */
  986. if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
  987. bad_area(regs, error_code, address);
  988. return;
  989. }
  990. }
  991. if (unlikely(expand_stack(vma, address))) {
  992. bad_area(regs, error_code, address);
  993. return;
  994. }
  995. /*
  996. * Ok, we have a good vm_area for this memory access, so
  997. * we can handle it..
  998. */
  999. good_area:
  1000. if (unlikely(access_error(error_code, vma))) {
  1001. bad_area_access_error(regs, error_code, address);
  1002. return;
  1003. }
  1004. /*
  1005. * If for any reason at all we couldn't handle the fault,
  1006. * make sure we exit gracefully rather than endlessly redo
  1007. * the fault:
  1008. */
  1009. fault = handle_mm_fault(mm, vma, address, flags);
  1010. /*
  1011. * If we need to retry but a fatal signal is pending, handle the
  1012. * signal first. We do not need to release the mmap_sem because it
  1013. * would already be released in __lock_page_or_retry in mm/filemap.c.
  1014. */
  1015. if (unlikely((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)))
  1016. return;
  1017. if (unlikely(fault & VM_FAULT_ERROR)) {
  1018. mm_fault_error(regs, error_code, address, fault);
  1019. return;
  1020. }
  1021. /*
  1022. * Major/minor page fault accounting is only done on the
  1023. * initial attempt. If we go through a retry, it is extremely
  1024. * likely that the page will be found in page cache at that point.
  1025. */
  1026. if (flags & FAULT_FLAG_ALLOW_RETRY) {
  1027. if (fault & VM_FAULT_MAJOR) {
  1028. tsk->maj_flt++;
  1029. perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
  1030. regs, address);
  1031. } else {
  1032. tsk->min_flt++;
  1033. perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
  1034. regs, address);
  1035. }
  1036. if (fault & VM_FAULT_RETRY) {
  1037. /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
  1038. * of starvation. */
  1039. flags &= ~FAULT_FLAG_ALLOW_RETRY;
  1040. flags |= FAULT_FLAG_TRIED;
  1041. goto retry;
  1042. }
  1043. }
  1044. check_v8086_mode(regs, address, tsk);
  1045. up_read(&mm->mmap_sem);
  1046. }
  1047. dotraplinkage void __kprobes
  1048. do_page_fault(struct pt_regs *regs, unsigned long error_code)
  1049. {
  1050. enum ctx_state prev_state;
  1051. prev_state = exception_enter();
  1052. __do_page_fault(regs, error_code);
  1053. exception_exit(prev_state);
  1054. }
  1055. static void trace_page_fault_entries(struct pt_regs *regs,
  1056. unsigned long error_code)
  1057. {
  1058. if (user_mode(regs))
  1059. trace_page_fault_user(read_cr2(), regs, error_code);
  1060. else
  1061. trace_page_fault_kernel(read_cr2(), regs, error_code);
  1062. }
  1063. dotraplinkage void __kprobes
  1064. trace_do_page_fault(struct pt_regs *regs, unsigned long error_code)
  1065. {
  1066. enum ctx_state prev_state;
  1067. prev_state = exception_enter();
  1068. trace_page_fault_entries(regs, error_code);
  1069. __do_page_fault(regs, error_code);
  1070. exception_exit(prev_state);
  1071. }