traps.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946
  1. /*
  2. * Copyright (C) 1991, 1992 Linus Torvalds
  3. * Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
  4. *
  5. * Pentium III FXSR, SSE support
  6. * Gareth Hughes <gareth@valinux.com>, May 2000
  7. */
  8. /*
  9. * Handle hardware traps and faults.
  10. */
  11. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12. #include <linux/context_tracking.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/kallsyms.h>
  15. #include <linux/spinlock.h>
  16. #include <linux/kprobes.h>
  17. #include <linux/uaccess.h>
  18. #include <linux/kdebug.h>
  19. #include <linux/kgdb.h>
  20. #include <linux/kernel.h>
  21. #include <linux/module.h>
  22. #include <linux/ptrace.h>
  23. #include <linux/uprobes.h>
  24. #include <linux/string.h>
  25. #include <linux/delay.h>
  26. #include <linux/errno.h>
  27. #include <linux/kexec.h>
  28. #include <linux/sched.h>
  29. #include <linux/timer.h>
  30. #include <linux/init.h>
  31. #include <linux/bug.h>
  32. #include <linux/nmi.h>
  33. #include <linux/mm.h>
  34. #include <linux/smp.h>
  35. #include <linux/io.h>
  36. #ifdef CONFIG_EISA
  37. #include <linux/ioport.h>
  38. #include <linux/eisa.h>
  39. #endif
  40. #if defined(CONFIG_EDAC)
  41. #include <linux/edac.h>
  42. #endif
  43. #include <asm/kmemcheck.h>
  44. #include <asm/stacktrace.h>
  45. #include <asm/processor.h>
  46. #include <asm/debugreg.h>
  47. #include <linux/atomic.h>
  48. #include <asm/ftrace.h>
  49. #include <asm/traps.h>
  50. #include <asm/desc.h>
  51. #include <asm/i387.h>
  52. #include <asm/fpu-internal.h>
  53. #include <asm/mce.h>
  54. #include <asm/fixmap.h>
  55. #include <asm/mach_traps.h>
  56. #include <asm/alternative.h>
  57. #include <asm/mpx.h>
  58. #ifdef CONFIG_X86_64
  59. #include <asm/x86_init.h>
  60. #include <asm/pgalloc.h>
  61. #include <asm/proto.h>
  62. /* No need to be aligned, but done to keep all IDTs defined the same way. */
  63. gate_desc debug_idt_table[NR_VECTORS] __page_aligned_bss;
  64. #else
  65. #include <asm/processor-flags.h>
  66. #include <asm/setup.h>
  67. asmlinkage int system_call(void);
  68. #endif
  69. /* Must be page-aligned because the real IDT is used in a fixmap. */
  70. gate_desc idt_table[NR_VECTORS] __page_aligned_bss;
  71. DECLARE_BITMAP(used_vectors, NR_VECTORS);
  72. EXPORT_SYMBOL_GPL(used_vectors);
  73. static inline void conditional_sti(struct pt_regs *regs)
  74. {
  75. if (regs->flags & X86_EFLAGS_IF)
  76. local_irq_enable();
  77. }
  78. static inline void preempt_conditional_sti(struct pt_regs *regs)
  79. {
  80. preempt_count_inc();
  81. if (regs->flags & X86_EFLAGS_IF)
  82. local_irq_enable();
  83. }
  84. static inline void conditional_cli(struct pt_regs *regs)
  85. {
  86. if (regs->flags & X86_EFLAGS_IF)
  87. local_irq_disable();
  88. }
  89. static inline void preempt_conditional_cli(struct pt_regs *regs)
  90. {
  91. if (regs->flags & X86_EFLAGS_IF)
  92. local_irq_disable();
  93. preempt_count_dec();
  94. }
  95. static nokprobe_inline int
  96. do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
  97. struct pt_regs *regs, long error_code)
  98. {
  99. #ifdef CONFIG_X86_32
  100. if (regs->flags & X86_VM_MASK) {
  101. /*
  102. * Traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
  103. * On nmi (interrupt 2), do_trap should not be called.
  104. */
  105. if (trapnr < X86_TRAP_UD) {
  106. if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,
  107. error_code, trapnr))
  108. return 0;
  109. }
  110. return -1;
  111. }
  112. #endif
  113. if (!user_mode(regs)) {
  114. if (!fixup_exception(regs)) {
  115. tsk->thread.error_code = error_code;
  116. tsk->thread.trap_nr = trapnr;
  117. die(str, regs, error_code);
  118. }
  119. return 0;
  120. }
  121. return -1;
  122. }
  123. static siginfo_t *fill_trap_info(struct pt_regs *regs, int signr, int trapnr,
  124. siginfo_t *info)
  125. {
  126. unsigned long siaddr;
  127. int sicode;
  128. switch (trapnr) {
  129. default:
  130. return SEND_SIG_PRIV;
  131. case X86_TRAP_DE:
  132. sicode = FPE_INTDIV;
  133. siaddr = uprobe_get_trap_addr(regs);
  134. break;
  135. case X86_TRAP_UD:
  136. sicode = ILL_ILLOPN;
  137. siaddr = uprobe_get_trap_addr(regs);
  138. break;
  139. case X86_TRAP_AC:
  140. sicode = BUS_ADRALN;
  141. siaddr = 0;
  142. break;
  143. }
  144. info->si_signo = signr;
  145. info->si_errno = 0;
  146. info->si_code = sicode;
  147. info->si_addr = (void __user *)siaddr;
  148. return info;
  149. }
  150. static void
  151. do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
  152. long error_code, siginfo_t *info)
  153. {
  154. struct task_struct *tsk = current;
  155. if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))
  156. return;
  157. /*
  158. * We want error_code and trap_nr set for userspace faults and
  159. * kernelspace faults which result in die(), but not
  160. * kernelspace faults which are fixed up. die() gives the
  161. * process no chance to handle the signal and notice the
  162. * kernel fault information, so that won't result in polluting
  163. * the information about previously queued, but not yet
  164. * delivered, faults. See also do_general_protection below.
  165. */
  166. tsk->thread.error_code = error_code;
  167. tsk->thread.trap_nr = trapnr;
  168. #ifdef CONFIG_X86_64
  169. if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
  170. printk_ratelimit()) {
  171. pr_info("%s[%d] trap %s ip:%lx sp:%lx error:%lx",
  172. tsk->comm, tsk->pid, str,
  173. regs->ip, regs->sp, error_code);
  174. print_vma_addr(" in ", regs->ip);
  175. pr_cont("\n");
  176. }
  177. #endif
  178. force_sig_info(signr, info ?: SEND_SIG_PRIV, tsk);
  179. }
  180. NOKPROBE_SYMBOL(do_trap);
  181. static void do_error_trap(struct pt_regs *regs, long error_code, char *str,
  182. unsigned long trapnr, int signr)
  183. {
  184. enum ctx_state prev_state = exception_enter();
  185. siginfo_t info;
  186. if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
  187. NOTIFY_STOP) {
  188. conditional_sti(regs);
  189. do_trap(trapnr, signr, str, regs, error_code,
  190. fill_trap_info(regs, signr, trapnr, &info));
  191. }
  192. exception_exit(prev_state);
  193. }
  194. #define DO_ERROR(trapnr, signr, str, name) \
  195. dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
  196. { \
  197. do_error_trap(regs, error_code, str, trapnr, signr); \
  198. }
  199. DO_ERROR(X86_TRAP_DE, SIGFPE, "divide error", divide_error)
  200. DO_ERROR(X86_TRAP_OF, SIGSEGV, "overflow", overflow)
  201. DO_ERROR(X86_TRAP_UD, SIGILL, "invalid opcode", invalid_op)
  202. DO_ERROR(X86_TRAP_OLD_MF, SIGFPE, "coprocessor segment overrun",coprocessor_segment_overrun)
  203. DO_ERROR(X86_TRAP_TS, SIGSEGV, "invalid TSS", invalid_TSS)
  204. DO_ERROR(X86_TRAP_NP, SIGBUS, "segment not present", segment_not_present)
  205. DO_ERROR(X86_TRAP_SS, SIGBUS, "stack segment", stack_segment)
  206. DO_ERROR(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check)
  207. #ifdef CONFIG_X86_64
  208. /* Runs on IST stack */
  209. dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
  210. {
  211. static const char str[] = "double fault";
  212. struct task_struct *tsk = current;
  213. #ifdef CONFIG_X86_ESPFIX64
  214. extern unsigned char native_irq_return_iret[];
  215. /*
  216. * If IRET takes a non-IST fault on the espfix64 stack, then we
  217. * end up promoting it to a doublefault. In that case, modify
  218. * the stack to make it look like we just entered the #GP
  219. * handler from user space, similar to bad_iret.
  220. */
  221. if (((long)regs->sp >> PGDIR_SHIFT) == ESPFIX_PGD_ENTRY &&
  222. regs->cs == __KERNEL_CS &&
  223. regs->ip == (unsigned long)native_irq_return_iret)
  224. {
  225. struct pt_regs *normal_regs = task_pt_regs(current);
  226. /* Fake a #GP(0) from userspace. */
  227. memmove(&normal_regs->ip, (void *)regs->sp, 5*8);
  228. normal_regs->orig_ax = 0; /* Missing (lost) #GP error code */
  229. regs->ip = (unsigned long)general_protection;
  230. regs->sp = (unsigned long)&normal_regs->orig_ax;
  231. return;
  232. }
  233. #endif
  234. exception_enter();
  235. /* Return not checked because double check cannot be ignored */
  236. notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
  237. tsk->thread.error_code = error_code;
  238. tsk->thread.trap_nr = X86_TRAP_DF;
  239. #ifdef CONFIG_DOUBLEFAULT
  240. df_debug(regs, error_code);
  241. #endif
  242. /*
  243. * This is always a kernel trap and never fixable (and thus must
  244. * never return).
  245. */
  246. for (;;)
  247. die(str, regs, error_code);
  248. }
  249. #endif
  250. dotraplinkage void do_bounds(struct pt_regs *regs, long error_code)
  251. {
  252. struct task_struct *tsk = current;
  253. struct xsave_struct *xsave_buf;
  254. enum ctx_state prev_state;
  255. struct bndcsr *bndcsr;
  256. siginfo_t *info;
  257. prev_state = exception_enter();
  258. if (notify_die(DIE_TRAP, "bounds", regs, error_code,
  259. X86_TRAP_BR, SIGSEGV) == NOTIFY_STOP)
  260. goto exit;
  261. conditional_sti(regs);
  262. if (!user_mode(regs))
  263. die("bounds", regs, error_code);
  264. if (!cpu_feature_enabled(X86_FEATURE_MPX)) {
  265. /* The exception is not from Intel MPX */
  266. goto exit_trap;
  267. }
  268. /*
  269. * We need to look at BNDSTATUS to resolve this exception.
  270. * It is not directly accessible, though, so we need to
  271. * do an xsave and then pull it out of the xsave buffer.
  272. */
  273. fpu_save_init(&tsk->thread.fpu);
  274. xsave_buf = &(tsk->thread.fpu.state->xsave);
  275. bndcsr = get_xsave_addr(xsave_buf, XSTATE_BNDCSR);
  276. if (!bndcsr)
  277. goto exit_trap;
  278. /*
  279. * The error code field of the BNDSTATUS register communicates status
  280. * information of a bound range exception #BR or operation involving
  281. * bound directory.
  282. */
  283. switch (bndcsr->bndstatus & MPX_BNDSTA_ERROR_CODE) {
  284. case 2: /* Bound directory has invalid entry. */
  285. if (mpx_handle_bd_fault(xsave_buf))
  286. goto exit_trap;
  287. break; /* Success, it was handled */
  288. case 1: /* Bound violation. */
  289. info = mpx_generate_siginfo(regs, xsave_buf);
  290. if (IS_ERR(info)) {
  291. /*
  292. * We failed to decode the MPX instruction. Act as if
  293. * the exception was not caused by MPX.
  294. */
  295. goto exit_trap;
  296. }
  297. /*
  298. * Success, we decoded the instruction and retrieved
  299. * an 'info' containing the address being accessed
  300. * which caused the exception. This information
  301. * allows and application to possibly handle the
  302. * #BR exception itself.
  303. */
  304. do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, info);
  305. kfree(info);
  306. break;
  307. case 0: /* No exception caused by Intel MPX operations. */
  308. goto exit_trap;
  309. default:
  310. die("bounds", regs, error_code);
  311. }
  312. exit:
  313. exception_exit(prev_state);
  314. return;
  315. exit_trap:
  316. /*
  317. * This path out is for all the cases where we could not
  318. * handle the exception in some way (like allocating a
  319. * table or telling userspace about it. We will also end
  320. * up here if the kernel has MPX turned off at compile
  321. * time..
  322. */
  323. do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, NULL);
  324. exception_exit(prev_state);
  325. }
  326. dotraplinkage void
  327. do_general_protection(struct pt_regs *regs, long error_code)
  328. {
  329. struct task_struct *tsk;
  330. enum ctx_state prev_state;
  331. prev_state = exception_enter();
  332. conditional_sti(regs);
  333. #ifdef CONFIG_X86_32
  334. if (regs->flags & X86_VM_MASK) {
  335. local_irq_enable();
  336. handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
  337. goto exit;
  338. }
  339. #endif
  340. tsk = current;
  341. if (!user_mode(regs)) {
  342. if (fixup_exception(regs))
  343. goto exit;
  344. tsk->thread.error_code = error_code;
  345. tsk->thread.trap_nr = X86_TRAP_GP;
  346. if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
  347. X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP)
  348. die("general protection fault", regs, error_code);
  349. goto exit;
  350. }
  351. tsk->thread.error_code = error_code;
  352. tsk->thread.trap_nr = X86_TRAP_GP;
  353. if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
  354. printk_ratelimit()) {
  355. pr_info("%s[%d] general protection ip:%lx sp:%lx error:%lx",
  356. tsk->comm, task_pid_nr(tsk),
  357. regs->ip, regs->sp, error_code);
  358. print_vma_addr(" in ", regs->ip);
  359. pr_cont("\n");
  360. }
  361. force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
  362. exit:
  363. exception_exit(prev_state);
  364. }
  365. NOKPROBE_SYMBOL(do_general_protection);
  366. /* May run on IST stack. */
  367. dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
  368. {
  369. enum ctx_state prev_state;
  370. #ifdef CONFIG_DYNAMIC_FTRACE
  371. /*
  372. * ftrace must be first, everything else may cause a recursive crash.
  373. * See note by declaration of modifying_ftrace_code in ftrace.c
  374. */
  375. if (unlikely(atomic_read(&modifying_ftrace_code)) &&
  376. ftrace_int3_handler(regs))
  377. return;
  378. #endif
  379. if (poke_int3_handler(regs))
  380. return;
  381. prev_state = exception_enter();
  382. #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
  383. if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
  384. SIGTRAP) == NOTIFY_STOP)
  385. goto exit;
  386. #endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
  387. #ifdef CONFIG_KPROBES
  388. if (kprobe_int3_handler(regs))
  389. goto exit;
  390. #endif
  391. if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
  392. SIGTRAP) == NOTIFY_STOP)
  393. goto exit;
  394. /*
  395. * Let others (NMI) know that the debug stack is in use
  396. * as we may switch to the interrupt stack.
  397. */
  398. debug_stack_usage_inc();
  399. preempt_conditional_sti(regs);
  400. do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
  401. preempt_conditional_cli(regs);
  402. debug_stack_usage_dec();
  403. exit:
  404. exception_exit(prev_state);
  405. }
  406. NOKPROBE_SYMBOL(do_int3);
  407. #ifdef CONFIG_X86_64
  408. /*
  409. * Help handler running on IST stack to switch back to user stack
  410. * for scheduling or signal handling. The actual stack switch is done in
  411. * entry.S
  412. */
  413. asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs)
  414. {
  415. struct pt_regs *regs = eregs;
  416. /* Did already sync */
  417. if (eregs == (struct pt_regs *)eregs->sp)
  418. ;
  419. /* Exception from user space */
  420. else if (user_mode(eregs))
  421. regs = task_pt_regs(current);
  422. /*
  423. * Exception from kernel and interrupts are enabled. Move to
  424. * kernel process stack.
  425. */
  426. else if (eregs->flags & X86_EFLAGS_IF)
  427. regs = (struct pt_regs *)(eregs->sp -= sizeof(struct pt_regs));
  428. if (eregs != regs)
  429. *regs = *eregs;
  430. return regs;
  431. }
  432. NOKPROBE_SYMBOL(sync_regs);
  433. struct bad_iret_stack {
  434. void *error_entry_ret;
  435. struct pt_regs regs;
  436. };
  437. asmlinkage __visible notrace
  438. struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s)
  439. {
  440. /*
  441. * This is called from entry_64.S early in handling a fault
  442. * caused by a bad iret to user mode. To handle the fault
  443. * correctly, we want move our stack frame to task_pt_regs
  444. * and we want to pretend that the exception came from the
  445. * iret target.
  446. */
  447. struct bad_iret_stack *new_stack =
  448. container_of(task_pt_regs(current),
  449. struct bad_iret_stack, regs);
  450. /* Copy the IRET target to the new stack. */
  451. memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8);
  452. /* Copy the remainder of the stack from the current stack. */
  453. memmove(new_stack, s, offsetof(struct bad_iret_stack, regs.ip));
  454. BUG_ON(!user_mode_vm(&new_stack->regs));
  455. return new_stack;
  456. }
  457. NOKPROBE_SYMBOL(fixup_bad_iret);
  458. #endif
  459. /*
  460. * Our handling of the processor debug registers is non-trivial.
  461. * We do not clear them on entry and exit from the kernel. Therefore
  462. * it is possible to get a watchpoint trap here from inside the kernel.
  463. * However, the code in ./ptrace.c has ensured that the user can
  464. * only set watchpoints on userspace addresses. Therefore the in-kernel
  465. * watchpoint trap can only occur in code which is reading/writing
  466. * from user space. Such code must not hold kernel locks (since it
  467. * can equally take a page fault), therefore it is safe to call
  468. * force_sig_info even though that claims and releases locks.
  469. *
  470. * Code in ./signal.c ensures that the debug control register
  471. * is restored before we deliver any signal, and therefore that
  472. * user code runs with the correct debug control register even though
  473. * we clear it here.
  474. *
  475. * Being careful here means that we don't have to be as careful in a
  476. * lot of more complicated places (task switching can be a bit lazy
  477. * about restoring all the debug state, and ptrace doesn't have to
  478. * find every occurrence of the TF bit that could be saved away even
  479. * by user code)
  480. *
  481. * May run on IST stack.
  482. */
  483. dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
  484. {
  485. struct task_struct *tsk = current;
  486. enum ctx_state prev_state;
  487. int user_icebp = 0;
  488. unsigned long dr6;
  489. int si_code;
  490. prev_state = exception_enter();
  491. get_debugreg(dr6, 6);
  492. /* Filter out all the reserved bits which are preset to 1 */
  493. dr6 &= ~DR6_RESERVED;
  494. /*
  495. * If dr6 has no reason to give us about the origin of this trap,
  496. * then it's very likely the result of an icebp/int01 trap.
  497. * User wants a sigtrap for that.
  498. */
  499. if (!dr6 && user_mode(regs))
  500. user_icebp = 1;
  501. /* Catch kmemcheck conditions first of all! */
  502. if ((dr6 & DR_STEP) && kmemcheck_trap(regs))
  503. goto exit;
  504. /* DR6 may or may not be cleared by the CPU */
  505. set_debugreg(0, 6);
  506. /*
  507. * The processor cleared BTF, so don't mark that we need it set.
  508. */
  509. clear_tsk_thread_flag(tsk, TIF_BLOCKSTEP);
  510. /* Store the virtualized DR6 value */
  511. tsk->thread.debugreg6 = dr6;
  512. #ifdef CONFIG_KPROBES
  513. if (kprobe_debug_handler(regs))
  514. goto exit;
  515. #endif
  516. if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, error_code,
  517. SIGTRAP) == NOTIFY_STOP)
  518. goto exit;
  519. /*
  520. * Let others (NMI) know that the debug stack is in use
  521. * as we may switch to the interrupt stack.
  522. */
  523. debug_stack_usage_inc();
  524. /* It's safe to allow irq's after DR6 has been saved */
  525. preempt_conditional_sti(regs);
  526. if (regs->flags & X86_VM_MASK) {
  527. handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code,
  528. X86_TRAP_DB);
  529. preempt_conditional_cli(regs);
  530. debug_stack_usage_dec();
  531. goto exit;
  532. }
  533. /*
  534. * Single-stepping through system calls: ignore any exceptions in
  535. * kernel space, but re-enable TF when returning to user mode.
  536. *
  537. * We already checked v86 mode above, so we can check for kernel mode
  538. * by just checking the CPL of CS.
  539. */
  540. if ((dr6 & DR_STEP) && !user_mode(regs)) {
  541. tsk->thread.debugreg6 &= ~DR_STEP;
  542. set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
  543. regs->flags &= ~X86_EFLAGS_TF;
  544. }
  545. si_code = get_si_code(tsk->thread.debugreg6);
  546. if (tsk->thread.debugreg6 & (DR_STEP | DR_TRAP_BITS) || user_icebp)
  547. send_sigtrap(tsk, regs, error_code, si_code);
  548. preempt_conditional_cli(regs);
  549. debug_stack_usage_dec();
  550. exit:
  551. exception_exit(prev_state);
  552. }
  553. NOKPROBE_SYMBOL(do_debug);
  554. /*
  555. * Note that we play around with the 'TS' bit in an attempt to get
  556. * the correct behaviour even in the presence of the asynchronous
  557. * IRQ13 behaviour
  558. */
  559. static void math_error(struct pt_regs *regs, int error_code, int trapnr)
  560. {
  561. struct task_struct *task = current;
  562. siginfo_t info;
  563. unsigned short err;
  564. char *str = (trapnr == X86_TRAP_MF) ? "fpu exception" :
  565. "simd exception";
  566. if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, SIGFPE) == NOTIFY_STOP)
  567. return;
  568. conditional_sti(regs);
  569. if (!user_mode_vm(regs))
  570. {
  571. if (!fixup_exception(regs)) {
  572. task->thread.error_code = error_code;
  573. task->thread.trap_nr = trapnr;
  574. die(str, regs, error_code);
  575. }
  576. return;
  577. }
  578. /*
  579. * Save the info for the exception handler and clear the error.
  580. */
  581. save_init_fpu(task);
  582. task->thread.trap_nr = trapnr;
  583. task->thread.error_code = error_code;
  584. info.si_signo = SIGFPE;
  585. info.si_errno = 0;
  586. info.si_addr = (void __user *)uprobe_get_trap_addr(regs);
  587. if (trapnr == X86_TRAP_MF) {
  588. unsigned short cwd, swd;
  589. /*
  590. * (~cwd & swd) will mask out exceptions that are not set to unmasked
  591. * status. 0x3f is the exception bits in these regs, 0x200 is the
  592. * C1 reg you need in case of a stack fault, 0x040 is the stack
  593. * fault bit. We should only be taking one exception at a time,
  594. * so if this combination doesn't produce any single exception,
  595. * then we have a bad program that isn't synchronizing its FPU usage
  596. * and it will suffer the consequences since we won't be able to
  597. * fully reproduce the context of the exception
  598. */
  599. cwd = get_fpu_cwd(task);
  600. swd = get_fpu_swd(task);
  601. err = swd & ~cwd;
  602. } else {
  603. /*
  604. * The SIMD FPU exceptions are handled a little differently, as there
  605. * is only a single status/control register. Thus, to determine which
  606. * unmasked exception was caught we must mask the exception mask bits
  607. * at 0x1f80, and then use these to mask the exception bits at 0x3f.
  608. */
  609. unsigned short mxcsr = get_fpu_mxcsr(task);
  610. err = ~(mxcsr >> 7) & mxcsr;
  611. }
  612. if (err & 0x001) { /* Invalid op */
  613. /*
  614. * swd & 0x240 == 0x040: Stack Underflow
  615. * swd & 0x240 == 0x240: Stack Overflow
  616. * User must clear the SF bit (0x40) if set
  617. */
  618. info.si_code = FPE_FLTINV;
  619. } else if (err & 0x004) { /* Divide by Zero */
  620. info.si_code = FPE_FLTDIV;
  621. } else if (err & 0x008) { /* Overflow */
  622. info.si_code = FPE_FLTOVF;
  623. } else if (err & 0x012) { /* Denormal, Underflow */
  624. info.si_code = FPE_FLTUND;
  625. } else if (err & 0x020) { /* Precision */
  626. info.si_code = FPE_FLTRES;
  627. } else {
  628. /*
  629. * If we're using IRQ 13, or supposedly even some trap
  630. * X86_TRAP_MF implementations, it's possible
  631. * we get a spurious trap, which is not an error.
  632. */
  633. return;
  634. }
  635. force_sig_info(SIGFPE, &info, task);
  636. }
  637. dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
  638. {
  639. enum ctx_state prev_state;
  640. prev_state = exception_enter();
  641. math_error(regs, error_code, X86_TRAP_MF);
  642. exception_exit(prev_state);
  643. }
  644. dotraplinkage void
  645. do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
  646. {
  647. enum ctx_state prev_state;
  648. prev_state = exception_enter();
  649. math_error(regs, error_code, X86_TRAP_XF);
  650. exception_exit(prev_state);
  651. }
  652. dotraplinkage void
  653. do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
  654. {
  655. conditional_sti(regs);
  656. #if 0
  657. /* No need to warn about this any longer. */
  658. pr_info("Ignoring P6 Local APIC Spurious Interrupt Bug...\n");
  659. #endif
  660. }
  661. asmlinkage __visible void __attribute__((weak)) smp_thermal_interrupt(void)
  662. {
  663. }
  664. asmlinkage __visible void __attribute__((weak)) smp_threshold_interrupt(void)
  665. {
  666. }
  667. /*
  668. * 'math_state_restore()' saves the current math information in the
  669. * old math state array, and gets the new ones from the current task
  670. *
  671. * Careful.. There are problems with IBM-designed IRQ13 behaviour.
  672. * Don't touch unless you *really* know how it works.
  673. *
  674. * Must be called with kernel preemption disabled (eg with local
  675. * local interrupts as in the case of do_device_not_available).
  676. */
  677. void math_state_restore(void)
  678. {
  679. struct task_struct *tsk = current;
  680. if (!tsk_used_math(tsk)) {
  681. local_irq_enable();
  682. /*
  683. * does a slab alloc which can sleep
  684. */
  685. if (init_fpu(tsk)) {
  686. /*
  687. * ran out of memory!
  688. */
  689. do_group_exit(SIGKILL);
  690. return;
  691. }
  692. local_irq_disable();
  693. }
  694. __thread_fpu_begin(tsk);
  695. /*
  696. * Paranoid restore. send a SIGSEGV if we fail to restore the state.
  697. */
  698. if (unlikely(restore_fpu_checking(tsk))) {
  699. drop_init_fpu(tsk);
  700. force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
  701. return;
  702. }
  703. tsk->thread.fpu_counter++;
  704. }
  705. EXPORT_SYMBOL_GPL(math_state_restore);
  706. dotraplinkage void
  707. do_device_not_available(struct pt_regs *regs, long error_code)
  708. {
  709. enum ctx_state prev_state;
  710. prev_state = exception_enter();
  711. BUG_ON(use_eager_fpu());
  712. #ifdef CONFIG_MATH_EMULATION
  713. if (read_cr0() & X86_CR0_EM) {
  714. struct math_emu_info info = { };
  715. conditional_sti(regs);
  716. info.regs = regs;
  717. math_emulate(&info);
  718. exception_exit(prev_state);
  719. return;
  720. }
  721. #endif
  722. math_state_restore(); /* interrupts still off */
  723. #ifdef CONFIG_X86_32
  724. conditional_sti(regs);
  725. #endif
  726. exception_exit(prev_state);
  727. }
  728. NOKPROBE_SYMBOL(do_device_not_available);
  729. #ifdef CONFIG_X86_32
  730. dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
  731. {
  732. siginfo_t info;
  733. enum ctx_state prev_state;
  734. prev_state = exception_enter();
  735. local_irq_enable();
  736. info.si_signo = SIGILL;
  737. info.si_errno = 0;
  738. info.si_code = ILL_BADSTK;
  739. info.si_addr = NULL;
  740. if (notify_die(DIE_TRAP, "iret exception", regs, error_code,
  741. X86_TRAP_IRET, SIGILL) != NOTIFY_STOP) {
  742. do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,
  743. &info);
  744. }
  745. exception_exit(prev_state);
  746. }
  747. #endif
  748. /* Set of traps needed for early debugging. */
  749. void __init early_trap_init(void)
  750. {
  751. set_intr_gate_ist(X86_TRAP_DB, &debug, DEBUG_STACK);
  752. /* int3 can be called from all */
  753. set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK);
  754. #ifdef CONFIG_X86_32
  755. set_intr_gate(X86_TRAP_PF, page_fault);
  756. #endif
  757. load_idt(&idt_descr);
  758. }
  759. void __init early_trap_pf_init(void)
  760. {
  761. #ifdef CONFIG_X86_64
  762. set_intr_gate(X86_TRAP_PF, page_fault);
  763. #endif
  764. }
  765. void __init trap_init(void)
  766. {
  767. int i;
  768. #ifdef CONFIG_EISA
  769. void __iomem *p = early_ioremap(0x0FFFD9, 4);
  770. if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24))
  771. EISA_bus = 1;
  772. early_iounmap(p, 4);
  773. #endif
  774. set_intr_gate(X86_TRAP_DE, divide_error);
  775. set_intr_gate_ist(X86_TRAP_NMI, &nmi, NMI_STACK);
  776. /* int4 can be called from all */
  777. set_system_intr_gate(X86_TRAP_OF, &overflow);
  778. set_intr_gate(X86_TRAP_BR, bounds);
  779. set_intr_gate(X86_TRAP_UD, invalid_op);
  780. set_intr_gate(X86_TRAP_NM, device_not_available);
  781. #ifdef CONFIG_X86_32
  782. set_task_gate(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS);
  783. #else
  784. set_intr_gate_ist(X86_TRAP_DF, &double_fault, DOUBLEFAULT_STACK);
  785. #endif
  786. set_intr_gate(X86_TRAP_OLD_MF, coprocessor_segment_overrun);
  787. set_intr_gate(X86_TRAP_TS, invalid_TSS);
  788. set_intr_gate(X86_TRAP_NP, segment_not_present);
  789. set_intr_gate(X86_TRAP_SS, stack_segment);
  790. set_intr_gate(X86_TRAP_GP, general_protection);
  791. set_intr_gate(X86_TRAP_SPURIOUS, spurious_interrupt_bug);
  792. set_intr_gate(X86_TRAP_MF, coprocessor_error);
  793. set_intr_gate(X86_TRAP_AC, alignment_check);
  794. #ifdef CONFIG_X86_MCE
  795. set_intr_gate_ist(X86_TRAP_MC, &machine_check, MCE_STACK);
  796. #endif
  797. set_intr_gate(X86_TRAP_XF, simd_coprocessor_error);
  798. /* Reserve all the builtin and the syscall vector: */
  799. for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
  800. set_bit(i, used_vectors);
  801. #ifdef CONFIG_IA32_EMULATION
  802. set_system_intr_gate(IA32_SYSCALL_VECTOR, ia32_syscall);
  803. set_bit(IA32_SYSCALL_VECTOR, used_vectors);
  804. #endif
  805. #ifdef CONFIG_X86_32
  806. set_system_trap_gate(SYSCALL_VECTOR, &system_call);
  807. set_bit(SYSCALL_VECTOR, used_vectors);
  808. #endif
  809. /*
  810. * Set the IDT descriptor to a fixed read-only location, so that the
  811. * "sidt" instruction will not leak the location of the kernel, and
  812. * to defend the IDT against arbitrary memory write vulnerabilities.
  813. * It will be reloaded in cpu_init() */
  814. __set_fixmap(FIX_RO_IDT, __pa_symbol(idt_table), PAGE_KERNEL_RO);
  815. idt_descr.address = fix_to_virt(FIX_RO_IDT);
  816. /*
  817. * Should be a barrier for any external CPU state:
  818. */
  819. cpu_init();
  820. x86_init.irqs.trap_init();
  821. #ifdef CONFIG_X86_64
  822. memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16);
  823. set_nmi_gate(X86_TRAP_DB, &debug);
  824. set_nmi_gate(X86_TRAP_BP, &int3);
  825. #endif
  826. }