process.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566
  1. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  2. #include <linux/errno.h>
  3. #include <linux/kernel.h>
  4. #include <linux/mm.h>
  5. #include <linux/smp.h>
  6. #include <linux/prctl.h>
  7. #include <linux/slab.h>
  8. #include <linux/sched.h>
  9. #include <linux/module.h>
  10. #include <linux/pm.h>
  11. #include <linux/tick.h>
  12. #include <linux/random.h>
  13. #include <linux/user-return-notifier.h>
  14. #include <linux/dmi.h>
  15. #include <linux/utsname.h>
  16. #include <linux/stackprotector.h>
  17. #include <linux/tick.h>
  18. #include <linux/cpuidle.h>
  19. #include <trace/events/power.h>
  20. #include <linux/hw_breakpoint.h>
  21. #include <asm/cpu.h>
  22. #include <asm/apic.h>
  23. #include <asm/syscalls.h>
  24. #include <asm/idle.h>
  25. #include <asm/uaccess.h>
  26. #include <asm/mwait.h>
  27. #include <asm/fpu/internal.h>
  28. #include <asm/debugreg.h>
  29. #include <asm/nmi.h>
  30. #include <asm/tlbflush.h>
  31. #include <asm/mce.h>
  32. #include <asm/vm86.h>
  33. /*
  34. * per-CPU TSS segments. Threads are completely 'soft' on Linux,
  35. * no more per-task TSS's. The TSS size is kept cacheline-aligned
  36. * so they are allowed to end up in the .data..cacheline_aligned
  37. * section. Since TSS's are completely CPU-local, we want them
  38. * on exact cacheline boundaries, to eliminate cacheline ping-pong.
  39. */
  40. __visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
  41. .x86_tss = {
  42. .sp0 = TOP_OF_INIT_STACK,
  43. #ifdef CONFIG_X86_32
  44. .ss0 = __KERNEL_DS,
  45. .ss1 = __KERNEL_CS,
  46. .io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
  47. #endif
  48. },
  49. #ifdef CONFIG_X86_32
  50. /*
  51. * Note that the .io_bitmap member must be extra-big. This is because
  52. * the CPU will access an additional byte beyond the end of the IO
  53. * permission bitmap. The extra byte must be all 1 bits, and must
  54. * be within the limit.
  55. */
  56. .io_bitmap = { [0 ... IO_BITMAP_LONGS] = ~0 },
  57. #endif
  58. };
  59. EXPORT_PER_CPU_SYMBOL(cpu_tss);
  60. #ifdef CONFIG_X86_64
  61. static DEFINE_PER_CPU(unsigned char, is_idle);
  62. static ATOMIC_NOTIFIER_HEAD(idle_notifier);
  63. void idle_notifier_register(struct notifier_block *n)
  64. {
  65. atomic_notifier_chain_register(&idle_notifier, n);
  66. }
  67. EXPORT_SYMBOL_GPL(idle_notifier_register);
  68. void idle_notifier_unregister(struct notifier_block *n)
  69. {
  70. atomic_notifier_chain_unregister(&idle_notifier, n);
  71. }
  72. EXPORT_SYMBOL_GPL(idle_notifier_unregister);
  73. #endif
  74. /*
  75. * this gets called so that we can store lazy state into memory and copy the
  76. * current task into the new thread.
  77. */
  78. int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
  79. {
  80. memcpy(dst, src, arch_task_struct_size);
  81. #ifdef CONFIG_VM86
  82. dst->thread.vm86 = NULL;
  83. #endif
  84. return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
  85. }
  86. /*
  87. * Free current thread data structures etc..
  88. */
  89. void exit_thread(void)
  90. {
  91. struct task_struct *me = current;
  92. struct thread_struct *t = &me->thread;
  93. unsigned long *bp = t->io_bitmap_ptr;
  94. struct fpu *fpu = &t->fpu;
  95. if (bp) {
  96. struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
  97. t->io_bitmap_ptr = NULL;
  98. clear_thread_flag(TIF_IO_BITMAP);
  99. /*
  100. * Careful, clear this in the TSS too:
  101. */
  102. memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
  103. t->io_bitmap_max = 0;
  104. put_cpu();
  105. kfree(bp);
  106. }
  107. free_vm86(t);
  108. fpu__drop(fpu);
  109. }
  110. void flush_thread(void)
  111. {
  112. struct task_struct *tsk = current;
  113. flush_ptrace_hw_breakpoint(tsk);
  114. memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
  115. fpu__clear(&tsk->thread.fpu);
  116. }
  117. static void hard_disable_TSC(void)
  118. {
  119. cr4_set_bits(X86_CR4_TSD);
  120. }
  121. void disable_TSC(void)
  122. {
  123. preempt_disable();
  124. if (!test_and_set_thread_flag(TIF_NOTSC))
  125. /*
  126. * Must flip the CPU state synchronously with
  127. * TIF_NOTSC in the current running context.
  128. */
  129. hard_disable_TSC();
  130. preempt_enable();
  131. }
  132. static void hard_enable_TSC(void)
  133. {
  134. cr4_clear_bits(X86_CR4_TSD);
  135. }
  136. static void enable_TSC(void)
  137. {
  138. preempt_disable();
  139. if (test_and_clear_thread_flag(TIF_NOTSC))
  140. /*
  141. * Must flip the CPU state synchronously with
  142. * TIF_NOTSC in the current running context.
  143. */
  144. hard_enable_TSC();
  145. preempt_enable();
  146. }
  147. int get_tsc_mode(unsigned long adr)
  148. {
  149. unsigned int val;
  150. if (test_thread_flag(TIF_NOTSC))
  151. val = PR_TSC_SIGSEGV;
  152. else
  153. val = PR_TSC_ENABLE;
  154. return put_user(val, (unsigned int __user *)adr);
  155. }
  156. int set_tsc_mode(unsigned int val)
  157. {
  158. if (val == PR_TSC_SIGSEGV)
  159. disable_TSC();
  160. else if (val == PR_TSC_ENABLE)
  161. enable_TSC();
  162. else
  163. return -EINVAL;
  164. return 0;
  165. }
  166. void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
  167. struct tss_struct *tss)
  168. {
  169. struct thread_struct *prev, *next;
  170. prev = &prev_p->thread;
  171. next = &next_p->thread;
  172. if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^
  173. test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) {
  174. unsigned long debugctl = get_debugctlmsr();
  175. debugctl &= ~DEBUGCTLMSR_BTF;
  176. if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP))
  177. debugctl |= DEBUGCTLMSR_BTF;
  178. update_debugctlmsr(debugctl);
  179. }
  180. if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
  181. test_tsk_thread_flag(next_p, TIF_NOTSC)) {
  182. /* prev and next are different */
  183. if (test_tsk_thread_flag(next_p, TIF_NOTSC))
  184. hard_disable_TSC();
  185. else
  186. hard_enable_TSC();
  187. }
  188. if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
  189. /*
  190. * Copy the relevant range of the IO bitmap.
  191. * Normally this is 128 bytes or less:
  192. */
  193. memcpy(tss->io_bitmap, next->io_bitmap_ptr,
  194. max(prev->io_bitmap_max, next->io_bitmap_max));
  195. } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
  196. /*
  197. * Clear any possible leftover bits:
  198. */
  199. memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
  200. }
  201. propagate_user_return_notify(prev_p, next_p);
  202. }
  203. /*
  204. * Idle related variables and functions
  205. */
  206. unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
  207. EXPORT_SYMBOL(boot_option_idle_override);
  208. static void (*x86_idle)(void);
  209. #ifndef CONFIG_SMP
  210. static inline void play_dead(void)
  211. {
  212. BUG();
  213. }
  214. #endif
  215. #ifdef CONFIG_X86_64
  216. void enter_idle(void)
  217. {
  218. this_cpu_write(is_idle, 1);
  219. atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
  220. }
  221. static void __exit_idle(void)
  222. {
  223. if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
  224. return;
  225. atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
  226. }
  227. /* Called from interrupts to signify idle end */
  228. void exit_idle(void)
  229. {
  230. /* idle loop has pid 0 */
  231. if (current->pid)
  232. return;
  233. __exit_idle();
  234. }
  235. #endif
  236. void arch_cpu_idle_enter(void)
  237. {
  238. local_touch_nmi();
  239. enter_idle();
  240. }
  241. void arch_cpu_idle_exit(void)
  242. {
  243. __exit_idle();
  244. }
  245. void arch_cpu_idle_dead(void)
  246. {
  247. play_dead();
  248. }
  249. /*
  250. * Called from the generic idle code.
  251. */
  252. void arch_cpu_idle(void)
  253. {
  254. x86_idle();
  255. }
  256. /*
  257. * We use this if we don't have any better idle routine..
  258. */
  259. void default_idle(void)
  260. {
  261. trace_cpu_idle_rcuidle(1, smp_processor_id());
  262. safe_halt();
  263. trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
  264. }
  265. #ifdef CONFIG_APM_MODULE
  266. EXPORT_SYMBOL(default_idle);
  267. #endif
  268. #ifdef CONFIG_XEN
  269. bool xen_set_default_idle(void)
  270. {
  271. bool ret = !!x86_idle;
  272. x86_idle = default_idle;
  273. return ret;
  274. }
  275. #endif
  276. void stop_this_cpu(void *dummy)
  277. {
  278. local_irq_disable();
  279. /*
  280. * Remove this CPU:
  281. */
  282. set_cpu_online(smp_processor_id(), false);
  283. disable_local_APIC();
  284. mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
  285. for (;;)
  286. halt();
  287. }
  288. bool amd_e400_c1e_detected;
  289. EXPORT_SYMBOL(amd_e400_c1e_detected);
  290. static cpumask_var_t amd_e400_c1e_mask;
  291. void amd_e400_remove_cpu(int cpu)
  292. {
  293. if (amd_e400_c1e_mask != NULL)
  294. cpumask_clear_cpu(cpu, amd_e400_c1e_mask);
  295. }
  296. /*
  297. * AMD Erratum 400 aware idle routine. We check for C1E active in the interrupt
  298. * pending message MSR. If we detect C1E, then we handle it the same
  299. * way as C3 power states (local apic timer and TSC stop)
  300. */
  301. static void amd_e400_idle(void)
  302. {
  303. if (!amd_e400_c1e_detected) {
  304. u32 lo, hi;
  305. rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
  306. if (lo & K8_INTP_C1E_ACTIVE_MASK) {
  307. amd_e400_c1e_detected = true;
  308. if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
  309. mark_tsc_unstable("TSC halt in AMD C1E");
  310. pr_info("System has AMD C1E enabled\n");
  311. }
  312. }
  313. if (amd_e400_c1e_detected) {
  314. int cpu = smp_processor_id();
  315. if (!cpumask_test_cpu(cpu, amd_e400_c1e_mask)) {
  316. cpumask_set_cpu(cpu, amd_e400_c1e_mask);
  317. /* Force broadcast so ACPI can not interfere. */
  318. tick_broadcast_force();
  319. pr_info("Switch to broadcast mode on CPU%d\n", cpu);
  320. }
  321. tick_broadcast_enter();
  322. default_idle();
  323. /*
  324. * The switch back from broadcast mode needs to be
  325. * called with interrupts disabled.
  326. */
  327. local_irq_disable();
  328. tick_broadcast_exit();
  329. local_irq_enable();
  330. } else
  331. default_idle();
  332. }
  333. /*
  334. * Intel Core2 and older machines prefer MWAIT over HALT for C1.
  335. * We can't rely on cpuidle installing MWAIT, because it will not load
  336. * on systems that support only C1 -- so the boot default must be MWAIT.
  337. *
  338. * Some AMD machines are the opposite, they depend on using HALT.
  339. *
  340. * So for default C1, which is used during boot until cpuidle loads,
  341. * use MWAIT-C1 on Intel HW that has it, else use HALT.
  342. */
  343. static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
  344. {
  345. if (c->x86_vendor != X86_VENDOR_INTEL)
  346. return 0;
  347. if (!cpu_has(c, X86_FEATURE_MWAIT))
  348. return 0;
  349. return 1;
  350. }
  351. /*
  352. * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
  353. * with interrupts enabled and no flags, which is backwards compatible with the
  354. * original MWAIT implementation.
  355. */
  356. static void mwait_idle(void)
  357. {
  358. if (!current_set_polling_and_test()) {
  359. trace_cpu_idle_rcuidle(1, smp_processor_id());
  360. if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
  361. smp_mb(); /* quirk */
  362. clflush((void *)&current_thread_info()->flags);
  363. smp_mb(); /* quirk */
  364. }
  365. __monitor((void *)&current_thread_info()->flags, 0, 0);
  366. if (!need_resched())
  367. __sti_mwait(0, 0);
  368. else
  369. local_irq_enable();
  370. trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
  371. } else {
  372. local_irq_enable();
  373. }
  374. __current_clr_polling();
  375. }
  376. void select_idle_routine(const struct cpuinfo_x86 *c)
  377. {
  378. #ifdef CONFIG_SMP
  379. if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
  380. pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
  381. #endif
  382. if (x86_idle || boot_option_idle_override == IDLE_POLL)
  383. return;
  384. if (cpu_has_bug(c, X86_BUG_AMD_APIC_C1E)) {
  385. /* E400: APIC timer interrupt does not wake up CPU from C1e */
  386. pr_info("using AMD E400 aware idle routine\n");
  387. x86_idle = amd_e400_idle;
  388. } else if (prefer_mwait_c1_over_halt(c)) {
  389. pr_info("using mwait in idle threads\n");
  390. x86_idle = mwait_idle;
  391. } else
  392. x86_idle = default_idle;
  393. }
  394. void __init init_amd_e400_c1e_mask(void)
  395. {
  396. /* If we're using amd_e400_idle, we need to allocate amd_e400_c1e_mask. */
  397. if (x86_idle == amd_e400_idle)
  398. zalloc_cpumask_var(&amd_e400_c1e_mask, GFP_KERNEL);
  399. }
  400. static int __init idle_setup(char *str)
  401. {
  402. if (!str)
  403. return -EINVAL;
  404. if (!strcmp(str, "poll")) {
  405. pr_info("using polling idle threads\n");
  406. boot_option_idle_override = IDLE_POLL;
  407. cpu_idle_poll_ctrl(true);
  408. } else if (!strcmp(str, "halt")) {
  409. /*
  410. * When the boot option of idle=halt is added, halt is
  411. * forced to be used for CPU idle. In such case CPU C2/C3
  412. * won't be used again.
  413. * To continue to load the CPU idle driver, don't touch
  414. * the boot_option_idle_override.
  415. */
  416. x86_idle = default_idle;
  417. boot_option_idle_override = IDLE_HALT;
  418. } else if (!strcmp(str, "nomwait")) {
  419. /*
  420. * If the boot option of "idle=nomwait" is added,
  421. * it means that mwait will be disabled for CPU C2/C3
  422. * states. In such case it won't touch the variable
  423. * of boot_option_idle_override.
  424. */
  425. boot_option_idle_override = IDLE_NOMWAIT;
  426. } else
  427. return -1;
  428. return 0;
  429. }
  430. early_param("idle", idle_setup);
  431. unsigned long arch_align_stack(unsigned long sp)
  432. {
  433. if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
  434. sp -= get_random_int() % 8192;
  435. return sp & ~0xf;
  436. }
  437. unsigned long arch_randomize_brk(struct mm_struct *mm)
  438. {
  439. unsigned long range_end = mm->brk + 0x02000000;
  440. return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
  441. }
  442. /*
  443. * Called from fs/proc with a reference on @p to find the function
  444. * which called into schedule(). This needs to be done carefully
  445. * because the task might wake up and we might look at a stack
  446. * changing under us.
  447. */
  448. unsigned long get_wchan(struct task_struct *p)
  449. {
  450. unsigned long start, bottom, top, sp, fp, ip;
  451. int count = 0;
  452. if (!p || p == current || p->state == TASK_RUNNING)
  453. return 0;
  454. start = (unsigned long)task_stack_page(p);
  455. if (!start)
  456. return 0;
  457. /*
  458. * Layout of the stack page:
  459. *
  460. * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
  461. * PADDING
  462. * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
  463. * stack
  464. * ----------- bottom = start + sizeof(thread_info)
  465. * thread_info
  466. * ----------- start
  467. *
  468. * The tasks stack pointer points at the location where the
  469. * framepointer is stored. The data on the stack is:
  470. * ... IP FP ... IP FP
  471. *
  472. * We need to read FP and IP, so we need to adjust the upper
  473. * bound by another unsigned long.
  474. */
  475. top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
  476. top -= 2 * sizeof(unsigned long);
  477. bottom = start + sizeof(struct thread_info);
  478. sp = READ_ONCE(p->thread.sp);
  479. if (sp < bottom || sp > top)
  480. return 0;
  481. fp = READ_ONCE_NOCHECK(*(unsigned long *)sp);
  482. do {
  483. if (fp < bottom || fp > top)
  484. return 0;
  485. ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
  486. if (!in_sched_functions(ip))
  487. return ip;
  488. fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
  489. } while (count++ < 16 && p->state != TASK_RUNNING);
  490. return 0;
  491. }