perf_event_intel_lbr.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974
  1. #include <linux/perf_event.h>
  2. #include <linux/types.h>
  3. #include <asm/perf_event.h>
  4. #include <asm/msr.h>
  5. #include <asm/insn.h>
  6. #include "perf_event.h"
  7. enum {
  8. LBR_FORMAT_32 = 0x00,
  9. LBR_FORMAT_LIP = 0x01,
  10. LBR_FORMAT_EIP = 0x02,
  11. LBR_FORMAT_EIP_FLAGS = 0x03,
  12. LBR_FORMAT_EIP_FLAGS2 = 0x04,
  13. LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_EIP_FLAGS2,
  14. };
  15. static enum {
  16. LBR_EIP_FLAGS = 1,
  17. LBR_TSX = 2,
  18. } lbr_desc[LBR_FORMAT_MAX_KNOWN + 1] = {
  19. [LBR_FORMAT_EIP_FLAGS] = LBR_EIP_FLAGS,
  20. [LBR_FORMAT_EIP_FLAGS2] = LBR_EIP_FLAGS | LBR_TSX,
  21. };
  22. /*
  23. * Intel LBR_SELECT bits
  24. * Intel Vol3a, April 2011, Section 16.7 Table 16-10
  25. *
  26. * Hardware branch filter (not available on all CPUs)
  27. */
  28. #define LBR_KERNEL_BIT 0 /* do not capture at ring0 */
  29. #define LBR_USER_BIT 1 /* do not capture at ring > 0 */
  30. #define LBR_JCC_BIT 2 /* do not capture conditional branches */
  31. #define LBR_REL_CALL_BIT 3 /* do not capture relative calls */
  32. #define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */
  33. #define LBR_RETURN_BIT 5 /* do not capture near returns */
  34. #define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */
  35. #define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */
  36. #define LBR_FAR_BIT 8 /* do not capture far branches */
  37. #define LBR_CALL_STACK_BIT 9 /* enable call stack */
  38. #define LBR_KERNEL (1 << LBR_KERNEL_BIT)
  39. #define LBR_USER (1 << LBR_USER_BIT)
  40. #define LBR_JCC (1 << LBR_JCC_BIT)
  41. #define LBR_REL_CALL (1 << LBR_REL_CALL_BIT)
  42. #define LBR_IND_CALL (1 << LBR_IND_CALL_BIT)
  43. #define LBR_RETURN (1 << LBR_RETURN_BIT)
  44. #define LBR_REL_JMP (1 << LBR_REL_JMP_BIT)
  45. #define LBR_IND_JMP (1 << LBR_IND_JMP_BIT)
  46. #define LBR_FAR (1 << LBR_FAR_BIT)
  47. #define LBR_CALL_STACK (1 << LBR_CALL_STACK_BIT)
  48. #define LBR_PLM (LBR_KERNEL | LBR_USER)
  49. #define LBR_SEL_MASK 0x1ff /* valid bits in LBR_SELECT */
  50. #define LBR_NOT_SUPP -1 /* LBR filter not supported */
  51. #define LBR_IGN 0 /* ignored */
  52. #define LBR_ANY \
  53. (LBR_JCC |\
  54. LBR_REL_CALL |\
  55. LBR_IND_CALL |\
  56. LBR_RETURN |\
  57. LBR_REL_JMP |\
  58. LBR_IND_JMP |\
  59. LBR_FAR)
  60. #define LBR_FROM_FLAG_MISPRED (1ULL << 63)
  61. #define LBR_FROM_FLAG_IN_TX (1ULL << 62)
  62. #define LBR_FROM_FLAG_ABORT (1ULL << 61)
  63. /*
  64. * x86control flow change classification
  65. * x86control flow changes include branches, interrupts, traps, faults
  66. */
  67. enum {
  68. X86_BR_NONE = 0, /* unknown */
  69. X86_BR_USER = 1 << 0, /* branch target is user */
  70. X86_BR_KERNEL = 1 << 1, /* branch target is kernel */
  71. X86_BR_CALL = 1 << 2, /* call */
  72. X86_BR_RET = 1 << 3, /* return */
  73. X86_BR_SYSCALL = 1 << 4, /* syscall */
  74. X86_BR_SYSRET = 1 << 5, /* syscall return */
  75. X86_BR_INT = 1 << 6, /* sw interrupt */
  76. X86_BR_IRET = 1 << 7, /* return from interrupt */
  77. X86_BR_JCC = 1 << 8, /* conditional */
  78. X86_BR_JMP = 1 << 9, /* jump */
  79. X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */
  80. X86_BR_IND_CALL = 1 << 11,/* indirect calls */
  81. X86_BR_ABORT = 1 << 12,/* transaction abort */
  82. X86_BR_IN_TX = 1 << 13,/* in transaction */
  83. X86_BR_NO_TX = 1 << 14,/* not in transaction */
  84. X86_BR_ZERO_CALL = 1 << 15,/* zero length call */
  85. X86_BR_CALL_STACK = 1 << 16,/* call stack */
  86. X86_BR_IND_JMP = 1 << 17,/* indirect jump */
  87. };
  88. #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
  89. #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX)
  90. #define X86_BR_ANY \
  91. (X86_BR_CALL |\
  92. X86_BR_RET |\
  93. X86_BR_SYSCALL |\
  94. X86_BR_SYSRET |\
  95. X86_BR_INT |\
  96. X86_BR_IRET |\
  97. X86_BR_JCC |\
  98. X86_BR_JMP |\
  99. X86_BR_IRQ |\
  100. X86_BR_ABORT |\
  101. X86_BR_IND_CALL |\
  102. X86_BR_IND_JMP |\
  103. X86_BR_ZERO_CALL)
  104. #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)
  105. #define X86_BR_ANY_CALL \
  106. (X86_BR_CALL |\
  107. X86_BR_IND_CALL |\
  108. X86_BR_ZERO_CALL |\
  109. X86_BR_SYSCALL |\
  110. X86_BR_IRQ |\
  111. X86_BR_INT)
  112. static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
  113. /*
  114. * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  115. * otherwise it becomes near impossible to get a reliable stack.
  116. */
  117. static void __intel_pmu_lbr_enable(bool pmi)
  118. {
  119. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  120. u64 debugctl, lbr_select = 0, orig_debugctl;
  121. /*
  122. * No need to reprogram LBR_SELECT in a PMI, as it
  123. * did not change.
  124. */
  125. if (cpuc->lbr_sel && !pmi) {
  126. lbr_select = cpuc->lbr_sel->config;
  127. wrmsrl(MSR_LBR_SELECT, lbr_select);
  128. }
  129. rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  130. orig_debugctl = debugctl;
  131. debugctl |= DEBUGCTLMSR_LBR;
  132. /*
  133. * LBR callstack does not work well with FREEZE_LBRS_ON_PMI.
  134. * If FREEZE_LBRS_ON_PMI is set, PMI near call/return instructions
  135. * may cause superfluous increase/decrease of LBR_TOS.
  136. */
  137. if (!(lbr_select & LBR_CALL_STACK))
  138. debugctl |= DEBUGCTLMSR_FREEZE_LBRS_ON_PMI;
  139. if (orig_debugctl != debugctl)
  140. wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  141. }
  142. static void __intel_pmu_lbr_disable(void)
  143. {
  144. u64 debugctl;
  145. rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  146. debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
  147. wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  148. }
  149. static void intel_pmu_lbr_reset_32(void)
  150. {
  151. int i;
  152. for (i = 0; i < x86_pmu.lbr_nr; i++)
  153. wrmsrl(x86_pmu.lbr_from + i, 0);
  154. }
  155. static void intel_pmu_lbr_reset_64(void)
  156. {
  157. int i;
  158. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  159. wrmsrl(x86_pmu.lbr_from + i, 0);
  160. wrmsrl(x86_pmu.lbr_to + i, 0);
  161. }
  162. }
  163. void intel_pmu_lbr_reset(void)
  164. {
  165. if (!x86_pmu.lbr_nr)
  166. return;
  167. if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
  168. intel_pmu_lbr_reset_32();
  169. else
  170. intel_pmu_lbr_reset_64();
  171. }
  172. /*
  173. * TOS = most recently recorded branch
  174. */
  175. static inline u64 intel_pmu_lbr_tos(void)
  176. {
  177. u64 tos;
  178. rdmsrl(x86_pmu.lbr_tos, tos);
  179. return tos;
  180. }
  181. enum {
  182. LBR_NONE,
  183. LBR_VALID,
  184. };
  185. static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
  186. {
  187. int i;
  188. unsigned lbr_idx, mask;
  189. u64 tos;
  190. if (task_ctx->lbr_callstack_users == 0 ||
  191. task_ctx->lbr_stack_state == LBR_NONE) {
  192. intel_pmu_lbr_reset();
  193. return;
  194. }
  195. mask = x86_pmu.lbr_nr - 1;
  196. tos = intel_pmu_lbr_tos();
  197. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  198. lbr_idx = (tos - i) & mask;
  199. wrmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
  200. wrmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
  201. }
  202. task_ctx->lbr_stack_state = LBR_NONE;
  203. }
  204. static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)
  205. {
  206. int i;
  207. unsigned lbr_idx, mask;
  208. u64 tos;
  209. if (task_ctx->lbr_callstack_users == 0) {
  210. task_ctx->lbr_stack_state = LBR_NONE;
  211. return;
  212. }
  213. mask = x86_pmu.lbr_nr - 1;
  214. tos = intel_pmu_lbr_tos();
  215. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  216. lbr_idx = (tos - i) & mask;
  217. rdmsrl(x86_pmu.lbr_from + lbr_idx, task_ctx->lbr_from[i]);
  218. rdmsrl(x86_pmu.lbr_to + lbr_idx, task_ctx->lbr_to[i]);
  219. }
  220. task_ctx->lbr_stack_state = LBR_VALID;
  221. }
  222. void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in)
  223. {
  224. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  225. struct x86_perf_task_context *task_ctx;
  226. /*
  227. * If LBR callstack feature is enabled and the stack was saved when
  228. * the task was scheduled out, restore the stack. Otherwise flush
  229. * the LBR stack.
  230. */
  231. task_ctx = ctx ? ctx->task_ctx_data : NULL;
  232. if (task_ctx) {
  233. if (sched_in) {
  234. __intel_pmu_lbr_restore(task_ctx);
  235. cpuc->lbr_context = ctx;
  236. } else {
  237. __intel_pmu_lbr_save(task_ctx);
  238. }
  239. return;
  240. }
  241. /*
  242. * When sampling the branck stack in system-wide, it may be
  243. * necessary to flush the stack on context switch. This happens
  244. * when the branch stack does not tag its entries with the pid
  245. * of the current task. Otherwise it becomes impossible to
  246. * associate a branch entry with a task. This ambiguity is more
  247. * likely to appear when the branch stack supports priv level
  248. * filtering and the user sets it to monitor only at the user
  249. * level (which could be a useful measurement in system-wide
  250. * mode). In that case, the risk is high of having a branch
  251. * stack with branch from multiple tasks.
  252. */
  253. if (sched_in) {
  254. intel_pmu_lbr_reset();
  255. cpuc->lbr_context = ctx;
  256. }
  257. }
  258. static inline bool branch_user_callstack(unsigned br_sel)
  259. {
  260. return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK);
  261. }
  262. void intel_pmu_lbr_enable(struct perf_event *event)
  263. {
  264. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  265. struct x86_perf_task_context *task_ctx;
  266. if (!x86_pmu.lbr_nr)
  267. return;
  268. /*
  269. * Reset the LBR stack if we changed task context to
  270. * avoid data leaks.
  271. */
  272. if (event->ctx->task && cpuc->lbr_context != event->ctx) {
  273. intel_pmu_lbr_reset();
  274. cpuc->lbr_context = event->ctx;
  275. }
  276. cpuc->br_sel = event->hw.branch_reg.reg;
  277. if (branch_user_callstack(cpuc->br_sel) && event->ctx &&
  278. event->ctx->task_ctx_data) {
  279. task_ctx = event->ctx->task_ctx_data;
  280. task_ctx->lbr_callstack_users++;
  281. }
  282. cpuc->lbr_users++;
  283. perf_sched_cb_inc(event->ctx->pmu);
  284. }
  285. void intel_pmu_lbr_disable(struct perf_event *event)
  286. {
  287. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  288. struct x86_perf_task_context *task_ctx;
  289. if (!x86_pmu.lbr_nr)
  290. return;
  291. if (branch_user_callstack(cpuc->br_sel) && event->ctx &&
  292. event->ctx->task_ctx_data) {
  293. task_ctx = event->ctx->task_ctx_data;
  294. task_ctx->lbr_callstack_users--;
  295. }
  296. cpuc->lbr_users--;
  297. WARN_ON_ONCE(cpuc->lbr_users < 0);
  298. perf_sched_cb_dec(event->ctx->pmu);
  299. if (cpuc->enabled && !cpuc->lbr_users) {
  300. __intel_pmu_lbr_disable();
  301. /* avoid stale pointer */
  302. cpuc->lbr_context = NULL;
  303. }
  304. }
  305. void intel_pmu_lbr_enable_all(bool pmi)
  306. {
  307. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  308. if (cpuc->lbr_users)
  309. __intel_pmu_lbr_enable(pmi);
  310. }
  311. void intel_pmu_lbr_disable_all(void)
  312. {
  313. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  314. if (cpuc->lbr_users)
  315. __intel_pmu_lbr_disable();
  316. }
  317. static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
  318. {
  319. unsigned long mask = x86_pmu.lbr_nr - 1;
  320. u64 tos = intel_pmu_lbr_tos();
  321. int i;
  322. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  323. unsigned long lbr_idx = (tos - i) & mask;
  324. union {
  325. struct {
  326. u32 from;
  327. u32 to;
  328. };
  329. u64 lbr;
  330. } msr_lastbranch;
  331. rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
  332. cpuc->lbr_entries[i].from = msr_lastbranch.from;
  333. cpuc->lbr_entries[i].to = msr_lastbranch.to;
  334. cpuc->lbr_entries[i].mispred = 0;
  335. cpuc->lbr_entries[i].predicted = 0;
  336. cpuc->lbr_entries[i].reserved = 0;
  337. }
  338. cpuc->lbr_stack.nr = i;
  339. }
  340. /*
  341. * Due to lack of segmentation in Linux the effective address (offset)
  342. * is the same as the linear address, allowing us to merge the LIP and EIP
  343. * LBR formats.
  344. */
  345. static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
  346. {
  347. unsigned long mask = x86_pmu.lbr_nr - 1;
  348. int lbr_format = x86_pmu.intel_cap.lbr_format;
  349. u64 tos = intel_pmu_lbr_tos();
  350. int i;
  351. int out = 0;
  352. for (i = 0; i < x86_pmu.lbr_nr; i++) {
  353. unsigned long lbr_idx = (tos - i) & mask;
  354. u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0;
  355. int skip = 0;
  356. int lbr_flags = lbr_desc[lbr_format];
  357. rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
  358. rdmsrl(x86_pmu.lbr_to + lbr_idx, to);
  359. if (lbr_flags & LBR_EIP_FLAGS) {
  360. mis = !!(from & LBR_FROM_FLAG_MISPRED);
  361. pred = !mis;
  362. skip = 1;
  363. }
  364. if (lbr_flags & LBR_TSX) {
  365. in_tx = !!(from & LBR_FROM_FLAG_IN_TX);
  366. abort = !!(from & LBR_FROM_FLAG_ABORT);
  367. skip = 3;
  368. }
  369. from = (u64)((((s64)from) << skip) >> skip);
  370. /*
  371. * Some CPUs report duplicated abort records,
  372. * with the second entry not having an abort bit set.
  373. * Skip them here. This loop runs backwards,
  374. * so we need to undo the previous record.
  375. * If the abort just happened outside the window
  376. * the extra entry cannot be removed.
  377. */
  378. if (abort && x86_pmu.lbr_double_abort && out > 0)
  379. out--;
  380. cpuc->lbr_entries[out].from = from;
  381. cpuc->lbr_entries[out].to = to;
  382. cpuc->lbr_entries[out].mispred = mis;
  383. cpuc->lbr_entries[out].predicted = pred;
  384. cpuc->lbr_entries[out].in_tx = in_tx;
  385. cpuc->lbr_entries[out].abort = abort;
  386. cpuc->lbr_entries[out].reserved = 0;
  387. out++;
  388. }
  389. cpuc->lbr_stack.nr = out;
  390. }
  391. void intel_pmu_lbr_read(void)
  392. {
  393. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  394. if (!cpuc->lbr_users)
  395. return;
  396. if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
  397. intel_pmu_lbr_read_32(cpuc);
  398. else
  399. intel_pmu_lbr_read_64(cpuc);
  400. intel_pmu_lbr_filter(cpuc);
  401. }
  402. /*
  403. * SW filter is used:
  404. * - in case there is no HW filter
  405. * - in case the HW filter has errata or limitations
  406. */
  407. static int intel_pmu_setup_sw_lbr_filter(struct perf_event *event)
  408. {
  409. u64 br_type = event->attr.branch_sample_type;
  410. int mask = 0;
  411. if (br_type & PERF_SAMPLE_BRANCH_USER)
  412. mask |= X86_BR_USER;
  413. if (br_type & PERF_SAMPLE_BRANCH_KERNEL)
  414. mask |= X86_BR_KERNEL;
  415. /* we ignore BRANCH_HV here */
  416. if (br_type & PERF_SAMPLE_BRANCH_ANY)
  417. mask |= X86_BR_ANY;
  418. if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL)
  419. mask |= X86_BR_ANY_CALL;
  420. if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
  421. mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET;
  422. if (br_type & PERF_SAMPLE_BRANCH_IND_CALL)
  423. mask |= X86_BR_IND_CALL;
  424. if (br_type & PERF_SAMPLE_BRANCH_ABORT_TX)
  425. mask |= X86_BR_ABORT;
  426. if (br_type & PERF_SAMPLE_BRANCH_IN_TX)
  427. mask |= X86_BR_IN_TX;
  428. if (br_type & PERF_SAMPLE_BRANCH_NO_TX)
  429. mask |= X86_BR_NO_TX;
  430. if (br_type & PERF_SAMPLE_BRANCH_COND)
  431. mask |= X86_BR_JCC;
  432. if (br_type & PERF_SAMPLE_BRANCH_CALL_STACK) {
  433. if (!x86_pmu_has_lbr_callstack())
  434. return -EOPNOTSUPP;
  435. if (mask & ~(X86_BR_USER | X86_BR_KERNEL))
  436. return -EINVAL;
  437. mask |= X86_BR_CALL | X86_BR_IND_CALL | X86_BR_RET |
  438. X86_BR_CALL_STACK;
  439. }
  440. if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP)
  441. mask |= X86_BR_IND_JMP;
  442. /*
  443. * stash actual user request into reg, it may
  444. * be used by fixup code for some CPU
  445. */
  446. event->hw.branch_reg.reg = mask;
  447. return 0;
  448. }
  449. /*
  450. * setup the HW LBR filter
  451. * Used only when available, may not be enough to disambiguate
  452. * all branches, may need the help of the SW filter
  453. */
  454. static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event)
  455. {
  456. struct hw_perf_event_extra *reg;
  457. u64 br_type = event->attr.branch_sample_type;
  458. u64 mask = 0, v;
  459. int i;
  460. for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) {
  461. if (!(br_type & (1ULL << i)))
  462. continue;
  463. v = x86_pmu.lbr_sel_map[i];
  464. if (v == LBR_NOT_SUPP)
  465. return -EOPNOTSUPP;
  466. if (v != LBR_IGN)
  467. mask |= v;
  468. }
  469. reg = &event->hw.branch_reg;
  470. reg->idx = EXTRA_REG_LBR;
  471. /*
  472. * The first 9 bits (LBR_SEL_MASK) in LBR_SELECT operate
  473. * in suppress mode. So LBR_SELECT should be set to
  474. * (~mask & LBR_SEL_MASK) | (mask & ~LBR_SEL_MASK)
  475. */
  476. reg->config = mask ^ x86_pmu.lbr_sel_mask;
  477. return 0;
  478. }
  479. int intel_pmu_setup_lbr_filter(struct perf_event *event)
  480. {
  481. int ret = 0;
  482. /*
  483. * no LBR on this PMU
  484. */
  485. if (!x86_pmu.lbr_nr)
  486. return -EOPNOTSUPP;
  487. /*
  488. * setup SW LBR filter
  489. */
  490. ret = intel_pmu_setup_sw_lbr_filter(event);
  491. if (ret)
  492. return ret;
  493. /*
  494. * setup HW LBR filter, if any
  495. */
  496. if (x86_pmu.lbr_sel_map)
  497. ret = intel_pmu_setup_hw_lbr_filter(event);
  498. return ret;
  499. }
  500. /*
  501. * return the type of control flow change at address "from"
  502. * intruction is not necessarily a branch (in case of interrupt).
  503. *
  504. * The branch type returned also includes the priv level of the
  505. * target of the control flow change (X86_BR_USER, X86_BR_KERNEL).
  506. *
  507. * If a branch type is unknown OR the instruction cannot be
  508. * decoded (e.g., text page not present), then X86_BR_NONE is
  509. * returned.
  510. */
  511. static int branch_type(unsigned long from, unsigned long to, int abort)
  512. {
  513. struct insn insn;
  514. void *addr;
  515. int bytes_read, bytes_left;
  516. int ret = X86_BR_NONE;
  517. int ext, to_plm, from_plm;
  518. u8 buf[MAX_INSN_SIZE];
  519. int is64 = 0;
  520. to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER;
  521. from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER;
  522. /*
  523. * maybe zero if lbr did not fill up after a reset by the time
  524. * we get a PMU interrupt
  525. */
  526. if (from == 0 || to == 0)
  527. return X86_BR_NONE;
  528. if (abort)
  529. return X86_BR_ABORT | to_plm;
  530. if (from_plm == X86_BR_USER) {
  531. /*
  532. * can happen if measuring at the user level only
  533. * and we interrupt in a kernel thread, e.g., idle.
  534. */
  535. if (!current->mm)
  536. return X86_BR_NONE;
  537. /* may fail if text not present */
  538. bytes_left = copy_from_user_nmi(buf, (void __user *)from,
  539. MAX_INSN_SIZE);
  540. bytes_read = MAX_INSN_SIZE - bytes_left;
  541. if (!bytes_read)
  542. return X86_BR_NONE;
  543. addr = buf;
  544. } else {
  545. /*
  546. * The LBR logs any address in the IP, even if the IP just
  547. * faulted. This means userspace can control the from address.
  548. * Ensure we don't blindy read any address by validating it is
  549. * a known text address.
  550. */
  551. if (kernel_text_address(from)) {
  552. addr = (void *)from;
  553. /*
  554. * Assume we can get the maximum possible size
  555. * when grabbing kernel data. This is not
  556. * _strictly_ true since we could possibly be
  557. * executing up next to a memory hole, but
  558. * it is very unlikely to be a problem.
  559. */
  560. bytes_read = MAX_INSN_SIZE;
  561. } else {
  562. return X86_BR_NONE;
  563. }
  564. }
  565. /*
  566. * decoder needs to know the ABI especially
  567. * on 64-bit systems running 32-bit apps
  568. */
  569. #ifdef CONFIG_X86_64
  570. is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32);
  571. #endif
  572. insn_init(&insn, addr, bytes_read, is64);
  573. insn_get_opcode(&insn);
  574. if (!insn.opcode.got)
  575. return X86_BR_ABORT;
  576. switch (insn.opcode.bytes[0]) {
  577. case 0xf:
  578. switch (insn.opcode.bytes[1]) {
  579. case 0x05: /* syscall */
  580. case 0x34: /* sysenter */
  581. ret = X86_BR_SYSCALL;
  582. break;
  583. case 0x07: /* sysret */
  584. case 0x35: /* sysexit */
  585. ret = X86_BR_SYSRET;
  586. break;
  587. case 0x80 ... 0x8f: /* conditional */
  588. ret = X86_BR_JCC;
  589. break;
  590. default:
  591. ret = X86_BR_NONE;
  592. }
  593. break;
  594. case 0x70 ... 0x7f: /* conditional */
  595. ret = X86_BR_JCC;
  596. break;
  597. case 0xc2: /* near ret */
  598. case 0xc3: /* near ret */
  599. case 0xca: /* far ret */
  600. case 0xcb: /* far ret */
  601. ret = X86_BR_RET;
  602. break;
  603. case 0xcf: /* iret */
  604. ret = X86_BR_IRET;
  605. break;
  606. case 0xcc ... 0xce: /* int */
  607. ret = X86_BR_INT;
  608. break;
  609. case 0xe8: /* call near rel */
  610. insn_get_immediate(&insn);
  611. if (insn.immediate1.value == 0) {
  612. /* zero length call */
  613. ret = X86_BR_ZERO_CALL;
  614. break;
  615. }
  616. case 0x9a: /* call far absolute */
  617. ret = X86_BR_CALL;
  618. break;
  619. case 0xe0 ... 0xe3: /* loop jmp */
  620. ret = X86_BR_JCC;
  621. break;
  622. case 0xe9 ... 0xeb: /* jmp */
  623. ret = X86_BR_JMP;
  624. break;
  625. case 0xff: /* call near absolute, call far absolute ind */
  626. insn_get_modrm(&insn);
  627. ext = (insn.modrm.bytes[0] >> 3) & 0x7;
  628. switch (ext) {
  629. case 2: /* near ind call */
  630. case 3: /* far ind call */
  631. ret = X86_BR_IND_CALL;
  632. break;
  633. case 4:
  634. case 5:
  635. ret = X86_BR_IND_JMP;
  636. break;
  637. }
  638. break;
  639. default:
  640. ret = X86_BR_NONE;
  641. }
  642. /*
  643. * interrupts, traps, faults (and thus ring transition) may
  644. * occur on any instructions. Thus, to classify them correctly,
  645. * we need to first look at the from and to priv levels. If they
  646. * are different and to is in the kernel, then it indicates
  647. * a ring transition. If the from instruction is not a ring
  648. * transition instr (syscall, systenter, int), then it means
  649. * it was a irq, trap or fault.
  650. *
  651. * we have no way of detecting kernel to kernel faults.
  652. */
  653. if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL
  654. && ret != X86_BR_SYSCALL && ret != X86_BR_INT)
  655. ret = X86_BR_IRQ;
  656. /*
  657. * branch priv level determined by target as
  658. * is done by HW when LBR_SELECT is implemented
  659. */
  660. if (ret != X86_BR_NONE)
  661. ret |= to_plm;
  662. return ret;
  663. }
  664. /*
  665. * implement actual branch filter based on user demand.
  666. * Hardware may not exactly satisfy that request, thus
  667. * we need to inspect opcodes. Mismatched branches are
  668. * discarded. Therefore, the number of branches returned
  669. * in PERF_SAMPLE_BRANCH_STACK sample may vary.
  670. */
  671. static void
  672. intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
  673. {
  674. u64 from, to;
  675. int br_sel = cpuc->br_sel;
  676. int i, j, type;
  677. bool compress = false;
  678. /* if sampling all branches, then nothing to filter */
  679. if ((br_sel & X86_BR_ALL) == X86_BR_ALL)
  680. return;
  681. for (i = 0; i < cpuc->lbr_stack.nr; i++) {
  682. from = cpuc->lbr_entries[i].from;
  683. to = cpuc->lbr_entries[i].to;
  684. type = branch_type(from, to, cpuc->lbr_entries[i].abort);
  685. if (type != X86_BR_NONE && (br_sel & X86_BR_ANYTX)) {
  686. if (cpuc->lbr_entries[i].in_tx)
  687. type |= X86_BR_IN_TX;
  688. else
  689. type |= X86_BR_NO_TX;
  690. }
  691. /* if type does not correspond, then discard */
  692. if (type == X86_BR_NONE || (br_sel & type) != type) {
  693. cpuc->lbr_entries[i].from = 0;
  694. compress = true;
  695. }
  696. }
  697. if (!compress)
  698. return;
  699. /* remove all entries with from=0 */
  700. for (i = 0; i < cpuc->lbr_stack.nr; ) {
  701. if (!cpuc->lbr_entries[i].from) {
  702. j = i;
  703. while (++j < cpuc->lbr_stack.nr)
  704. cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];
  705. cpuc->lbr_stack.nr--;
  706. if (!cpuc->lbr_entries[i].from)
  707. continue;
  708. }
  709. i++;
  710. }
  711. }
  712. /*
  713. * Map interface branch filters onto LBR filters
  714. */
  715. static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = {
  716. [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY,
  717. [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER,
  718. [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL,
  719. [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN,
  720. [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_REL_JMP
  721. | LBR_IND_JMP | LBR_FAR,
  722. /*
  723. * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches
  724. */
  725. [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] =
  726. LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR,
  727. /*
  728. * NHM/WSM erratum: must include IND_JMP to capture IND_CALL
  729. */
  730. [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL | LBR_IND_JMP,
  731. [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC,
  732. [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP,
  733. };
  734. static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = {
  735. [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY,
  736. [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER,
  737. [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL,
  738. [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN,
  739. [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR,
  740. [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL
  741. | LBR_FAR,
  742. [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL,
  743. [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC,
  744. [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP,
  745. };
  746. static const int hsw_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = {
  747. [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY,
  748. [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER,
  749. [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL,
  750. [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN,
  751. [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR,
  752. [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL
  753. | LBR_FAR,
  754. [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL,
  755. [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC,
  756. [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_REL_CALL | LBR_IND_CALL
  757. | LBR_RETURN | LBR_CALL_STACK,
  758. [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP,
  759. };
  760. /* core */
  761. void __init intel_pmu_lbr_init_core(void)
  762. {
  763. x86_pmu.lbr_nr = 4;
  764. x86_pmu.lbr_tos = MSR_LBR_TOS;
  765. x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
  766. x86_pmu.lbr_to = MSR_LBR_CORE_TO;
  767. /*
  768. * SW branch filter usage:
  769. * - compensate for lack of HW filter
  770. */
  771. pr_cont("4-deep LBR, ");
  772. }
  773. /* nehalem/westmere */
  774. void __init intel_pmu_lbr_init_nhm(void)
  775. {
  776. x86_pmu.lbr_nr = 16;
  777. x86_pmu.lbr_tos = MSR_LBR_TOS;
  778. x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
  779. x86_pmu.lbr_to = MSR_LBR_NHM_TO;
  780. x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
  781. x86_pmu.lbr_sel_map = nhm_lbr_sel_map;
  782. /*
  783. * SW branch filter usage:
  784. * - workaround LBR_SEL errata (see above)
  785. * - support syscall, sysret capture.
  786. * That requires LBR_FAR but that means far
  787. * jmp need to be filtered out
  788. */
  789. pr_cont("16-deep LBR, ");
  790. }
  791. /* sandy bridge */
  792. void __init intel_pmu_lbr_init_snb(void)
  793. {
  794. x86_pmu.lbr_nr = 16;
  795. x86_pmu.lbr_tos = MSR_LBR_TOS;
  796. x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
  797. x86_pmu.lbr_to = MSR_LBR_NHM_TO;
  798. x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
  799. x86_pmu.lbr_sel_map = snb_lbr_sel_map;
  800. /*
  801. * SW branch filter usage:
  802. * - support syscall, sysret capture.
  803. * That requires LBR_FAR but that means far
  804. * jmp need to be filtered out
  805. */
  806. pr_cont("16-deep LBR, ");
  807. }
  808. /* haswell */
  809. void intel_pmu_lbr_init_hsw(void)
  810. {
  811. x86_pmu.lbr_nr = 16;
  812. x86_pmu.lbr_tos = MSR_LBR_TOS;
  813. x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
  814. x86_pmu.lbr_to = MSR_LBR_NHM_TO;
  815. x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
  816. x86_pmu.lbr_sel_map = hsw_lbr_sel_map;
  817. pr_cont("16-deep LBR, ");
  818. }
  819. /* atom */
  820. void __init intel_pmu_lbr_init_atom(void)
  821. {
  822. /*
  823. * only models starting at stepping 10 seems
  824. * to have an operational LBR which can freeze
  825. * on PMU interrupt
  826. */
  827. if (boot_cpu_data.x86_model == 28
  828. && boot_cpu_data.x86_mask < 10) {
  829. pr_cont("LBR disabled due to erratum");
  830. return;
  831. }
  832. x86_pmu.lbr_nr = 8;
  833. x86_pmu.lbr_tos = MSR_LBR_TOS;
  834. x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
  835. x86_pmu.lbr_to = MSR_LBR_CORE_TO;
  836. /*
  837. * SW branch filter usage:
  838. * - compensate for lack of HW filter
  839. */
  840. pr_cont("8-deep LBR, ");
  841. }