perf_event.c 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172
  1. /*
  2. * Performance events x86 architecture code
  3. *
  4. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  6. * Copyright (C) 2009 Jaswinder Singh Rajput
  7. * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  8. * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  9. * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10. * Copyright (C) 2009 Google, Inc., Stephane Eranian
  11. *
  12. * For licencing details see kernel-base/COPYING
  13. */
  14. #include <linux/perf_event.h>
  15. #include <linux/capability.h>
  16. #include <linux/notifier.h>
  17. #include <linux/hardirq.h>
  18. #include <linux/kprobes.h>
  19. #include <linux/module.h>
  20. #include <linux/kdebug.h>
  21. #include <linux/sched.h>
  22. #include <linux/uaccess.h>
  23. #include <linux/slab.h>
  24. #include <linux/cpu.h>
  25. #include <linux/bitops.h>
  26. #include <linux/device.h>
  27. #include <asm/apic.h>
  28. #include <asm/stacktrace.h>
  29. #include <asm/nmi.h>
  30. #include <asm/smp.h>
  31. #include <asm/alternative.h>
  32. #include <asm/timer.h>
  33. #include <asm/desc.h>
  34. #include <asm/ldt.h>
  35. #include "perf_event.h"
  36. struct x86_pmu x86_pmu __read_mostly;
  37. DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  38. .enabled = 1,
  39. };
  40. u64 __read_mostly hw_cache_event_ids
  41. [PERF_COUNT_HW_CACHE_MAX]
  42. [PERF_COUNT_HW_CACHE_OP_MAX]
  43. [PERF_COUNT_HW_CACHE_RESULT_MAX];
  44. u64 __read_mostly hw_cache_extra_regs
  45. [PERF_COUNT_HW_CACHE_MAX]
  46. [PERF_COUNT_HW_CACHE_OP_MAX]
  47. [PERF_COUNT_HW_CACHE_RESULT_MAX];
  48. /*
  49. * Propagate event elapsed time into the generic event.
  50. * Can only be executed on the CPU where the event is active.
  51. * Returns the delta events processed.
  52. */
  53. u64 x86_perf_event_update(struct perf_event *event)
  54. {
  55. struct hw_perf_event *hwc = &event->hw;
  56. int shift = 64 - x86_pmu.cntval_bits;
  57. u64 prev_raw_count, new_raw_count;
  58. int idx = hwc->idx;
  59. s64 delta;
  60. if (idx == INTEL_PMC_IDX_FIXED_BTS)
  61. return 0;
  62. /*
  63. * Careful: an NMI might modify the previous event value.
  64. *
  65. * Our tactic to handle this is to first atomically read and
  66. * exchange a new raw count - then add that new-prev delta
  67. * count to the generic event atomically:
  68. */
  69. again:
  70. prev_raw_count = local64_read(&hwc->prev_count);
  71. rdpmcl(hwc->event_base_rdpmc, new_raw_count);
  72. if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  73. new_raw_count) != prev_raw_count)
  74. goto again;
  75. /*
  76. * Now we have the new raw value and have updated the prev
  77. * timestamp already. We can now calculate the elapsed delta
  78. * (event-)time and add that to the generic event.
  79. *
  80. * Careful, not all hw sign-extends above the physical width
  81. * of the count.
  82. */
  83. delta = (new_raw_count << shift) - (prev_raw_count << shift);
  84. delta >>= shift;
  85. local64_add(delta, &event->count);
  86. local64_sub(delta, &hwc->period_left);
  87. return new_raw_count;
  88. }
  89. /*
  90. * Find and validate any extra registers to set up.
  91. */
  92. static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
  93. {
  94. struct hw_perf_event_extra *reg;
  95. struct extra_reg *er;
  96. reg = &event->hw.extra_reg;
  97. if (!x86_pmu.extra_regs)
  98. return 0;
  99. for (er = x86_pmu.extra_regs; er->msr; er++) {
  100. if (er->event != (config & er->config_mask))
  101. continue;
  102. if (event->attr.config1 & ~er->valid_mask)
  103. return -EINVAL;
  104. /* Check if the extra msrs can be safely accessed*/
  105. if (!er->extra_msr_access)
  106. return -ENXIO;
  107. reg->idx = er->idx;
  108. reg->config = event->attr.config1;
  109. reg->reg = er->msr;
  110. break;
  111. }
  112. return 0;
  113. }
  114. static atomic_t active_events;
  115. static DEFINE_MUTEX(pmc_reserve_mutex);
  116. #ifdef CONFIG_X86_LOCAL_APIC
  117. static bool reserve_pmc_hardware(void)
  118. {
  119. int i;
  120. for (i = 0; i < x86_pmu.num_counters; i++) {
  121. if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
  122. goto perfctr_fail;
  123. }
  124. for (i = 0; i < x86_pmu.num_counters; i++) {
  125. if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
  126. goto eventsel_fail;
  127. }
  128. return true;
  129. eventsel_fail:
  130. for (i--; i >= 0; i--)
  131. release_evntsel_nmi(x86_pmu_config_addr(i));
  132. i = x86_pmu.num_counters;
  133. perfctr_fail:
  134. for (i--; i >= 0; i--)
  135. release_perfctr_nmi(x86_pmu_event_addr(i));
  136. return false;
  137. }
  138. static void release_pmc_hardware(void)
  139. {
  140. int i;
  141. for (i = 0; i < x86_pmu.num_counters; i++) {
  142. release_perfctr_nmi(x86_pmu_event_addr(i));
  143. release_evntsel_nmi(x86_pmu_config_addr(i));
  144. }
  145. }
  146. #else
  147. static bool reserve_pmc_hardware(void) { return true; }
  148. static void release_pmc_hardware(void) {}
  149. #endif
  150. static bool check_hw_exists(void)
  151. {
  152. u64 val, val_fail, val_new= ~0;
  153. int i, reg, reg_fail, ret = 0;
  154. int bios_fail = 0;
  155. /*
  156. * Check to see if the BIOS enabled any of the counters, if so
  157. * complain and bail.
  158. */
  159. for (i = 0; i < x86_pmu.num_counters; i++) {
  160. reg = x86_pmu_config_addr(i);
  161. ret = rdmsrl_safe(reg, &val);
  162. if (ret)
  163. goto msr_fail;
  164. if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
  165. bios_fail = 1;
  166. val_fail = val;
  167. reg_fail = reg;
  168. }
  169. }
  170. if (x86_pmu.num_counters_fixed) {
  171. reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
  172. ret = rdmsrl_safe(reg, &val);
  173. if (ret)
  174. goto msr_fail;
  175. for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
  176. if (val & (0x03 << i*4)) {
  177. bios_fail = 1;
  178. val_fail = val;
  179. reg_fail = reg;
  180. }
  181. }
  182. }
  183. /*
  184. * Read the current value, change it and read it back to see if it
  185. * matches, this is needed to detect certain hardware emulators
  186. * (qemu/kvm) that don't trap on the MSR access and always return 0s.
  187. */
  188. reg = x86_pmu_event_addr(0);
  189. if (rdmsrl_safe(reg, &val))
  190. goto msr_fail;
  191. val ^= 0xffffUL;
  192. ret = wrmsrl_safe(reg, val);
  193. ret |= rdmsrl_safe(reg, &val_new);
  194. if (ret || val != val_new)
  195. goto msr_fail;
  196. /*
  197. * We still allow the PMU driver to operate:
  198. */
  199. if (bios_fail) {
  200. printk(KERN_CONT "Broken BIOS detected, complain to your hardware vendor.\n");
  201. printk(KERN_ERR FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", reg_fail, val_fail);
  202. }
  203. return true;
  204. msr_fail:
  205. printk(KERN_CONT "Broken PMU hardware detected, using software events only.\n");
  206. printk("%sFailed to access perfctr msr (MSR %x is %Lx)\n",
  207. boot_cpu_has(X86_FEATURE_HYPERVISOR) ? KERN_INFO : KERN_ERR,
  208. reg, val_new);
  209. return false;
  210. }
  211. static void hw_perf_event_destroy(struct perf_event *event)
  212. {
  213. if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
  214. release_pmc_hardware();
  215. release_ds_buffers();
  216. mutex_unlock(&pmc_reserve_mutex);
  217. }
  218. }
  219. static inline int x86_pmu_initialized(void)
  220. {
  221. return x86_pmu.handle_irq != NULL;
  222. }
  223. static inline int
  224. set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
  225. {
  226. struct perf_event_attr *attr = &event->attr;
  227. unsigned int cache_type, cache_op, cache_result;
  228. u64 config, val;
  229. config = attr->config;
  230. cache_type = (config >> 0) & 0xff;
  231. if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  232. return -EINVAL;
  233. cache_op = (config >> 8) & 0xff;
  234. if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  235. return -EINVAL;
  236. cache_result = (config >> 16) & 0xff;
  237. if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  238. return -EINVAL;
  239. val = hw_cache_event_ids[cache_type][cache_op][cache_result];
  240. if (val == 0)
  241. return -ENOENT;
  242. if (val == -1)
  243. return -EINVAL;
  244. hwc->config |= val;
  245. attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
  246. return x86_pmu_extra_regs(val, event);
  247. }
  248. int x86_setup_perfctr(struct perf_event *event)
  249. {
  250. struct perf_event_attr *attr = &event->attr;
  251. struct hw_perf_event *hwc = &event->hw;
  252. u64 config;
  253. if (!is_sampling_event(event)) {
  254. hwc->sample_period = x86_pmu.max_period;
  255. hwc->last_period = hwc->sample_period;
  256. local64_set(&hwc->period_left, hwc->sample_period);
  257. }
  258. if (attr->type == PERF_TYPE_RAW)
  259. return x86_pmu_extra_regs(event->attr.config, event);
  260. if (attr->type == PERF_TYPE_HW_CACHE)
  261. return set_ext_hw_attr(hwc, event);
  262. if (attr->config >= x86_pmu.max_events)
  263. return -EINVAL;
  264. /*
  265. * The generic map:
  266. */
  267. config = x86_pmu.event_map(attr->config);
  268. if (config == 0)
  269. return -ENOENT;
  270. if (config == -1LL)
  271. return -EINVAL;
  272. /*
  273. * Branch tracing:
  274. */
  275. if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS &&
  276. !attr->freq && hwc->sample_period == 1) {
  277. /* BTS is not supported by this architecture. */
  278. if (!x86_pmu.bts_active)
  279. return -EOPNOTSUPP;
  280. /* BTS is currently only allowed for user-mode. */
  281. if (!attr->exclude_kernel)
  282. return -EOPNOTSUPP;
  283. }
  284. hwc->config |= config;
  285. return 0;
  286. }
  287. /*
  288. * check that branch_sample_type is compatible with
  289. * settings needed for precise_ip > 1 which implies
  290. * using the LBR to capture ALL taken branches at the
  291. * priv levels of the measurement
  292. */
  293. static inline int precise_br_compat(struct perf_event *event)
  294. {
  295. u64 m = event->attr.branch_sample_type;
  296. u64 b = 0;
  297. /* must capture all branches */
  298. if (!(m & PERF_SAMPLE_BRANCH_ANY))
  299. return 0;
  300. m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
  301. if (!event->attr.exclude_user)
  302. b |= PERF_SAMPLE_BRANCH_USER;
  303. if (!event->attr.exclude_kernel)
  304. b |= PERF_SAMPLE_BRANCH_KERNEL;
  305. /*
  306. * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
  307. */
  308. return m == b;
  309. }
  310. int x86_pmu_hw_config(struct perf_event *event)
  311. {
  312. if (event->attr.precise_ip) {
  313. int precise = 0;
  314. /* Support for constant skid */
  315. if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
  316. precise++;
  317. /* Support for IP fixup */
  318. if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
  319. precise++;
  320. }
  321. if (event->attr.precise_ip > precise)
  322. return -EOPNOTSUPP;
  323. /*
  324. * check that PEBS LBR correction does not conflict with
  325. * whatever the user is asking with attr->branch_sample_type
  326. */
  327. if (event->attr.precise_ip > 1 &&
  328. x86_pmu.intel_cap.pebs_format < 2) {
  329. u64 *br_type = &event->attr.branch_sample_type;
  330. if (has_branch_stack(event)) {
  331. if (!precise_br_compat(event))
  332. return -EOPNOTSUPP;
  333. /* branch_sample_type is compatible */
  334. } else {
  335. /*
  336. * user did not specify branch_sample_type
  337. *
  338. * For PEBS fixups, we capture all
  339. * the branches at the priv level of the
  340. * event.
  341. */
  342. *br_type = PERF_SAMPLE_BRANCH_ANY;
  343. if (!event->attr.exclude_user)
  344. *br_type |= PERF_SAMPLE_BRANCH_USER;
  345. if (!event->attr.exclude_kernel)
  346. *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
  347. }
  348. }
  349. }
  350. /*
  351. * Generate PMC IRQs:
  352. * (keep 'enabled' bit clear for now)
  353. */
  354. event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
  355. /*
  356. * Count user and OS events unless requested not to
  357. */
  358. if (!event->attr.exclude_user)
  359. event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
  360. if (!event->attr.exclude_kernel)
  361. event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
  362. if (event->attr.type == PERF_TYPE_RAW)
  363. event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
  364. return x86_setup_perfctr(event);
  365. }
  366. /*
  367. * Setup the hardware configuration for a given attr_type
  368. */
  369. static int __x86_pmu_event_init(struct perf_event *event)
  370. {
  371. int err;
  372. if (!x86_pmu_initialized())
  373. return -ENODEV;
  374. err = 0;
  375. if (!atomic_inc_not_zero(&active_events)) {
  376. mutex_lock(&pmc_reserve_mutex);
  377. if (atomic_read(&active_events) == 0) {
  378. if (!reserve_pmc_hardware())
  379. err = -EBUSY;
  380. else
  381. reserve_ds_buffers();
  382. }
  383. if (!err)
  384. atomic_inc(&active_events);
  385. mutex_unlock(&pmc_reserve_mutex);
  386. }
  387. if (err)
  388. return err;
  389. event->destroy = hw_perf_event_destroy;
  390. event->hw.idx = -1;
  391. event->hw.last_cpu = -1;
  392. event->hw.last_tag = ~0ULL;
  393. /* mark unused */
  394. event->hw.extra_reg.idx = EXTRA_REG_NONE;
  395. event->hw.branch_reg.idx = EXTRA_REG_NONE;
  396. return x86_pmu.hw_config(event);
  397. }
  398. void x86_pmu_disable_all(void)
  399. {
  400. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  401. int idx;
  402. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  403. u64 val;
  404. if (!test_bit(idx, cpuc->active_mask))
  405. continue;
  406. rdmsrl(x86_pmu_config_addr(idx), val);
  407. if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
  408. continue;
  409. val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
  410. wrmsrl(x86_pmu_config_addr(idx), val);
  411. }
  412. }
  413. static void x86_pmu_disable(struct pmu *pmu)
  414. {
  415. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  416. if (!x86_pmu_initialized())
  417. return;
  418. if (!cpuc->enabled)
  419. return;
  420. cpuc->n_added = 0;
  421. cpuc->enabled = 0;
  422. barrier();
  423. x86_pmu.disable_all();
  424. }
  425. void x86_pmu_enable_all(int added)
  426. {
  427. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  428. int idx;
  429. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  430. struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
  431. if (!test_bit(idx, cpuc->active_mask))
  432. continue;
  433. __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
  434. }
  435. }
  436. static struct pmu pmu;
  437. static inline int is_x86_event(struct perf_event *event)
  438. {
  439. return event->pmu == &pmu;
  440. }
  441. /*
  442. * Event scheduler state:
  443. *
  444. * Assign events iterating over all events and counters, beginning
  445. * with events with least weights first. Keep the current iterator
  446. * state in struct sched_state.
  447. */
  448. struct sched_state {
  449. int weight;
  450. int event; /* event index */
  451. int counter; /* counter index */
  452. int unassigned; /* number of events to be assigned left */
  453. unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  454. };
  455. /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
  456. #define SCHED_STATES_MAX 2
  457. struct perf_sched {
  458. int max_weight;
  459. int max_events;
  460. struct perf_event **events;
  461. struct sched_state state;
  462. int saved_states;
  463. struct sched_state saved[SCHED_STATES_MAX];
  464. };
  465. /*
  466. * Initialize interator that runs through all events and counters.
  467. */
  468. static void perf_sched_init(struct perf_sched *sched, struct perf_event **events,
  469. int num, int wmin, int wmax)
  470. {
  471. int idx;
  472. memset(sched, 0, sizeof(*sched));
  473. sched->max_events = num;
  474. sched->max_weight = wmax;
  475. sched->events = events;
  476. for (idx = 0; idx < num; idx++) {
  477. if (events[idx]->hw.constraint->weight == wmin)
  478. break;
  479. }
  480. sched->state.event = idx; /* start with min weight */
  481. sched->state.weight = wmin;
  482. sched->state.unassigned = num;
  483. }
  484. static void perf_sched_save_state(struct perf_sched *sched)
  485. {
  486. if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
  487. return;
  488. sched->saved[sched->saved_states] = sched->state;
  489. sched->saved_states++;
  490. }
  491. static bool perf_sched_restore_state(struct perf_sched *sched)
  492. {
  493. if (!sched->saved_states)
  494. return false;
  495. sched->saved_states--;
  496. sched->state = sched->saved[sched->saved_states];
  497. /* continue with next counter: */
  498. clear_bit(sched->state.counter++, sched->state.used);
  499. return true;
  500. }
  501. /*
  502. * Select a counter for the current event to schedule. Return true on
  503. * success.
  504. */
  505. static bool __perf_sched_find_counter(struct perf_sched *sched)
  506. {
  507. struct event_constraint *c;
  508. int idx;
  509. if (!sched->state.unassigned)
  510. return false;
  511. if (sched->state.event >= sched->max_events)
  512. return false;
  513. c = sched->events[sched->state.event]->hw.constraint;
  514. /* Prefer fixed purpose counters */
  515. if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
  516. idx = INTEL_PMC_IDX_FIXED;
  517. for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
  518. if (!__test_and_set_bit(idx, sched->state.used))
  519. goto done;
  520. }
  521. }
  522. /* Grab the first unused counter starting with idx */
  523. idx = sched->state.counter;
  524. for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
  525. if (!__test_and_set_bit(idx, sched->state.used))
  526. goto done;
  527. }
  528. return false;
  529. done:
  530. sched->state.counter = idx;
  531. if (c->overlap)
  532. perf_sched_save_state(sched);
  533. return true;
  534. }
  535. static bool perf_sched_find_counter(struct perf_sched *sched)
  536. {
  537. while (!__perf_sched_find_counter(sched)) {
  538. if (!perf_sched_restore_state(sched))
  539. return false;
  540. }
  541. return true;
  542. }
  543. /*
  544. * Go through all unassigned events and find the next one to schedule.
  545. * Take events with the least weight first. Return true on success.
  546. */
  547. static bool perf_sched_next_event(struct perf_sched *sched)
  548. {
  549. struct event_constraint *c;
  550. if (!sched->state.unassigned || !--sched->state.unassigned)
  551. return false;
  552. do {
  553. /* next event */
  554. sched->state.event++;
  555. if (sched->state.event >= sched->max_events) {
  556. /* next weight */
  557. sched->state.event = 0;
  558. sched->state.weight++;
  559. if (sched->state.weight > sched->max_weight)
  560. return false;
  561. }
  562. c = sched->events[sched->state.event]->hw.constraint;
  563. } while (c->weight != sched->state.weight);
  564. sched->state.counter = 0; /* start with first counter */
  565. return true;
  566. }
  567. /*
  568. * Assign a counter for each event.
  569. */
  570. int perf_assign_events(struct perf_event **events, int n,
  571. int wmin, int wmax, int *assign)
  572. {
  573. struct perf_sched sched;
  574. perf_sched_init(&sched, events, n, wmin, wmax);
  575. do {
  576. if (!perf_sched_find_counter(&sched))
  577. break; /* failed */
  578. if (assign)
  579. assign[sched.state.event] = sched.state.counter;
  580. } while (perf_sched_next_event(&sched));
  581. return sched.state.unassigned;
  582. }
  583. EXPORT_SYMBOL_GPL(perf_assign_events);
  584. int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
  585. {
  586. struct event_constraint *c;
  587. unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  588. struct perf_event *e;
  589. int i, wmin, wmax, num = 0;
  590. struct hw_perf_event *hwc;
  591. bitmap_zero(used_mask, X86_PMC_IDX_MAX);
  592. for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
  593. hwc = &cpuc->event_list[i]->hw;
  594. c = x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]);
  595. hwc->constraint = c;
  596. wmin = min(wmin, c->weight);
  597. wmax = max(wmax, c->weight);
  598. }
  599. /*
  600. * fastpath, try to reuse previous register
  601. */
  602. for (i = 0; i < n; i++) {
  603. hwc = &cpuc->event_list[i]->hw;
  604. c = hwc->constraint;
  605. /* never assigned */
  606. if (hwc->idx == -1)
  607. break;
  608. /* constraint still honored */
  609. if (!test_bit(hwc->idx, c->idxmsk))
  610. break;
  611. /* not already used */
  612. if (test_bit(hwc->idx, used_mask))
  613. break;
  614. __set_bit(hwc->idx, used_mask);
  615. if (assign)
  616. assign[i] = hwc->idx;
  617. }
  618. /* slow path */
  619. if (i != n)
  620. num = perf_assign_events(cpuc->event_list, n, wmin,
  621. wmax, assign);
  622. /*
  623. * Mark the event as committed, so we do not put_constraint()
  624. * in case new events are added and fail scheduling.
  625. */
  626. if (!num && assign) {
  627. for (i = 0; i < n; i++) {
  628. e = cpuc->event_list[i];
  629. e->hw.flags |= PERF_X86_EVENT_COMMITTED;
  630. }
  631. }
  632. /*
  633. * scheduling failed or is just a simulation,
  634. * free resources if necessary
  635. */
  636. if (!assign || num) {
  637. for (i = 0; i < n; i++) {
  638. e = cpuc->event_list[i];
  639. /*
  640. * do not put_constraint() on comitted events,
  641. * because they are good to go
  642. */
  643. if ((e->hw.flags & PERF_X86_EVENT_COMMITTED))
  644. continue;
  645. if (x86_pmu.put_event_constraints)
  646. x86_pmu.put_event_constraints(cpuc, e);
  647. }
  648. }
  649. return num ? -EINVAL : 0;
  650. }
  651. /*
  652. * dogrp: true if must collect siblings events (group)
  653. * returns total number of events and error code
  654. */
  655. static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
  656. {
  657. struct perf_event *event;
  658. int n, max_count;
  659. max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
  660. /* current number of events already accepted */
  661. n = cpuc->n_events;
  662. if (is_x86_event(leader)) {
  663. if (n >= max_count)
  664. return -EINVAL;
  665. cpuc->event_list[n] = leader;
  666. n++;
  667. }
  668. if (!dogrp)
  669. return n;
  670. list_for_each_entry(event, &leader->sibling_list, group_entry) {
  671. if (!is_x86_event(event) ||
  672. event->state <= PERF_EVENT_STATE_OFF)
  673. continue;
  674. if (n >= max_count)
  675. return -EINVAL;
  676. cpuc->event_list[n] = event;
  677. n++;
  678. }
  679. return n;
  680. }
  681. static inline void x86_assign_hw_event(struct perf_event *event,
  682. struct cpu_hw_events *cpuc, int i)
  683. {
  684. struct hw_perf_event *hwc = &event->hw;
  685. hwc->idx = cpuc->assign[i];
  686. hwc->last_cpu = smp_processor_id();
  687. hwc->last_tag = ++cpuc->tags[i];
  688. if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
  689. hwc->config_base = 0;
  690. hwc->event_base = 0;
  691. } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
  692. hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
  693. hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
  694. hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
  695. } else {
  696. hwc->config_base = x86_pmu_config_addr(hwc->idx);
  697. hwc->event_base = x86_pmu_event_addr(hwc->idx);
  698. hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
  699. }
  700. }
  701. static inline int match_prev_assignment(struct hw_perf_event *hwc,
  702. struct cpu_hw_events *cpuc,
  703. int i)
  704. {
  705. return hwc->idx == cpuc->assign[i] &&
  706. hwc->last_cpu == smp_processor_id() &&
  707. hwc->last_tag == cpuc->tags[i];
  708. }
  709. static void x86_pmu_start(struct perf_event *event, int flags);
  710. static void x86_pmu_enable(struct pmu *pmu)
  711. {
  712. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  713. struct perf_event *event;
  714. struct hw_perf_event *hwc;
  715. int i, added = cpuc->n_added;
  716. if (!x86_pmu_initialized())
  717. return;
  718. if (cpuc->enabled)
  719. return;
  720. if (cpuc->n_added) {
  721. int n_running = cpuc->n_events - cpuc->n_added;
  722. /*
  723. * apply assignment obtained either from
  724. * hw_perf_group_sched_in() or x86_pmu_enable()
  725. *
  726. * step1: save events moving to new counters
  727. */
  728. for (i = 0; i < n_running; i++) {
  729. event = cpuc->event_list[i];
  730. hwc = &event->hw;
  731. /*
  732. * we can avoid reprogramming counter if:
  733. * - assigned same counter as last time
  734. * - running on same CPU as last time
  735. * - no other event has used the counter since
  736. */
  737. if (hwc->idx == -1 ||
  738. match_prev_assignment(hwc, cpuc, i))
  739. continue;
  740. /*
  741. * Ensure we don't accidentally enable a stopped
  742. * counter simply because we rescheduled.
  743. */
  744. if (hwc->state & PERF_HES_STOPPED)
  745. hwc->state |= PERF_HES_ARCH;
  746. x86_pmu_stop(event, PERF_EF_UPDATE);
  747. }
  748. /*
  749. * step2: reprogram moved events into new counters
  750. */
  751. for (i = 0; i < cpuc->n_events; i++) {
  752. event = cpuc->event_list[i];
  753. hwc = &event->hw;
  754. if (!match_prev_assignment(hwc, cpuc, i))
  755. x86_assign_hw_event(event, cpuc, i);
  756. else if (i < n_running)
  757. continue;
  758. if (hwc->state & PERF_HES_ARCH)
  759. continue;
  760. x86_pmu_start(event, PERF_EF_RELOAD);
  761. }
  762. cpuc->n_added = 0;
  763. perf_events_lapic_init();
  764. }
  765. cpuc->enabled = 1;
  766. barrier();
  767. x86_pmu.enable_all(added);
  768. }
  769. static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
  770. /*
  771. * Set the next IRQ period, based on the hwc->period_left value.
  772. * To be called with the event disabled in hw:
  773. */
  774. int x86_perf_event_set_period(struct perf_event *event)
  775. {
  776. struct hw_perf_event *hwc = &event->hw;
  777. s64 left = local64_read(&hwc->period_left);
  778. s64 period = hwc->sample_period;
  779. int ret = 0, idx = hwc->idx;
  780. if (idx == INTEL_PMC_IDX_FIXED_BTS)
  781. return 0;
  782. /*
  783. * If we are way outside a reasonable range then just skip forward:
  784. */
  785. if (unlikely(left <= -period)) {
  786. left = period;
  787. local64_set(&hwc->period_left, left);
  788. hwc->last_period = period;
  789. ret = 1;
  790. }
  791. if (unlikely(left <= 0)) {
  792. left += period;
  793. local64_set(&hwc->period_left, left);
  794. hwc->last_period = period;
  795. ret = 1;
  796. }
  797. /*
  798. * Quirk: certain CPUs dont like it if just 1 hw_event is left:
  799. */
  800. if (unlikely(left < 2))
  801. left = 2;
  802. if (left > x86_pmu.max_period)
  803. left = x86_pmu.max_period;
  804. per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
  805. /*
  806. * The hw event starts counting from this event offset,
  807. * mark it to be able to extra future deltas:
  808. */
  809. local64_set(&hwc->prev_count, (u64)-left);
  810. wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
  811. /*
  812. * Due to erratum on certan cpu we need
  813. * a second write to be sure the register
  814. * is updated properly
  815. */
  816. if (x86_pmu.perfctr_second_write) {
  817. wrmsrl(hwc->event_base,
  818. (u64)(-left) & x86_pmu.cntval_mask);
  819. }
  820. perf_event_update_userpage(event);
  821. return ret;
  822. }
  823. void x86_pmu_enable_event(struct perf_event *event)
  824. {
  825. if (__this_cpu_read(cpu_hw_events.enabled))
  826. __x86_pmu_enable_event(&event->hw,
  827. ARCH_PERFMON_EVENTSEL_ENABLE);
  828. }
  829. /*
  830. * Add a single event to the PMU.
  831. *
  832. * The event is added to the group of enabled events
  833. * but only if it can be scehduled with existing events.
  834. */
  835. static int x86_pmu_add(struct perf_event *event, int flags)
  836. {
  837. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  838. struct hw_perf_event *hwc;
  839. int assign[X86_PMC_IDX_MAX];
  840. int n, n0, ret;
  841. hwc = &event->hw;
  842. perf_pmu_disable(event->pmu);
  843. n0 = cpuc->n_events;
  844. ret = n = collect_events(cpuc, event, false);
  845. if (ret < 0)
  846. goto out;
  847. hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
  848. if (!(flags & PERF_EF_START))
  849. hwc->state |= PERF_HES_ARCH;
  850. /*
  851. * If group events scheduling transaction was started,
  852. * skip the schedulability test here, it will be performed
  853. * at commit time (->commit_txn) as a whole.
  854. */
  855. if (cpuc->group_flag & PERF_EVENT_TXN)
  856. goto done_collect;
  857. ret = x86_pmu.schedule_events(cpuc, n, assign);
  858. if (ret)
  859. goto out;
  860. /*
  861. * copy new assignment, now we know it is possible
  862. * will be used by hw_perf_enable()
  863. */
  864. memcpy(cpuc->assign, assign, n*sizeof(int));
  865. done_collect:
  866. /*
  867. * Commit the collect_events() state. See x86_pmu_del() and
  868. * x86_pmu_*_txn().
  869. */
  870. cpuc->n_events = n;
  871. cpuc->n_added += n - n0;
  872. cpuc->n_txn += n - n0;
  873. ret = 0;
  874. out:
  875. perf_pmu_enable(event->pmu);
  876. return ret;
  877. }
  878. static void x86_pmu_start(struct perf_event *event, int flags)
  879. {
  880. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  881. int idx = event->hw.idx;
  882. if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
  883. return;
  884. if (WARN_ON_ONCE(idx == -1))
  885. return;
  886. if (flags & PERF_EF_RELOAD) {
  887. WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
  888. x86_perf_event_set_period(event);
  889. }
  890. event->hw.state = 0;
  891. cpuc->events[idx] = event;
  892. __set_bit(idx, cpuc->active_mask);
  893. __set_bit(idx, cpuc->running);
  894. x86_pmu.enable(event);
  895. perf_event_update_userpage(event);
  896. }
  897. void perf_event_print_debug(void)
  898. {
  899. u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
  900. u64 pebs;
  901. struct cpu_hw_events *cpuc;
  902. unsigned long flags;
  903. int cpu, idx;
  904. if (!x86_pmu.num_counters)
  905. return;
  906. local_irq_save(flags);
  907. cpu = smp_processor_id();
  908. cpuc = &per_cpu(cpu_hw_events, cpu);
  909. if (x86_pmu.version >= 2) {
  910. rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
  911. rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
  912. rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
  913. rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
  914. rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
  915. pr_info("\n");
  916. pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
  917. pr_info("CPU#%d: status: %016llx\n", cpu, status);
  918. pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
  919. pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
  920. pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs);
  921. }
  922. pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
  923. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  924. rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
  925. rdmsrl(x86_pmu_event_addr(idx), pmc_count);
  926. prev_left = per_cpu(pmc_prev_left[idx], cpu);
  927. pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
  928. cpu, idx, pmc_ctrl);
  929. pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
  930. cpu, idx, pmc_count);
  931. pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
  932. cpu, idx, prev_left);
  933. }
  934. for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
  935. rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
  936. pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
  937. cpu, idx, pmc_count);
  938. }
  939. local_irq_restore(flags);
  940. }
  941. void x86_pmu_stop(struct perf_event *event, int flags)
  942. {
  943. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  944. struct hw_perf_event *hwc = &event->hw;
  945. if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
  946. x86_pmu.disable(event);
  947. cpuc->events[hwc->idx] = NULL;
  948. WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
  949. hwc->state |= PERF_HES_STOPPED;
  950. }
  951. if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
  952. /*
  953. * Drain the remaining delta count out of a event
  954. * that we are disabling:
  955. */
  956. x86_perf_event_update(event);
  957. hwc->state |= PERF_HES_UPTODATE;
  958. }
  959. }
  960. static void x86_pmu_del(struct perf_event *event, int flags)
  961. {
  962. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  963. int i;
  964. /*
  965. * event is descheduled
  966. */
  967. event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;
  968. /*
  969. * If we're called during a txn, we don't need to do anything.
  970. * The events never got scheduled and ->cancel_txn will truncate
  971. * the event_list.
  972. *
  973. * XXX assumes any ->del() called during a TXN will only be on
  974. * an event added during that same TXN.
  975. */
  976. if (cpuc->group_flag & PERF_EVENT_TXN)
  977. return;
  978. /*
  979. * Not a TXN, therefore cleanup properly.
  980. */
  981. x86_pmu_stop(event, PERF_EF_UPDATE);
  982. for (i = 0; i < cpuc->n_events; i++) {
  983. if (event == cpuc->event_list[i])
  984. break;
  985. }
  986. if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
  987. return;
  988. /* If we have a newly added event; make sure to decrease n_added. */
  989. if (i >= cpuc->n_events - cpuc->n_added)
  990. --cpuc->n_added;
  991. if (x86_pmu.put_event_constraints)
  992. x86_pmu.put_event_constraints(cpuc, event);
  993. /* Delete the array entry. */
  994. while (++i < cpuc->n_events)
  995. cpuc->event_list[i-1] = cpuc->event_list[i];
  996. --cpuc->n_events;
  997. perf_event_update_userpage(event);
  998. }
  999. int x86_pmu_handle_irq(struct pt_regs *regs)
  1000. {
  1001. struct perf_sample_data data;
  1002. struct cpu_hw_events *cpuc;
  1003. struct perf_event *event;
  1004. int idx, handled = 0;
  1005. u64 val;
  1006. cpuc = this_cpu_ptr(&cpu_hw_events);
  1007. /*
  1008. * Some chipsets need to unmask the LVTPC in a particular spot
  1009. * inside the nmi handler. As a result, the unmasking was pushed
  1010. * into all the nmi handlers.
  1011. *
  1012. * This generic handler doesn't seem to have any issues where the
  1013. * unmasking occurs so it was left at the top.
  1014. */
  1015. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1016. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  1017. if (!test_bit(idx, cpuc->active_mask)) {
  1018. /*
  1019. * Though we deactivated the counter some cpus
  1020. * might still deliver spurious interrupts still
  1021. * in flight. Catch them:
  1022. */
  1023. if (__test_and_clear_bit(idx, cpuc->running))
  1024. handled++;
  1025. continue;
  1026. }
  1027. event = cpuc->events[idx];
  1028. val = x86_perf_event_update(event);
  1029. if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
  1030. continue;
  1031. /*
  1032. * event overflow
  1033. */
  1034. handled++;
  1035. perf_sample_data_init(&data, 0, event->hw.last_period);
  1036. if (!x86_perf_event_set_period(event))
  1037. continue;
  1038. if (perf_event_overflow(event, &data, regs))
  1039. x86_pmu_stop(event, 0);
  1040. }
  1041. if (handled)
  1042. inc_irq_stat(apic_perf_irqs);
  1043. return handled;
  1044. }
  1045. void perf_events_lapic_init(void)
  1046. {
  1047. if (!x86_pmu.apic || !x86_pmu_initialized())
  1048. return;
  1049. /*
  1050. * Always use NMI for PMU
  1051. */
  1052. apic_write(APIC_LVTPC, APIC_DM_NMI);
  1053. }
  1054. static int
  1055. perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
  1056. {
  1057. u64 start_clock;
  1058. u64 finish_clock;
  1059. int ret;
  1060. if (!atomic_read(&active_events))
  1061. return NMI_DONE;
  1062. start_clock = sched_clock();
  1063. ret = x86_pmu.handle_irq(regs);
  1064. finish_clock = sched_clock();
  1065. perf_sample_event_took(finish_clock - start_clock);
  1066. return ret;
  1067. }
  1068. NOKPROBE_SYMBOL(perf_event_nmi_handler);
  1069. struct event_constraint emptyconstraint;
  1070. struct event_constraint unconstrained;
  1071. static int
  1072. x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
  1073. {
  1074. unsigned int cpu = (long)hcpu;
  1075. struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
  1076. int ret = NOTIFY_OK;
  1077. switch (action & ~CPU_TASKS_FROZEN) {
  1078. case CPU_UP_PREPARE:
  1079. cpuc->kfree_on_online = NULL;
  1080. if (x86_pmu.cpu_prepare)
  1081. ret = x86_pmu.cpu_prepare(cpu);
  1082. break;
  1083. case CPU_STARTING:
  1084. if (x86_pmu.attr_rdpmc)
  1085. set_in_cr4(X86_CR4_PCE);
  1086. if (x86_pmu.cpu_starting)
  1087. x86_pmu.cpu_starting(cpu);
  1088. break;
  1089. case CPU_ONLINE:
  1090. kfree(cpuc->kfree_on_online);
  1091. break;
  1092. case CPU_DYING:
  1093. if (x86_pmu.cpu_dying)
  1094. x86_pmu.cpu_dying(cpu);
  1095. break;
  1096. case CPU_UP_CANCELED:
  1097. case CPU_DEAD:
  1098. if (x86_pmu.cpu_dead)
  1099. x86_pmu.cpu_dead(cpu);
  1100. break;
  1101. default:
  1102. break;
  1103. }
  1104. return ret;
  1105. }
  1106. static void __init pmu_check_apic(void)
  1107. {
  1108. if (cpu_has_apic)
  1109. return;
  1110. x86_pmu.apic = 0;
  1111. pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
  1112. pr_info("no hardware sampling interrupt available.\n");
  1113. /*
  1114. * If we have a PMU initialized but no APIC
  1115. * interrupts, we cannot sample hardware
  1116. * events (user-space has to fall back and
  1117. * sample via a hrtimer based software event):
  1118. */
  1119. pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
  1120. }
  1121. static struct attribute_group x86_pmu_format_group = {
  1122. .name = "format",
  1123. .attrs = NULL,
  1124. };
  1125. /*
  1126. * Remove all undefined events (x86_pmu.event_map(id) == 0)
  1127. * out of events_attr attributes.
  1128. */
  1129. static void __init filter_events(struct attribute **attrs)
  1130. {
  1131. struct device_attribute *d;
  1132. struct perf_pmu_events_attr *pmu_attr;
  1133. int i, j;
  1134. for (i = 0; attrs[i]; i++) {
  1135. d = (struct device_attribute *)attrs[i];
  1136. pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
  1137. /* str trumps id */
  1138. if (pmu_attr->event_str)
  1139. continue;
  1140. if (x86_pmu.event_map(i))
  1141. continue;
  1142. for (j = i; attrs[j]; j++)
  1143. attrs[j] = attrs[j + 1];
  1144. /* Check the shifted attr. */
  1145. i--;
  1146. }
  1147. }
  1148. /* Merge two pointer arrays */
  1149. static __init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
  1150. {
  1151. struct attribute **new;
  1152. int j, i;
  1153. for (j = 0; a[j]; j++)
  1154. ;
  1155. for (i = 0; b[i]; i++)
  1156. j++;
  1157. j++;
  1158. new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL);
  1159. if (!new)
  1160. return NULL;
  1161. j = 0;
  1162. for (i = 0; a[i]; i++)
  1163. new[j++] = a[i];
  1164. for (i = 0; b[i]; i++)
  1165. new[j++] = b[i];
  1166. new[j] = NULL;
  1167. return new;
  1168. }
  1169. ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
  1170. char *page)
  1171. {
  1172. struct perf_pmu_events_attr *pmu_attr = \
  1173. container_of(attr, struct perf_pmu_events_attr, attr);
  1174. u64 config = x86_pmu.event_map(pmu_attr->id);
  1175. /* string trumps id */
  1176. if (pmu_attr->event_str)
  1177. return sprintf(page, "%s", pmu_attr->event_str);
  1178. return x86_pmu.events_sysfs_show(page, config);
  1179. }
  1180. EVENT_ATTR(cpu-cycles, CPU_CYCLES );
  1181. EVENT_ATTR(instructions, INSTRUCTIONS );
  1182. EVENT_ATTR(cache-references, CACHE_REFERENCES );
  1183. EVENT_ATTR(cache-misses, CACHE_MISSES );
  1184. EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS );
  1185. EVENT_ATTR(branch-misses, BRANCH_MISSES );
  1186. EVENT_ATTR(bus-cycles, BUS_CYCLES );
  1187. EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND );
  1188. EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND );
  1189. EVENT_ATTR(ref-cycles, REF_CPU_CYCLES );
  1190. static struct attribute *empty_attrs;
  1191. static struct attribute *events_attr[] = {
  1192. EVENT_PTR(CPU_CYCLES),
  1193. EVENT_PTR(INSTRUCTIONS),
  1194. EVENT_PTR(CACHE_REFERENCES),
  1195. EVENT_PTR(CACHE_MISSES),
  1196. EVENT_PTR(BRANCH_INSTRUCTIONS),
  1197. EVENT_PTR(BRANCH_MISSES),
  1198. EVENT_PTR(BUS_CYCLES),
  1199. EVENT_PTR(STALLED_CYCLES_FRONTEND),
  1200. EVENT_PTR(STALLED_CYCLES_BACKEND),
  1201. EVENT_PTR(REF_CPU_CYCLES),
  1202. NULL,
  1203. };
  1204. static struct attribute_group x86_pmu_events_group = {
  1205. .name = "events",
  1206. .attrs = events_attr,
  1207. };
  1208. ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
  1209. {
  1210. u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
  1211. u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
  1212. bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE);
  1213. bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
  1214. bool any = (config & ARCH_PERFMON_EVENTSEL_ANY);
  1215. bool inv = (config & ARCH_PERFMON_EVENTSEL_INV);
  1216. ssize_t ret;
  1217. /*
  1218. * We have whole page size to spend and just little data
  1219. * to write, so we can safely use sprintf.
  1220. */
  1221. ret = sprintf(page, "event=0x%02llx", event);
  1222. if (umask)
  1223. ret += sprintf(page + ret, ",umask=0x%02llx", umask);
  1224. if (edge)
  1225. ret += sprintf(page + ret, ",edge");
  1226. if (pc)
  1227. ret += sprintf(page + ret, ",pc");
  1228. if (any)
  1229. ret += sprintf(page + ret, ",any");
  1230. if (inv)
  1231. ret += sprintf(page + ret, ",inv");
  1232. if (cmask)
  1233. ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
  1234. ret += sprintf(page + ret, "\n");
  1235. return ret;
  1236. }
  1237. static int __init init_hw_perf_events(void)
  1238. {
  1239. struct x86_pmu_quirk *quirk;
  1240. int err;
  1241. pr_info("Performance Events: ");
  1242. switch (boot_cpu_data.x86_vendor) {
  1243. case X86_VENDOR_INTEL:
  1244. err = intel_pmu_init();
  1245. break;
  1246. case X86_VENDOR_AMD:
  1247. err = amd_pmu_init();
  1248. break;
  1249. default:
  1250. err = -ENOTSUPP;
  1251. }
  1252. if (err != 0) {
  1253. pr_cont("no PMU driver, software events only.\n");
  1254. return 0;
  1255. }
  1256. pmu_check_apic();
  1257. /* sanity check that the hardware exists or is emulated */
  1258. if (!check_hw_exists())
  1259. return 0;
  1260. pr_cont("%s PMU driver.\n", x86_pmu.name);
  1261. x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
  1262. for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
  1263. quirk->func();
  1264. if (!x86_pmu.intel_ctrl)
  1265. x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
  1266. perf_events_lapic_init();
  1267. register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
  1268. unconstrained = (struct event_constraint)
  1269. __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
  1270. 0, x86_pmu.num_counters, 0, 0);
  1271. x86_pmu_format_group.attrs = x86_pmu.format_attrs;
  1272. if (x86_pmu.event_attrs)
  1273. x86_pmu_events_group.attrs = x86_pmu.event_attrs;
  1274. if (!x86_pmu.events_sysfs_show)
  1275. x86_pmu_events_group.attrs = &empty_attrs;
  1276. else
  1277. filter_events(x86_pmu_events_group.attrs);
  1278. if (x86_pmu.cpu_events) {
  1279. struct attribute **tmp;
  1280. tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
  1281. if (!WARN_ON(!tmp))
  1282. x86_pmu_events_group.attrs = tmp;
  1283. }
  1284. pr_info("... version: %d\n", x86_pmu.version);
  1285. pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
  1286. pr_info("... generic registers: %d\n", x86_pmu.num_counters);
  1287. pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);
  1288. pr_info("... max period: %016Lx\n", x86_pmu.max_period);
  1289. pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed);
  1290. pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl);
  1291. perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
  1292. perf_cpu_notifier(x86_pmu_notifier);
  1293. return 0;
  1294. }
  1295. early_initcall(init_hw_perf_events);
  1296. static inline void x86_pmu_read(struct perf_event *event)
  1297. {
  1298. x86_perf_event_update(event);
  1299. }
  1300. /*
  1301. * Start group events scheduling transaction
  1302. * Set the flag to make pmu::enable() not perform the
  1303. * schedulability test, it will be performed at commit time
  1304. */
  1305. static void x86_pmu_start_txn(struct pmu *pmu)
  1306. {
  1307. perf_pmu_disable(pmu);
  1308. __this_cpu_or(cpu_hw_events.group_flag, PERF_EVENT_TXN);
  1309. __this_cpu_write(cpu_hw_events.n_txn, 0);
  1310. }
  1311. /*
  1312. * Stop group events scheduling transaction
  1313. * Clear the flag and pmu::enable() will perform the
  1314. * schedulability test.
  1315. */
  1316. static void x86_pmu_cancel_txn(struct pmu *pmu)
  1317. {
  1318. __this_cpu_and(cpu_hw_events.group_flag, ~PERF_EVENT_TXN);
  1319. /*
  1320. * Truncate collected array by the number of events added in this
  1321. * transaction. See x86_pmu_add() and x86_pmu_*_txn().
  1322. */
  1323. __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
  1324. __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
  1325. perf_pmu_enable(pmu);
  1326. }
  1327. /*
  1328. * Commit group events scheduling transaction
  1329. * Perform the group schedulability test as a whole
  1330. * Return 0 if success
  1331. *
  1332. * Does not cancel the transaction on failure; expects the caller to do this.
  1333. */
  1334. static int x86_pmu_commit_txn(struct pmu *pmu)
  1335. {
  1336. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  1337. int assign[X86_PMC_IDX_MAX];
  1338. int n, ret;
  1339. n = cpuc->n_events;
  1340. if (!x86_pmu_initialized())
  1341. return -EAGAIN;
  1342. ret = x86_pmu.schedule_events(cpuc, n, assign);
  1343. if (ret)
  1344. return ret;
  1345. /*
  1346. * copy new assignment, now we know it is possible
  1347. * will be used by hw_perf_enable()
  1348. */
  1349. memcpy(cpuc->assign, assign, n*sizeof(int));
  1350. cpuc->group_flag &= ~PERF_EVENT_TXN;
  1351. perf_pmu_enable(pmu);
  1352. return 0;
  1353. }
  1354. /*
  1355. * a fake_cpuc is used to validate event groups. Due to
  1356. * the extra reg logic, we need to also allocate a fake
  1357. * per_core and per_cpu structure. Otherwise, group events
  1358. * using extra reg may conflict without the kernel being
  1359. * able to catch this when the last event gets added to
  1360. * the group.
  1361. */
  1362. static void free_fake_cpuc(struct cpu_hw_events *cpuc)
  1363. {
  1364. kfree(cpuc->shared_regs);
  1365. kfree(cpuc);
  1366. }
  1367. static struct cpu_hw_events *allocate_fake_cpuc(void)
  1368. {
  1369. struct cpu_hw_events *cpuc;
  1370. int cpu = raw_smp_processor_id();
  1371. cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
  1372. if (!cpuc)
  1373. return ERR_PTR(-ENOMEM);
  1374. /* only needed, if we have extra_regs */
  1375. if (x86_pmu.extra_regs) {
  1376. cpuc->shared_regs = allocate_shared_regs(cpu);
  1377. if (!cpuc->shared_regs)
  1378. goto error;
  1379. }
  1380. cpuc->is_fake = 1;
  1381. return cpuc;
  1382. error:
  1383. free_fake_cpuc(cpuc);
  1384. return ERR_PTR(-ENOMEM);
  1385. }
  1386. /*
  1387. * validate that we can schedule this event
  1388. */
  1389. static int validate_event(struct perf_event *event)
  1390. {
  1391. struct cpu_hw_events *fake_cpuc;
  1392. struct event_constraint *c;
  1393. int ret = 0;
  1394. fake_cpuc = allocate_fake_cpuc();
  1395. if (IS_ERR(fake_cpuc))
  1396. return PTR_ERR(fake_cpuc);
  1397. c = x86_pmu.get_event_constraints(fake_cpuc, event);
  1398. if (!c || !c->weight)
  1399. ret = -EINVAL;
  1400. if (x86_pmu.put_event_constraints)
  1401. x86_pmu.put_event_constraints(fake_cpuc, event);
  1402. free_fake_cpuc(fake_cpuc);
  1403. return ret;
  1404. }
  1405. /*
  1406. * validate a single event group
  1407. *
  1408. * validation include:
  1409. * - check events are compatible which each other
  1410. * - events do not compete for the same counter
  1411. * - number of events <= number of counters
  1412. *
  1413. * validation ensures the group can be loaded onto the
  1414. * PMU if it was the only group available.
  1415. */
  1416. static int validate_group(struct perf_event *event)
  1417. {
  1418. struct perf_event *leader = event->group_leader;
  1419. struct cpu_hw_events *fake_cpuc;
  1420. int ret = -EINVAL, n;
  1421. fake_cpuc = allocate_fake_cpuc();
  1422. if (IS_ERR(fake_cpuc))
  1423. return PTR_ERR(fake_cpuc);
  1424. /*
  1425. * the event is not yet connected with its
  1426. * siblings therefore we must first collect
  1427. * existing siblings, then add the new event
  1428. * before we can simulate the scheduling
  1429. */
  1430. n = collect_events(fake_cpuc, leader, true);
  1431. if (n < 0)
  1432. goto out;
  1433. fake_cpuc->n_events = n;
  1434. n = collect_events(fake_cpuc, event, false);
  1435. if (n < 0)
  1436. goto out;
  1437. fake_cpuc->n_events = n;
  1438. ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
  1439. out:
  1440. free_fake_cpuc(fake_cpuc);
  1441. return ret;
  1442. }
  1443. static int x86_pmu_event_init(struct perf_event *event)
  1444. {
  1445. struct pmu *tmp;
  1446. int err;
  1447. switch (event->attr.type) {
  1448. case PERF_TYPE_RAW:
  1449. case PERF_TYPE_HARDWARE:
  1450. case PERF_TYPE_HW_CACHE:
  1451. break;
  1452. default:
  1453. return -ENOENT;
  1454. }
  1455. err = __x86_pmu_event_init(event);
  1456. if (!err) {
  1457. /*
  1458. * we temporarily connect event to its pmu
  1459. * such that validate_group() can classify
  1460. * it as an x86 event using is_x86_event()
  1461. */
  1462. tmp = event->pmu;
  1463. event->pmu = &pmu;
  1464. if (event->group_leader != event)
  1465. err = validate_group(event);
  1466. else
  1467. err = validate_event(event);
  1468. event->pmu = tmp;
  1469. }
  1470. if (err) {
  1471. if (event->destroy)
  1472. event->destroy(event);
  1473. }
  1474. return err;
  1475. }
  1476. static int x86_pmu_event_idx(struct perf_event *event)
  1477. {
  1478. int idx = event->hw.idx;
  1479. if (!x86_pmu.attr_rdpmc)
  1480. return 0;
  1481. if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
  1482. idx -= INTEL_PMC_IDX_FIXED;
  1483. idx |= 1 << 30;
  1484. }
  1485. return idx + 1;
  1486. }
  1487. static ssize_t get_attr_rdpmc(struct device *cdev,
  1488. struct device_attribute *attr,
  1489. char *buf)
  1490. {
  1491. return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
  1492. }
  1493. static void change_rdpmc(void *info)
  1494. {
  1495. bool enable = !!(unsigned long)info;
  1496. if (enable)
  1497. set_in_cr4(X86_CR4_PCE);
  1498. else
  1499. clear_in_cr4(X86_CR4_PCE);
  1500. }
  1501. static ssize_t set_attr_rdpmc(struct device *cdev,
  1502. struct device_attribute *attr,
  1503. const char *buf, size_t count)
  1504. {
  1505. unsigned long val;
  1506. ssize_t ret;
  1507. ret = kstrtoul(buf, 0, &val);
  1508. if (ret)
  1509. return ret;
  1510. if (x86_pmu.attr_rdpmc_broken)
  1511. return -ENOTSUPP;
  1512. if (!!val != !!x86_pmu.attr_rdpmc) {
  1513. x86_pmu.attr_rdpmc = !!val;
  1514. on_each_cpu(change_rdpmc, (void *)val, 1);
  1515. }
  1516. return count;
  1517. }
  1518. static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
  1519. static struct attribute *x86_pmu_attrs[] = {
  1520. &dev_attr_rdpmc.attr,
  1521. NULL,
  1522. };
  1523. static struct attribute_group x86_pmu_attr_group = {
  1524. .attrs = x86_pmu_attrs,
  1525. };
  1526. static const struct attribute_group *x86_pmu_attr_groups[] = {
  1527. &x86_pmu_attr_group,
  1528. &x86_pmu_format_group,
  1529. &x86_pmu_events_group,
  1530. NULL,
  1531. };
  1532. static void x86_pmu_flush_branch_stack(void)
  1533. {
  1534. if (x86_pmu.flush_branch_stack)
  1535. x86_pmu.flush_branch_stack();
  1536. }
  1537. void perf_check_microcode(void)
  1538. {
  1539. if (x86_pmu.check_microcode)
  1540. x86_pmu.check_microcode();
  1541. }
  1542. EXPORT_SYMBOL_GPL(perf_check_microcode);
  1543. static struct pmu pmu = {
  1544. .pmu_enable = x86_pmu_enable,
  1545. .pmu_disable = x86_pmu_disable,
  1546. .attr_groups = x86_pmu_attr_groups,
  1547. .event_init = x86_pmu_event_init,
  1548. .add = x86_pmu_add,
  1549. .del = x86_pmu_del,
  1550. .start = x86_pmu_start,
  1551. .stop = x86_pmu_stop,
  1552. .read = x86_pmu_read,
  1553. .start_txn = x86_pmu_start_txn,
  1554. .cancel_txn = x86_pmu_cancel_txn,
  1555. .commit_txn = x86_pmu_commit_txn,
  1556. .event_idx = x86_pmu_event_idx,
  1557. .flush_branch_stack = x86_pmu_flush_branch_stack,
  1558. };
  1559. void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
  1560. {
  1561. struct cyc2ns_data *data;
  1562. userpg->cap_user_time = 0;
  1563. userpg->cap_user_time_zero = 0;
  1564. userpg->cap_user_rdpmc = x86_pmu.attr_rdpmc;
  1565. userpg->pmc_width = x86_pmu.cntval_bits;
  1566. if (!sched_clock_stable())
  1567. return;
  1568. data = cyc2ns_read_begin();
  1569. userpg->cap_user_time = 1;
  1570. userpg->time_mult = data->cyc2ns_mul;
  1571. userpg->time_shift = data->cyc2ns_shift;
  1572. userpg->time_offset = data->cyc2ns_offset - now;
  1573. userpg->cap_user_time_zero = 1;
  1574. userpg->time_zero = data->cyc2ns_offset;
  1575. cyc2ns_read_end(data);
  1576. }
  1577. /*
  1578. * callchain support
  1579. */
  1580. static int backtrace_stack(void *data, char *name)
  1581. {
  1582. return 0;
  1583. }
  1584. static void backtrace_address(void *data, unsigned long addr, int reliable)
  1585. {
  1586. struct perf_callchain_entry *entry = data;
  1587. perf_callchain_store(entry, addr);
  1588. }
  1589. static const struct stacktrace_ops backtrace_ops = {
  1590. .stack = backtrace_stack,
  1591. .address = backtrace_address,
  1592. .walk_stack = print_context_stack_bp,
  1593. };
  1594. void
  1595. perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
  1596. {
  1597. if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
  1598. /* TODO: We don't support guest os callchain now */
  1599. return;
  1600. }
  1601. perf_callchain_store(entry, regs->ip);
  1602. dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry);
  1603. }
  1604. static inline int
  1605. valid_user_frame(const void __user *fp, unsigned long size)
  1606. {
  1607. return (__range_not_ok(fp, size, TASK_SIZE) == 0);
  1608. }
  1609. static unsigned long get_segment_base(unsigned int segment)
  1610. {
  1611. struct desc_struct *desc;
  1612. int idx = segment >> 3;
  1613. if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
  1614. if (idx > LDT_ENTRIES)
  1615. return 0;
  1616. if (idx > current->active_mm->context.size)
  1617. return 0;
  1618. desc = current->active_mm->context.ldt;
  1619. } else {
  1620. if (idx > GDT_ENTRIES)
  1621. return 0;
  1622. desc = raw_cpu_ptr(gdt_page.gdt);
  1623. }
  1624. return get_desc_base(desc + idx);
  1625. }
  1626. #ifdef CONFIG_COMPAT
  1627. #include <asm/compat.h>
  1628. static inline int
  1629. perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
  1630. {
  1631. /* 32-bit process in 64-bit kernel. */
  1632. unsigned long ss_base, cs_base;
  1633. struct stack_frame_ia32 frame;
  1634. const void __user *fp;
  1635. if (!test_thread_flag(TIF_IA32))
  1636. return 0;
  1637. cs_base = get_segment_base(regs->cs);
  1638. ss_base = get_segment_base(regs->ss);
  1639. fp = compat_ptr(ss_base + regs->bp);
  1640. while (entry->nr < PERF_MAX_STACK_DEPTH) {
  1641. unsigned long bytes;
  1642. frame.next_frame = 0;
  1643. frame.return_address = 0;
  1644. bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
  1645. if (bytes != 0)
  1646. break;
  1647. if (!valid_user_frame(fp, sizeof(frame)))
  1648. break;
  1649. perf_callchain_store(entry, cs_base + frame.return_address);
  1650. fp = compat_ptr(ss_base + frame.next_frame);
  1651. }
  1652. return 1;
  1653. }
  1654. #else
  1655. static inline int
  1656. perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
  1657. {
  1658. return 0;
  1659. }
  1660. #endif
  1661. void
  1662. perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
  1663. {
  1664. struct stack_frame frame;
  1665. const void __user *fp;
  1666. if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
  1667. /* TODO: We don't support guest os callchain now */
  1668. return;
  1669. }
  1670. /*
  1671. * We don't know what to do with VM86 stacks.. ignore them for now.
  1672. */
  1673. if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
  1674. return;
  1675. fp = (void __user *)regs->bp;
  1676. perf_callchain_store(entry, regs->ip);
  1677. if (!current->mm)
  1678. return;
  1679. if (perf_callchain_user32(regs, entry))
  1680. return;
  1681. while (entry->nr < PERF_MAX_STACK_DEPTH) {
  1682. unsigned long bytes;
  1683. frame.next_frame = NULL;
  1684. frame.return_address = 0;
  1685. bytes = copy_from_user_nmi(&frame, fp, sizeof(frame));
  1686. if (bytes != 0)
  1687. break;
  1688. if (!valid_user_frame(fp, sizeof(frame)))
  1689. break;
  1690. perf_callchain_store(entry, frame.return_address);
  1691. fp = frame.next_frame;
  1692. }
  1693. }
  1694. /*
  1695. * Deal with code segment offsets for the various execution modes:
  1696. *
  1697. * VM86 - the good olde 16 bit days, where the linear address is
  1698. * 20 bits and we use regs->ip + 0x10 * regs->cs.
  1699. *
  1700. * IA32 - Where we need to look at GDT/LDT segment descriptor tables
  1701. * to figure out what the 32bit base address is.
  1702. *
  1703. * X32 - has TIF_X32 set, but is running in x86_64
  1704. *
  1705. * X86_64 - CS,DS,SS,ES are all zero based.
  1706. */
  1707. static unsigned long code_segment_base(struct pt_regs *regs)
  1708. {
  1709. /*
  1710. * If we are in VM86 mode, add the segment offset to convert to a
  1711. * linear address.
  1712. */
  1713. if (regs->flags & X86_VM_MASK)
  1714. return 0x10 * regs->cs;
  1715. /*
  1716. * For IA32 we look at the GDT/LDT segment base to convert the
  1717. * effective IP to a linear address.
  1718. */
  1719. #ifdef CONFIG_X86_32
  1720. if (user_mode(regs) && regs->cs != __USER_CS)
  1721. return get_segment_base(regs->cs);
  1722. #else
  1723. if (test_thread_flag(TIF_IA32)) {
  1724. if (user_mode(regs) && regs->cs != __USER32_CS)
  1725. return get_segment_base(regs->cs);
  1726. }
  1727. #endif
  1728. return 0;
  1729. }
  1730. unsigned long perf_instruction_pointer(struct pt_regs *regs)
  1731. {
  1732. if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
  1733. return perf_guest_cbs->get_guest_ip();
  1734. return regs->ip + code_segment_base(regs);
  1735. }
  1736. unsigned long perf_misc_flags(struct pt_regs *regs)
  1737. {
  1738. int misc = 0;
  1739. if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
  1740. if (perf_guest_cbs->is_user_mode())
  1741. misc |= PERF_RECORD_MISC_GUEST_USER;
  1742. else
  1743. misc |= PERF_RECORD_MISC_GUEST_KERNEL;
  1744. } else {
  1745. if (user_mode(regs))
  1746. misc |= PERF_RECORD_MISC_USER;
  1747. else
  1748. misc |= PERF_RECORD_MISC_KERNEL;
  1749. }
  1750. if (regs->flags & PERF_EFLAGS_EXACT)
  1751. misc |= PERF_RECORD_MISC_EXACT_IP;
  1752. return misc;
  1753. }
  1754. void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
  1755. {
  1756. cap->version = x86_pmu.version;
  1757. cap->num_counters_gp = x86_pmu.num_counters;
  1758. cap->num_counters_fixed = x86_pmu.num_counters_fixed;
  1759. cap->bit_width_gp = x86_pmu.cntval_bits;
  1760. cap->bit_width_fixed = x86_pmu.cntval_bits;
  1761. cap->events_mask = (unsigned int)x86_pmu.events_maskl;
  1762. cap->events_mask_len = x86_pmu.events_mask_len;
  1763. }
  1764. EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);