hw_breakpoint.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697
  1. /*
  2. * This program is free software; you can redistribute it and/or modify
  3. * it under the terms of the GNU General Public License as published by
  4. * the Free Software Foundation; either version 2 of the License, or
  5. * (at your option) any later version.
  6. *
  7. * This program is distributed in the hope that it will be useful,
  8. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. * GNU General Public License for more details.
  11. *
  12. * You should have received a copy of the GNU General Public License
  13. * along with this program; if not, write to the Free Software
  14. * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  15. *
  16. * Copyright (C) 2007 Alan Stern
  17. * Copyright (C) IBM Corporation, 2009
  18. * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com>
  19. *
  20. * Thanks to Ingo Molnar for his many suggestions.
  21. *
  22. * Authors: Alan Stern <stern@rowland.harvard.edu>
  23. * K.Prasad <prasad@linux.vnet.ibm.com>
  24. * Frederic Weisbecker <fweisbec@gmail.com>
  25. */
  26. /*
  27. * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
  28. * using the CPU's debug registers.
  29. * This file contains the arch-independent routines.
  30. */
  31. #include <linux/irqflags.h>
  32. #include <linux/kallsyms.h>
  33. #include <linux/notifier.h>
  34. #include <linux/kprobes.h>
  35. #include <linux/kdebug.h>
  36. #include <linux/kernel.h>
  37. #include <linux/module.h>
  38. #include <linux/percpu.h>
  39. #include <linux/sched.h>
  40. #include <linux/init.h>
  41. #include <linux/slab.h>
  42. #include <linux/list.h>
  43. #include <linux/cpu.h>
  44. #include <linux/smp.h>
  45. #include <linux/bug.h>
  46. #include <linux/hw_breakpoint.h>
  47. /*
  48. * Constraints data
  49. */
  50. struct bp_cpuinfo {
  51. /* Number of pinned cpu breakpoints in a cpu */
  52. unsigned int cpu_pinned;
  53. /* tsk_pinned[n] is the number of tasks having n+1 breakpoints */
  54. unsigned int *tsk_pinned;
  55. /* Number of non-pinned cpu/task breakpoints in a cpu */
  56. unsigned int flexible; /* XXX: placeholder, see fetch_this_slot() */
  57. };
  58. static DEFINE_PER_CPU(struct bp_cpuinfo, bp_cpuinfo[TYPE_MAX]);
  59. static int nr_slots[TYPE_MAX];
  60. static struct bp_cpuinfo *get_bp_info(int cpu, enum bp_type_idx type)
  61. {
  62. return per_cpu_ptr(bp_cpuinfo + type, cpu);
  63. }
  64. /* Keep track of the breakpoints attached to tasks */
  65. static LIST_HEAD(bp_task_head);
  66. static int constraints_initialized;
  67. /* Gather the number of total pinned and un-pinned bp in a cpuset */
  68. struct bp_busy_slots {
  69. unsigned int pinned;
  70. unsigned int flexible;
  71. };
  72. /* Serialize accesses to the above constraints */
  73. static DEFINE_MUTEX(nr_bp_mutex);
  74. __weak int hw_breakpoint_weight(struct perf_event *bp)
  75. {
  76. return 1;
  77. }
  78. static inline enum bp_type_idx find_slot_idx(u64 bp_type)
  79. {
  80. if (bp_type & HW_BREAKPOINT_RW)
  81. return TYPE_DATA;
  82. return TYPE_INST;
  83. }
  84. /*
  85. * Report the maximum number of pinned breakpoints a task
  86. * have in this cpu
  87. */
  88. static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
  89. {
  90. unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
  91. int i;
  92. for (i = nr_slots[type] - 1; i >= 0; i--) {
  93. if (tsk_pinned[i] > 0)
  94. return i + 1;
  95. }
  96. return 0;
  97. }
  98. /*
  99. * Count the number of breakpoints of the same type and same task.
  100. * The given event must be not on the list.
  101. */
  102. static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)
  103. {
  104. struct task_struct *tsk = bp->hw.target;
  105. struct perf_event *iter;
  106. int count = 0;
  107. list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
  108. if (iter->hw.target == tsk &&
  109. find_slot_idx(iter->attr.bp_type) == type &&
  110. (iter->cpu < 0 || cpu == iter->cpu))
  111. count += hw_breakpoint_weight(iter);
  112. }
  113. return count;
  114. }
  115. static const struct cpumask *cpumask_of_bp(struct perf_event *bp)
  116. {
  117. if (bp->cpu >= 0)
  118. return cpumask_of(bp->cpu);
  119. return cpu_possible_mask;
  120. }
  121. /*
  122. * Report the number of pinned/un-pinned breakpoints we have in
  123. * a given cpu (cpu > -1) or in all of them (cpu = -1).
  124. */
  125. static void
  126. fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
  127. enum bp_type_idx type)
  128. {
  129. const struct cpumask *cpumask = cpumask_of_bp(bp);
  130. int cpu;
  131. for_each_cpu(cpu, cpumask) {
  132. struct bp_cpuinfo *info = get_bp_info(cpu, type);
  133. int nr;
  134. nr = info->cpu_pinned;
  135. if (!bp->hw.target)
  136. nr += max_task_bp_pinned(cpu, type);
  137. else
  138. nr += task_bp_pinned(cpu, bp, type);
  139. if (nr > slots->pinned)
  140. slots->pinned = nr;
  141. nr = info->flexible;
  142. if (nr > slots->flexible)
  143. slots->flexible = nr;
  144. }
  145. }
  146. /*
  147. * For now, continue to consider flexible as pinned, until we can
  148. * ensure no flexible event can ever be scheduled before a pinned event
  149. * in a same cpu.
  150. */
  151. static void
  152. fetch_this_slot(struct bp_busy_slots *slots, int weight)
  153. {
  154. slots->pinned += weight;
  155. }
  156. /*
  157. * Add a pinned breakpoint for the given task in our constraint table
  158. */
  159. static void toggle_bp_task_slot(struct perf_event *bp, int cpu,
  160. enum bp_type_idx type, int weight)
  161. {
  162. unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
  163. int old_idx, new_idx;
  164. old_idx = task_bp_pinned(cpu, bp, type) - 1;
  165. new_idx = old_idx + weight;
  166. if (old_idx >= 0)
  167. tsk_pinned[old_idx]--;
  168. if (new_idx >= 0)
  169. tsk_pinned[new_idx]++;
  170. }
  171. /*
  172. * Add/remove the given breakpoint in our constraint table
  173. */
  174. static void
  175. toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
  176. int weight)
  177. {
  178. const struct cpumask *cpumask = cpumask_of_bp(bp);
  179. int cpu;
  180. if (!enable)
  181. weight = -weight;
  182. /* Pinned counter cpu profiling */
  183. if (!bp->hw.target) {
  184. get_bp_info(bp->cpu, type)->cpu_pinned += weight;
  185. return;
  186. }
  187. /* Pinned counter task profiling */
  188. for_each_cpu(cpu, cpumask)
  189. toggle_bp_task_slot(bp, cpu, type, weight);
  190. if (enable)
  191. list_add_tail(&bp->hw.bp_list, &bp_task_head);
  192. else
  193. list_del(&bp->hw.bp_list);
  194. }
  195. /*
  196. * Function to perform processor-specific cleanup during unregistration
  197. */
  198. __weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
  199. {
  200. /*
  201. * A weak stub function here for those archs that don't define
  202. * it inside arch/.../kernel/hw_breakpoint.c
  203. */
  204. }
  205. /*
  206. * Contraints to check before allowing this new breakpoint counter:
  207. *
  208. * == Non-pinned counter == (Considered as pinned for now)
  209. *
  210. * - If attached to a single cpu, check:
  211. *
  212. * (per_cpu(info->flexible, cpu) || (per_cpu(info->cpu_pinned, cpu)
  213. * + max(per_cpu(info->tsk_pinned, cpu)))) < HBP_NUM
  214. *
  215. * -> If there are already non-pinned counters in this cpu, it means
  216. * there is already a free slot for them.
  217. * Otherwise, we check that the maximum number of per task
  218. * breakpoints (for this cpu) plus the number of per cpu breakpoint
  219. * (for this cpu) doesn't cover every registers.
  220. *
  221. * - If attached to every cpus, check:
  222. *
  223. * (per_cpu(info->flexible, *) || (max(per_cpu(info->cpu_pinned, *))
  224. * + max(per_cpu(info->tsk_pinned, *)))) < HBP_NUM
  225. *
  226. * -> This is roughly the same, except we check the number of per cpu
  227. * bp for every cpu and we keep the max one. Same for the per tasks
  228. * breakpoints.
  229. *
  230. *
  231. * == Pinned counter ==
  232. *
  233. * - If attached to a single cpu, check:
  234. *
  235. * ((per_cpu(info->flexible, cpu) > 1) + per_cpu(info->cpu_pinned, cpu)
  236. * + max(per_cpu(info->tsk_pinned, cpu))) < HBP_NUM
  237. *
  238. * -> Same checks as before. But now the info->flexible, if any, must keep
  239. * one register at least (or they will never be fed).
  240. *
  241. * - If attached to every cpus, check:
  242. *
  243. * ((per_cpu(info->flexible, *) > 1) + max(per_cpu(info->cpu_pinned, *))
  244. * + max(per_cpu(info->tsk_pinned, *))) < HBP_NUM
  245. */
  246. static int __reserve_bp_slot(struct perf_event *bp, u64 bp_type)
  247. {
  248. struct bp_busy_slots slots = {0};
  249. enum bp_type_idx type;
  250. int weight;
  251. /* We couldn't initialize breakpoint constraints on boot */
  252. if (!constraints_initialized)
  253. return -ENOMEM;
  254. /* Basic checks */
  255. if (bp_type == HW_BREAKPOINT_EMPTY ||
  256. bp_type == HW_BREAKPOINT_INVALID)
  257. return -EINVAL;
  258. type = find_slot_idx(bp_type);
  259. weight = hw_breakpoint_weight(bp);
  260. fetch_bp_busy_slots(&slots, bp, type);
  261. /*
  262. * Simulate the addition of this breakpoint to the constraints
  263. * and see the result.
  264. */
  265. fetch_this_slot(&slots, weight);
  266. /* Flexible counters need to keep at least one slot */
  267. if (slots.pinned + (!!slots.flexible) > nr_slots[type])
  268. return -ENOSPC;
  269. toggle_bp_slot(bp, true, type, weight);
  270. return 0;
  271. }
  272. int reserve_bp_slot(struct perf_event *bp)
  273. {
  274. int ret;
  275. mutex_lock(&nr_bp_mutex);
  276. ret = __reserve_bp_slot(bp, bp->attr.bp_type);
  277. mutex_unlock(&nr_bp_mutex);
  278. return ret;
  279. }
  280. static void __release_bp_slot(struct perf_event *bp, u64 bp_type)
  281. {
  282. enum bp_type_idx type;
  283. int weight;
  284. type = find_slot_idx(bp_type);
  285. weight = hw_breakpoint_weight(bp);
  286. toggle_bp_slot(bp, false, type, weight);
  287. }
  288. void release_bp_slot(struct perf_event *bp)
  289. {
  290. mutex_lock(&nr_bp_mutex);
  291. arch_unregister_hw_breakpoint(bp);
  292. __release_bp_slot(bp, bp->attr.bp_type);
  293. mutex_unlock(&nr_bp_mutex);
  294. }
  295. static int __modify_bp_slot(struct perf_event *bp, u64 old_type)
  296. {
  297. int err;
  298. __release_bp_slot(bp, old_type);
  299. err = __reserve_bp_slot(bp, bp->attr.bp_type);
  300. if (err) {
  301. /*
  302. * Reserve the old_type slot back in case
  303. * there's no space for the new type.
  304. *
  305. * This must succeed, because we just released
  306. * the old_type slot in the __release_bp_slot
  307. * call above. If not, something is broken.
  308. */
  309. WARN_ON(__reserve_bp_slot(bp, old_type));
  310. }
  311. return err;
  312. }
  313. static int modify_bp_slot(struct perf_event *bp, u64 old_type)
  314. {
  315. int ret;
  316. mutex_lock(&nr_bp_mutex);
  317. ret = __modify_bp_slot(bp, old_type);
  318. mutex_unlock(&nr_bp_mutex);
  319. return ret;
  320. }
  321. /*
  322. * Allow the kernel debugger to reserve breakpoint slots without
  323. * taking a lock using the dbg_* variant of for the reserve and
  324. * release breakpoint slots.
  325. */
  326. int dbg_reserve_bp_slot(struct perf_event *bp)
  327. {
  328. if (mutex_is_locked(&nr_bp_mutex))
  329. return -1;
  330. return __reserve_bp_slot(bp, bp->attr.bp_type);
  331. }
  332. int dbg_release_bp_slot(struct perf_event *bp)
  333. {
  334. if (mutex_is_locked(&nr_bp_mutex))
  335. return -1;
  336. __release_bp_slot(bp, bp->attr.bp_type);
  337. return 0;
  338. }
  339. static int validate_hw_breakpoint(struct perf_event *bp)
  340. {
  341. int ret;
  342. ret = arch_validate_hwbkpt_settings(bp);
  343. if (ret)
  344. return ret;
  345. if (arch_check_bp_in_kernelspace(bp)) {
  346. if (bp->attr.exclude_kernel)
  347. return -EINVAL;
  348. /*
  349. * Don't let unprivileged users set a breakpoint in the trap
  350. * path to avoid trap recursion attacks.
  351. */
  352. if (!capable(CAP_SYS_ADMIN))
  353. return -EPERM;
  354. }
  355. return 0;
  356. }
  357. int register_perf_hw_breakpoint(struct perf_event *bp)
  358. {
  359. int ret;
  360. ret = reserve_bp_slot(bp);
  361. if (ret)
  362. return ret;
  363. ret = validate_hw_breakpoint(bp);
  364. /* if arch_validate_hwbkpt_settings() fails then release bp slot */
  365. if (ret)
  366. release_bp_slot(bp);
  367. return ret;
  368. }
  369. /**
  370. * register_user_hw_breakpoint - register a hardware breakpoint for user space
  371. * @attr: breakpoint attributes
  372. * @triggered: callback to trigger when we hit the breakpoint
  373. * @tsk: pointer to 'task_struct' of the process to which the address belongs
  374. */
  375. struct perf_event *
  376. register_user_hw_breakpoint(struct perf_event_attr *attr,
  377. perf_overflow_handler_t triggered,
  378. void *context,
  379. struct task_struct *tsk)
  380. {
  381. return perf_event_create_kernel_counter(attr, -1, tsk, triggered,
  382. context);
  383. }
  384. EXPORT_SYMBOL_GPL(register_user_hw_breakpoint);
  385. int
  386. modify_user_hw_breakpoint_check(struct perf_event *bp, struct perf_event_attr *attr,
  387. bool check)
  388. {
  389. u64 old_addr = bp->attr.bp_addr;
  390. u64 old_len = bp->attr.bp_len;
  391. int old_type = bp->attr.bp_type;
  392. bool modify = attr->bp_type != old_type;
  393. int err = 0;
  394. bp->attr.bp_addr = attr->bp_addr;
  395. bp->attr.bp_type = attr->bp_type;
  396. bp->attr.bp_len = attr->bp_len;
  397. if (check && memcmp(&bp->attr, attr, sizeof(*attr)))
  398. return -EINVAL;
  399. err = validate_hw_breakpoint(bp);
  400. if (!err && modify)
  401. err = modify_bp_slot(bp, old_type);
  402. if (err) {
  403. bp->attr.bp_addr = old_addr;
  404. bp->attr.bp_type = old_type;
  405. bp->attr.bp_len = old_len;
  406. return err;
  407. }
  408. bp->attr.disabled = attr->disabled;
  409. return 0;
  410. }
  411. /**
  412. * modify_user_hw_breakpoint - modify a user-space hardware breakpoint
  413. * @bp: the breakpoint structure to modify
  414. * @attr: new breakpoint attributes
  415. */
  416. int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)
  417. {
  418. /*
  419. * modify_user_hw_breakpoint can be invoked with IRQs disabled and hence it
  420. * will not be possible to raise IPIs that invoke __perf_event_disable.
  421. * So call the function directly after making sure we are targeting the
  422. * current task.
  423. */
  424. if (irqs_disabled() && bp->ctx && bp->ctx->task == current)
  425. perf_event_disable_local(bp);
  426. else
  427. perf_event_disable(bp);
  428. if (!attr->disabled) {
  429. int err = modify_user_hw_breakpoint_check(bp, attr, false);
  430. if (err)
  431. return err;
  432. perf_event_enable(bp);
  433. bp->attr.disabled = 0;
  434. }
  435. return 0;
  436. }
  437. EXPORT_SYMBOL_GPL(modify_user_hw_breakpoint);
  438. /**
  439. * unregister_hw_breakpoint - unregister a user-space hardware breakpoint
  440. * @bp: the breakpoint structure to unregister
  441. */
  442. void unregister_hw_breakpoint(struct perf_event *bp)
  443. {
  444. if (!bp)
  445. return;
  446. perf_event_release_kernel(bp);
  447. }
  448. EXPORT_SYMBOL_GPL(unregister_hw_breakpoint);
  449. /**
  450. * register_wide_hw_breakpoint - register a wide breakpoint in the kernel
  451. * @attr: breakpoint attributes
  452. * @triggered: callback to trigger when we hit the breakpoint
  453. *
  454. * @return a set of per_cpu pointers to perf events
  455. */
  456. struct perf_event * __percpu *
  457. register_wide_hw_breakpoint(struct perf_event_attr *attr,
  458. perf_overflow_handler_t triggered,
  459. void *context)
  460. {
  461. struct perf_event * __percpu *cpu_events, *bp;
  462. long err = 0;
  463. int cpu;
  464. cpu_events = alloc_percpu(typeof(*cpu_events));
  465. if (!cpu_events)
  466. return (void __percpu __force *)ERR_PTR(-ENOMEM);
  467. get_online_cpus();
  468. for_each_online_cpu(cpu) {
  469. bp = perf_event_create_kernel_counter(attr, cpu, NULL,
  470. triggered, context);
  471. if (IS_ERR(bp)) {
  472. err = PTR_ERR(bp);
  473. break;
  474. }
  475. per_cpu(*cpu_events, cpu) = bp;
  476. }
  477. put_online_cpus();
  478. if (likely(!err))
  479. return cpu_events;
  480. unregister_wide_hw_breakpoint(cpu_events);
  481. return (void __percpu __force *)ERR_PTR(err);
  482. }
  483. EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
  484. /**
  485. * unregister_wide_hw_breakpoint - unregister a wide breakpoint in the kernel
  486. * @cpu_events: the per cpu set of events to unregister
  487. */
  488. void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)
  489. {
  490. int cpu;
  491. for_each_possible_cpu(cpu)
  492. unregister_hw_breakpoint(per_cpu(*cpu_events, cpu));
  493. free_percpu(cpu_events);
  494. }
  495. EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint);
  496. static struct notifier_block hw_breakpoint_exceptions_nb = {
  497. .notifier_call = hw_breakpoint_exceptions_notify,
  498. /* we need to be notified first */
  499. .priority = 0x7fffffff
  500. };
  501. static void bp_perf_event_destroy(struct perf_event *event)
  502. {
  503. release_bp_slot(event);
  504. }
  505. static int hw_breakpoint_event_init(struct perf_event *bp)
  506. {
  507. int err;
  508. if (bp->attr.type != PERF_TYPE_BREAKPOINT)
  509. return -ENOENT;
  510. /*
  511. * no branch sampling for breakpoint events
  512. */
  513. if (has_branch_stack(bp))
  514. return -EOPNOTSUPP;
  515. err = register_perf_hw_breakpoint(bp);
  516. if (err)
  517. return err;
  518. bp->destroy = bp_perf_event_destroy;
  519. return 0;
  520. }
  521. static int hw_breakpoint_add(struct perf_event *bp, int flags)
  522. {
  523. if (!(flags & PERF_EF_START))
  524. bp->hw.state = PERF_HES_STOPPED;
  525. if (is_sampling_event(bp)) {
  526. bp->hw.last_period = bp->hw.sample_period;
  527. perf_swevent_set_period(bp);
  528. }
  529. return arch_install_hw_breakpoint(bp);
  530. }
  531. static void hw_breakpoint_del(struct perf_event *bp, int flags)
  532. {
  533. arch_uninstall_hw_breakpoint(bp);
  534. }
  535. static void hw_breakpoint_start(struct perf_event *bp, int flags)
  536. {
  537. bp->hw.state = 0;
  538. }
  539. static void hw_breakpoint_stop(struct perf_event *bp, int flags)
  540. {
  541. bp->hw.state = PERF_HES_STOPPED;
  542. }
  543. static struct pmu perf_breakpoint = {
  544. .task_ctx_nr = perf_sw_context, /* could eventually get its own */
  545. .event_init = hw_breakpoint_event_init,
  546. .add = hw_breakpoint_add,
  547. .del = hw_breakpoint_del,
  548. .start = hw_breakpoint_start,
  549. .stop = hw_breakpoint_stop,
  550. .read = hw_breakpoint_pmu_read,
  551. };
  552. int __init init_hw_breakpoint(void)
  553. {
  554. int cpu, err_cpu;
  555. int i;
  556. for (i = 0; i < TYPE_MAX; i++)
  557. nr_slots[i] = hw_breakpoint_slots(i);
  558. for_each_possible_cpu(cpu) {
  559. for (i = 0; i < TYPE_MAX; i++) {
  560. struct bp_cpuinfo *info = get_bp_info(cpu, i);
  561. info->tsk_pinned = kcalloc(nr_slots[i], sizeof(int),
  562. GFP_KERNEL);
  563. if (!info->tsk_pinned)
  564. goto err_alloc;
  565. }
  566. }
  567. constraints_initialized = 1;
  568. perf_pmu_register(&perf_breakpoint, "breakpoint", PERF_TYPE_BREAKPOINT);
  569. return register_die_notifier(&hw_breakpoint_exceptions_nb);
  570. err_alloc:
  571. for_each_possible_cpu(err_cpu) {
  572. for (i = 0; i < TYPE_MAX; i++)
  573. kfree(get_bp_info(err_cpu, i)->tsk_pinned);
  574. if (err_cpu == cpu)
  575. break;
  576. }
  577. return -ENOMEM;
  578. }