trace_kprobe.c 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521
  1. /*
  2. * Kprobes-based tracing events
  3. *
  4. * Created by Masami Hiramatsu <mhiramat@redhat.com>
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18. */
  19. #include <linux/module.h>
  20. #include <linux/uaccess.h>
  21. #include "trace_probe.h"
  22. #define KPROBE_EVENT_SYSTEM "kprobes"
  23. /**
  24. * Kprobe event core functions
  25. */
  26. struct trace_kprobe {
  27. struct list_head list;
  28. struct kretprobe rp; /* Use rp.kp for kprobe use */
  29. unsigned long __percpu *nhit;
  30. const char *symbol; /* symbol name */
  31. struct trace_probe tp;
  32. };
  33. #define SIZEOF_TRACE_KPROBE(n) \
  34. (offsetof(struct trace_kprobe, tp.args) + \
  35. (sizeof(struct probe_arg) * (n)))
  36. static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
  37. {
  38. return tk->rp.handler != NULL;
  39. }
  40. static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
  41. {
  42. return tk->symbol ? tk->symbol : "unknown";
  43. }
  44. static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
  45. {
  46. return tk->rp.kp.offset;
  47. }
  48. static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
  49. {
  50. return !!(kprobe_gone(&tk->rp.kp));
  51. }
  52. static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
  53. struct module *mod)
  54. {
  55. int len = strlen(mod->name);
  56. const char *name = trace_kprobe_symbol(tk);
  57. return strncmp(mod->name, name, len) == 0 && name[len] == ':';
  58. }
  59. static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
  60. {
  61. return !!strchr(trace_kprobe_symbol(tk), ':');
  62. }
  63. static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
  64. {
  65. unsigned long nhit = 0;
  66. int cpu;
  67. for_each_possible_cpu(cpu)
  68. nhit += *per_cpu_ptr(tk->nhit, cpu);
  69. return nhit;
  70. }
  71. static int register_kprobe_event(struct trace_kprobe *tk);
  72. static int unregister_kprobe_event(struct trace_kprobe *tk);
  73. static DEFINE_MUTEX(probe_lock);
  74. static LIST_HEAD(probe_list);
  75. static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
  76. static int kretprobe_dispatcher(struct kretprobe_instance *ri,
  77. struct pt_regs *regs);
  78. /* Memory fetching by symbol */
  79. struct symbol_cache {
  80. char *symbol;
  81. long offset;
  82. unsigned long addr;
  83. };
  84. unsigned long update_symbol_cache(struct symbol_cache *sc)
  85. {
  86. sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
  87. if (sc->addr)
  88. sc->addr += sc->offset;
  89. return sc->addr;
  90. }
  91. void free_symbol_cache(struct symbol_cache *sc)
  92. {
  93. kfree(sc->symbol);
  94. kfree(sc);
  95. }
  96. struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
  97. {
  98. struct symbol_cache *sc;
  99. if (!sym || strlen(sym) == 0)
  100. return NULL;
  101. sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
  102. if (!sc)
  103. return NULL;
  104. sc->symbol = kstrdup(sym, GFP_KERNEL);
  105. if (!sc->symbol) {
  106. kfree(sc);
  107. return NULL;
  108. }
  109. sc->offset = offset;
  110. update_symbol_cache(sc);
  111. return sc;
  112. }
  113. /*
  114. * Kprobes-specific fetch functions
  115. */
  116. #define DEFINE_FETCH_stack(type) \
  117. static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
  118. void *offset, void *dest) \
  119. { \
  120. *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
  121. (unsigned int)((unsigned long)offset)); \
  122. } \
  123. NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
  124. DEFINE_BASIC_FETCH_FUNCS(stack)
  125. /* No string on the stack entry */
  126. #define fetch_stack_string NULL
  127. #define fetch_stack_string_size NULL
  128. #define DEFINE_FETCH_memory(type) \
  129. static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
  130. void *addr, void *dest) \
  131. { \
  132. type retval; \
  133. if (probe_kernel_address(addr, retval)) \
  134. *(type *)dest = 0; \
  135. else \
  136. *(type *)dest = retval; \
  137. } \
  138. NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
  139. DEFINE_BASIC_FETCH_FUNCS(memory)
  140. /*
  141. * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
  142. * length and relative data location.
  143. */
  144. static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
  145. void *addr, void *dest)
  146. {
  147. int maxlen = get_rloc_len(*(u32 *)dest);
  148. u8 *dst = get_rloc_data(dest);
  149. long ret;
  150. if (!maxlen)
  151. return;
  152. /*
  153. * Try to get string again, since the string can be changed while
  154. * probing.
  155. */
  156. ret = strncpy_from_unsafe(dst, addr, maxlen);
  157. if (ret < 0) { /* Failed to fetch string */
  158. dst[0] = '\0';
  159. *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
  160. } else {
  161. *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
  162. }
  163. }
  164. NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
  165. /* Return the length of string -- including null terminal byte */
  166. static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
  167. void *addr, void *dest)
  168. {
  169. mm_segment_t old_fs;
  170. int ret, len = 0;
  171. u8 c;
  172. old_fs = get_fs();
  173. set_fs(KERNEL_DS);
  174. pagefault_disable();
  175. do {
  176. ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
  177. len++;
  178. } while (c && ret == 0 && len < MAX_STRING_SIZE);
  179. pagefault_enable();
  180. set_fs(old_fs);
  181. if (ret < 0) /* Failed to check the length */
  182. *(u32 *)dest = 0;
  183. else
  184. *(u32 *)dest = len;
  185. }
  186. NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
  187. #define DEFINE_FETCH_symbol(type) \
  188. void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
  189. { \
  190. struct symbol_cache *sc = data; \
  191. if (sc->addr) \
  192. fetch_memory_##type(regs, (void *)sc->addr, dest); \
  193. else \
  194. *(type *)dest = 0; \
  195. } \
  196. NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
  197. DEFINE_BASIC_FETCH_FUNCS(symbol)
  198. DEFINE_FETCH_symbol(string)
  199. DEFINE_FETCH_symbol(string_size)
  200. /* kprobes don't support file_offset fetch methods */
  201. #define fetch_file_offset_u8 NULL
  202. #define fetch_file_offset_u16 NULL
  203. #define fetch_file_offset_u32 NULL
  204. #define fetch_file_offset_u64 NULL
  205. #define fetch_file_offset_string NULL
  206. #define fetch_file_offset_string_size NULL
  207. /* Fetch type information table */
  208. static const struct fetch_type kprobes_fetch_type_table[] = {
  209. /* Special types */
  210. [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
  211. sizeof(u32), 1, "__data_loc char[]"),
  212. [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
  213. string_size, sizeof(u32), 0, "u32"),
  214. /* Basic types */
  215. ASSIGN_FETCH_TYPE(u8, u8, 0),
  216. ASSIGN_FETCH_TYPE(u16, u16, 0),
  217. ASSIGN_FETCH_TYPE(u32, u32, 0),
  218. ASSIGN_FETCH_TYPE(u64, u64, 0),
  219. ASSIGN_FETCH_TYPE(s8, u8, 1),
  220. ASSIGN_FETCH_TYPE(s16, u16, 1),
  221. ASSIGN_FETCH_TYPE(s32, u32, 1),
  222. ASSIGN_FETCH_TYPE(s64, u64, 1),
  223. ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
  224. ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
  225. ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
  226. ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
  227. ASSIGN_FETCH_TYPE_END
  228. };
  229. /*
  230. * Allocate new trace_probe and initialize it (including kprobes).
  231. */
  232. static struct trace_kprobe *alloc_trace_kprobe(const char *group,
  233. const char *event,
  234. void *addr,
  235. const char *symbol,
  236. unsigned long offs,
  237. int nargs, bool is_return)
  238. {
  239. struct trace_kprobe *tk;
  240. int ret = -ENOMEM;
  241. tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);
  242. if (!tk)
  243. return ERR_PTR(ret);
  244. tk->nhit = alloc_percpu(unsigned long);
  245. if (!tk->nhit)
  246. goto error;
  247. if (symbol) {
  248. tk->symbol = kstrdup(symbol, GFP_KERNEL);
  249. if (!tk->symbol)
  250. goto error;
  251. tk->rp.kp.symbol_name = tk->symbol;
  252. tk->rp.kp.offset = offs;
  253. } else
  254. tk->rp.kp.addr = addr;
  255. if (is_return)
  256. tk->rp.handler = kretprobe_dispatcher;
  257. else
  258. tk->rp.kp.pre_handler = kprobe_dispatcher;
  259. if (!event || !is_good_name(event)) {
  260. ret = -EINVAL;
  261. goto error;
  262. }
  263. tk->tp.call.class = &tk->tp.class;
  264. tk->tp.call.name = kstrdup(event, GFP_KERNEL);
  265. if (!tk->tp.call.name)
  266. goto error;
  267. if (!group || !is_good_name(group)) {
  268. ret = -EINVAL;
  269. goto error;
  270. }
  271. tk->tp.class.system = kstrdup(group, GFP_KERNEL);
  272. if (!tk->tp.class.system)
  273. goto error;
  274. INIT_LIST_HEAD(&tk->list);
  275. INIT_LIST_HEAD(&tk->tp.files);
  276. return tk;
  277. error:
  278. kfree(tk->tp.call.name);
  279. kfree(tk->symbol);
  280. free_percpu(tk->nhit);
  281. kfree(tk);
  282. return ERR_PTR(ret);
  283. }
  284. static void free_trace_kprobe(struct trace_kprobe *tk)
  285. {
  286. int i;
  287. for (i = 0; i < tk->tp.nr_args; i++)
  288. traceprobe_free_probe_arg(&tk->tp.args[i]);
  289. kfree(tk->tp.call.class->system);
  290. kfree(tk->tp.call.name);
  291. kfree(tk->symbol);
  292. free_percpu(tk->nhit);
  293. kfree(tk);
  294. }
  295. static struct trace_kprobe *find_trace_kprobe(const char *event,
  296. const char *group)
  297. {
  298. struct trace_kprobe *tk;
  299. list_for_each_entry(tk, &probe_list, list)
  300. if (strcmp(trace_event_name(&tk->tp.call), event) == 0 &&
  301. strcmp(tk->tp.call.class->system, group) == 0)
  302. return tk;
  303. return NULL;
  304. }
  305. /*
  306. * Enable trace_probe
  307. * if the file is NULL, enable "perf" handler, or enable "trace" handler.
  308. */
  309. static int
  310. enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
  311. {
  312. int ret = 0;
  313. if (file) {
  314. struct event_file_link *link;
  315. link = kmalloc(sizeof(*link), GFP_KERNEL);
  316. if (!link) {
  317. ret = -ENOMEM;
  318. goto out;
  319. }
  320. link->file = file;
  321. list_add_tail_rcu(&link->list, &tk->tp.files);
  322. tk->tp.flags |= TP_FLAG_TRACE;
  323. } else
  324. tk->tp.flags |= TP_FLAG_PROFILE;
  325. if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {
  326. if (trace_kprobe_is_return(tk))
  327. ret = enable_kretprobe(&tk->rp);
  328. else
  329. ret = enable_kprobe(&tk->rp.kp);
  330. }
  331. out:
  332. return ret;
  333. }
  334. /*
  335. * Disable trace_probe
  336. * if the file is NULL, disable "perf" handler, or disable "trace" handler.
  337. */
  338. static int
  339. disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
  340. {
  341. struct event_file_link *link = NULL;
  342. int wait = 0;
  343. int ret = 0;
  344. if (file) {
  345. link = find_event_file_link(&tk->tp, file);
  346. if (!link) {
  347. ret = -EINVAL;
  348. goto out;
  349. }
  350. list_del_rcu(&link->list);
  351. wait = 1;
  352. if (!list_empty(&tk->tp.files))
  353. goto out;
  354. tk->tp.flags &= ~TP_FLAG_TRACE;
  355. } else
  356. tk->tp.flags &= ~TP_FLAG_PROFILE;
  357. if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {
  358. if (trace_kprobe_is_return(tk))
  359. disable_kretprobe(&tk->rp);
  360. else
  361. disable_kprobe(&tk->rp.kp);
  362. wait = 1;
  363. }
  364. out:
  365. if (wait) {
  366. /*
  367. * Synchronize with kprobe_trace_func/kretprobe_trace_func
  368. * to ensure disabled (all running handlers are finished).
  369. * This is not only for kfree(), but also the caller,
  370. * trace_remove_event_call() supposes it for releasing
  371. * event_call related objects, which will be accessed in
  372. * the kprobe_trace_func/kretprobe_trace_func.
  373. */
  374. synchronize_sched();
  375. kfree(link); /* Ignored if link == NULL */
  376. }
  377. return ret;
  378. }
  379. /* Internal register function - just handle k*probes and flags */
  380. static int __register_trace_kprobe(struct trace_kprobe *tk)
  381. {
  382. int i, ret;
  383. if (trace_probe_is_registered(&tk->tp))
  384. return -EINVAL;
  385. for (i = 0; i < tk->tp.nr_args; i++)
  386. traceprobe_update_arg(&tk->tp.args[i]);
  387. /* Set/clear disabled flag according to tp->flag */
  388. if (trace_probe_is_enabled(&tk->tp))
  389. tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
  390. else
  391. tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
  392. if (trace_kprobe_is_return(tk))
  393. ret = register_kretprobe(&tk->rp);
  394. else
  395. ret = register_kprobe(&tk->rp.kp);
  396. if (ret == 0)
  397. tk->tp.flags |= TP_FLAG_REGISTERED;
  398. else {
  399. pr_warn("Could not insert probe at %s+%lu: %d\n",
  400. trace_kprobe_symbol(tk), trace_kprobe_offset(tk), ret);
  401. if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) {
  402. pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
  403. ret = 0;
  404. } else if (ret == -EILSEQ) {
  405. pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
  406. tk->rp.kp.addr);
  407. ret = -EINVAL;
  408. }
  409. }
  410. return ret;
  411. }
  412. /* Internal unregister function - just handle k*probes and flags */
  413. static void __unregister_trace_kprobe(struct trace_kprobe *tk)
  414. {
  415. if (trace_probe_is_registered(&tk->tp)) {
  416. if (trace_kprobe_is_return(tk))
  417. unregister_kretprobe(&tk->rp);
  418. else
  419. unregister_kprobe(&tk->rp.kp);
  420. tk->tp.flags &= ~TP_FLAG_REGISTERED;
  421. /* Cleanup kprobe for reuse */
  422. if (tk->rp.kp.symbol_name)
  423. tk->rp.kp.addr = NULL;
  424. }
  425. }
  426. /* Unregister a trace_probe and probe_event: call with locking probe_lock */
  427. static int unregister_trace_kprobe(struct trace_kprobe *tk)
  428. {
  429. /* Enabled event can not be unregistered */
  430. if (trace_probe_is_enabled(&tk->tp))
  431. return -EBUSY;
  432. /* Will fail if probe is being used by ftrace or perf */
  433. if (unregister_kprobe_event(tk))
  434. return -EBUSY;
  435. __unregister_trace_kprobe(tk);
  436. list_del(&tk->list);
  437. return 0;
  438. }
  439. /* Register a trace_probe and probe_event */
  440. static int register_trace_kprobe(struct trace_kprobe *tk)
  441. {
  442. struct trace_kprobe *old_tk;
  443. int ret;
  444. mutex_lock(&probe_lock);
  445. /* Delete old (same name) event if exist */
  446. old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call),
  447. tk->tp.call.class->system);
  448. if (old_tk) {
  449. ret = unregister_trace_kprobe(old_tk);
  450. if (ret < 0)
  451. goto end;
  452. free_trace_kprobe(old_tk);
  453. }
  454. /* Register new event */
  455. ret = register_kprobe_event(tk);
  456. if (ret) {
  457. pr_warn("Failed to register probe event(%d)\n", ret);
  458. goto end;
  459. }
  460. /* Register k*probe */
  461. ret = __register_trace_kprobe(tk);
  462. if (ret < 0)
  463. unregister_kprobe_event(tk);
  464. else
  465. list_add_tail(&tk->list, &probe_list);
  466. end:
  467. mutex_unlock(&probe_lock);
  468. return ret;
  469. }
  470. /* Module notifier call back, checking event on the module */
  471. static int trace_kprobe_module_callback(struct notifier_block *nb,
  472. unsigned long val, void *data)
  473. {
  474. struct module *mod = data;
  475. struct trace_kprobe *tk;
  476. int ret;
  477. if (val != MODULE_STATE_COMING)
  478. return NOTIFY_DONE;
  479. /* Update probes on coming module */
  480. mutex_lock(&probe_lock);
  481. list_for_each_entry(tk, &probe_list, list) {
  482. if (trace_kprobe_within_module(tk, mod)) {
  483. /* Don't need to check busy - this should have gone. */
  484. __unregister_trace_kprobe(tk);
  485. ret = __register_trace_kprobe(tk);
  486. if (ret)
  487. pr_warn("Failed to re-register probe %s on %s: %d\n",
  488. trace_event_name(&tk->tp.call),
  489. mod->name, ret);
  490. }
  491. }
  492. mutex_unlock(&probe_lock);
  493. return NOTIFY_DONE;
  494. }
  495. static struct notifier_block trace_kprobe_module_nb = {
  496. .notifier_call = trace_kprobe_module_callback,
  497. .priority = 1 /* Invoked after kprobe module callback */
  498. };
  499. static int create_trace_kprobe(int argc, char **argv)
  500. {
  501. /*
  502. * Argument syntax:
  503. * - Add kprobe: p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
  504. * - Add kretprobe: r[:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
  505. * Fetch args:
  506. * $retval : fetch return value
  507. * $stack : fetch stack address
  508. * $stackN : fetch Nth of stack (N:0-)
  509. * $comm : fetch current task comm
  510. * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
  511. * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
  512. * %REG : fetch register REG
  513. * Dereferencing memory fetch:
  514. * +|-offs(ARG) : fetch memory at ARG +|- offs address.
  515. * Alias name of args:
  516. * NAME=FETCHARG : set NAME as alias of FETCHARG.
  517. * Type of args:
  518. * FETCHARG:TYPE : use TYPE instead of unsigned long.
  519. */
  520. struct trace_kprobe *tk;
  521. int i, ret = 0;
  522. bool is_return = false, is_delete = false;
  523. char *symbol = NULL, *event = NULL, *group = NULL;
  524. char *arg;
  525. unsigned long offset = 0;
  526. void *addr = NULL;
  527. char buf[MAX_EVENT_NAME_LEN];
  528. /* argc must be >= 1 */
  529. if (argv[0][0] == 'p')
  530. is_return = false;
  531. else if (argv[0][0] == 'r')
  532. is_return = true;
  533. else if (argv[0][0] == '-')
  534. is_delete = true;
  535. else {
  536. pr_info("Probe definition must be started with 'p', 'r' or"
  537. " '-'.\n");
  538. return -EINVAL;
  539. }
  540. if (argv[0][1] == ':') {
  541. event = &argv[0][2];
  542. if (strchr(event, '/')) {
  543. group = event;
  544. event = strchr(group, '/') + 1;
  545. event[-1] = '\0';
  546. if (strlen(group) == 0) {
  547. pr_info("Group name is not specified\n");
  548. return -EINVAL;
  549. }
  550. }
  551. if (strlen(event) == 0) {
  552. pr_info("Event name is not specified\n");
  553. return -EINVAL;
  554. }
  555. }
  556. if (!group)
  557. group = KPROBE_EVENT_SYSTEM;
  558. if (is_delete) {
  559. if (!event) {
  560. pr_info("Delete command needs an event name.\n");
  561. return -EINVAL;
  562. }
  563. mutex_lock(&probe_lock);
  564. tk = find_trace_kprobe(event, group);
  565. if (!tk) {
  566. mutex_unlock(&probe_lock);
  567. pr_info("Event %s/%s doesn't exist.\n", group, event);
  568. return -ENOENT;
  569. }
  570. /* delete an event */
  571. ret = unregister_trace_kprobe(tk);
  572. if (ret == 0)
  573. free_trace_kprobe(tk);
  574. mutex_unlock(&probe_lock);
  575. return ret;
  576. }
  577. if (argc < 2) {
  578. pr_info("Probe point is not specified.\n");
  579. return -EINVAL;
  580. }
  581. if (isdigit(argv[1][0])) {
  582. if (is_return) {
  583. pr_info("Return probe point must be a symbol.\n");
  584. return -EINVAL;
  585. }
  586. /* an address specified */
  587. ret = kstrtoul(&argv[1][0], 0, (unsigned long *)&addr);
  588. if (ret) {
  589. pr_info("Failed to parse address.\n");
  590. return ret;
  591. }
  592. } else {
  593. /* a symbol specified */
  594. symbol = argv[1];
  595. /* TODO: support .init module functions */
  596. ret = traceprobe_split_symbol_offset(symbol, &offset);
  597. if (ret) {
  598. pr_info("Failed to parse symbol.\n");
  599. return ret;
  600. }
  601. if (offset && is_return) {
  602. pr_info("Return probe must be used without offset.\n");
  603. return -EINVAL;
  604. }
  605. }
  606. argc -= 2; argv += 2;
  607. /* setup a probe */
  608. if (!event) {
  609. /* Make a new event name */
  610. if (symbol)
  611. snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
  612. is_return ? 'r' : 'p', symbol, offset);
  613. else
  614. snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
  615. is_return ? 'r' : 'p', addr);
  616. event = buf;
  617. }
  618. tk = alloc_trace_kprobe(group, event, addr, symbol, offset, argc,
  619. is_return);
  620. if (IS_ERR(tk)) {
  621. pr_info("Failed to allocate trace_probe.(%d)\n",
  622. (int)PTR_ERR(tk));
  623. return PTR_ERR(tk);
  624. }
  625. /* parse arguments */
  626. ret = 0;
  627. for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
  628. struct probe_arg *parg = &tk->tp.args[i];
  629. /* Increment count for freeing args in error case */
  630. tk->tp.nr_args++;
  631. /* Parse argument name */
  632. arg = strchr(argv[i], '=');
  633. if (arg) {
  634. *arg++ = '\0';
  635. parg->name = kstrdup(argv[i], GFP_KERNEL);
  636. } else {
  637. arg = argv[i];
  638. /* If argument name is omitted, set "argN" */
  639. snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
  640. parg->name = kstrdup(buf, GFP_KERNEL);
  641. }
  642. if (!parg->name) {
  643. pr_info("Failed to allocate argument[%d] name.\n", i);
  644. ret = -ENOMEM;
  645. goto error;
  646. }
  647. if (!is_good_name(parg->name)) {
  648. pr_info("Invalid argument[%d] name: %s\n",
  649. i, parg->name);
  650. ret = -EINVAL;
  651. goto error;
  652. }
  653. if (traceprobe_conflict_field_name(parg->name,
  654. tk->tp.args, i)) {
  655. pr_info("Argument[%d] name '%s' conflicts with "
  656. "another field.\n", i, argv[i]);
  657. ret = -EINVAL;
  658. goto error;
  659. }
  660. /* Parse fetch argument */
  661. ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
  662. is_return, true,
  663. kprobes_fetch_type_table);
  664. if (ret) {
  665. pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
  666. goto error;
  667. }
  668. }
  669. ret = register_trace_kprobe(tk);
  670. if (ret)
  671. goto error;
  672. return 0;
  673. error:
  674. free_trace_kprobe(tk);
  675. return ret;
  676. }
  677. static int release_all_trace_kprobes(void)
  678. {
  679. struct trace_kprobe *tk;
  680. int ret = 0;
  681. mutex_lock(&probe_lock);
  682. /* Ensure no probe is in use. */
  683. list_for_each_entry(tk, &probe_list, list)
  684. if (trace_probe_is_enabled(&tk->tp)) {
  685. ret = -EBUSY;
  686. goto end;
  687. }
  688. /* TODO: Use batch unregistration */
  689. while (!list_empty(&probe_list)) {
  690. tk = list_entry(probe_list.next, struct trace_kprobe, list);
  691. ret = unregister_trace_kprobe(tk);
  692. if (ret)
  693. goto end;
  694. free_trace_kprobe(tk);
  695. }
  696. end:
  697. mutex_unlock(&probe_lock);
  698. return ret;
  699. }
  700. /* Probes listing interfaces */
  701. static void *probes_seq_start(struct seq_file *m, loff_t *pos)
  702. {
  703. mutex_lock(&probe_lock);
  704. return seq_list_start(&probe_list, *pos);
  705. }
  706. static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
  707. {
  708. return seq_list_next(v, &probe_list, pos);
  709. }
  710. static void probes_seq_stop(struct seq_file *m, void *v)
  711. {
  712. mutex_unlock(&probe_lock);
  713. }
  714. static int probes_seq_show(struct seq_file *m, void *v)
  715. {
  716. struct trace_kprobe *tk = v;
  717. int i;
  718. seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
  719. seq_printf(m, ":%s/%s", tk->tp.call.class->system,
  720. trace_event_name(&tk->tp.call));
  721. if (!tk->symbol)
  722. seq_printf(m, " 0x%p", tk->rp.kp.addr);
  723. else if (tk->rp.kp.offset)
  724. seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
  725. tk->rp.kp.offset);
  726. else
  727. seq_printf(m, " %s", trace_kprobe_symbol(tk));
  728. for (i = 0; i < tk->tp.nr_args; i++)
  729. seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
  730. seq_putc(m, '\n');
  731. return 0;
  732. }
  733. static const struct seq_operations probes_seq_op = {
  734. .start = probes_seq_start,
  735. .next = probes_seq_next,
  736. .stop = probes_seq_stop,
  737. .show = probes_seq_show
  738. };
  739. static int probes_open(struct inode *inode, struct file *file)
  740. {
  741. int ret;
  742. if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
  743. ret = release_all_trace_kprobes();
  744. if (ret < 0)
  745. return ret;
  746. }
  747. return seq_open(file, &probes_seq_op);
  748. }
  749. static ssize_t probes_write(struct file *file, const char __user *buffer,
  750. size_t count, loff_t *ppos)
  751. {
  752. return traceprobe_probes_write(file, buffer, count, ppos,
  753. create_trace_kprobe);
  754. }
  755. static const struct file_operations kprobe_events_ops = {
  756. .owner = THIS_MODULE,
  757. .open = probes_open,
  758. .read = seq_read,
  759. .llseek = seq_lseek,
  760. .release = seq_release,
  761. .write = probes_write,
  762. };
  763. /* Probes profiling interfaces */
  764. static int probes_profile_seq_show(struct seq_file *m, void *v)
  765. {
  766. struct trace_kprobe *tk = v;
  767. seq_printf(m, " %-44s %15lu %15lu\n",
  768. trace_event_name(&tk->tp.call),
  769. trace_kprobe_nhit(tk),
  770. tk->rp.kp.nmissed);
  771. return 0;
  772. }
  773. static const struct seq_operations profile_seq_op = {
  774. .start = probes_seq_start,
  775. .next = probes_seq_next,
  776. .stop = probes_seq_stop,
  777. .show = probes_profile_seq_show
  778. };
  779. static int profile_open(struct inode *inode, struct file *file)
  780. {
  781. return seq_open(file, &profile_seq_op);
  782. }
  783. static const struct file_operations kprobe_profile_ops = {
  784. .owner = THIS_MODULE,
  785. .open = profile_open,
  786. .read = seq_read,
  787. .llseek = seq_lseek,
  788. .release = seq_release,
  789. };
  790. /* Kprobe handler */
  791. static nokprobe_inline void
  792. __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
  793. struct trace_event_file *trace_file)
  794. {
  795. struct kprobe_trace_entry_head *entry;
  796. struct ring_buffer_event *event;
  797. struct ring_buffer *buffer;
  798. int size, dsize, pc;
  799. unsigned long irq_flags;
  800. struct trace_event_call *call = &tk->tp.call;
  801. WARN_ON(call != trace_file->event_call);
  802. if (trace_trigger_soft_disabled(trace_file))
  803. return;
  804. local_save_flags(irq_flags);
  805. pc = preempt_count();
  806. dsize = __get_data_size(&tk->tp, regs);
  807. size = sizeof(*entry) + tk->tp.size + dsize;
  808. event = trace_event_buffer_lock_reserve(&buffer, trace_file,
  809. call->event.type,
  810. size, irq_flags, pc);
  811. if (!event)
  812. return;
  813. entry = ring_buffer_event_data(event);
  814. entry->ip = (unsigned long)tk->rp.kp.addr;
  815. store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
  816. event_trigger_unlock_commit_regs(trace_file, buffer, event,
  817. entry, irq_flags, pc, regs);
  818. }
  819. static void
  820. kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
  821. {
  822. struct event_file_link *link;
  823. list_for_each_entry_rcu(link, &tk->tp.files, list)
  824. __kprobe_trace_func(tk, regs, link->file);
  825. }
  826. NOKPROBE_SYMBOL(kprobe_trace_func);
  827. /* Kretprobe handler */
  828. static nokprobe_inline void
  829. __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
  830. struct pt_regs *regs,
  831. struct trace_event_file *trace_file)
  832. {
  833. struct kretprobe_trace_entry_head *entry;
  834. struct ring_buffer_event *event;
  835. struct ring_buffer *buffer;
  836. int size, pc, dsize;
  837. unsigned long irq_flags;
  838. struct trace_event_call *call = &tk->tp.call;
  839. WARN_ON(call != trace_file->event_call);
  840. if (trace_trigger_soft_disabled(trace_file))
  841. return;
  842. local_save_flags(irq_flags);
  843. pc = preempt_count();
  844. dsize = __get_data_size(&tk->tp, regs);
  845. size = sizeof(*entry) + tk->tp.size + dsize;
  846. event = trace_event_buffer_lock_reserve(&buffer, trace_file,
  847. call->event.type,
  848. size, irq_flags, pc);
  849. if (!event)
  850. return;
  851. entry = ring_buffer_event_data(event);
  852. entry->func = (unsigned long)tk->rp.kp.addr;
  853. entry->ret_ip = (unsigned long)ri->ret_addr;
  854. store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
  855. event_trigger_unlock_commit_regs(trace_file, buffer, event,
  856. entry, irq_flags, pc, regs);
  857. }
  858. static void
  859. kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
  860. struct pt_regs *regs)
  861. {
  862. struct event_file_link *link;
  863. list_for_each_entry_rcu(link, &tk->tp.files, list)
  864. __kretprobe_trace_func(tk, ri, regs, link->file);
  865. }
  866. NOKPROBE_SYMBOL(kretprobe_trace_func);
  867. /* Event entry printers */
  868. static enum print_line_t
  869. print_kprobe_event(struct trace_iterator *iter, int flags,
  870. struct trace_event *event)
  871. {
  872. struct kprobe_trace_entry_head *field;
  873. struct trace_seq *s = &iter->seq;
  874. struct trace_probe *tp;
  875. u8 *data;
  876. int i;
  877. field = (struct kprobe_trace_entry_head *)iter->ent;
  878. tp = container_of(event, struct trace_probe, call.event);
  879. trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
  880. if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
  881. goto out;
  882. trace_seq_putc(s, ')');
  883. data = (u8 *)&field[1];
  884. for (i = 0; i < tp->nr_args; i++)
  885. if (!tp->args[i].type->print(s, tp->args[i].name,
  886. data + tp->args[i].offset, field))
  887. goto out;
  888. trace_seq_putc(s, '\n');
  889. out:
  890. return trace_handle_return(s);
  891. }
  892. static enum print_line_t
  893. print_kretprobe_event(struct trace_iterator *iter, int flags,
  894. struct trace_event *event)
  895. {
  896. struct kretprobe_trace_entry_head *field;
  897. struct trace_seq *s = &iter->seq;
  898. struct trace_probe *tp;
  899. u8 *data;
  900. int i;
  901. field = (struct kretprobe_trace_entry_head *)iter->ent;
  902. tp = container_of(event, struct trace_probe, call.event);
  903. trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
  904. if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
  905. goto out;
  906. trace_seq_puts(s, " <- ");
  907. if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
  908. goto out;
  909. trace_seq_putc(s, ')');
  910. data = (u8 *)&field[1];
  911. for (i = 0; i < tp->nr_args; i++)
  912. if (!tp->args[i].type->print(s, tp->args[i].name,
  913. data + tp->args[i].offset, field))
  914. goto out;
  915. trace_seq_putc(s, '\n');
  916. out:
  917. return trace_handle_return(s);
  918. }
  919. static int kprobe_event_define_fields(struct trace_event_call *event_call)
  920. {
  921. int ret, i;
  922. struct kprobe_trace_entry_head field;
  923. struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
  924. DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
  925. /* Set argument names as fields */
  926. for (i = 0; i < tk->tp.nr_args; i++) {
  927. struct probe_arg *parg = &tk->tp.args[i];
  928. ret = trace_define_field(event_call, parg->type->fmttype,
  929. parg->name,
  930. sizeof(field) + parg->offset,
  931. parg->type->size,
  932. parg->type->is_signed,
  933. FILTER_OTHER);
  934. if (ret)
  935. return ret;
  936. }
  937. return 0;
  938. }
  939. static int kretprobe_event_define_fields(struct trace_event_call *event_call)
  940. {
  941. int ret, i;
  942. struct kretprobe_trace_entry_head field;
  943. struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
  944. DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
  945. DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
  946. /* Set argument names as fields */
  947. for (i = 0; i < tk->tp.nr_args; i++) {
  948. struct probe_arg *parg = &tk->tp.args[i];
  949. ret = trace_define_field(event_call, parg->type->fmttype,
  950. parg->name,
  951. sizeof(field) + parg->offset,
  952. parg->type->size,
  953. parg->type->is_signed,
  954. FILTER_OTHER);
  955. if (ret)
  956. return ret;
  957. }
  958. return 0;
  959. }
  960. #ifdef CONFIG_PERF_EVENTS
  961. /* Kprobe profile handler */
  962. static void
  963. kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
  964. {
  965. struct trace_event_call *call = &tk->tp.call;
  966. struct bpf_prog *prog = call->prog;
  967. struct kprobe_trace_entry_head *entry;
  968. struct hlist_head *head;
  969. int size, __size, dsize;
  970. int rctx;
  971. if (prog && !trace_call_bpf(prog, regs))
  972. return;
  973. head = this_cpu_ptr(call->perf_events);
  974. if (hlist_empty(head))
  975. return;
  976. dsize = __get_data_size(&tk->tp, regs);
  977. __size = sizeof(*entry) + tk->tp.size + dsize;
  978. size = ALIGN(__size + sizeof(u32), sizeof(u64));
  979. size -= sizeof(u32);
  980. entry = perf_trace_buf_alloc(size, NULL, &rctx);
  981. if (!entry)
  982. return;
  983. entry->ip = (unsigned long)tk->rp.kp.addr;
  984. memset(&entry[1], 0, dsize);
  985. store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
  986. perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
  987. head, NULL);
  988. }
  989. NOKPROBE_SYMBOL(kprobe_perf_func);
  990. /* Kretprobe profile handler */
  991. static void
  992. kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
  993. struct pt_regs *regs)
  994. {
  995. struct trace_event_call *call = &tk->tp.call;
  996. struct bpf_prog *prog = call->prog;
  997. struct kretprobe_trace_entry_head *entry;
  998. struct hlist_head *head;
  999. int size, __size, dsize;
  1000. int rctx;
  1001. if (prog && !trace_call_bpf(prog, regs))
  1002. return;
  1003. head = this_cpu_ptr(call->perf_events);
  1004. if (hlist_empty(head))
  1005. return;
  1006. dsize = __get_data_size(&tk->tp, regs);
  1007. __size = sizeof(*entry) + tk->tp.size + dsize;
  1008. size = ALIGN(__size + sizeof(u32), sizeof(u64));
  1009. size -= sizeof(u32);
  1010. entry = perf_trace_buf_alloc(size, NULL, &rctx);
  1011. if (!entry)
  1012. return;
  1013. entry->func = (unsigned long)tk->rp.kp.addr;
  1014. entry->ret_ip = (unsigned long)ri->ret_addr;
  1015. store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
  1016. perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
  1017. head, NULL);
  1018. }
  1019. NOKPROBE_SYMBOL(kretprobe_perf_func);
  1020. #endif /* CONFIG_PERF_EVENTS */
  1021. /*
  1022. * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
  1023. *
  1024. * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
  1025. * lockless, but we can't race with this __init function.
  1026. */
  1027. static int kprobe_register(struct trace_event_call *event,
  1028. enum trace_reg type, void *data)
  1029. {
  1030. struct trace_kprobe *tk = (struct trace_kprobe *)event->data;
  1031. struct trace_event_file *file = data;
  1032. switch (type) {
  1033. case TRACE_REG_REGISTER:
  1034. return enable_trace_kprobe(tk, file);
  1035. case TRACE_REG_UNREGISTER:
  1036. return disable_trace_kprobe(tk, file);
  1037. #ifdef CONFIG_PERF_EVENTS
  1038. case TRACE_REG_PERF_REGISTER:
  1039. return enable_trace_kprobe(tk, NULL);
  1040. case TRACE_REG_PERF_UNREGISTER:
  1041. return disable_trace_kprobe(tk, NULL);
  1042. case TRACE_REG_PERF_OPEN:
  1043. case TRACE_REG_PERF_CLOSE:
  1044. case TRACE_REG_PERF_ADD:
  1045. case TRACE_REG_PERF_DEL:
  1046. return 0;
  1047. #endif
  1048. }
  1049. return 0;
  1050. }
  1051. static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
  1052. {
  1053. struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
  1054. raw_cpu_inc(*tk->nhit);
  1055. if (tk->tp.flags & TP_FLAG_TRACE)
  1056. kprobe_trace_func(tk, regs);
  1057. #ifdef CONFIG_PERF_EVENTS
  1058. if (tk->tp.flags & TP_FLAG_PROFILE)
  1059. kprobe_perf_func(tk, regs);
  1060. #endif
  1061. return 0; /* We don't tweek kernel, so just return 0 */
  1062. }
  1063. NOKPROBE_SYMBOL(kprobe_dispatcher);
  1064. static int
  1065. kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
  1066. {
  1067. struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
  1068. raw_cpu_inc(*tk->nhit);
  1069. if (tk->tp.flags & TP_FLAG_TRACE)
  1070. kretprobe_trace_func(tk, ri, regs);
  1071. #ifdef CONFIG_PERF_EVENTS
  1072. if (tk->tp.flags & TP_FLAG_PROFILE)
  1073. kretprobe_perf_func(tk, ri, regs);
  1074. #endif
  1075. return 0; /* We don't tweek kernel, so just return 0 */
  1076. }
  1077. NOKPROBE_SYMBOL(kretprobe_dispatcher);
  1078. static struct trace_event_functions kretprobe_funcs = {
  1079. .trace = print_kretprobe_event
  1080. };
  1081. static struct trace_event_functions kprobe_funcs = {
  1082. .trace = print_kprobe_event
  1083. };
  1084. static int register_kprobe_event(struct trace_kprobe *tk)
  1085. {
  1086. struct trace_event_call *call = &tk->tp.call;
  1087. int ret;
  1088. /* Initialize trace_event_call */
  1089. INIT_LIST_HEAD(&call->class->fields);
  1090. if (trace_kprobe_is_return(tk)) {
  1091. call->event.funcs = &kretprobe_funcs;
  1092. call->class->define_fields = kretprobe_event_define_fields;
  1093. } else {
  1094. call->event.funcs = &kprobe_funcs;
  1095. call->class->define_fields = kprobe_event_define_fields;
  1096. }
  1097. if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)
  1098. return -ENOMEM;
  1099. ret = register_trace_event(&call->event);
  1100. if (!ret) {
  1101. kfree(call->print_fmt);
  1102. return -ENODEV;
  1103. }
  1104. call->flags = TRACE_EVENT_FL_KPROBE;
  1105. call->class->reg = kprobe_register;
  1106. call->data = tk;
  1107. ret = trace_add_event_call(call);
  1108. if (ret) {
  1109. pr_info("Failed to register kprobe event: %s\n",
  1110. trace_event_name(call));
  1111. kfree(call->print_fmt);
  1112. unregister_trace_event(&call->event);
  1113. }
  1114. return ret;
  1115. }
  1116. static int unregister_kprobe_event(struct trace_kprobe *tk)
  1117. {
  1118. int ret;
  1119. /* tp->event is unregistered in trace_remove_event_call() */
  1120. ret = trace_remove_event_call(&tk->tp.call);
  1121. if (!ret)
  1122. kfree(tk->tp.call.print_fmt);
  1123. return ret;
  1124. }
  1125. /* Make a tracefs interface for controlling probe points */
  1126. static __init int init_kprobe_trace(void)
  1127. {
  1128. struct dentry *d_tracer;
  1129. struct dentry *entry;
  1130. if (register_module_notifier(&trace_kprobe_module_nb))
  1131. return -EINVAL;
  1132. d_tracer = tracing_init_dentry();
  1133. if (IS_ERR(d_tracer))
  1134. return 0;
  1135. entry = tracefs_create_file("kprobe_events", 0644, d_tracer,
  1136. NULL, &kprobe_events_ops);
  1137. /* Event list interface */
  1138. if (!entry)
  1139. pr_warn("Could not create tracefs 'kprobe_events' entry\n");
  1140. /* Profile interface */
  1141. entry = tracefs_create_file("kprobe_profile", 0444, d_tracer,
  1142. NULL, &kprobe_profile_ops);
  1143. if (!entry)
  1144. pr_warn("Could not create tracefs 'kprobe_profile' entry\n");
  1145. return 0;
  1146. }
  1147. fs_initcall(init_kprobe_trace);
  1148. #ifdef CONFIG_FTRACE_STARTUP_TEST
  1149. /*
  1150. * The "__used" keeps gcc from removing the function symbol
  1151. * from the kallsyms table. 'noinline' makes sure that there
  1152. * isn't an inlined version used by the test method below
  1153. */
  1154. static __used __init noinline int
  1155. kprobe_trace_selftest_target(int a1, int a2, int a3, int a4, int a5, int a6)
  1156. {
  1157. return a1 + a2 + a3 + a4 + a5 + a6;
  1158. }
  1159. static struct __init trace_event_file *
  1160. find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
  1161. {
  1162. struct trace_event_file *file;
  1163. list_for_each_entry(file, &tr->events, list)
  1164. if (file->event_call == &tk->tp.call)
  1165. return file;
  1166. return NULL;
  1167. }
  1168. /*
  1169. * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
  1170. * stage, we can do this lockless.
  1171. */
  1172. static __init int kprobe_trace_self_tests_init(void)
  1173. {
  1174. int ret, warn = 0;
  1175. int (*target)(int, int, int, int, int, int);
  1176. struct trace_kprobe *tk;
  1177. struct trace_event_file *file;
  1178. if (tracing_is_disabled())
  1179. return -ENODEV;
  1180. target = kprobe_trace_selftest_target;
  1181. pr_info("Testing kprobe tracing: ");
  1182. ret = traceprobe_command("p:testprobe kprobe_trace_selftest_target "
  1183. "$stack $stack0 +0($stack)",
  1184. create_trace_kprobe);
  1185. if (WARN_ON_ONCE(ret)) {
  1186. pr_warn("error on probing function entry.\n");
  1187. warn++;
  1188. } else {
  1189. /* Enable trace point */
  1190. tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
  1191. if (WARN_ON_ONCE(tk == NULL)) {
  1192. pr_warn("error on getting new probe.\n");
  1193. warn++;
  1194. } else {
  1195. file = find_trace_probe_file(tk, top_trace_array());
  1196. if (WARN_ON_ONCE(file == NULL)) {
  1197. pr_warn("error on getting probe file.\n");
  1198. warn++;
  1199. } else
  1200. enable_trace_kprobe(tk, file);
  1201. }
  1202. }
  1203. ret = traceprobe_command("r:testprobe2 kprobe_trace_selftest_target "
  1204. "$retval", create_trace_kprobe);
  1205. if (WARN_ON_ONCE(ret)) {
  1206. pr_warn("error on probing function return.\n");
  1207. warn++;
  1208. } else {
  1209. /* Enable trace point */
  1210. tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
  1211. if (WARN_ON_ONCE(tk == NULL)) {
  1212. pr_warn("error on getting 2nd new probe.\n");
  1213. warn++;
  1214. } else {
  1215. file = find_trace_probe_file(tk, top_trace_array());
  1216. if (WARN_ON_ONCE(file == NULL)) {
  1217. pr_warn("error on getting probe file.\n");
  1218. warn++;
  1219. } else
  1220. enable_trace_kprobe(tk, file);
  1221. }
  1222. }
  1223. if (warn)
  1224. goto end;
  1225. ret = target(1, 2, 3, 4, 5, 6);
  1226. /*
  1227. * Not expecting an error here, the check is only to prevent the
  1228. * optimizer from removing the call to target() as otherwise there
  1229. * are no side-effects and the call is never performed.
  1230. */
  1231. if (ret != 21)
  1232. warn++;
  1233. /* Disable trace points before removing it */
  1234. tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
  1235. if (WARN_ON_ONCE(tk == NULL)) {
  1236. pr_warn("error on getting test probe.\n");
  1237. warn++;
  1238. } else {
  1239. if (trace_kprobe_nhit(tk) != 1) {
  1240. pr_warn("incorrect number of testprobe hits\n");
  1241. warn++;
  1242. }
  1243. file = find_trace_probe_file(tk, top_trace_array());
  1244. if (WARN_ON_ONCE(file == NULL)) {
  1245. pr_warn("error on getting probe file.\n");
  1246. warn++;
  1247. } else
  1248. disable_trace_kprobe(tk, file);
  1249. }
  1250. tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
  1251. if (WARN_ON_ONCE(tk == NULL)) {
  1252. pr_warn("error on getting 2nd test probe.\n");
  1253. warn++;
  1254. } else {
  1255. if (trace_kprobe_nhit(tk) != 1) {
  1256. pr_warn("incorrect number of testprobe2 hits\n");
  1257. warn++;
  1258. }
  1259. file = find_trace_probe_file(tk, top_trace_array());
  1260. if (WARN_ON_ONCE(file == NULL)) {
  1261. pr_warn("error on getting probe file.\n");
  1262. warn++;
  1263. } else
  1264. disable_trace_kprobe(tk, file);
  1265. }
  1266. ret = traceprobe_command("-:testprobe", create_trace_kprobe);
  1267. if (WARN_ON_ONCE(ret)) {
  1268. pr_warn("error on deleting a probe.\n");
  1269. warn++;
  1270. }
  1271. ret = traceprobe_command("-:testprobe2", create_trace_kprobe);
  1272. if (WARN_ON_ONCE(ret)) {
  1273. pr_warn("error on deleting a probe.\n");
  1274. warn++;
  1275. }
  1276. end:
  1277. release_all_trace_kprobes();
  1278. if (warn)
  1279. pr_cont("NG: Some tests are failed. Please check them.\n");
  1280. else
  1281. pr_cont("OK\n");
  1282. return 0;
  1283. }
  1284. late_initcall(kprobe_trace_self_tests_init);
  1285. #endif