ftrace.h 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733
  1. /*
  2. * Stage 1 of the trace events.
  3. *
  4. * Override the macros in <trace/trace_events.h> to include the following:
  5. *
  6. * struct ftrace_raw_<call> {
  7. * struct trace_entry ent;
  8. * <type> <item>;
  9. * <type2> <item2>[<len>];
  10. * [...]
  11. * };
  12. *
  13. * The <type> <item> is created by the __field(type, item) macro or
  14. * the __array(type2, item2, len) macro.
  15. * We simply do "type item;", and that will create the fields
  16. * in the structure.
  17. */
  18. #include <linux/ftrace_event.h>
  19. /*
  20. * DECLARE_EVENT_CLASS can be used to add a generic function
  21. * handlers for events. That is, if all events have the same
  22. * parameters and just have distinct trace points.
  23. * Each tracepoint can be defined with DEFINE_EVENT and that
  24. * will map the DECLARE_EVENT_CLASS to the tracepoint.
  25. *
  26. * TRACE_EVENT is a one to one mapping between tracepoint and template.
  27. */
  28. #undef TRACE_EVENT
  29. #define TRACE_EVENT(name, proto, args, tstruct, assign, print) \
  30. DECLARE_EVENT_CLASS(name, \
  31. PARAMS(proto), \
  32. PARAMS(args), \
  33. PARAMS(tstruct), \
  34. PARAMS(assign), \
  35. PARAMS(print)); \
  36. DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args));
  37. #undef __field
  38. #define __field(type, item) type item;
  39. #undef __field_ext
  40. #define __field_ext(type, item, filter_type) type item;
  41. #undef __array
  42. #define __array(type, item, len) type item[len];
  43. #undef __dynamic_array
  44. #define __dynamic_array(type, item, len) u32 __data_loc_##item;
  45. #undef __string
  46. #define __string(item, src) __dynamic_array(char, item, -1)
  47. #undef TP_STRUCT__entry
  48. #define TP_STRUCT__entry(args...) args
  49. #undef DECLARE_EVENT_CLASS
  50. #define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print) \
  51. struct ftrace_raw_##name { \
  52. struct trace_entry ent; \
  53. tstruct \
  54. char __data[0]; \
  55. }; \
  56. \
  57. static struct ftrace_event_class event_class_##name;
  58. #undef DEFINE_EVENT
  59. #define DEFINE_EVENT(template, name, proto, args) \
  60. static struct ftrace_event_call __used \
  61. __attribute__((__aligned__(4))) event_##name
  62. #undef DEFINE_EVENT_FN
  63. #define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
  64. DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
  65. #undef DEFINE_EVENT_PRINT
  66. #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
  67. DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
  68. /* Callbacks are meaningless to ftrace. */
  69. #undef TRACE_EVENT_FN
  70. #define TRACE_EVENT_FN(name, proto, args, tstruct, \
  71. assign, print, reg, unreg) \
  72. TRACE_EVENT(name, PARAMS(proto), PARAMS(args), \
  73. PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \
  74. #undef TRACE_EVENT_FLAGS
  75. #define TRACE_EVENT_FLAGS(name, value) \
  76. __TRACE_EVENT_FLAGS(name, value)
  77. #undef TRACE_EVENT_PERF_PERM
  78. #define TRACE_EVENT_PERF_PERM(name, expr...) \
  79. __TRACE_EVENT_PERF_PERM(name, expr)
  80. #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
  81. /*
  82. * Stage 2 of the trace events.
  83. *
  84. * Include the following:
  85. *
  86. * struct ftrace_data_offsets_<call> {
  87. * u32 <item1>;
  88. * u32 <item2>;
  89. * [...]
  90. * };
  91. *
  92. * The __dynamic_array() macro will create each u32 <item>, this is
  93. * to keep the offset of each array from the beginning of the event.
  94. * The size of an array is also encoded, in the higher 16 bits of <item>.
  95. */
  96. #undef __field
  97. #define __field(type, item)
  98. #undef __field_ext
  99. #define __field_ext(type, item, filter_type)
  100. #undef __array
  101. #define __array(type, item, len)
  102. #undef __dynamic_array
  103. #define __dynamic_array(type, item, len) u32 item;
  104. #undef __string
  105. #define __string(item, src) __dynamic_array(char, item, -1)
  106. #undef DECLARE_EVENT_CLASS
  107. #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
  108. struct ftrace_data_offsets_##call { \
  109. tstruct; \
  110. };
  111. #undef DEFINE_EVENT
  112. #define DEFINE_EVENT(template, name, proto, args)
  113. #undef DEFINE_EVENT_PRINT
  114. #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
  115. DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
  116. #undef TRACE_EVENT_FLAGS
  117. #define TRACE_EVENT_FLAGS(event, flag)
  118. #undef TRACE_EVENT_PERF_PERM
  119. #define TRACE_EVENT_PERF_PERM(event, expr...)
  120. #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
  121. /*
  122. * Stage 3 of the trace events.
  123. *
  124. * Override the macros in <trace/trace_events.h> to include the following:
  125. *
  126. * enum print_line_t
  127. * ftrace_raw_output_<call>(struct trace_iterator *iter, int flags)
  128. * {
  129. * struct trace_seq *s = &iter->seq;
  130. * struct ftrace_raw_<call> *field; <-- defined in stage 1
  131. * struct trace_entry *entry;
  132. * struct trace_seq *p = &iter->tmp_seq;
  133. * int ret;
  134. *
  135. * entry = iter->ent;
  136. *
  137. * if (entry->type != event_<call>->event.type) {
  138. * WARN_ON_ONCE(1);
  139. * return TRACE_TYPE_UNHANDLED;
  140. * }
  141. *
  142. * field = (typeof(field))entry;
  143. *
  144. * trace_seq_init(p);
  145. * ret = trace_seq_printf(s, "%s: ", <call>);
  146. * if (ret)
  147. * ret = trace_seq_printf(s, <TP_printk> "\n");
  148. * if (!ret)
  149. * return TRACE_TYPE_PARTIAL_LINE;
  150. *
  151. * return TRACE_TYPE_HANDLED;
  152. * }
  153. *
  154. * This is the method used to print the raw event to the trace
  155. * output format. Note, this is not needed if the data is read
  156. * in binary.
  157. */
  158. #undef __entry
  159. #define __entry field
  160. #undef TP_printk
  161. #define TP_printk(fmt, args...) fmt "\n", args
  162. #undef __get_dynamic_array
  163. #define __get_dynamic_array(field) \
  164. ((void *)__entry + (__entry->__data_loc_##field & 0xffff))
  165. #undef __get_str
  166. #define __get_str(field) (char *)__get_dynamic_array(field)
  167. #undef __print_flags
  168. #define __print_flags(flag, delim, flag_array...) \
  169. ({ \
  170. static const struct trace_print_flags __flags[] = \
  171. { flag_array, { -1, NULL }}; \
  172. ftrace_print_flags_seq(p, delim, flag, __flags); \
  173. })
  174. #undef __print_symbolic
  175. #define __print_symbolic(value, symbol_array...) \
  176. ({ \
  177. static const struct trace_print_flags symbols[] = \
  178. { symbol_array, { -1, NULL }}; \
  179. ftrace_print_symbols_seq(p, value, symbols); \
  180. })
  181. #undef __print_symbolic_u64
  182. #if BITS_PER_LONG == 32
  183. #define __print_symbolic_u64(value, symbol_array...) \
  184. ({ \
  185. static const struct trace_print_flags_u64 symbols[] = \
  186. { symbol_array, { -1, NULL } }; \
  187. ftrace_print_symbols_seq_u64(p, value, symbols); \
  188. })
  189. #else
  190. #define __print_symbolic_u64(value, symbol_array...) \
  191. __print_symbolic(value, symbol_array)
  192. #endif
  193. #undef __print_hex
  194. #define __print_hex(buf, buf_len) ftrace_print_hex_seq(p, buf, buf_len)
  195. #undef DECLARE_EVENT_CLASS
  196. #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
  197. static notrace enum print_line_t \
  198. ftrace_raw_output_##call(struct trace_iterator *iter, int flags, \
  199. struct trace_event *trace_event) \
  200. { \
  201. struct trace_seq *s = &iter->seq; \
  202. struct trace_seq __maybe_unused *p = &iter->tmp_seq; \
  203. struct ftrace_raw_##call *field; \
  204. int ret; \
  205. \
  206. field = (typeof(field))iter->ent; \
  207. \
  208. ret = ftrace_raw_output_prep(iter, trace_event); \
  209. if (ret) \
  210. return ret; \
  211. \
  212. ret = trace_seq_printf(s, print); \
  213. if (!ret) \
  214. return TRACE_TYPE_PARTIAL_LINE; \
  215. \
  216. return TRACE_TYPE_HANDLED; \
  217. } \
  218. static struct trace_event_functions ftrace_event_type_funcs_##call = { \
  219. .trace = ftrace_raw_output_##call, \
  220. };
  221. #undef DEFINE_EVENT_PRINT
  222. #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \
  223. static notrace enum print_line_t \
  224. ftrace_raw_output_##call(struct trace_iterator *iter, int flags, \
  225. struct trace_event *event) \
  226. { \
  227. struct trace_seq *s = &iter->seq; \
  228. struct ftrace_raw_##template *field; \
  229. struct trace_entry *entry; \
  230. struct trace_seq *p = &iter->tmp_seq; \
  231. int ret; \
  232. \
  233. entry = iter->ent; \
  234. \
  235. if (entry->type != event_##call.event.type) { \
  236. WARN_ON_ONCE(1); \
  237. return TRACE_TYPE_UNHANDLED; \
  238. } \
  239. \
  240. field = (typeof(field))entry; \
  241. \
  242. trace_seq_init(p); \
  243. ret = trace_seq_printf(s, "%s: ", #call); \
  244. if (ret) \
  245. ret = trace_seq_printf(s, print); \
  246. if (!ret) \
  247. return TRACE_TYPE_PARTIAL_LINE; \
  248. \
  249. return TRACE_TYPE_HANDLED; \
  250. } \
  251. static struct trace_event_functions ftrace_event_type_funcs_##call = { \
  252. .trace = ftrace_raw_output_##call, \
  253. };
  254. #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
  255. #undef __field_ext
  256. #define __field_ext(type, item, filter_type) \
  257. ret = trace_define_field(event_call, #type, #item, \
  258. offsetof(typeof(field), item), \
  259. sizeof(field.item), \
  260. is_signed_type(type), filter_type); \
  261. if (ret) \
  262. return ret;
  263. #undef __field
  264. #define __field(type, item) __field_ext(type, item, FILTER_OTHER)
  265. #undef __array
  266. #define __array(type, item, len) \
  267. do { \
  268. mutex_lock(&event_storage_mutex); \
  269. BUILD_BUG_ON(len > MAX_FILTER_STR_VAL); \
  270. snprintf(event_storage, sizeof(event_storage), \
  271. "%s[%d]", #type, len); \
  272. ret = trace_define_field(event_call, event_storage, #item, \
  273. offsetof(typeof(field), item), \
  274. sizeof(field.item), \
  275. is_signed_type(type), FILTER_OTHER); \
  276. mutex_unlock(&event_storage_mutex); \
  277. if (ret) \
  278. return ret; \
  279. } while (0);
  280. #undef __dynamic_array
  281. #define __dynamic_array(type, item, len) \
  282. ret = trace_define_field(event_call, "__data_loc " #type "[]", #item, \
  283. offsetof(typeof(field), __data_loc_##item), \
  284. sizeof(field.__data_loc_##item), \
  285. is_signed_type(type), FILTER_OTHER);
  286. #undef __string
  287. #define __string(item, src) __dynamic_array(char, item, -1)
  288. #undef DECLARE_EVENT_CLASS
  289. #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \
  290. static int notrace __init \
  291. ftrace_define_fields_##call(struct ftrace_event_call *event_call) \
  292. { \
  293. struct ftrace_raw_##call field; \
  294. int ret; \
  295. \
  296. tstruct; \
  297. \
  298. return ret; \
  299. }
  300. #undef DEFINE_EVENT
  301. #define DEFINE_EVENT(template, name, proto, args)
  302. #undef DEFINE_EVENT_PRINT
  303. #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
  304. DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
  305. #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
  306. /*
  307. * remember the offset of each array from the beginning of the event.
  308. */
  309. #undef __entry
  310. #define __entry entry
  311. #undef __field
  312. #define __field(type, item)
  313. #undef __field_ext
  314. #define __field_ext(type, item, filter_type)
  315. #undef __array
  316. #define __array(type, item, len)
  317. #undef __dynamic_array
  318. #define __dynamic_array(type, item, len) \
  319. __data_offsets->item = __data_size + \
  320. offsetof(typeof(*entry), __data); \
  321. __data_offsets->item |= (len * sizeof(type)) << 16; \
  322. __data_size += (len) * sizeof(type);
  323. #undef __string
  324. #define __string(item, src) __dynamic_array(char, item, \
  325. strlen((src) ? (const char *)(src) : "(null)") + 1)
  326. #undef DECLARE_EVENT_CLASS
  327. #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
  328. static inline notrace int ftrace_get_offsets_##call( \
  329. struct ftrace_data_offsets_##call *__data_offsets, proto) \
  330. { \
  331. int __data_size = 0; \
  332. struct ftrace_raw_##call __maybe_unused *entry; \
  333. \
  334. tstruct; \
  335. \
  336. return __data_size; \
  337. }
  338. #undef DEFINE_EVENT
  339. #define DEFINE_EVENT(template, name, proto, args)
  340. #undef DEFINE_EVENT_PRINT
  341. #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
  342. DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
  343. #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
  344. /*
  345. * Stage 4 of the trace events.
  346. *
  347. * Override the macros in <trace/trace_events.h> to include the following:
  348. *
  349. * For those macros defined with TRACE_EVENT:
  350. *
  351. * static struct ftrace_event_call event_<call>;
  352. *
  353. * static void ftrace_raw_event_<call>(void *__data, proto)
  354. * {
  355. * struct ftrace_event_file *ftrace_file = __data;
  356. * struct ftrace_event_call *event_call = ftrace_file->event_call;
  357. * struct ftrace_data_offsets_<call> __maybe_unused __data_offsets;
  358. * unsigned long eflags = ftrace_file->flags;
  359. * enum event_trigger_type __tt = ETT_NONE;
  360. * struct ring_buffer_event *event;
  361. * struct ftrace_raw_<call> *entry; <-- defined in stage 1
  362. * struct ring_buffer *buffer;
  363. * unsigned long irq_flags;
  364. * int __data_size;
  365. * int pc;
  366. *
  367. * if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) {
  368. * if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE)
  369. * event_triggers_call(ftrace_file, NULL);
  370. * if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED)
  371. * return;
  372. * }
  373. *
  374. * local_save_flags(irq_flags);
  375. * pc = preempt_count();
  376. *
  377. * __data_size = ftrace_get_offsets_<call>(&__data_offsets, args);
  378. *
  379. * event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
  380. * event_<call>->event.type,
  381. * sizeof(*entry) + __data_size,
  382. * irq_flags, pc);
  383. * if (!event)
  384. * return;
  385. * entry = ring_buffer_event_data(event);
  386. *
  387. * { <assign>; } <-- Here we assign the entries by the __field and
  388. * __array macros.
  389. *
  390. * if (eflags & FTRACE_EVENT_FL_TRIGGER_COND)
  391. * __tt = event_triggers_call(ftrace_file, entry);
  392. *
  393. * if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT,
  394. * &ftrace_file->flags))
  395. * ring_buffer_discard_commit(buffer, event);
  396. * else if (!filter_check_discard(ftrace_file, entry, buffer, event))
  397. * trace_buffer_unlock_commit(buffer, event, irq_flags, pc);
  398. *
  399. * if (__tt)
  400. * event_triggers_post_call(ftrace_file, __tt);
  401. * }
  402. *
  403. * static struct trace_event ftrace_event_type_<call> = {
  404. * .trace = ftrace_raw_output_<call>, <-- stage 2
  405. * };
  406. *
  407. * static const char print_fmt_<call>[] = <TP_printk>;
  408. *
  409. * static struct ftrace_event_class __used event_class_<template> = {
  410. * .system = "<system>",
  411. * .define_fields = ftrace_define_fields_<call>,
  412. * .fields = LIST_HEAD_INIT(event_class_##call.fields),
  413. * .raw_init = trace_event_raw_init,
  414. * .probe = ftrace_raw_event_##call,
  415. * .reg = ftrace_event_reg,
  416. * };
  417. *
  418. * static struct ftrace_event_call event_<call> = {
  419. * .name = "<call>",
  420. * .class = event_class_<template>,
  421. * .event = &ftrace_event_type_<call>,
  422. * .print_fmt = print_fmt_<call>,
  423. * };
  424. * // its only safe to use pointers when doing linker tricks to
  425. * // create an array.
  426. * static struct ftrace_event_call __used
  427. * __attribute__((section("_ftrace_events"))) *__event_<call> = &event_<call>;
  428. *
  429. */
  430. #ifdef CONFIG_PERF_EVENTS
  431. #define _TRACE_PERF_PROTO(call, proto) \
  432. static notrace void \
  433. perf_trace_##call(void *__data, proto);
  434. #define _TRACE_PERF_INIT(call) \
  435. .perf_probe = perf_trace_##call,
  436. #else
  437. #define _TRACE_PERF_PROTO(call, proto)
  438. #define _TRACE_PERF_INIT(call)
  439. #endif /* CONFIG_PERF_EVENTS */
  440. #undef __entry
  441. #define __entry entry
  442. #undef __field
  443. #define __field(type, item)
  444. #undef __array
  445. #define __array(type, item, len)
  446. #undef __dynamic_array
  447. #define __dynamic_array(type, item, len) \
  448. __entry->__data_loc_##item = __data_offsets.item;
  449. #undef __string
  450. #define __string(item, src) __dynamic_array(char, item, -1) \
  451. #undef __assign_str
  452. #define __assign_str(dst, src) \
  453. strcpy(__get_str(dst), (src) ? (const char *)(src) : "(null)");
  454. #undef TP_fast_assign
  455. #define TP_fast_assign(args...) args
  456. #undef __perf_addr
  457. #define __perf_addr(a) (a)
  458. #undef __perf_count
  459. #define __perf_count(c) (c)
  460. #undef __perf_task
  461. #define __perf_task(t) (t)
  462. #undef DECLARE_EVENT_CLASS
  463. #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
  464. \
  465. static notrace void \
  466. ftrace_raw_event_##call(void *__data, proto) \
  467. { \
  468. struct ftrace_event_file *ftrace_file = __data; \
  469. struct ftrace_event_call *event_call = ftrace_file->event_call; \
  470. struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
  471. struct ring_buffer_event *event; \
  472. struct ftrace_raw_##call *entry; \
  473. struct ring_buffer *buffer; \
  474. unsigned long irq_flags; \
  475. int __data_size; \
  476. int pc; \
  477. \
  478. if (ftrace_trigger_soft_disabled(ftrace_file)) \
  479. return; \
  480. \
  481. local_save_flags(irq_flags); \
  482. pc = preempt_count(); \
  483. \
  484. __data_size = ftrace_get_offsets_##call(&__data_offsets, args); \
  485. \
  486. event = trace_event_buffer_lock_reserve(&buffer, ftrace_file, \
  487. event_call->event.type, \
  488. sizeof(*entry) + __data_size, \
  489. irq_flags, pc); \
  490. if (!event) \
  491. return; \
  492. entry = ring_buffer_event_data(event); \
  493. \
  494. tstruct \
  495. \
  496. { assign; } \
  497. \
  498. event_trigger_unlock_commit(ftrace_file, buffer, event, entry, \
  499. irq_flags, pc); \
  500. }
  501. /*
  502. * The ftrace_test_probe is compiled out, it is only here as a build time check
  503. * to make sure that if the tracepoint handling changes, the ftrace probe will
  504. * fail to compile unless it too is updated.
  505. */
  506. #undef DEFINE_EVENT
  507. #define DEFINE_EVENT(template, call, proto, args) \
  508. static inline void ftrace_test_probe_##call(void) \
  509. { \
  510. check_trace_callback_type_##call(ftrace_raw_event_##template); \
  511. }
  512. #undef DEFINE_EVENT_PRINT
  513. #define DEFINE_EVENT_PRINT(template, name, proto, args, print)
  514. #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
  515. #undef __entry
  516. #define __entry REC
  517. #undef __print_flags
  518. #undef __print_symbolic
  519. #undef __print_hex
  520. #undef __get_dynamic_array
  521. #undef __get_str
  522. #undef TP_printk
  523. #define TP_printk(fmt, args...) "\"" fmt "\", " __stringify(args)
  524. #undef DECLARE_EVENT_CLASS
  525. #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
  526. _TRACE_PERF_PROTO(call, PARAMS(proto)); \
  527. static const char print_fmt_##call[] = print; \
  528. static struct ftrace_event_class __used __refdata event_class_##call = { \
  529. .system = __stringify(TRACE_SYSTEM), \
  530. .define_fields = ftrace_define_fields_##call, \
  531. .fields = LIST_HEAD_INIT(event_class_##call.fields),\
  532. .raw_init = trace_event_raw_init, \
  533. .probe = ftrace_raw_event_##call, \
  534. .reg = ftrace_event_reg, \
  535. _TRACE_PERF_INIT(call) \
  536. };
  537. #undef DEFINE_EVENT
  538. #define DEFINE_EVENT(template, call, proto, args) \
  539. \
  540. static struct ftrace_event_call __used event_##call = { \
  541. .name = #call, \
  542. .class = &event_class_##template, \
  543. .event.funcs = &ftrace_event_type_funcs_##template, \
  544. .print_fmt = print_fmt_##template, \
  545. }; \
  546. static struct ftrace_event_call __used \
  547. __attribute__((section("_ftrace_events"))) *__event_##call = &event_##call
  548. #undef DEFINE_EVENT_PRINT
  549. #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \
  550. \
  551. static const char print_fmt_##call[] = print; \
  552. \
  553. static struct ftrace_event_call __used event_##call = { \
  554. .name = #call, \
  555. .class = &event_class_##template, \
  556. .event.funcs = &ftrace_event_type_funcs_##call, \
  557. .print_fmt = print_fmt_##call, \
  558. }; \
  559. static struct ftrace_event_call __used \
  560. __attribute__((section("_ftrace_events"))) *__event_##call = &event_##call
  561. #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
  562. #ifdef CONFIG_PERF_EVENTS
  563. #undef __entry
  564. #define __entry entry
  565. #undef __get_dynamic_array
  566. #define __get_dynamic_array(field) \
  567. ((void *)__entry + (__entry->__data_loc_##field & 0xffff))
  568. #undef __get_str
  569. #define __get_str(field) (char *)__get_dynamic_array(field)
  570. #undef __perf_addr
  571. #define __perf_addr(a) (__addr = (a))
  572. #undef __perf_count
  573. #define __perf_count(c) (__count = (c))
  574. #undef __perf_task
  575. #define __perf_task(t) (__task = (t))
  576. #undef DECLARE_EVENT_CLASS
  577. #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
  578. static notrace void \
  579. perf_trace_##call(void *__data, proto) \
  580. { \
  581. struct ftrace_event_call *event_call = __data; \
  582. struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
  583. struct ftrace_raw_##call *entry; \
  584. struct pt_regs __regs; \
  585. u64 __addr = 0, __count = 1; \
  586. struct task_struct *__task = NULL; \
  587. struct hlist_head *head; \
  588. int __entry_size; \
  589. int __data_size; \
  590. int rctx; \
  591. \
  592. __data_size = ftrace_get_offsets_##call(&__data_offsets, args); \
  593. \
  594. head = this_cpu_ptr(event_call->perf_events); \
  595. if (__builtin_constant_p(!__task) && !__task && \
  596. hlist_empty(head)) \
  597. return; \
  598. \
  599. __entry_size = ALIGN(__data_size + sizeof(*entry) + sizeof(u32),\
  600. sizeof(u64)); \
  601. __entry_size -= sizeof(u32); \
  602. \
  603. perf_fetch_caller_regs(&__regs); \
  604. entry = perf_trace_buf_prepare(__entry_size, \
  605. event_call->event.type, &__regs, &rctx); \
  606. if (!entry) \
  607. return; \
  608. \
  609. tstruct \
  610. \
  611. { assign; } \
  612. \
  613. perf_trace_buf_submit(entry, __entry_size, rctx, __addr, \
  614. __count, &__regs, head, __task); \
  615. }
  616. /*
  617. * This part is compiled out, it is only here as a build time check
  618. * to make sure that if the tracepoint handling changes, the
  619. * perf probe will fail to compile unless it too is updated.
  620. */
  621. #undef DEFINE_EVENT
  622. #define DEFINE_EVENT(template, call, proto, args) \
  623. static inline void perf_test_probe_##call(void) \
  624. { \
  625. check_trace_callback_type_##call(perf_trace_##template); \
  626. }
  627. #undef DEFINE_EVENT_PRINT
  628. #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
  629. DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
  630. #include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
  631. #endif /* CONFIG_PERF_EVENTS */