builtin-diff.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362
  1. /*
  2. * builtin-diff.c
  3. *
  4. * Builtin diff command: Analyze two perf.data input files, look up and read
  5. * DSOs and symbol information, sort them and produce a diff.
  6. */
  7. #include "builtin.h"
  8. #include "util/debug.h"
  9. #include "util/event.h"
  10. #include "util/hist.h"
  11. #include "util/evsel.h"
  12. #include "util/evlist.h"
  13. #include "util/session.h"
  14. #include "util/tool.h"
  15. #include "util/sort.h"
  16. #include "util/symbol.h"
  17. #include "util/util.h"
  18. #include "util/data.h"
  19. #include "util/config.h"
  20. #include <errno.h>
  21. #include <inttypes.h>
  22. #include <stdlib.h>
  23. #include <math.h>
  24. /* Diff command specific HPP columns. */
  25. enum {
  26. PERF_HPP_DIFF__BASELINE,
  27. PERF_HPP_DIFF__PERIOD,
  28. PERF_HPP_DIFF__PERIOD_BASELINE,
  29. PERF_HPP_DIFF__DELTA,
  30. PERF_HPP_DIFF__RATIO,
  31. PERF_HPP_DIFF__WEIGHTED_DIFF,
  32. PERF_HPP_DIFF__FORMULA,
  33. PERF_HPP_DIFF__DELTA_ABS,
  34. PERF_HPP_DIFF__MAX_INDEX
  35. };
  36. struct diff_hpp_fmt {
  37. struct perf_hpp_fmt fmt;
  38. int idx;
  39. char *header;
  40. int header_width;
  41. };
  42. struct data__file {
  43. struct perf_session *session;
  44. struct perf_data data;
  45. int idx;
  46. struct hists *hists;
  47. struct diff_hpp_fmt fmt[PERF_HPP_DIFF__MAX_INDEX];
  48. };
  49. static struct data__file *data__files;
  50. static int data__files_cnt;
  51. #define data__for_each_file_start(i, d, s) \
  52. for (i = s, d = &data__files[s]; \
  53. i < data__files_cnt; \
  54. i++, d = &data__files[i])
  55. #define data__for_each_file(i, d) data__for_each_file_start(i, d, 0)
  56. #define data__for_each_file_new(i, d) data__for_each_file_start(i, d, 1)
  57. static bool force;
  58. static bool show_period;
  59. static bool show_formula;
  60. static bool show_baseline_only;
  61. static unsigned int sort_compute = 1;
  62. static s64 compute_wdiff_w1;
  63. static s64 compute_wdiff_w2;
  64. enum {
  65. COMPUTE_DELTA,
  66. COMPUTE_RATIO,
  67. COMPUTE_WEIGHTED_DIFF,
  68. COMPUTE_DELTA_ABS,
  69. COMPUTE_MAX,
  70. };
  71. const char *compute_names[COMPUTE_MAX] = {
  72. [COMPUTE_DELTA] = "delta",
  73. [COMPUTE_DELTA_ABS] = "delta-abs",
  74. [COMPUTE_RATIO] = "ratio",
  75. [COMPUTE_WEIGHTED_DIFF] = "wdiff",
  76. };
  77. static int compute = COMPUTE_DELTA_ABS;
  78. static int compute_2_hpp[COMPUTE_MAX] = {
  79. [COMPUTE_DELTA] = PERF_HPP_DIFF__DELTA,
  80. [COMPUTE_DELTA_ABS] = PERF_HPP_DIFF__DELTA_ABS,
  81. [COMPUTE_RATIO] = PERF_HPP_DIFF__RATIO,
  82. [COMPUTE_WEIGHTED_DIFF] = PERF_HPP_DIFF__WEIGHTED_DIFF,
  83. };
  84. #define MAX_COL_WIDTH 70
  85. static struct header_column {
  86. const char *name;
  87. int width;
  88. } columns[PERF_HPP_DIFF__MAX_INDEX] = {
  89. [PERF_HPP_DIFF__BASELINE] = {
  90. .name = "Baseline",
  91. },
  92. [PERF_HPP_DIFF__PERIOD] = {
  93. .name = "Period",
  94. .width = 14,
  95. },
  96. [PERF_HPP_DIFF__PERIOD_BASELINE] = {
  97. .name = "Base period",
  98. .width = 14,
  99. },
  100. [PERF_HPP_DIFF__DELTA] = {
  101. .name = "Delta",
  102. .width = 7,
  103. },
  104. [PERF_HPP_DIFF__DELTA_ABS] = {
  105. .name = "Delta Abs",
  106. .width = 7,
  107. },
  108. [PERF_HPP_DIFF__RATIO] = {
  109. .name = "Ratio",
  110. .width = 14,
  111. },
  112. [PERF_HPP_DIFF__WEIGHTED_DIFF] = {
  113. .name = "Weighted diff",
  114. .width = 14,
  115. },
  116. [PERF_HPP_DIFF__FORMULA] = {
  117. .name = "Formula",
  118. .width = MAX_COL_WIDTH,
  119. }
  120. };
  121. static int setup_compute_opt_wdiff(char *opt)
  122. {
  123. char *w1_str = opt;
  124. char *w2_str;
  125. int ret = -EINVAL;
  126. if (!opt)
  127. goto out;
  128. w2_str = strchr(opt, ',');
  129. if (!w2_str)
  130. goto out;
  131. *w2_str++ = 0x0;
  132. if (!*w2_str)
  133. goto out;
  134. compute_wdiff_w1 = strtol(w1_str, NULL, 10);
  135. compute_wdiff_w2 = strtol(w2_str, NULL, 10);
  136. if (!compute_wdiff_w1 || !compute_wdiff_w2)
  137. goto out;
  138. pr_debug("compute wdiff w1(%" PRId64 ") w2(%" PRId64 ")\n",
  139. compute_wdiff_w1, compute_wdiff_w2);
  140. ret = 0;
  141. out:
  142. if (ret)
  143. pr_err("Failed: wrong weight data, use 'wdiff:w1,w2'\n");
  144. return ret;
  145. }
  146. static int setup_compute_opt(char *opt)
  147. {
  148. if (compute == COMPUTE_WEIGHTED_DIFF)
  149. return setup_compute_opt_wdiff(opt);
  150. if (opt) {
  151. pr_err("Failed: extra option specified '%s'", opt);
  152. return -EINVAL;
  153. }
  154. return 0;
  155. }
  156. static int setup_compute(const struct option *opt, const char *str,
  157. int unset __maybe_unused)
  158. {
  159. int *cp = (int *) opt->value;
  160. char *cstr = (char *) str;
  161. char buf[50];
  162. unsigned i;
  163. char *option;
  164. if (!str) {
  165. *cp = COMPUTE_DELTA;
  166. return 0;
  167. }
  168. option = strchr(str, ':');
  169. if (option) {
  170. unsigned len = option++ - str;
  171. /*
  172. * The str data are not writeable, so we need
  173. * to use another buffer.
  174. */
  175. /* No option value is longer. */
  176. if (len >= sizeof(buf))
  177. return -EINVAL;
  178. strncpy(buf, str, len);
  179. buf[len] = 0x0;
  180. cstr = buf;
  181. }
  182. for (i = 0; i < COMPUTE_MAX; i++)
  183. if (!strcmp(cstr, compute_names[i])) {
  184. *cp = i;
  185. return setup_compute_opt(option);
  186. }
  187. pr_err("Failed: '%s' is not computation method "
  188. "(use 'delta','ratio' or 'wdiff')\n", str);
  189. return -EINVAL;
  190. }
  191. static double period_percent(struct hist_entry *he, u64 period)
  192. {
  193. u64 total = hists__total_period(he->hists);
  194. return (period * 100.0) / total;
  195. }
  196. static double compute_delta(struct hist_entry *he, struct hist_entry *pair)
  197. {
  198. double old_percent = period_percent(he, he->stat.period);
  199. double new_percent = period_percent(pair, pair->stat.period);
  200. pair->diff.period_ratio_delta = new_percent - old_percent;
  201. pair->diff.computed = true;
  202. return pair->diff.period_ratio_delta;
  203. }
  204. static double compute_ratio(struct hist_entry *he, struct hist_entry *pair)
  205. {
  206. double old_period = he->stat.period ?: 1;
  207. double new_period = pair->stat.period;
  208. pair->diff.computed = true;
  209. pair->diff.period_ratio = new_period / old_period;
  210. return pair->diff.period_ratio;
  211. }
  212. static s64 compute_wdiff(struct hist_entry *he, struct hist_entry *pair)
  213. {
  214. u64 old_period = he->stat.period;
  215. u64 new_period = pair->stat.period;
  216. pair->diff.computed = true;
  217. pair->diff.wdiff = new_period * compute_wdiff_w2 -
  218. old_period * compute_wdiff_w1;
  219. return pair->diff.wdiff;
  220. }
  221. static int formula_delta(struct hist_entry *he, struct hist_entry *pair,
  222. char *buf, size_t size)
  223. {
  224. u64 he_total = he->hists->stats.total_period;
  225. u64 pair_total = pair->hists->stats.total_period;
  226. if (symbol_conf.filter_relative) {
  227. he_total = he->hists->stats.total_non_filtered_period;
  228. pair_total = pair->hists->stats.total_non_filtered_period;
  229. }
  230. return scnprintf(buf, size,
  231. "(%" PRIu64 " * 100 / %" PRIu64 ") - "
  232. "(%" PRIu64 " * 100 / %" PRIu64 ")",
  233. pair->stat.period, pair_total,
  234. he->stat.period, he_total);
  235. }
  236. static int formula_ratio(struct hist_entry *he, struct hist_entry *pair,
  237. char *buf, size_t size)
  238. {
  239. double old_period = he->stat.period;
  240. double new_period = pair->stat.period;
  241. return scnprintf(buf, size, "%.0F / %.0F", new_period, old_period);
  242. }
  243. static int formula_wdiff(struct hist_entry *he, struct hist_entry *pair,
  244. char *buf, size_t size)
  245. {
  246. u64 old_period = he->stat.period;
  247. u64 new_period = pair->stat.period;
  248. return scnprintf(buf, size,
  249. "(%" PRIu64 " * " "%" PRId64 ") - (%" PRIu64 " * " "%" PRId64 ")",
  250. new_period, compute_wdiff_w2, old_period, compute_wdiff_w1);
  251. }
  252. static int formula_fprintf(struct hist_entry *he, struct hist_entry *pair,
  253. char *buf, size_t size)
  254. {
  255. switch (compute) {
  256. case COMPUTE_DELTA:
  257. case COMPUTE_DELTA_ABS:
  258. return formula_delta(he, pair, buf, size);
  259. case COMPUTE_RATIO:
  260. return formula_ratio(he, pair, buf, size);
  261. case COMPUTE_WEIGHTED_DIFF:
  262. return formula_wdiff(he, pair, buf, size);
  263. default:
  264. BUG_ON(1);
  265. }
  266. return -1;
  267. }
  268. static int diff__process_sample_event(struct perf_tool *tool __maybe_unused,
  269. union perf_event *event,
  270. struct perf_sample *sample,
  271. struct perf_evsel *evsel,
  272. struct machine *machine)
  273. {
  274. struct addr_location al;
  275. struct hists *hists = evsel__hists(evsel);
  276. int ret = -1;
  277. if (machine__resolve(machine, &al, sample) < 0) {
  278. pr_warning("problem processing %d event, skipping it.\n",
  279. event->header.type);
  280. return -1;
  281. }
  282. if (!hists__add_entry(hists, &al, NULL, NULL, NULL, sample, true)) {
  283. pr_warning("problem incrementing symbol period, skipping event\n");
  284. goto out_put;
  285. }
  286. /*
  287. * The total_period is updated here before going to the output
  288. * tree since normally only the baseline hists will call
  289. * hists__output_resort() and precompute needs the total
  290. * period in order to sort entries by percentage delta.
  291. */
  292. hists->stats.total_period += sample->period;
  293. if (!al.filtered)
  294. hists->stats.total_non_filtered_period += sample->period;
  295. ret = 0;
  296. out_put:
  297. addr_location__put(&al);
  298. return ret;
  299. }
  300. static struct perf_tool tool = {
  301. .sample = diff__process_sample_event,
  302. .mmap = perf_event__process_mmap,
  303. .mmap2 = perf_event__process_mmap2,
  304. .comm = perf_event__process_comm,
  305. .exit = perf_event__process_exit,
  306. .fork = perf_event__process_fork,
  307. .lost = perf_event__process_lost,
  308. .namespaces = perf_event__process_namespaces,
  309. .ordered_events = true,
  310. .ordering_requires_timestamps = true,
  311. };
  312. static struct perf_evsel *evsel_match(struct perf_evsel *evsel,
  313. struct perf_evlist *evlist)
  314. {
  315. struct perf_evsel *e;
  316. evlist__for_each_entry(evlist, e) {
  317. if (perf_evsel__match2(evsel, e))
  318. return e;
  319. }
  320. return NULL;
  321. }
  322. static void perf_evlist__collapse_resort(struct perf_evlist *evlist)
  323. {
  324. struct perf_evsel *evsel;
  325. evlist__for_each_entry(evlist, evsel) {
  326. struct hists *hists = evsel__hists(evsel);
  327. hists__collapse_resort(hists, NULL);
  328. }
  329. }
  330. static struct data__file *fmt_to_data_file(struct perf_hpp_fmt *fmt)
  331. {
  332. struct diff_hpp_fmt *dfmt = container_of(fmt, struct diff_hpp_fmt, fmt);
  333. void *ptr = dfmt - dfmt->idx;
  334. struct data__file *d = container_of(ptr, struct data__file, fmt);
  335. return d;
  336. }
  337. static struct hist_entry*
  338. get_pair_data(struct hist_entry *he, struct data__file *d)
  339. {
  340. if (hist_entry__has_pairs(he)) {
  341. struct hist_entry *pair;
  342. list_for_each_entry(pair, &he->pairs.head, pairs.node)
  343. if (pair->hists == d->hists)
  344. return pair;
  345. }
  346. return NULL;
  347. }
  348. static struct hist_entry*
  349. get_pair_fmt(struct hist_entry *he, struct diff_hpp_fmt *dfmt)
  350. {
  351. struct data__file *d = fmt_to_data_file(&dfmt->fmt);
  352. return get_pair_data(he, d);
  353. }
  354. static void hists__baseline_only(struct hists *hists)
  355. {
  356. struct rb_root *root;
  357. struct rb_node *next;
  358. if (hists__has(hists, need_collapse))
  359. root = &hists->entries_collapsed;
  360. else
  361. root = hists->entries_in;
  362. next = rb_first(root);
  363. while (next != NULL) {
  364. struct hist_entry *he = rb_entry(next, struct hist_entry, rb_node_in);
  365. next = rb_next(&he->rb_node_in);
  366. if (!hist_entry__next_pair(he)) {
  367. rb_erase(&he->rb_node_in, root);
  368. hist_entry__delete(he);
  369. }
  370. }
  371. }
  372. static void hists__precompute(struct hists *hists)
  373. {
  374. struct rb_root *root;
  375. struct rb_node *next;
  376. if (hists__has(hists, need_collapse))
  377. root = &hists->entries_collapsed;
  378. else
  379. root = hists->entries_in;
  380. next = rb_first(root);
  381. while (next != NULL) {
  382. struct hist_entry *he, *pair;
  383. struct data__file *d;
  384. int i;
  385. he = rb_entry(next, struct hist_entry, rb_node_in);
  386. next = rb_next(&he->rb_node_in);
  387. data__for_each_file_new(i, d) {
  388. pair = get_pair_data(he, d);
  389. if (!pair)
  390. continue;
  391. switch (compute) {
  392. case COMPUTE_DELTA:
  393. case COMPUTE_DELTA_ABS:
  394. compute_delta(he, pair);
  395. break;
  396. case COMPUTE_RATIO:
  397. compute_ratio(he, pair);
  398. break;
  399. case COMPUTE_WEIGHTED_DIFF:
  400. compute_wdiff(he, pair);
  401. break;
  402. default:
  403. BUG_ON(1);
  404. }
  405. }
  406. }
  407. }
  408. static int64_t cmp_doubles(double l, double r)
  409. {
  410. if (l > r)
  411. return -1;
  412. else if (l < r)
  413. return 1;
  414. else
  415. return 0;
  416. }
  417. static int64_t
  418. __hist_entry__cmp_compute(struct hist_entry *left, struct hist_entry *right,
  419. int c)
  420. {
  421. switch (c) {
  422. case COMPUTE_DELTA:
  423. {
  424. double l = left->diff.period_ratio_delta;
  425. double r = right->diff.period_ratio_delta;
  426. return cmp_doubles(l, r);
  427. }
  428. case COMPUTE_DELTA_ABS:
  429. {
  430. double l = fabs(left->diff.period_ratio_delta);
  431. double r = fabs(right->diff.period_ratio_delta);
  432. return cmp_doubles(l, r);
  433. }
  434. case COMPUTE_RATIO:
  435. {
  436. double l = left->diff.period_ratio;
  437. double r = right->diff.period_ratio;
  438. return cmp_doubles(l, r);
  439. }
  440. case COMPUTE_WEIGHTED_DIFF:
  441. {
  442. s64 l = left->diff.wdiff;
  443. s64 r = right->diff.wdiff;
  444. return r - l;
  445. }
  446. default:
  447. BUG_ON(1);
  448. }
  449. return 0;
  450. }
  451. static int64_t
  452. hist_entry__cmp_compute(struct hist_entry *left, struct hist_entry *right,
  453. int c, int sort_idx)
  454. {
  455. bool pairs_left = hist_entry__has_pairs(left);
  456. bool pairs_right = hist_entry__has_pairs(right);
  457. struct hist_entry *p_right, *p_left;
  458. if (!pairs_left && !pairs_right)
  459. return 0;
  460. if (!pairs_left || !pairs_right)
  461. return pairs_left ? -1 : 1;
  462. p_left = get_pair_data(left, &data__files[sort_idx]);
  463. p_right = get_pair_data(right, &data__files[sort_idx]);
  464. if (!p_left && !p_right)
  465. return 0;
  466. if (!p_left || !p_right)
  467. return p_left ? -1 : 1;
  468. /*
  469. * We have 2 entries of same kind, let's
  470. * make the data comparison.
  471. */
  472. return __hist_entry__cmp_compute(p_left, p_right, c);
  473. }
  474. static int64_t
  475. hist_entry__cmp_compute_idx(struct hist_entry *left, struct hist_entry *right,
  476. int c, int sort_idx)
  477. {
  478. struct hist_entry *p_right, *p_left;
  479. p_left = get_pair_data(left, &data__files[sort_idx]);
  480. p_right = get_pair_data(right, &data__files[sort_idx]);
  481. if (!p_left && !p_right)
  482. return 0;
  483. if (!p_left || !p_right)
  484. return p_left ? -1 : 1;
  485. if (c != COMPUTE_DELTA && c != COMPUTE_DELTA_ABS) {
  486. /*
  487. * The delta can be computed without the baseline, but
  488. * others are not. Put those entries which have no
  489. * values below.
  490. */
  491. if (left->dummy && right->dummy)
  492. return 0;
  493. if (left->dummy || right->dummy)
  494. return left->dummy ? 1 : -1;
  495. }
  496. return __hist_entry__cmp_compute(p_left, p_right, c);
  497. }
  498. static int64_t
  499. hist_entry__cmp_nop(struct perf_hpp_fmt *fmt __maybe_unused,
  500. struct hist_entry *left __maybe_unused,
  501. struct hist_entry *right __maybe_unused)
  502. {
  503. return 0;
  504. }
  505. static int64_t
  506. hist_entry__cmp_baseline(struct perf_hpp_fmt *fmt __maybe_unused,
  507. struct hist_entry *left, struct hist_entry *right)
  508. {
  509. if (left->stat.period == right->stat.period)
  510. return 0;
  511. return left->stat.period > right->stat.period ? 1 : -1;
  512. }
  513. static int64_t
  514. hist_entry__cmp_delta(struct perf_hpp_fmt *fmt,
  515. struct hist_entry *left, struct hist_entry *right)
  516. {
  517. struct data__file *d = fmt_to_data_file(fmt);
  518. return hist_entry__cmp_compute(right, left, COMPUTE_DELTA, d->idx);
  519. }
  520. static int64_t
  521. hist_entry__cmp_delta_abs(struct perf_hpp_fmt *fmt,
  522. struct hist_entry *left, struct hist_entry *right)
  523. {
  524. struct data__file *d = fmt_to_data_file(fmt);
  525. return hist_entry__cmp_compute(right, left, COMPUTE_DELTA_ABS, d->idx);
  526. }
  527. static int64_t
  528. hist_entry__cmp_ratio(struct perf_hpp_fmt *fmt,
  529. struct hist_entry *left, struct hist_entry *right)
  530. {
  531. struct data__file *d = fmt_to_data_file(fmt);
  532. return hist_entry__cmp_compute(right, left, COMPUTE_RATIO, d->idx);
  533. }
  534. static int64_t
  535. hist_entry__cmp_wdiff(struct perf_hpp_fmt *fmt,
  536. struct hist_entry *left, struct hist_entry *right)
  537. {
  538. struct data__file *d = fmt_to_data_file(fmt);
  539. return hist_entry__cmp_compute(right, left, COMPUTE_WEIGHTED_DIFF, d->idx);
  540. }
  541. static int64_t
  542. hist_entry__cmp_delta_idx(struct perf_hpp_fmt *fmt __maybe_unused,
  543. struct hist_entry *left, struct hist_entry *right)
  544. {
  545. return hist_entry__cmp_compute_idx(right, left, COMPUTE_DELTA,
  546. sort_compute);
  547. }
  548. static int64_t
  549. hist_entry__cmp_delta_abs_idx(struct perf_hpp_fmt *fmt __maybe_unused,
  550. struct hist_entry *left, struct hist_entry *right)
  551. {
  552. return hist_entry__cmp_compute_idx(right, left, COMPUTE_DELTA_ABS,
  553. sort_compute);
  554. }
  555. static int64_t
  556. hist_entry__cmp_ratio_idx(struct perf_hpp_fmt *fmt __maybe_unused,
  557. struct hist_entry *left, struct hist_entry *right)
  558. {
  559. return hist_entry__cmp_compute_idx(right, left, COMPUTE_RATIO,
  560. sort_compute);
  561. }
  562. static int64_t
  563. hist_entry__cmp_wdiff_idx(struct perf_hpp_fmt *fmt __maybe_unused,
  564. struct hist_entry *left, struct hist_entry *right)
  565. {
  566. return hist_entry__cmp_compute_idx(right, left, COMPUTE_WEIGHTED_DIFF,
  567. sort_compute);
  568. }
  569. static void hists__process(struct hists *hists)
  570. {
  571. if (show_baseline_only)
  572. hists__baseline_only(hists);
  573. hists__precompute(hists);
  574. hists__output_resort(hists, NULL);
  575. hists__fprintf(hists, !quiet, 0, 0, 0, stdout,
  576. symbol_conf.use_callchain);
  577. }
  578. static void data__fprintf(void)
  579. {
  580. struct data__file *d;
  581. int i;
  582. fprintf(stdout, "# Data files:\n");
  583. data__for_each_file(i, d)
  584. fprintf(stdout, "# [%d] %s %s\n",
  585. d->idx, d->data.file.path,
  586. !d->idx ? "(Baseline)" : "");
  587. fprintf(stdout, "#\n");
  588. }
  589. static void data_process(void)
  590. {
  591. struct perf_evlist *evlist_base = data__files[0].session->evlist;
  592. struct perf_evsel *evsel_base;
  593. bool first = true;
  594. evlist__for_each_entry(evlist_base, evsel_base) {
  595. struct hists *hists_base = evsel__hists(evsel_base);
  596. struct data__file *d;
  597. int i;
  598. data__for_each_file_new(i, d) {
  599. struct perf_evlist *evlist = d->session->evlist;
  600. struct perf_evsel *evsel;
  601. struct hists *hists;
  602. evsel = evsel_match(evsel_base, evlist);
  603. if (!evsel)
  604. continue;
  605. hists = evsel__hists(evsel);
  606. d->hists = hists;
  607. hists__match(hists_base, hists);
  608. if (!show_baseline_only)
  609. hists__link(hists_base, hists);
  610. }
  611. if (!quiet) {
  612. fprintf(stdout, "%s# Event '%s'\n#\n", first ? "" : "\n",
  613. perf_evsel__name(evsel_base));
  614. }
  615. first = false;
  616. if (verbose > 0 || ((data__files_cnt > 2) && !quiet))
  617. data__fprintf();
  618. /* Don't sort callchain for perf diff */
  619. perf_evsel__reset_sample_bit(evsel_base, CALLCHAIN);
  620. hists__process(hists_base);
  621. }
  622. }
  623. static void data__free(struct data__file *d)
  624. {
  625. int col;
  626. for (col = 0; col < PERF_HPP_DIFF__MAX_INDEX; col++) {
  627. struct diff_hpp_fmt *fmt = &d->fmt[col];
  628. zfree(&fmt->header);
  629. }
  630. }
  631. static int __cmd_diff(void)
  632. {
  633. struct data__file *d;
  634. int ret = -EINVAL, i;
  635. data__for_each_file(i, d) {
  636. d->session = perf_session__new(&d->data, false, &tool);
  637. if (!d->session) {
  638. pr_err("Failed to open %s\n", d->data.file.path);
  639. ret = -1;
  640. goto out_delete;
  641. }
  642. ret = perf_session__process_events(d->session);
  643. if (ret) {
  644. pr_err("Failed to process %s\n", d->data.file.path);
  645. goto out_delete;
  646. }
  647. perf_evlist__collapse_resort(d->session->evlist);
  648. }
  649. data_process();
  650. out_delete:
  651. data__for_each_file(i, d) {
  652. perf_session__delete(d->session);
  653. data__free(d);
  654. }
  655. free(data__files);
  656. return ret;
  657. }
  658. static const char * const diff_usage[] = {
  659. "perf diff [<options>] [old_file] [new_file]",
  660. NULL,
  661. };
  662. static const struct option options[] = {
  663. OPT_INCR('v', "verbose", &verbose,
  664. "be more verbose (show symbol address, etc)"),
  665. OPT_BOOLEAN('q', "quiet", &quiet, "Do not show any message"),
  666. OPT_BOOLEAN('b', "baseline-only", &show_baseline_only,
  667. "Show only items with match in baseline"),
  668. OPT_CALLBACK('c', "compute", &compute,
  669. "delta,delta-abs,ratio,wdiff:w1,w2 (default delta-abs)",
  670. "Entries differential computation selection",
  671. setup_compute),
  672. OPT_BOOLEAN('p', "period", &show_period,
  673. "Show period values."),
  674. OPT_BOOLEAN('F', "formula", &show_formula,
  675. "Show formula."),
  676. OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
  677. "dump raw trace in ASCII"),
  678. OPT_BOOLEAN('f', "force", &force, "don't complain, do it"),
  679. OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name,
  680. "file", "kallsyms pathname"),
  681. OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules,
  682. "load module symbols - WARNING: use only with -k and LIVE kernel"),
  683. OPT_STRING('d', "dsos", &symbol_conf.dso_list_str, "dso[,dso...]",
  684. "only consider symbols in these dsos"),
  685. OPT_STRING('C', "comms", &symbol_conf.comm_list_str, "comm[,comm...]",
  686. "only consider symbols in these comms"),
  687. OPT_STRING('S', "symbols", &symbol_conf.sym_list_str, "symbol[,symbol...]",
  688. "only consider these symbols"),
  689. OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
  690. "sort by key(s): pid, comm, dso, symbol, parent, cpu, srcline, ..."
  691. " Please refer the man page for the complete list."),
  692. OPT_STRING_NOEMPTY('t', "field-separator", &symbol_conf.field_sep, "separator",
  693. "separator for columns, no spaces will be added between "
  694. "columns '.' is reserved."),
  695. OPT_CALLBACK(0, "symfs", NULL, "directory",
  696. "Look for files with symbols relative to this directory",
  697. symbol__config_symfs),
  698. OPT_UINTEGER('o', "order", &sort_compute, "Specify compute sorting."),
  699. OPT_CALLBACK(0, "percentage", NULL, "relative|absolute",
  700. "How to display percentage of filtered entries", parse_filter_percentage),
  701. OPT_END()
  702. };
  703. static double baseline_percent(struct hist_entry *he)
  704. {
  705. u64 total = hists__total_period(he->hists);
  706. return 100.0 * he->stat.period / total;
  707. }
  708. static int hpp__color_baseline(struct perf_hpp_fmt *fmt,
  709. struct perf_hpp *hpp, struct hist_entry *he)
  710. {
  711. struct diff_hpp_fmt *dfmt =
  712. container_of(fmt, struct diff_hpp_fmt, fmt);
  713. double percent = baseline_percent(he);
  714. char pfmt[20] = " ";
  715. if (!he->dummy) {
  716. scnprintf(pfmt, 20, "%%%d.2f%%%%", dfmt->header_width - 1);
  717. return percent_color_snprintf(hpp->buf, hpp->size,
  718. pfmt, percent);
  719. } else
  720. return scnprintf(hpp->buf, hpp->size, "%*s",
  721. dfmt->header_width, pfmt);
  722. }
  723. static int hpp__entry_baseline(struct hist_entry *he, char *buf, size_t size)
  724. {
  725. double percent = baseline_percent(he);
  726. const char *fmt = symbol_conf.field_sep ? "%.2f" : "%6.2f%%";
  727. int ret = 0;
  728. if (!he->dummy)
  729. ret = scnprintf(buf, size, fmt, percent);
  730. return ret;
  731. }
  732. static int __hpp__color_compare(struct perf_hpp_fmt *fmt,
  733. struct perf_hpp *hpp, struct hist_entry *he,
  734. int comparison_method)
  735. {
  736. struct diff_hpp_fmt *dfmt =
  737. container_of(fmt, struct diff_hpp_fmt, fmt);
  738. struct hist_entry *pair = get_pair_fmt(he, dfmt);
  739. double diff;
  740. s64 wdiff;
  741. char pfmt[20] = " ";
  742. if (!pair)
  743. goto no_print;
  744. switch (comparison_method) {
  745. case COMPUTE_DELTA:
  746. if (pair->diff.computed)
  747. diff = pair->diff.period_ratio_delta;
  748. else
  749. diff = compute_delta(he, pair);
  750. scnprintf(pfmt, 20, "%%%+d.2f%%%%", dfmt->header_width - 1);
  751. return percent_color_snprintf(hpp->buf, hpp->size,
  752. pfmt, diff);
  753. case COMPUTE_RATIO:
  754. if (he->dummy)
  755. goto dummy_print;
  756. if (pair->diff.computed)
  757. diff = pair->diff.period_ratio;
  758. else
  759. diff = compute_ratio(he, pair);
  760. scnprintf(pfmt, 20, "%%%d.6f", dfmt->header_width);
  761. return value_color_snprintf(hpp->buf, hpp->size,
  762. pfmt, diff);
  763. case COMPUTE_WEIGHTED_DIFF:
  764. if (he->dummy)
  765. goto dummy_print;
  766. if (pair->diff.computed)
  767. wdiff = pair->diff.wdiff;
  768. else
  769. wdiff = compute_wdiff(he, pair);
  770. scnprintf(pfmt, 20, "%%14ld", dfmt->header_width);
  771. return color_snprintf(hpp->buf, hpp->size,
  772. get_percent_color(wdiff),
  773. pfmt, wdiff);
  774. default:
  775. BUG_ON(1);
  776. }
  777. dummy_print:
  778. return scnprintf(hpp->buf, hpp->size, "%*s",
  779. dfmt->header_width, "N/A");
  780. no_print:
  781. return scnprintf(hpp->buf, hpp->size, "%*s",
  782. dfmt->header_width, pfmt);
  783. }
  784. static int hpp__color_delta(struct perf_hpp_fmt *fmt,
  785. struct perf_hpp *hpp, struct hist_entry *he)
  786. {
  787. return __hpp__color_compare(fmt, hpp, he, COMPUTE_DELTA);
  788. }
  789. static int hpp__color_ratio(struct perf_hpp_fmt *fmt,
  790. struct perf_hpp *hpp, struct hist_entry *he)
  791. {
  792. return __hpp__color_compare(fmt, hpp, he, COMPUTE_RATIO);
  793. }
  794. static int hpp__color_wdiff(struct perf_hpp_fmt *fmt,
  795. struct perf_hpp *hpp, struct hist_entry *he)
  796. {
  797. return __hpp__color_compare(fmt, hpp, he, COMPUTE_WEIGHTED_DIFF);
  798. }
  799. static void
  800. hpp__entry_unpair(struct hist_entry *he, int idx, char *buf, size_t size)
  801. {
  802. switch (idx) {
  803. case PERF_HPP_DIFF__PERIOD_BASELINE:
  804. scnprintf(buf, size, "%" PRIu64, he->stat.period);
  805. break;
  806. default:
  807. break;
  808. }
  809. }
  810. static void
  811. hpp__entry_pair(struct hist_entry *he, struct hist_entry *pair,
  812. int idx, char *buf, size_t size)
  813. {
  814. double diff;
  815. double ratio;
  816. s64 wdiff;
  817. switch (idx) {
  818. case PERF_HPP_DIFF__DELTA:
  819. case PERF_HPP_DIFF__DELTA_ABS:
  820. if (pair->diff.computed)
  821. diff = pair->diff.period_ratio_delta;
  822. else
  823. diff = compute_delta(he, pair);
  824. scnprintf(buf, size, "%+4.2F%%", diff);
  825. break;
  826. case PERF_HPP_DIFF__RATIO:
  827. /* No point for ratio number if we are dummy.. */
  828. if (he->dummy) {
  829. scnprintf(buf, size, "N/A");
  830. break;
  831. }
  832. if (pair->diff.computed)
  833. ratio = pair->diff.period_ratio;
  834. else
  835. ratio = compute_ratio(he, pair);
  836. if (ratio > 0.0)
  837. scnprintf(buf, size, "%14.6F", ratio);
  838. break;
  839. case PERF_HPP_DIFF__WEIGHTED_DIFF:
  840. /* No point for wdiff number if we are dummy.. */
  841. if (he->dummy) {
  842. scnprintf(buf, size, "N/A");
  843. break;
  844. }
  845. if (pair->diff.computed)
  846. wdiff = pair->diff.wdiff;
  847. else
  848. wdiff = compute_wdiff(he, pair);
  849. if (wdiff != 0)
  850. scnprintf(buf, size, "%14ld", wdiff);
  851. break;
  852. case PERF_HPP_DIFF__FORMULA:
  853. formula_fprintf(he, pair, buf, size);
  854. break;
  855. case PERF_HPP_DIFF__PERIOD:
  856. scnprintf(buf, size, "%" PRIu64, pair->stat.period);
  857. break;
  858. default:
  859. BUG_ON(1);
  860. };
  861. }
  862. static void
  863. __hpp__entry_global(struct hist_entry *he, struct diff_hpp_fmt *dfmt,
  864. char *buf, size_t size)
  865. {
  866. struct hist_entry *pair = get_pair_fmt(he, dfmt);
  867. int idx = dfmt->idx;
  868. /* baseline is special */
  869. if (idx == PERF_HPP_DIFF__BASELINE)
  870. hpp__entry_baseline(he, buf, size);
  871. else {
  872. if (pair)
  873. hpp__entry_pair(he, pair, idx, buf, size);
  874. else
  875. hpp__entry_unpair(he, idx, buf, size);
  876. }
  877. }
  878. static int hpp__entry_global(struct perf_hpp_fmt *_fmt, struct perf_hpp *hpp,
  879. struct hist_entry *he)
  880. {
  881. struct diff_hpp_fmt *dfmt =
  882. container_of(_fmt, struct diff_hpp_fmt, fmt);
  883. char buf[MAX_COL_WIDTH] = " ";
  884. __hpp__entry_global(he, dfmt, buf, MAX_COL_WIDTH);
  885. if (symbol_conf.field_sep)
  886. return scnprintf(hpp->buf, hpp->size, "%s", buf);
  887. else
  888. return scnprintf(hpp->buf, hpp->size, "%*s",
  889. dfmt->header_width, buf);
  890. }
  891. static int hpp__header(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
  892. struct hists *hists __maybe_unused,
  893. int line __maybe_unused,
  894. int *span __maybe_unused)
  895. {
  896. struct diff_hpp_fmt *dfmt =
  897. container_of(fmt, struct diff_hpp_fmt, fmt);
  898. BUG_ON(!dfmt->header);
  899. return scnprintf(hpp->buf, hpp->size, dfmt->header);
  900. }
  901. static int hpp__width(struct perf_hpp_fmt *fmt,
  902. struct perf_hpp *hpp __maybe_unused,
  903. struct hists *hists __maybe_unused)
  904. {
  905. struct diff_hpp_fmt *dfmt =
  906. container_of(fmt, struct diff_hpp_fmt, fmt);
  907. BUG_ON(dfmt->header_width <= 0);
  908. return dfmt->header_width;
  909. }
  910. static void init_header(struct data__file *d, struct diff_hpp_fmt *dfmt)
  911. {
  912. #define MAX_HEADER_NAME 100
  913. char buf_indent[MAX_HEADER_NAME];
  914. char buf[MAX_HEADER_NAME];
  915. const char *header = NULL;
  916. int width = 0;
  917. BUG_ON(dfmt->idx >= PERF_HPP_DIFF__MAX_INDEX);
  918. header = columns[dfmt->idx].name;
  919. width = columns[dfmt->idx].width;
  920. /* Only our defined HPP fmts should appear here. */
  921. BUG_ON(!header);
  922. if (data__files_cnt > 2)
  923. scnprintf(buf, MAX_HEADER_NAME, "%s/%d", header, d->idx);
  924. #define NAME (data__files_cnt > 2 ? buf : header)
  925. dfmt->header_width = width;
  926. width = (int) strlen(NAME);
  927. if (dfmt->header_width < width)
  928. dfmt->header_width = width;
  929. scnprintf(buf_indent, MAX_HEADER_NAME, "%*s",
  930. dfmt->header_width, NAME);
  931. dfmt->header = strdup(buf_indent);
  932. #undef MAX_HEADER_NAME
  933. #undef NAME
  934. }
  935. static void data__hpp_register(struct data__file *d, int idx)
  936. {
  937. struct diff_hpp_fmt *dfmt = &d->fmt[idx];
  938. struct perf_hpp_fmt *fmt = &dfmt->fmt;
  939. dfmt->idx = idx;
  940. fmt->header = hpp__header;
  941. fmt->width = hpp__width;
  942. fmt->entry = hpp__entry_global;
  943. fmt->cmp = hist_entry__cmp_nop;
  944. fmt->collapse = hist_entry__cmp_nop;
  945. /* TODO more colors */
  946. switch (idx) {
  947. case PERF_HPP_DIFF__BASELINE:
  948. fmt->color = hpp__color_baseline;
  949. fmt->sort = hist_entry__cmp_baseline;
  950. break;
  951. case PERF_HPP_DIFF__DELTA:
  952. fmt->color = hpp__color_delta;
  953. fmt->sort = hist_entry__cmp_delta;
  954. break;
  955. case PERF_HPP_DIFF__RATIO:
  956. fmt->color = hpp__color_ratio;
  957. fmt->sort = hist_entry__cmp_ratio;
  958. break;
  959. case PERF_HPP_DIFF__WEIGHTED_DIFF:
  960. fmt->color = hpp__color_wdiff;
  961. fmt->sort = hist_entry__cmp_wdiff;
  962. break;
  963. case PERF_HPP_DIFF__DELTA_ABS:
  964. fmt->color = hpp__color_delta;
  965. fmt->sort = hist_entry__cmp_delta_abs;
  966. break;
  967. default:
  968. fmt->sort = hist_entry__cmp_nop;
  969. break;
  970. }
  971. init_header(d, dfmt);
  972. perf_hpp__column_register(fmt);
  973. perf_hpp__register_sort_field(fmt);
  974. }
  975. static int ui_init(void)
  976. {
  977. struct data__file *d;
  978. struct perf_hpp_fmt *fmt;
  979. int i;
  980. data__for_each_file(i, d) {
  981. /*
  982. * Baseline or compute realted columns:
  983. *
  984. * PERF_HPP_DIFF__BASELINE
  985. * PERF_HPP_DIFF__DELTA
  986. * PERF_HPP_DIFF__RATIO
  987. * PERF_HPP_DIFF__WEIGHTED_DIFF
  988. */
  989. data__hpp_register(d, i ? compute_2_hpp[compute] :
  990. PERF_HPP_DIFF__BASELINE);
  991. /*
  992. * And the rest:
  993. *
  994. * PERF_HPP_DIFF__FORMULA
  995. * PERF_HPP_DIFF__PERIOD
  996. * PERF_HPP_DIFF__PERIOD_BASELINE
  997. */
  998. if (show_formula && i)
  999. data__hpp_register(d, PERF_HPP_DIFF__FORMULA);
  1000. if (show_period)
  1001. data__hpp_register(d, i ? PERF_HPP_DIFF__PERIOD :
  1002. PERF_HPP_DIFF__PERIOD_BASELINE);
  1003. }
  1004. if (!sort_compute)
  1005. return 0;
  1006. /*
  1007. * Prepend an fmt to sort on columns at 'sort_compute' first.
  1008. * This fmt is added only to the sort list but not to the
  1009. * output fields list.
  1010. *
  1011. * Note that this column (data) can be compared twice - one
  1012. * for this 'sort_compute' fmt and another for the normal
  1013. * diff_hpp_fmt. But it shouldn't a problem as most entries
  1014. * will be sorted out by first try or baseline and comparing
  1015. * is not a costly operation.
  1016. */
  1017. fmt = zalloc(sizeof(*fmt));
  1018. if (fmt == NULL) {
  1019. pr_err("Memory allocation failed\n");
  1020. return -1;
  1021. }
  1022. fmt->cmp = hist_entry__cmp_nop;
  1023. fmt->collapse = hist_entry__cmp_nop;
  1024. switch (compute) {
  1025. case COMPUTE_DELTA:
  1026. fmt->sort = hist_entry__cmp_delta_idx;
  1027. break;
  1028. case COMPUTE_RATIO:
  1029. fmt->sort = hist_entry__cmp_ratio_idx;
  1030. break;
  1031. case COMPUTE_WEIGHTED_DIFF:
  1032. fmt->sort = hist_entry__cmp_wdiff_idx;
  1033. break;
  1034. case COMPUTE_DELTA_ABS:
  1035. fmt->sort = hist_entry__cmp_delta_abs_idx;
  1036. break;
  1037. default:
  1038. BUG_ON(1);
  1039. }
  1040. perf_hpp__prepend_sort_field(fmt);
  1041. return 0;
  1042. }
  1043. static int data_init(int argc, const char **argv)
  1044. {
  1045. struct data__file *d;
  1046. static const char *defaults[] = {
  1047. "perf.data.old",
  1048. "perf.data",
  1049. };
  1050. bool use_default = true;
  1051. int i;
  1052. data__files_cnt = 2;
  1053. if (argc) {
  1054. if (argc == 1)
  1055. defaults[1] = argv[0];
  1056. else {
  1057. data__files_cnt = argc;
  1058. use_default = false;
  1059. }
  1060. } else if (perf_guest) {
  1061. defaults[0] = "perf.data.host";
  1062. defaults[1] = "perf.data.guest";
  1063. }
  1064. if (sort_compute >= (unsigned int) data__files_cnt) {
  1065. pr_err("Order option out of limit.\n");
  1066. return -EINVAL;
  1067. }
  1068. data__files = zalloc(sizeof(*data__files) * data__files_cnt);
  1069. if (!data__files)
  1070. return -ENOMEM;
  1071. data__for_each_file(i, d) {
  1072. struct perf_data *data = &d->data;
  1073. data->file.path = use_default ? defaults[i] : argv[i];
  1074. data->mode = PERF_DATA_MODE_READ,
  1075. data->force = force,
  1076. d->idx = i;
  1077. }
  1078. return 0;
  1079. }
  1080. static int diff__config(const char *var, const char *value,
  1081. void *cb __maybe_unused)
  1082. {
  1083. if (!strcmp(var, "diff.order")) {
  1084. int ret;
  1085. if (perf_config_int(&ret, var, value) < 0)
  1086. return -1;
  1087. sort_compute = ret;
  1088. return 0;
  1089. }
  1090. if (!strcmp(var, "diff.compute")) {
  1091. if (!strcmp(value, "delta")) {
  1092. compute = COMPUTE_DELTA;
  1093. } else if (!strcmp(value, "delta-abs")) {
  1094. compute = COMPUTE_DELTA_ABS;
  1095. } else if (!strcmp(value, "ratio")) {
  1096. compute = COMPUTE_RATIO;
  1097. } else if (!strcmp(value, "wdiff")) {
  1098. compute = COMPUTE_WEIGHTED_DIFF;
  1099. } else {
  1100. pr_err("Invalid compute method: %s\n", value);
  1101. return -1;
  1102. }
  1103. }
  1104. return 0;
  1105. }
  1106. int cmd_diff(int argc, const char **argv)
  1107. {
  1108. int ret = hists__init();
  1109. if (ret < 0)
  1110. return ret;
  1111. perf_config(diff__config, NULL);
  1112. argc = parse_options(argc, argv, options, diff_usage, 0);
  1113. if (quiet)
  1114. perf_quiet_option();
  1115. if (symbol__init(NULL) < 0)
  1116. return -1;
  1117. if (data_init(argc, argv) < 0)
  1118. return -1;
  1119. if (ui_init() < 0)
  1120. return -1;
  1121. sort__mode = SORT_MODE__DIFF;
  1122. if (setup_sorting(NULL) < 0)
  1123. usage_with_options(diff_usage, options);
  1124. setup_pager();
  1125. sort__setup_elide(NULL);
  1126. return __cmd_diff();
  1127. }