libbpf.c 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401
  1. /*
  2. * Common eBPF ELF object loading operations.
  3. *
  4. * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
  5. * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
  6. * Copyright (C) 2015 Huawei Inc.
  7. */
  8. #include <stdlib.h>
  9. #include <stdio.h>
  10. #include <stdarg.h>
  11. #include <inttypes.h>
  12. #include <string.h>
  13. #include <unistd.h>
  14. #include <fcntl.h>
  15. #include <errno.h>
  16. #include <asm/unistd.h>
  17. #include <linux/kernel.h>
  18. #include <linux/bpf.h>
  19. #include <linux/list.h>
  20. #include <libelf.h>
  21. #include <gelf.h>
  22. #include "libbpf.h"
  23. #include "bpf.h"
  24. #define __printf(a, b) __attribute__((format(printf, a, b)))
  25. __printf(1, 2)
  26. static int __base_pr(const char *format, ...)
  27. {
  28. va_list args;
  29. int err;
  30. va_start(args, format);
  31. err = vfprintf(stderr, format, args);
  32. va_end(args);
  33. return err;
  34. }
  35. static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
  36. static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
  37. static __printf(1, 2) libbpf_print_fn_t __pr_debug;
  38. #define __pr(func, fmt, ...) \
  39. do { \
  40. if ((func)) \
  41. (func)("libbpf: " fmt, ##__VA_ARGS__); \
  42. } while (0)
  43. #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__)
  44. #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__)
  45. #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__)
  46. void libbpf_set_print(libbpf_print_fn_t warn,
  47. libbpf_print_fn_t info,
  48. libbpf_print_fn_t debug)
  49. {
  50. __pr_warning = warn;
  51. __pr_info = info;
  52. __pr_debug = debug;
  53. }
  54. #define STRERR_BUFSIZE 128
  55. #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START)
  56. #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c)
  57. #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START)
  58. static const char *libbpf_strerror_table[NR_ERRNO] = {
  59. [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf",
  60. [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid",
  61. [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost",
  62. [ERRCODE_OFFSET(ENDIAN)] = "Endian missmatch",
  63. [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf",
  64. [ERRCODE_OFFSET(RELOC)] = "Relocation failed",
  65. [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading",
  66. [ERRCODE_OFFSET(PROG2BIG)] = "Program too big",
  67. [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version",
  68. };
  69. int libbpf_strerror(int err, char *buf, size_t size)
  70. {
  71. if (!buf || !size)
  72. return -1;
  73. err = err > 0 ? err : -err;
  74. if (err < __LIBBPF_ERRNO__START) {
  75. int ret;
  76. ret = strerror_r(err, buf, size);
  77. buf[size - 1] = '\0';
  78. return ret;
  79. }
  80. if (err < __LIBBPF_ERRNO__END) {
  81. const char *msg;
  82. msg = libbpf_strerror_table[ERRNO_OFFSET(err)];
  83. snprintf(buf, size, "%s", msg);
  84. buf[size - 1] = '\0';
  85. return 0;
  86. }
  87. snprintf(buf, size, "Unknown libbpf error %d", err);
  88. buf[size - 1] = '\0';
  89. return -1;
  90. }
  91. #define CHECK_ERR(action, err, out) do { \
  92. err = action; \
  93. if (err) \
  94. goto out; \
  95. } while(0)
  96. /* Copied from tools/perf/util/util.h */
  97. #ifndef zfree
  98. # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
  99. #endif
  100. #ifndef zclose
  101. # define zclose(fd) ({ \
  102. int ___err = 0; \
  103. if ((fd) >= 0) \
  104. ___err = close((fd)); \
  105. fd = -1; \
  106. ___err; })
  107. #endif
  108. #ifdef HAVE_LIBELF_MMAP_SUPPORT
  109. # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
  110. #else
  111. # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
  112. #endif
  113. /*
  114. * bpf_prog should be a better name but it has been used in
  115. * linux/filter.h.
  116. */
  117. struct bpf_program {
  118. /* Index in elf obj file, for relocation use. */
  119. int idx;
  120. char *section_name;
  121. struct bpf_insn *insns;
  122. size_t insns_cnt;
  123. struct {
  124. int insn_idx;
  125. int map_idx;
  126. } *reloc_desc;
  127. int nr_reloc;
  128. struct {
  129. int nr;
  130. int *fds;
  131. } instances;
  132. bpf_program_prep_t preprocessor;
  133. struct bpf_object *obj;
  134. void *priv;
  135. bpf_program_clear_priv_t clear_priv;
  136. };
  137. struct bpf_map {
  138. int fd;
  139. char *name;
  140. struct bpf_map_def def;
  141. void *priv;
  142. bpf_map_clear_priv_t clear_priv;
  143. };
  144. static LIST_HEAD(bpf_objects_list);
  145. struct bpf_object {
  146. char license[64];
  147. u32 kern_version;
  148. struct bpf_program *programs;
  149. size_t nr_programs;
  150. struct bpf_map *maps;
  151. size_t nr_maps;
  152. bool loaded;
  153. /*
  154. * Information when doing elf related work. Only valid if fd
  155. * is valid.
  156. */
  157. struct {
  158. int fd;
  159. void *obj_buf;
  160. size_t obj_buf_sz;
  161. Elf *elf;
  162. GElf_Ehdr ehdr;
  163. Elf_Data *symbols;
  164. size_t strtabidx;
  165. struct {
  166. GElf_Shdr shdr;
  167. Elf_Data *data;
  168. } *reloc;
  169. int nr_reloc;
  170. int maps_shndx;
  171. } efile;
  172. /*
  173. * All loaded bpf_object is linked in a list, which is
  174. * hidden to caller. bpf_objects__<func> handlers deal with
  175. * all objects.
  176. */
  177. struct list_head list;
  178. char path[];
  179. };
  180. #define obj_elf_valid(o) ((o)->efile.elf)
  181. static void bpf_program__unload(struct bpf_program *prog)
  182. {
  183. int i;
  184. if (!prog)
  185. return;
  186. /*
  187. * If the object is opened but the program was never loaded,
  188. * it is possible that prog->instances.nr == -1.
  189. */
  190. if (prog->instances.nr > 0) {
  191. for (i = 0; i < prog->instances.nr; i++)
  192. zclose(prog->instances.fds[i]);
  193. } else if (prog->instances.nr != -1) {
  194. pr_warning("Internal error: instances.nr is %d\n",
  195. prog->instances.nr);
  196. }
  197. prog->instances.nr = -1;
  198. zfree(&prog->instances.fds);
  199. }
  200. static void bpf_program__exit(struct bpf_program *prog)
  201. {
  202. if (!prog)
  203. return;
  204. if (prog->clear_priv)
  205. prog->clear_priv(prog, prog->priv);
  206. prog->priv = NULL;
  207. prog->clear_priv = NULL;
  208. bpf_program__unload(prog);
  209. zfree(&prog->section_name);
  210. zfree(&prog->insns);
  211. zfree(&prog->reloc_desc);
  212. prog->nr_reloc = 0;
  213. prog->insns_cnt = 0;
  214. prog->idx = -1;
  215. }
  216. static int
  217. bpf_program__init(void *data, size_t size, char *name, int idx,
  218. struct bpf_program *prog)
  219. {
  220. if (size < sizeof(struct bpf_insn)) {
  221. pr_warning("corrupted section '%s'\n", name);
  222. return -EINVAL;
  223. }
  224. bzero(prog, sizeof(*prog));
  225. prog->section_name = strdup(name);
  226. if (!prog->section_name) {
  227. pr_warning("failed to alloc name for prog %s\n",
  228. name);
  229. goto errout;
  230. }
  231. prog->insns = malloc(size);
  232. if (!prog->insns) {
  233. pr_warning("failed to alloc insns for %s\n", name);
  234. goto errout;
  235. }
  236. prog->insns_cnt = size / sizeof(struct bpf_insn);
  237. memcpy(prog->insns, data,
  238. prog->insns_cnt * sizeof(struct bpf_insn));
  239. prog->idx = idx;
  240. prog->instances.fds = NULL;
  241. prog->instances.nr = -1;
  242. return 0;
  243. errout:
  244. bpf_program__exit(prog);
  245. return -ENOMEM;
  246. }
  247. static int
  248. bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
  249. char *name, int idx)
  250. {
  251. struct bpf_program prog, *progs;
  252. int nr_progs, err;
  253. err = bpf_program__init(data, size, name, idx, &prog);
  254. if (err)
  255. return err;
  256. progs = obj->programs;
  257. nr_progs = obj->nr_programs;
  258. progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1));
  259. if (!progs) {
  260. /*
  261. * In this case the original obj->programs
  262. * is still valid, so don't need special treat for
  263. * bpf_close_object().
  264. */
  265. pr_warning("failed to alloc a new program '%s'\n",
  266. name);
  267. bpf_program__exit(&prog);
  268. return -ENOMEM;
  269. }
  270. pr_debug("found program %s\n", prog.section_name);
  271. obj->programs = progs;
  272. obj->nr_programs = nr_progs + 1;
  273. prog.obj = obj;
  274. progs[nr_progs] = prog;
  275. return 0;
  276. }
  277. static struct bpf_object *bpf_object__new(const char *path,
  278. void *obj_buf,
  279. size_t obj_buf_sz)
  280. {
  281. struct bpf_object *obj;
  282. obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
  283. if (!obj) {
  284. pr_warning("alloc memory failed for %s\n", path);
  285. return ERR_PTR(-ENOMEM);
  286. }
  287. strcpy(obj->path, path);
  288. obj->efile.fd = -1;
  289. /*
  290. * Caller of this function should also calls
  291. * bpf_object__elf_finish() after data collection to return
  292. * obj_buf to user. If not, we should duplicate the buffer to
  293. * avoid user freeing them before elf finish.
  294. */
  295. obj->efile.obj_buf = obj_buf;
  296. obj->efile.obj_buf_sz = obj_buf_sz;
  297. obj->efile.maps_shndx = -1;
  298. obj->loaded = false;
  299. INIT_LIST_HEAD(&obj->list);
  300. list_add(&obj->list, &bpf_objects_list);
  301. return obj;
  302. }
  303. static void bpf_object__elf_finish(struct bpf_object *obj)
  304. {
  305. if (!obj_elf_valid(obj))
  306. return;
  307. if (obj->efile.elf) {
  308. elf_end(obj->efile.elf);
  309. obj->efile.elf = NULL;
  310. }
  311. obj->efile.symbols = NULL;
  312. zfree(&obj->efile.reloc);
  313. obj->efile.nr_reloc = 0;
  314. zclose(obj->efile.fd);
  315. obj->efile.obj_buf = NULL;
  316. obj->efile.obj_buf_sz = 0;
  317. }
  318. static int bpf_object__elf_init(struct bpf_object *obj)
  319. {
  320. int err = 0;
  321. GElf_Ehdr *ep;
  322. if (obj_elf_valid(obj)) {
  323. pr_warning("elf init: internal error\n");
  324. return -LIBBPF_ERRNO__LIBELF;
  325. }
  326. if (obj->efile.obj_buf_sz > 0) {
  327. /*
  328. * obj_buf should have been validated by
  329. * bpf_object__open_buffer().
  330. */
  331. obj->efile.elf = elf_memory(obj->efile.obj_buf,
  332. obj->efile.obj_buf_sz);
  333. } else {
  334. obj->efile.fd = open(obj->path, O_RDONLY);
  335. if (obj->efile.fd < 0) {
  336. pr_warning("failed to open %s: %s\n", obj->path,
  337. strerror(errno));
  338. return -errno;
  339. }
  340. obj->efile.elf = elf_begin(obj->efile.fd,
  341. LIBBPF_ELF_C_READ_MMAP,
  342. NULL);
  343. }
  344. if (!obj->efile.elf) {
  345. pr_warning("failed to open %s as ELF file\n",
  346. obj->path);
  347. err = -LIBBPF_ERRNO__LIBELF;
  348. goto errout;
  349. }
  350. if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
  351. pr_warning("failed to get EHDR from %s\n",
  352. obj->path);
  353. err = -LIBBPF_ERRNO__FORMAT;
  354. goto errout;
  355. }
  356. ep = &obj->efile.ehdr;
  357. if ((ep->e_type != ET_REL) || (ep->e_machine != 0)) {
  358. pr_warning("%s is not an eBPF object file\n",
  359. obj->path);
  360. err = -LIBBPF_ERRNO__FORMAT;
  361. goto errout;
  362. }
  363. return 0;
  364. errout:
  365. bpf_object__elf_finish(obj);
  366. return err;
  367. }
  368. static int
  369. bpf_object__check_endianness(struct bpf_object *obj)
  370. {
  371. static unsigned int const endian = 1;
  372. switch (obj->efile.ehdr.e_ident[EI_DATA]) {
  373. case ELFDATA2LSB:
  374. /* We are big endian, BPF obj is little endian. */
  375. if (*(unsigned char const *)&endian != 1)
  376. goto mismatch;
  377. break;
  378. case ELFDATA2MSB:
  379. /* We are little endian, BPF obj is big endian. */
  380. if (*(unsigned char const *)&endian != 0)
  381. goto mismatch;
  382. break;
  383. default:
  384. return -LIBBPF_ERRNO__ENDIAN;
  385. }
  386. return 0;
  387. mismatch:
  388. pr_warning("Error: endianness mismatch.\n");
  389. return -LIBBPF_ERRNO__ENDIAN;
  390. }
  391. static int
  392. bpf_object__init_license(struct bpf_object *obj,
  393. void *data, size_t size)
  394. {
  395. memcpy(obj->license, data,
  396. min(size, sizeof(obj->license) - 1));
  397. pr_debug("license of %s is %s\n", obj->path, obj->license);
  398. return 0;
  399. }
  400. static int
  401. bpf_object__init_kversion(struct bpf_object *obj,
  402. void *data, size_t size)
  403. {
  404. u32 kver;
  405. if (size != sizeof(kver)) {
  406. pr_warning("invalid kver section in %s\n", obj->path);
  407. return -LIBBPF_ERRNO__FORMAT;
  408. }
  409. memcpy(&kver, data, sizeof(kver));
  410. obj->kern_version = kver;
  411. pr_debug("kernel version of %s is %x\n", obj->path,
  412. obj->kern_version);
  413. return 0;
  414. }
  415. static int
  416. bpf_object__init_maps(struct bpf_object *obj, void *data,
  417. size_t size)
  418. {
  419. size_t nr_maps;
  420. int i;
  421. nr_maps = size / sizeof(struct bpf_map_def);
  422. if (!data || !nr_maps) {
  423. pr_debug("%s doesn't need map definition\n",
  424. obj->path);
  425. return 0;
  426. }
  427. pr_debug("maps in %s: %zd bytes\n", obj->path, size);
  428. obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
  429. if (!obj->maps) {
  430. pr_warning("alloc maps for object failed\n");
  431. return -ENOMEM;
  432. }
  433. obj->nr_maps = nr_maps;
  434. for (i = 0; i < nr_maps; i++) {
  435. struct bpf_map_def *def = &obj->maps[i].def;
  436. /*
  437. * fill all fd with -1 so won't close incorrect
  438. * fd (fd=0 is stdin) when failure (zclose won't close
  439. * negative fd)).
  440. */
  441. obj->maps[i].fd = -1;
  442. /* Save map definition into obj->maps */
  443. *def = ((struct bpf_map_def *)data)[i];
  444. }
  445. return 0;
  446. }
  447. static int
  448. bpf_object__init_maps_name(struct bpf_object *obj)
  449. {
  450. int i;
  451. Elf_Data *symbols = obj->efile.symbols;
  452. if (!symbols || obj->efile.maps_shndx < 0)
  453. return -EINVAL;
  454. for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
  455. GElf_Sym sym;
  456. size_t map_idx;
  457. const char *map_name;
  458. if (!gelf_getsym(symbols, i, &sym))
  459. continue;
  460. if (sym.st_shndx != obj->efile.maps_shndx)
  461. continue;
  462. map_name = elf_strptr(obj->efile.elf,
  463. obj->efile.strtabidx,
  464. sym.st_name);
  465. map_idx = sym.st_value / sizeof(struct bpf_map_def);
  466. if (map_idx >= obj->nr_maps) {
  467. pr_warning("index of map \"%s\" is buggy: %zu > %zu\n",
  468. map_name, map_idx, obj->nr_maps);
  469. continue;
  470. }
  471. obj->maps[map_idx].name = strdup(map_name);
  472. if (!obj->maps[map_idx].name) {
  473. pr_warning("failed to alloc map name\n");
  474. return -ENOMEM;
  475. }
  476. pr_debug("map %zu is \"%s\"\n", map_idx,
  477. obj->maps[map_idx].name);
  478. }
  479. return 0;
  480. }
  481. static int bpf_object__elf_collect(struct bpf_object *obj)
  482. {
  483. Elf *elf = obj->efile.elf;
  484. GElf_Ehdr *ep = &obj->efile.ehdr;
  485. Elf_Scn *scn = NULL;
  486. int idx = 0, err = 0;
  487. /* Elf is corrupted/truncated, avoid calling elf_strptr. */
  488. if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
  489. pr_warning("failed to get e_shstrndx from %s\n",
  490. obj->path);
  491. return -LIBBPF_ERRNO__FORMAT;
  492. }
  493. while ((scn = elf_nextscn(elf, scn)) != NULL) {
  494. char *name;
  495. GElf_Shdr sh;
  496. Elf_Data *data;
  497. idx++;
  498. if (gelf_getshdr(scn, &sh) != &sh) {
  499. pr_warning("failed to get section header from %s\n",
  500. obj->path);
  501. err = -LIBBPF_ERRNO__FORMAT;
  502. goto out;
  503. }
  504. name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
  505. if (!name) {
  506. pr_warning("failed to get section name from %s\n",
  507. obj->path);
  508. err = -LIBBPF_ERRNO__FORMAT;
  509. goto out;
  510. }
  511. data = elf_getdata(scn, 0);
  512. if (!data) {
  513. pr_warning("failed to get section data from %s(%s)\n",
  514. name, obj->path);
  515. err = -LIBBPF_ERRNO__FORMAT;
  516. goto out;
  517. }
  518. pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n",
  519. name, (unsigned long)data->d_size,
  520. (int)sh.sh_link, (unsigned long)sh.sh_flags,
  521. (int)sh.sh_type);
  522. if (strcmp(name, "license") == 0)
  523. err = bpf_object__init_license(obj,
  524. data->d_buf,
  525. data->d_size);
  526. else if (strcmp(name, "version") == 0)
  527. err = bpf_object__init_kversion(obj,
  528. data->d_buf,
  529. data->d_size);
  530. else if (strcmp(name, "maps") == 0) {
  531. err = bpf_object__init_maps(obj, data->d_buf,
  532. data->d_size);
  533. obj->efile.maps_shndx = idx;
  534. } else if (sh.sh_type == SHT_SYMTAB) {
  535. if (obj->efile.symbols) {
  536. pr_warning("bpf: multiple SYMTAB in %s\n",
  537. obj->path);
  538. err = -LIBBPF_ERRNO__FORMAT;
  539. } else {
  540. obj->efile.symbols = data;
  541. obj->efile.strtabidx = sh.sh_link;
  542. }
  543. } else if ((sh.sh_type == SHT_PROGBITS) &&
  544. (sh.sh_flags & SHF_EXECINSTR) &&
  545. (data->d_size > 0)) {
  546. err = bpf_object__add_program(obj, data->d_buf,
  547. data->d_size, name, idx);
  548. if (err) {
  549. char errmsg[STRERR_BUFSIZE];
  550. strerror_r(-err, errmsg, sizeof(errmsg));
  551. pr_warning("failed to alloc program %s (%s): %s",
  552. name, obj->path, errmsg);
  553. }
  554. } else if (sh.sh_type == SHT_REL) {
  555. void *reloc = obj->efile.reloc;
  556. int nr_reloc = obj->efile.nr_reloc + 1;
  557. reloc = realloc(reloc,
  558. sizeof(*obj->efile.reloc) * nr_reloc);
  559. if (!reloc) {
  560. pr_warning("realloc failed\n");
  561. err = -ENOMEM;
  562. } else {
  563. int n = nr_reloc - 1;
  564. obj->efile.reloc = reloc;
  565. obj->efile.nr_reloc = nr_reloc;
  566. obj->efile.reloc[n].shdr = sh;
  567. obj->efile.reloc[n].data = data;
  568. }
  569. }
  570. if (err)
  571. goto out;
  572. }
  573. if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
  574. pr_warning("Corrupted ELF file: index of strtab invalid\n");
  575. return LIBBPF_ERRNO__FORMAT;
  576. }
  577. if (obj->efile.maps_shndx >= 0)
  578. err = bpf_object__init_maps_name(obj);
  579. out:
  580. return err;
  581. }
  582. static struct bpf_program *
  583. bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
  584. {
  585. struct bpf_program *prog;
  586. size_t i;
  587. for (i = 0; i < obj->nr_programs; i++) {
  588. prog = &obj->programs[i];
  589. if (prog->idx == idx)
  590. return prog;
  591. }
  592. return NULL;
  593. }
  594. static int
  595. bpf_program__collect_reloc(struct bpf_program *prog,
  596. size_t nr_maps, GElf_Shdr *shdr,
  597. Elf_Data *data, Elf_Data *symbols,
  598. int maps_shndx)
  599. {
  600. int i, nrels;
  601. pr_debug("collecting relocating info for: '%s'\n",
  602. prog->section_name);
  603. nrels = shdr->sh_size / shdr->sh_entsize;
  604. prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
  605. if (!prog->reloc_desc) {
  606. pr_warning("failed to alloc memory in relocation\n");
  607. return -ENOMEM;
  608. }
  609. prog->nr_reloc = nrels;
  610. for (i = 0; i < nrels; i++) {
  611. GElf_Sym sym;
  612. GElf_Rel rel;
  613. unsigned int insn_idx;
  614. struct bpf_insn *insns = prog->insns;
  615. size_t map_idx;
  616. if (!gelf_getrel(data, i, &rel)) {
  617. pr_warning("relocation: failed to get %d reloc\n", i);
  618. return -LIBBPF_ERRNO__FORMAT;
  619. }
  620. if (!gelf_getsym(symbols,
  621. GELF_R_SYM(rel.r_info),
  622. &sym)) {
  623. pr_warning("relocation: symbol %"PRIx64" not found\n",
  624. GELF_R_SYM(rel.r_info));
  625. return -LIBBPF_ERRNO__FORMAT;
  626. }
  627. if (sym.st_shndx != maps_shndx) {
  628. pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
  629. prog->section_name, sym.st_shndx);
  630. return -LIBBPF_ERRNO__RELOC;
  631. }
  632. insn_idx = rel.r_offset / sizeof(struct bpf_insn);
  633. pr_debug("relocation: insn_idx=%u\n", insn_idx);
  634. if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
  635. pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
  636. insn_idx, insns[insn_idx].code);
  637. return -LIBBPF_ERRNO__RELOC;
  638. }
  639. map_idx = sym.st_value / sizeof(struct bpf_map_def);
  640. if (map_idx >= nr_maps) {
  641. pr_warning("bpf relocation: map_idx %d large than %d\n",
  642. (int)map_idx, (int)nr_maps - 1);
  643. return -LIBBPF_ERRNO__RELOC;
  644. }
  645. prog->reloc_desc[i].insn_idx = insn_idx;
  646. prog->reloc_desc[i].map_idx = map_idx;
  647. }
  648. return 0;
  649. }
  650. static int
  651. bpf_object__create_maps(struct bpf_object *obj)
  652. {
  653. unsigned int i;
  654. for (i = 0; i < obj->nr_maps; i++) {
  655. struct bpf_map_def *def = &obj->maps[i].def;
  656. int *pfd = &obj->maps[i].fd;
  657. *pfd = bpf_create_map(def->type,
  658. def->key_size,
  659. def->value_size,
  660. def->max_entries);
  661. if (*pfd < 0) {
  662. size_t j;
  663. int err = *pfd;
  664. pr_warning("failed to create map: %s\n",
  665. strerror(errno));
  666. for (j = 0; j < i; j++)
  667. zclose(obj->maps[j].fd);
  668. return err;
  669. }
  670. pr_debug("create map: fd=%d\n", *pfd);
  671. }
  672. return 0;
  673. }
  674. static int
  675. bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
  676. {
  677. int i;
  678. if (!prog || !prog->reloc_desc)
  679. return 0;
  680. for (i = 0; i < prog->nr_reloc; i++) {
  681. int insn_idx, map_idx;
  682. struct bpf_insn *insns = prog->insns;
  683. insn_idx = prog->reloc_desc[i].insn_idx;
  684. map_idx = prog->reloc_desc[i].map_idx;
  685. if (insn_idx >= (int)prog->insns_cnt) {
  686. pr_warning("relocation out of range: '%s'\n",
  687. prog->section_name);
  688. return -LIBBPF_ERRNO__RELOC;
  689. }
  690. insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
  691. insns[insn_idx].imm = obj->maps[map_idx].fd;
  692. }
  693. zfree(&prog->reloc_desc);
  694. prog->nr_reloc = 0;
  695. return 0;
  696. }
  697. static int
  698. bpf_object__relocate(struct bpf_object *obj)
  699. {
  700. struct bpf_program *prog;
  701. size_t i;
  702. int err;
  703. for (i = 0; i < obj->nr_programs; i++) {
  704. prog = &obj->programs[i];
  705. err = bpf_program__relocate(prog, obj);
  706. if (err) {
  707. pr_warning("failed to relocate '%s'\n",
  708. prog->section_name);
  709. return err;
  710. }
  711. }
  712. return 0;
  713. }
  714. static int bpf_object__collect_reloc(struct bpf_object *obj)
  715. {
  716. int i, err;
  717. if (!obj_elf_valid(obj)) {
  718. pr_warning("Internal error: elf object is closed\n");
  719. return -LIBBPF_ERRNO__INTERNAL;
  720. }
  721. for (i = 0; i < obj->efile.nr_reloc; i++) {
  722. GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
  723. Elf_Data *data = obj->efile.reloc[i].data;
  724. int idx = shdr->sh_info;
  725. struct bpf_program *prog;
  726. size_t nr_maps = obj->nr_maps;
  727. if (shdr->sh_type != SHT_REL) {
  728. pr_warning("internal error at %d\n", __LINE__);
  729. return -LIBBPF_ERRNO__INTERNAL;
  730. }
  731. prog = bpf_object__find_prog_by_idx(obj, idx);
  732. if (!prog) {
  733. pr_warning("relocation failed: no %d section\n",
  734. idx);
  735. return -LIBBPF_ERRNO__RELOC;
  736. }
  737. err = bpf_program__collect_reloc(prog, nr_maps,
  738. shdr, data,
  739. obj->efile.symbols,
  740. obj->efile.maps_shndx);
  741. if (err)
  742. return err;
  743. }
  744. return 0;
  745. }
  746. static int
  747. load_program(struct bpf_insn *insns, int insns_cnt,
  748. char *license, u32 kern_version, int *pfd)
  749. {
  750. int ret;
  751. char *log_buf;
  752. if (!insns || !insns_cnt)
  753. return -EINVAL;
  754. log_buf = malloc(BPF_LOG_BUF_SIZE);
  755. if (!log_buf)
  756. pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
  757. ret = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns,
  758. insns_cnt, license, kern_version,
  759. log_buf, BPF_LOG_BUF_SIZE);
  760. if (ret >= 0) {
  761. *pfd = ret;
  762. ret = 0;
  763. goto out;
  764. }
  765. ret = -LIBBPF_ERRNO__LOAD;
  766. pr_warning("load bpf program failed: %s\n", strerror(errno));
  767. if (log_buf && log_buf[0] != '\0') {
  768. ret = -LIBBPF_ERRNO__VERIFY;
  769. pr_warning("-- BEGIN DUMP LOG ---\n");
  770. pr_warning("\n%s\n", log_buf);
  771. pr_warning("-- END LOG --\n");
  772. } else {
  773. if (insns_cnt >= BPF_MAXINSNS) {
  774. pr_warning("Program too large (%d insns), at most %d insns\n",
  775. insns_cnt, BPF_MAXINSNS);
  776. ret = -LIBBPF_ERRNO__PROG2BIG;
  777. } else if (log_buf) {
  778. pr_warning("log buffer is empty\n");
  779. ret = -LIBBPF_ERRNO__KVER;
  780. }
  781. }
  782. out:
  783. free(log_buf);
  784. return ret;
  785. }
  786. static int
  787. bpf_program__load(struct bpf_program *prog,
  788. char *license, u32 kern_version)
  789. {
  790. int err = 0, fd, i;
  791. if (prog->instances.nr < 0 || !prog->instances.fds) {
  792. if (prog->preprocessor) {
  793. pr_warning("Internal error: can't load program '%s'\n",
  794. prog->section_name);
  795. return -LIBBPF_ERRNO__INTERNAL;
  796. }
  797. prog->instances.fds = malloc(sizeof(int));
  798. if (!prog->instances.fds) {
  799. pr_warning("Not enough memory for BPF fds\n");
  800. return -ENOMEM;
  801. }
  802. prog->instances.nr = 1;
  803. prog->instances.fds[0] = -1;
  804. }
  805. if (!prog->preprocessor) {
  806. if (prog->instances.nr != 1) {
  807. pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
  808. prog->section_name, prog->instances.nr);
  809. }
  810. err = load_program(prog->insns, prog->insns_cnt,
  811. license, kern_version, &fd);
  812. if (!err)
  813. prog->instances.fds[0] = fd;
  814. goto out;
  815. }
  816. for (i = 0; i < prog->instances.nr; i++) {
  817. struct bpf_prog_prep_result result;
  818. bpf_program_prep_t preprocessor = prog->preprocessor;
  819. bzero(&result, sizeof(result));
  820. err = preprocessor(prog, i, prog->insns,
  821. prog->insns_cnt, &result);
  822. if (err) {
  823. pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
  824. i, prog->section_name);
  825. goto out;
  826. }
  827. if (!result.new_insn_ptr || !result.new_insn_cnt) {
  828. pr_debug("Skip loading the %dth instance of program '%s'\n",
  829. i, prog->section_name);
  830. prog->instances.fds[i] = -1;
  831. if (result.pfd)
  832. *result.pfd = -1;
  833. continue;
  834. }
  835. err = load_program(result.new_insn_ptr,
  836. result.new_insn_cnt,
  837. license, kern_version, &fd);
  838. if (err) {
  839. pr_warning("Loading the %dth instance of program '%s' failed\n",
  840. i, prog->section_name);
  841. goto out;
  842. }
  843. if (result.pfd)
  844. *result.pfd = fd;
  845. prog->instances.fds[i] = fd;
  846. }
  847. out:
  848. if (err)
  849. pr_warning("failed to load program '%s'\n",
  850. prog->section_name);
  851. zfree(&prog->insns);
  852. prog->insns_cnt = 0;
  853. return err;
  854. }
  855. static int
  856. bpf_object__load_progs(struct bpf_object *obj)
  857. {
  858. size_t i;
  859. int err;
  860. for (i = 0; i < obj->nr_programs; i++) {
  861. err = bpf_program__load(&obj->programs[i],
  862. obj->license,
  863. obj->kern_version);
  864. if (err)
  865. return err;
  866. }
  867. return 0;
  868. }
  869. static int bpf_object__validate(struct bpf_object *obj)
  870. {
  871. if (obj->kern_version == 0) {
  872. pr_warning("%s doesn't provide kernel version\n",
  873. obj->path);
  874. return -LIBBPF_ERRNO__KVERSION;
  875. }
  876. return 0;
  877. }
  878. static struct bpf_object *
  879. __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz)
  880. {
  881. struct bpf_object *obj;
  882. int err;
  883. if (elf_version(EV_CURRENT) == EV_NONE) {
  884. pr_warning("failed to init libelf for %s\n", path);
  885. return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
  886. }
  887. obj = bpf_object__new(path, obj_buf, obj_buf_sz);
  888. if (IS_ERR(obj))
  889. return obj;
  890. CHECK_ERR(bpf_object__elf_init(obj), err, out);
  891. CHECK_ERR(bpf_object__check_endianness(obj), err, out);
  892. CHECK_ERR(bpf_object__elf_collect(obj), err, out);
  893. CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
  894. CHECK_ERR(bpf_object__validate(obj), err, out);
  895. bpf_object__elf_finish(obj);
  896. return obj;
  897. out:
  898. bpf_object__close(obj);
  899. return ERR_PTR(err);
  900. }
  901. struct bpf_object *bpf_object__open(const char *path)
  902. {
  903. /* param validation */
  904. if (!path)
  905. return NULL;
  906. pr_debug("loading %s\n", path);
  907. return __bpf_object__open(path, NULL, 0);
  908. }
  909. struct bpf_object *bpf_object__open_buffer(void *obj_buf,
  910. size_t obj_buf_sz,
  911. const char *name)
  912. {
  913. char tmp_name[64];
  914. /* param validation */
  915. if (!obj_buf || obj_buf_sz <= 0)
  916. return NULL;
  917. if (!name) {
  918. snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
  919. (unsigned long)obj_buf,
  920. (unsigned long)obj_buf_sz);
  921. tmp_name[sizeof(tmp_name) - 1] = '\0';
  922. name = tmp_name;
  923. }
  924. pr_debug("loading object '%s' from buffer\n",
  925. name);
  926. return __bpf_object__open(name, obj_buf, obj_buf_sz);
  927. }
  928. int bpf_object__unload(struct bpf_object *obj)
  929. {
  930. size_t i;
  931. if (!obj)
  932. return -EINVAL;
  933. for (i = 0; i < obj->nr_maps; i++)
  934. zclose(obj->maps[i].fd);
  935. for (i = 0; i < obj->nr_programs; i++)
  936. bpf_program__unload(&obj->programs[i]);
  937. return 0;
  938. }
  939. int bpf_object__load(struct bpf_object *obj)
  940. {
  941. int err;
  942. if (!obj)
  943. return -EINVAL;
  944. if (obj->loaded) {
  945. pr_warning("object should not be loaded twice\n");
  946. return -EINVAL;
  947. }
  948. obj->loaded = true;
  949. CHECK_ERR(bpf_object__create_maps(obj), err, out);
  950. CHECK_ERR(bpf_object__relocate(obj), err, out);
  951. CHECK_ERR(bpf_object__load_progs(obj), err, out);
  952. return 0;
  953. out:
  954. bpf_object__unload(obj);
  955. pr_warning("failed to load object '%s'\n", obj->path);
  956. return err;
  957. }
  958. void bpf_object__close(struct bpf_object *obj)
  959. {
  960. size_t i;
  961. if (!obj)
  962. return;
  963. bpf_object__elf_finish(obj);
  964. bpf_object__unload(obj);
  965. for (i = 0; i < obj->nr_maps; i++) {
  966. zfree(&obj->maps[i].name);
  967. if (obj->maps[i].clear_priv)
  968. obj->maps[i].clear_priv(&obj->maps[i],
  969. obj->maps[i].priv);
  970. obj->maps[i].priv = NULL;
  971. obj->maps[i].clear_priv = NULL;
  972. }
  973. zfree(&obj->maps);
  974. obj->nr_maps = 0;
  975. if (obj->programs && obj->nr_programs) {
  976. for (i = 0; i < obj->nr_programs; i++)
  977. bpf_program__exit(&obj->programs[i]);
  978. }
  979. zfree(&obj->programs);
  980. list_del(&obj->list);
  981. free(obj);
  982. }
  983. struct bpf_object *
  984. bpf_object__next(struct bpf_object *prev)
  985. {
  986. struct bpf_object *next;
  987. if (!prev)
  988. next = list_first_entry(&bpf_objects_list,
  989. struct bpf_object,
  990. list);
  991. else
  992. next = list_next_entry(prev, list);
  993. /* Empty list is noticed here so don't need checking on entry. */
  994. if (&next->list == &bpf_objects_list)
  995. return NULL;
  996. return next;
  997. }
  998. const char *
  999. bpf_object__get_name(struct bpf_object *obj)
  1000. {
  1001. if (!obj)
  1002. return ERR_PTR(-EINVAL);
  1003. return obj->path;
  1004. }
  1005. unsigned int
  1006. bpf_object__get_kversion(struct bpf_object *obj)
  1007. {
  1008. if (!obj)
  1009. return 0;
  1010. return obj->kern_version;
  1011. }
  1012. struct bpf_program *
  1013. bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
  1014. {
  1015. size_t idx;
  1016. if (!obj->programs)
  1017. return NULL;
  1018. /* First handler */
  1019. if (prev == NULL)
  1020. return &obj->programs[0];
  1021. if (prev->obj != obj) {
  1022. pr_warning("error: program handler doesn't match object\n");
  1023. return NULL;
  1024. }
  1025. idx = (prev - obj->programs) + 1;
  1026. if (idx >= obj->nr_programs)
  1027. return NULL;
  1028. return &obj->programs[idx];
  1029. }
  1030. int bpf_program__set_private(struct bpf_program *prog,
  1031. void *priv,
  1032. bpf_program_clear_priv_t clear_priv)
  1033. {
  1034. if (prog->priv && prog->clear_priv)
  1035. prog->clear_priv(prog, prog->priv);
  1036. prog->priv = priv;
  1037. prog->clear_priv = clear_priv;
  1038. return 0;
  1039. }
  1040. int bpf_program__get_private(struct bpf_program *prog, void **ppriv)
  1041. {
  1042. *ppriv = prog->priv;
  1043. return 0;
  1044. }
  1045. const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
  1046. {
  1047. const char *title;
  1048. title = prog->section_name;
  1049. if (needs_copy) {
  1050. title = strdup(title);
  1051. if (!title) {
  1052. pr_warning("failed to strdup program title\n");
  1053. return ERR_PTR(-ENOMEM);
  1054. }
  1055. }
  1056. return title;
  1057. }
  1058. int bpf_program__fd(struct bpf_program *prog)
  1059. {
  1060. return bpf_program__nth_fd(prog, 0);
  1061. }
  1062. int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
  1063. bpf_program_prep_t prep)
  1064. {
  1065. int *instances_fds;
  1066. if (nr_instances <= 0 || !prep)
  1067. return -EINVAL;
  1068. if (prog->instances.nr > 0 || prog->instances.fds) {
  1069. pr_warning("Can't set pre-processor after loading\n");
  1070. return -EINVAL;
  1071. }
  1072. instances_fds = malloc(sizeof(int) * nr_instances);
  1073. if (!instances_fds) {
  1074. pr_warning("alloc memory failed for fds\n");
  1075. return -ENOMEM;
  1076. }
  1077. /* fill all fd with -1 */
  1078. memset(instances_fds, -1, sizeof(int) * nr_instances);
  1079. prog->instances.nr = nr_instances;
  1080. prog->instances.fds = instances_fds;
  1081. prog->preprocessor = prep;
  1082. return 0;
  1083. }
  1084. int bpf_program__nth_fd(struct bpf_program *prog, int n)
  1085. {
  1086. int fd;
  1087. if (n >= prog->instances.nr || n < 0) {
  1088. pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
  1089. n, prog->section_name, prog->instances.nr);
  1090. return -EINVAL;
  1091. }
  1092. fd = prog->instances.fds[n];
  1093. if (fd < 0) {
  1094. pr_warning("%dth instance of program '%s' is invalid\n",
  1095. n, prog->section_name);
  1096. return -ENOENT;
  1097. }
  1098. return fd;
  1099. }
  1100. int bpf_map__get_fd(struct bpf_map *map)
  1101. {
  1102. if (!map)
  1103. return -EINVAL;
  1104. return map->fd;
  1105. }
  1106. int bpf_map__get_def(struct bpf_map *map, struct bpf_map_def *pdef)
  1107. {
  1108. if (!map || !pdef)
  1109. return -EINVAL;
  1110. *pdef = map->def;
  1111. return 0;
  1112. }
  1113. const char *bpf_map__get_name(struct bpf_map *map)
  1114. {
  1115. if (!map)
  1116. return NULL;
  1117. return map->name;
  1118. }
  1119. int bpf_map__set_private(struct bpf_map *map, void *priv,
  1120. bpf_map_clear_priv_t clear_priv)
  1121. {
  1122. if (!map)
  1123. return -EINVAL;
  1124. if (map->priv) {
  1125. if (map->clear_priv)
  1126. map->clear_priv(map, map->priv);
  1127. }
  1128. map->priv = priv;
  1129. map->clear_priv = clear_priv;
  1130. return 0;
  1131. }
  1132. int bpf_map__get_private(struct bpf_map *map, void **ppriv)
  1133. {
  1134. if (!map)
  1135. return -EINVAL;
  1136. if (ppriv)
  1137. *ppriv = map->priv;
  1138. return 0;
  1139. }
  1140. struct bpf_map *
  1141. bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
  1142. {
  1143. size_t idx;
  1144. struct bpf_map *s, *e;
  1145. if (!obj || !obj->maps)
  1146. return NULL;
  1147. s = obj->maps;
  1148. e = obj->maps + obj->nr_maps;
  1149. if (prev == NULL)
  1150. return s;
  1151. if ((prev < s) || (prev >= e)) {
  1152. pr_warning("error in %s: map handler doesn't belong to object\n",
  1153. __func__);
  1154. return NULL;
  1155. }
  1156. idx = (prev - obj->maps) + 1;
  1157. if (idx >= obj->nr_maps)
  1158. return NULL;
  1159. return &obj->maps[idx];
  1160. }
  1161. struct bpf_map *
  1162. bpf_object__get_map_by_name(struct bpf_object *obj, const char *name)
  1163. {
  1164. struct bpf_map *pos;
  1165. bpf_map__for_each(pos, obj) {
  1166. if (pos->name && !strcmp(pos->name, name))
  1167. return pos;
  1168. }
  1169. return NULL;
  1170. }