syscall.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857
  1. /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  2. *
  3. * This program is free software; you can redistribute it and/or
  4. * modify it under the terms of version 2 of the GNU General Public
  5. * License as published by the Free Software Foundation.
  6. *
  7. * This program is distributed in the hope that it will be useful, but
  8. * WITHOUT ANY WARRANTY; without even the implied warranty of
  9. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  10. * General Public License for more details.
  11. */
  12. #include <linux/bpf.h>
  13. #include <linux/syscalls.h>
  14. #include <linux/slab.h>
  15. #include <linux/anon_inodes.h>
  16. #include <linux/file.h>
  17. #include <linux/license.h>
  18. #include <linux/filter.h>
  19. #include <linux/version.h>
  20. DEFINE_PER_CPU(int, bpf_prog_active);
  21. int sysctl_unprivileged_bpf_disabled __read_mostly;
  22. static LIST_HEAD(bpf_map_types);
  23. static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
  24. {
  25. struct bpf_map_type_list *tl;
  26. struct bpf_map *map;
  27. list_for_each_entry(tl, &bpf_map_types, list_node) {
  28. if (tl->type == attr->map_type) {
  29. map = tl->ops->map_alloc(attr);
  30. if (IS_ERR(map))
  31. return map;
  32. map->ops = tl->ops;
  33. map->map_type = attr->map_type;
  34. return map;
  35. }
  36. }
  37. return ERR_PTR(-EINVAL);
  38. }
  39. /* boot time registration of different map implementations */
  40. void bpf_register_map_type(struct bpf_map_type_list *tl)
  41. {
  42. list_add(&tl->list_node, &bpf_map_types);
  43. }
  44. int bpf_map_precharge_memlock(u32 pages)
  45. {
  46. struct user_struct *user = get_current_user();
  47. unsigned long memlock_limit, cur;
  48. memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  49. cur = atomic_long_read(&user->locked_vm);
  50. free_uid(user);
  51. if (cur + pages > memlock_limit)
  52. return -EPERM;
  53. return 0;
  54. }
  55. static int bpf_map_charge_memlock(struct bpf_map *map)
  56. {
  57. struct user_struct *user = get_current_user();
  58. unsigned long memlock_limit;
  59. memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  60. atomic_long_add(map->pages, &user->locked_vm);
  61. if (atomic_long_read(&user->locked_vm) > memlock_limit) {
  62. atomic_long_sub(map->pages, &user->locked_vm);
  63. free_uid(user);
  64. return -EPERM;
  65. }
  66. map->user = user;
  67. return 0;
  68. }
  69. static void bpf_map_uncharge_memlock(struct bpf_map *map)
  70. {
  71. struct user_struct *user = map->user;
  72. atomic_long_sub(map->pages, &user->locked_vm);
  73. free_uid(user);
  74. }
  75. /* called from workqueue */
  76. static void bpf_map_free_deferred(struct work_struct *work)
  77. {
  78. struct bpf_map *map = container_of(work, struct bpf_map, work);
  79. bpf_map_uncharge_memlock(map);
  80. /* implementation dependent freeing */
  81. map->ops->map_free(map);
  82. }
  83. static void bpf_map_put_uref(struct bpf_map *map)
  84. {
  85. if (atomic_dec_and_test(&map->usercnt)) {
  86. if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
  87. bpf_fd_array_map_clear(map);
  88. }
  89. }
  90. /* decrement map refcnt and schedule it for freeing via workqueue
  91. * (unrelying map implementation ops->map_free() might sleep)
  92. */
  93. void bpf_map_put(struct bpf_map *map)
  94. {
  95. if (atomic_dec_and_test(&map->refcnt)) {
  96. INIT_WORK(&map->work, bpf_map_free_deferred);
  97. schedule_work(&map->work);
  98. }
  99. }
  100. void bpf_map_put_with_uref(struct bpf_map *map)
  101. {
  102. bpf_map_put_uref(map);
  103. bpf_map_put(map);
  104. }
  105. static int bpf_map_release(struct inode *inode, struct file *filp)
  106. {
  107. bpf_map_put_with_uref(filp->private_data);
  108. return 0;
  109. }
  110. #ifdef CONFIG_PROC_FS
  111. static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
  112. {
  113. const struct bpf_map *map = filp->private_data;
  114. seq_printf(m,
  115. "map_type:\t%u\n"
  116. "key_size:\t%u\n"
  117. "value_size:\t%u\n"
  118. "max_entries:\t%u\n",
  119. map->map_type,
  120. map->key_size,
  121. map->value_size,
  122. map->max_entries);
  123. }
  124. #endif
  125. static const struct file_operations bpf_map_fops = {
  126. #ifdef CONFIG_PROC_FS
  127. .show_fdinfo = bpf_map_show_fdinfo,
  128. #endif
  129. .release = bpf_map_release,
  130. };
  131. int bpf_map_new_fd(struct bpf_map *map)
  132. {
  133. return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
  134. O_RDWR | O_CLOEXEC);
  135. }
  136. /* helper macro to check that unused fields 'union bpf_attr' are zero */
  137. #define CHECK_ATTR(CMD) \
  138. memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
  139. sizeof(attr->CMD##_LAST_FIELD), 0, \
  140. sizeof(*attr) - \
  141. offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
  142. sizeof(attr->CMD##_LAST_FIELD)) != NULL
  143. #define BPF_MAP_CREATE_LAST_FIELD map_flags
  144. /* called via syscall */
  145. static int map_create(union bpf_attr *attr)
  146. {
  147. struct bpf_map *map;
  148. int err;
  149. err = CHECK_ATTR(BPF_MAP_CREATE);
  150. if (err)
  151. return -EINVAL;
  152. /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
  153. map = find_and_alloc_map(attr);
  154. if (IS_ERR(map))
  155. return PTR_ERR(map);
  156. atomic_set(&map->refcnt, 1);
  157. atomic_set(&map->usercnt, 1);
  158. err = bpf_map_charge_memlock(map);
  159. if (err)
  160. goto free_map;
  161. err = bpf_map_new_fd(map);
  162. if (err < 0)
  163. /* failed to allocate fd */
  164. goto free_map;
  165. return err;
  166. free_map:
  167. map->ops->map_free(map);
  168. return err;
  169. }
  170. /* if error is returned, fd is released.
  171. * On success caller should complete fd access with matching fdput()
  172. */
  173. struct bpf_map *__bpf_map_get(struct fd f)
  174. {
  175. if (!f.file)
  176. return ERR_PTR(-EBADF);
  177. if (f.file->f_op != &bpf_map_fops) {
  178. fdput(f);
  179. return ERR_PTR(-EINVAL);
  180. }
  181. return f.file->private_data;
  182. }
  183. void bpf_map_inc(struct bpf_map *map, bool uref)
  184. {
  185. atomic_inc(&map->refcnt);
  186. if (uref)
  187. atomic_inc(&map->usercnt);
  188. }
  189. struct bpf_map *bpf_map_get_with_uref(u32 ufd)
  190. {
  191. struct fd f = fdget(ufd);
  192. struct bpf_map *map;
  193. map = __bpf_map_get(f);
  194. if (IS_ERR(map))
  195. return map;
  196. bpf_map_inc(map, true);
  197. fdput(f);
  198. return map;
  199. }
  200. /* helper to convert user pointers passed inside __aligned_u64 fields */
  201. static void __user *u64_to_ptr(__u64 val)
  202. {
  203. return (void __user *) (unsigned long) val;
  204. }
  205. int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
  206. {
  207. return -ENOTSUPP;
  208. }
  209. /* last field in 'union bpf_attr' used by this command */
  210. #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
  211. static int map_lookup_elem(union bpf_attr *attr)
  212. {
  213. void __user *ukey = u64_to_ptr(attr->key);
  214. void __user *uvalue = u64_to_ptr(attr->value);
  215. int ufd = attr->map_fd;
  216. struct bpf_map *map;
  217. void *key, *value, *ptr;
  218. u32 value_size;
  219. struct fd f;
  220. int err;
  221. if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
  222. return -EINVAL;
  223. f = fdget(ufd);
  224. map = __bpf_map_get(f);
  225. if (IS_ERR(map))
  226. return PTR_ERR(map);
  227. err = -ENOMEM;
  228. key = kmalloc(map->key_size, GFP_USER);
  229. if (!key)
  230. goto err_put;
  231. err = -EFAULT;
  232. if (copy_from_user(key, ukey, map->key_size) != 0)
  233. goto free_key;
  234. if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
  235. map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
  236. value_size = round_up(map->value_size, 8) * num_possible_cpus();
  237. else
  238. value_size = map->value_size;
  239. err = -ENOMEM;
  240. value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
  241. if (!value)
  242. goto free_key;
  243. if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH) {
  244. err = bpf_percpu_hash_copy(map, key, value);
  245. } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
  246. err = bpf_percpu_array_copy(map, key, value);
  247. } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
  248. err = bpf_stackmap_copy(map, key, value);
  249. } else {
  250. rcu_read_lock();
  251. ptr = map->ops->map_lookup_elem(map, key);
  252. if (ptr)
  253. memcpy(value, ptr, value_size);
  254. rcu_read_unlock();
  255. err = ptr ? 0 : -ENOENT;
  256. }
  257. if (err)
  258. goto free_value;
  259. err = -EFAULT;
  260. if (copy_to_user(uvalue, value, value_size) != 0)
  261. goto free_value;
  262. err = 0;
  263. free_value:
  264. kfree(value);
  265. free_key:
  266. kfree(key);
  267. err_put:
  268. fdput(f);
  269. return err;
  270. }
  271. #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
  272. static int map_update_elem(union bpf_attr *attr)
  273. {
  274. void __user *ukey = u64_to_ptr(attr->key);
  275. void __user *uvalue = u64_to_ptr(attr->value);
  276. int ufd = attr->map_fd;
  277. struct bpf_map *map;
  278. void *key, *value;
  279. u32 value_size;
  280. struct fd f;
  281. int err;
  282. if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
  283. return -EINVAL;
  284. f = fdget(ufd);
  285. map = __bpf_map_get(f);
  286. if (IS_ERR(map))
  287. return PTR_ERR(map);
  288. err = -ENOMEM;
  289. key = kmalloc(map->key_size, GFP_USER);
  290. if (!key)
  291. goto err_put;
  292. err = -EFAULT;
  293. if (copy_from_user(key, ukey, map->key_size) != 0)
  294. goto free_key;
  295. if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
  296. map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
  297. value_size = round_up(map->value_size, 8) * num_possible_cpus();
  298. else
  299. value_size = map->value_size;
  300. err = -ENOMEM;
  301. value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
  302. if (!value)
  303. goto free_key;
  304. err = -EFAULT;
  305. if (copy_from_user(value, uvalue, value_size) != 0)
  306. goto free_value;
  307. /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
  308. * inside bpf map update or delete otherwise deadlocks are possible
  309. */
  310. preempt_disable();
  311. __this_cpu_inc(bpf_prog_active);
  312. if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH) {
  313. err = bpf_percpu_hash_update(map, key, value, attr->flags);
  314. } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
  315. err = bpf_percpu_array_update(map, key, value, attr->flags);
  316. } else {
  317. rcu_read_lock();
  318. err = map->ops->map_update_elem(map, key, value, attr->flags);
  319. rcu_read_unlock();
  320. }
  321. __this_cpu_dec(bpf_prog_active);
  322. preempt_enable();
  323. free_value:
  324. kfree(value);
  325. free_key:
  326. kfree(key);
  327. err_put:
  328. fdput(f);
  329. return err;
  330. }
  331. #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
  332. static int map_delete_elem(union bpf_attr *attr)
  333. {
  334. void __user *ukey = u64_to_ptr(attr->key);
  335. int ufd = attr->map_fd;
  336. struct bpf_map *map;
  337. struct fd f;
  338. void *key;
  339. int err;
  340. if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
  341. return -EINVAL;
  342. f = fdget(ufd);
  343. map = __bpf_map_get(f);
  344. if (IS_ERR(map))
  345. return PTR_ERR(map);
  346. err = -ENOMEM;
  347. key = kmalloc(map->key_size, GFP_USER);
  348. if (!key)
  349. goto err_put;
  350. err = -EFAULT;
  351. if (copy_from_user(key, ukey, map->key_size) != 0)
  352. goto free_key;
  353. preempt_disable();
  354. __this_cpu_inc(bpf_prog_active);
  355. rcu_read_lock();
  356. err = map->ops->map_delete_elem(map, key);
  357. rcu_read_unlock();
  358. __this_cpu_dec(bpf_prog_active);
  359. preempt_enable();
  360. free_key:
  361. kfree(key);
  362. err_put:
  363. fdput(f);
  364. return err;
  365. }
  366. /* last field in 'union bpf_attr' used by this command */
  367. #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
  368. static int map_get_next_key(union bpf_attr *attr)
  369. {
  370. void __user *ukey = u64_to_ptr(attr->key);
  371. void __user *unext_key = u64_to_ptr(attr->next_key);
  372. int ufd = attr->map_fd;
  373. struct bpf_map *map;
  374. void *key, *next_key;
  375. struct fd f;
  376. int err;
  377. if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
  378. return -EINVAL;
  379. f = fdget(ufd);
  380. map = __bpf_map_get(f);
  381. if (IS_ERR(map))
  382. return PTR_ERR(map);
  383. err = -ENOMEM;
  384. key = kmalloc(map->key_size, GFP_USER);
  385. if (!key)
  386. goto err_put;
  387. err = -EFAULT;
  388. if (copy_from_user(key, ukey, map->key_size) != 0)
  389. goto free_key;
  390. err = -ENOMEM;
  391. next_key = kmalloc(map->key_size, GFP_USER);
  392. if (!next_key)
  393. goto free_key;
  394. rcu_read_lock();
  395. err = map->ops->map_get_next_key(map, key, next_key);
  396. rcu_read_unlock();
  397. if (err)
  398. goto free_next_key;
  399. err = -EFAULT;
  400. if (copy_to_user(unext_key, next_key, map->key_size) != 0)
  401. goto free_next_key;
  402. err = 0;
  403. free_next_key:
  404. kfree(next_key);
  405. free_key:
  406. kfree(key);
  407. err_put:
  408. fdput(f);
  409. return err;
  410. }
  411. static LIST_HEAD(bpf_prog_types);
  412. static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
  413. {
  414. struct bpf_prog_type_list *tl;
  415. list_for_each_entry(tl, &bpf_prog_types, list_node) {
  416. if (tl->type == type) {
  417. prog->aux->ops = tl->ops;
  418. prog->type = type;
  419. return 0;
  420. }
  421. }
  422. return -EINVAL;
  423. }
  424. void bpf_register_prog_type(struct bpf_prog_type_list *tl)
  425. {
  426. list_add(&tl->list_node, &bpf_prog_types);
  427. }
  428. /* fixup insn->imm field of bpf_call instructions:
  429. * if (insn->imm == BPF_FUNC_map_lookup_elem)
  430. * insn->imm = bpf_map_lookup_elem - __bpf_call_base;
  431. * else if (insn->imm == BPF_FUNC_map_update_elem)
  432. * insn->imm = bpf_map_update_elem - __bpf_call_base;
  433. * else ...
  434. *
  435. * this function is called after eBPF program passed verification
  436. */
  437. static void fixup_bpf_calls(struct bpf_prog *prog)
  438. {
  439. const struct bpf_func_proto *fn;
  440. int i;
  441. for (i = 0; i < prog->len; i++) {
  442. struct bpf_insn *insn = &prog->insnsi[i];
  443. if (insn->code == (BPF_JMP | BPF_CALL)) {
  444. /* we reach here when program has bpf_call instructions
  445. * and it passed bpf_check(), means that
  446. * ops->get_func_proto must have been supplied, check it
  447. */
  448. BUG_ON(!prog->aux->ops->get_func_proto);
  449. if (insn->imm == BPF_FUNC_get_route_realm)
  450. prog->dst_needed = 1;
  451. if (insn->imm == BPF_FUNC_get_prandom_u32)
  452. bpf_user_rnd_init_once();
  453. if (insn->imm == BPF_FUNC_tail_call) {
  454. /* mark bpf_tail_call as different opcode
  455. * to avoid conditional branch in
  456. * interpeter for every normal call
  457. * and to prevent accidental JITing by
  458. * JIT compiler that doesn't support
  459. * bpf_tail_call yet
  460. */
  461. insn->imm = 0;
  462. insn->code |= BPF_X;
  463. continue;
  464. }
  465. fn = prog->aux->ops->get_func_proto(insn->imm);
  466. /* all functions that have prototype and verifier allowed
  467. * programs to call them, must be real in-kernel functions
  468. */
  469. BUG_ON(!fn->func);
  470. insn->imm = fn->func - __bpf_call_base;
  471. }
  472. }
  473. }
  474. /* drop refcnt on maps used by eBPF program and free auxilary data */
  475. static void free_used_maps(struct bpf_prog_aux *aux)
  476. {
  477. int i;
  478. for (i = 0; i < aux->used_map_cnt; i++)
  479. bpf_map_put(aux->used_maps[i]);
  480. kfree(aux->used_maps);
  481. }
  482. static int bpf_prog_charge_memlock(struct bpf_prog *prog)
  483. {
  484. struct user_struct *user = get_current_user();
  485. unsigned long memlock_limit;
  486. memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  487. atomic_long_add(prog->pages, &user->locked_vm);
  488. if (atomic_long_read(&user->locked_vm) > memlock_limit) {
  489. atomic_long_sub(prog->pages, &user->locked_vm);
  490. free_uid(user);
  491. return -EPERM;
  492. }
  493. prog->aux->user = user;
  494. return 0;
  495. }
  496. static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
  497. {
  498. struct user_struct *user = prog->aux->user;
  499. atomic_long_sub(prog->pages, &user->locked_vm);
  500. free_uid(user);
  501. }
  502. static void __prog_put_common(struct rcu_head *rcu)
  503. {
  504. struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
  505. free_used_maps(aux);
  506. bpf_prog_uncharge_memlock(aux->prog);
  507. bpf_prog_free(aux->prog);
  508. }
  509. /* version of bpf_prog_put() that is called after a grace period */
  510. void bpf_prog_put_rcu(struct bpf_prog *prog)
  511. {
  512. if (atomic_dec_and_test(&prog->aux->refcnt))
  513. call_rcu(&prog->aux->rcu, __prog_put_common);
  514. }
  515. void bpf_prog_put(struct bpf_prog *prog)
  516. {
  517. if (atomic_dec_and_test(&prog->aux->refcnt))
  518. __prog_put_common(&prog->aux->rcu);
  519. }
  520. EXPORT_SYMBOL_GPL(bpf_prog_put);
  521. static int bpf_prog_release(struct inode *inode, struct file *filp)
  522. {
  523. struct bpf_prog *prog = filp->private_data;
  524. bpf_prog_put_rcu(prog);
  525. return 0;
  526. }
  527. static const struct file_operations bpf_prog_fops = {
  528. .release = bpf_prog_release,
  529. };
  530. int bpf_prog_new_fd(struct bpf_prog *prog)
  531. {
  532. return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
  533. O_RDWR | O_CLOEXEC);
  534. }
  535. static struct bpf_prog *__bpf_prog_get(struct fd f)
  536. {
  537. if (!f.file)
  538. return ERR_PTR(-EBADF);
  539. if (f.file->f_op != &bpf_prog_fops) {
  540. fdput(f);
  541. return ERR_PTR(-EINVAL);
  542. }
  543. return f.file->private_data;
  544. }
  545. /* called by sockets/tracing/seccomp before attaching program to an event
  546. * pairs with bpf_prog_put()
  547. */
  548. struct bpf_prog *bpf_prog_get(u32 ufd)
  549. {
  550. struct fd f = fdget(ufd);
  551. struct bpf_prog *prog;
  552. prog = __bpf_prog_get(f);
  553. if (IS_ERR(prog))
  554. return prog;
  555. atomic_inc(&prog->aux->refcnt);
  556. fdput(f);
  557. return prog;
  558. }
  559. EXPORT_SYMBOL_GPL(bpf_prog_get);
  560. /* last field in 'union bpf_attr' used by this command */
  561. #define BPF_PROG_LOAD_LAST_FIELD kern_version
  562. static int bpf_prog_load(union bpf_attr *attr)
  563. {
  564. enum bpf_prog_type type = attr->prog_type;
  565. struct bpf_prog *prog;
  566. int err;
  567. char license[128];
  568. bool is_gpl;
  569. if (CHECK_ATTR(BPF_PROG_LOAD))
  570. return -EINVAL;
  571. /* copy eBPF program license from user space */
  572. if (strncpy_from_user(license, u64_to_ptr(attr->license),
  573. sizeof(license) - 1) < 0)
  574. return -EFAULT;
  575. license[sizeof(license) - 1] = 0;
  576. /* eBPF programs must be GPL compatible to use GPL-ed functions */
  577. is_gpl = license_is_gpl_compatible(license);
  578. if (attr->insn_cnt >= BPF_MAXINSNS)
  579. return -EINVAL;
  580. if (type == BPF_PROG_TYPE_KPROBE &&
  581. attr->kern_version != LINUX_VERSION_CODE)
  582. return -EINVAL;
  583. if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN))
  584. return -EPERM;
  585. /* plain bpf_prog allocation */
  586. prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
  587. if (!prog)
  588. return -ENOMEM;
  589. err = bpf_prog_charge_memlock(prog);
  590. if (err)
  591. goto free_prog_nouncharge;
  592. prog->len = attr->insn_cnt;
  593. err = -EFAULT;
  594. if (copy_from_user(prog->insns, u64_to_ptr(attr->insns),
  595. prog->len * sizeof(struct bpf_insn)) != 0)
  596. goto free_prog;
  597. prog->orig_prog = NULL;
  598. prog->jited = 0;
  599. atomic_set(&prog->aux->refcnt, 1);
  600. prog->gpl_compatible = is_gpl ? 1 : 0;
  601. /* find program type: socket_filter vs tracing_filter */
  602. err = find_prog_type(type, prog);
  603. if (err < 0)
  604. goto free_prog;
  605. /* run eBPF verifier */
  606. err = bpf_check(&prog, attr);
  607. if (err < 0)
  608. goto free_used_maps;
  609. /* fixup BPF_CALL->imm field */
  610. fixup_bpf_calls(prog);
  611. /* eBPF program is ready to be JITed */
  612. err = bpf_prog_select_runtime(prog);
  613. if (err < 0)
  614. goto free_used_maps;
  615. err = bpf_prog_new_fd(prog);
  616. if (err < 0)
  617. /* failed to allocate fd */
  618. goto free_used_maps;
  619. return err;
  620. free_used_maps:
  621. free_used_maps(prog->aux);
  622. free_prog:
  623. bpf_prog_uncharge_memlock(prog);
  624. free_prog_nouncharge:
  625. bpf_prog_free(prog);
  626. return err;
  627. }
  628. #define BPF_OBJ_LAST_FIELD bpf_fd
  629. static int bpf_obj_pin(const union bpf_attr *attr)
  630. {
  631. if (CHECK_ATTR(BPF_OBJ))
  632. return -EINVAL;
  633. return bpf_obj_pin_user(attr->bpf_fd, u64_to_ptr(attr->pathname));
  634. }
  635. static int bpf_obj_get(const union bpf_attr *attr)
  636. {
  637. if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0)
  638. return -EINVAL;
  639. return bpf_obj_get_user(u64_to_ptr(attr->pathname));
  640. }
  641. SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
  642. {
  643. union bpf_attr attr = {};
  644. int err;
  645. if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
  646. return -EPERM;
  647. if (!access_ok(VERIFY_READ, uattr, 1))
  648. return -EFAULT;
  649. if (size > PAGE_SIZE) /* silly large */
  650. return -E2BIG;
  651. /* If we're handed a bigger struct than we know of,
  652. * ensure all the unknown bits are 0 - i.e. new
  653. * user-space does not rely on any kernel feature
  654. * extensions we dont know about yet.
  655. */
  656. if (size > sizeof(attr)) {
  657. unsigned char __user *addr;
  658. unsigned char __user *end;
  659. unsigned char val;
  660. addr = (void __user *)uattr + sizeof(attr);
  661. end = (void __user *)uattr + size;
  662. for (; addr < end; addr++) {
  663. err = get_user(val, addr);
  664. if (err)
  665. return err;
  666. if (val)
  667. return -E2BIG;
  668. }
  669. size = sizeof(attr);
  670. }
  671. /* copy attributes from user space, may be less than sizeof(bpf_attr) */
  672. if (copy_from_user(&attr, uattr, size) != 0)
  673. return -EFAULT;
  674. switch (cmd) {
  675. case BPF_MAP_CREATE:
  676. err = map_create(&attr);
  677. break;
  678. case BPF_MAP_LOOKUP_ELEM:
  679. err = map_lookup_elem(&attr);
  680. break;
  681. case BPF_MAP_UPDATE_ELEM:
  682. err = map_update_elem(&attr);
  683. break;
  684. case BPF_MAP_DELETE_ELEM:
  685. err = map_delete_elem(&attr);
  686. break;
  687. case BPF_MAP_GET_NEXT_KEY:
  688. err = map_get_next_key(&attr);
  689. break;
  690. case BPF_PROG_LOAD:
  691. err = bpf_prog_load(&attr);
  692. break;
  693. case BPF_OBJ_PIN:
  694. err = bpf_obj_pin(&attr);
  695. break;
  696. case BPF_OBJ_GET:
  697. err = bpf_obj_get(&attr);
  698. break;
  699. default:
  700. err = -EINVAL;
  701. break;
  702. }
  703. return err;
  704. }