algif_aead.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821
  1. /*
  2. * algif_aead: User-space interface for AEAD algorithms
  3. *
  4. * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
  5. *
  6. * This file provides the user-space API for AEAD ciphers.
  7. *
  8. * This file is derived from algif_skcipher.c.
  9. *
  10. * This program is free software; you can redistribute it and/or modify it
  11. * under the terms of the GNU General Public License as published by the Free
  12. * Software Foundation; either version 2 of the License, or (at your option)
  13. * any later version.
  14. */
  15. #include <crypto/internal/aead.h>
  16. #include <crypto/scatterwalk.h>
  17. #include <crypto/if_alg.h>
  18. #include <linux/init.h>
  19. #include <linux/list.h>
  20. #include <linux/kernel.h>
  21. #include <linux/mm.h>
  22. #include <linux/module.h>
  23. #include <linux/net.h>
  24. #include <net/sock.h>
  25. struct aead_sg_list {
  26. unsigned int cur;
  27. struct scatterlist sg[ALG_MAX_PAGES];
  28. };
  29. struct aead_async_rsgl {
  30. struct af_alg_sgl sgl;
  31. struct list_head list;
  32. };
  33. struct aead_async_req {
  34. struct scatterlist *tsgl;
  35. struct aead_async_rsgl first_rsgl;
  36. struct list_head list;
  37. struct kiocb *iocb;
  38. unsigned int tsgls;
  39. char iv[];
  40. };
  41. struct aead_ctx {
  42. struct aead_sg_list tsgl;
  43. struct aead_async_rsgl first_rsgl;
  44. struct list_head list;
  45. void *iv;
  46. struct af_alg_completion completion;
  47. unsigned long used;
  48. unsigned int len;
  49. bool more;
  50. bool merge;
  51. bool enc;
  52. size_t aead_assoclen;
  53. struct aead_request aead_req;
  54. };
  55. static inline int aead_sndbuf(struct sock *sk)
  56. {
  57. struct alg_sock *ask = alg_sk(sk);
  58. struct aead_ctx *ctx = ask->private;
  59. return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) -
  60. ctx->used, 0);
  61. }
  62. static inline bool aead_writable(struct sock *sk)
  63. {
  64. return PAGE_SIZE <= aead_sndbuf(sk);
  65. }
  66. static inline bool aead_sufficient_data(struct aead_ctx *ctx)
  67. {
  68. unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
  69. return ctx->used >= ctx->aead_assoclen + as;
  70. }
  71. static void aead_reset_ctx(struct aead_ctx *ctx)
  72. {
  73. struct aead_sg_list *sgl = &ctx->tsgl;
  74. sg_init_table(sgl->sg, ALG_MAX_PAGES);
  75. sgl->cur = 0;
  76. ctx->used = 0;
  77. ctx->more = 0;
  78. ctx->merge = 0;
  79. }
  80. static void aead_put_sgl(struct sock *sk)
  81. {
  82. struct alg_sock *ask = alg_sk(sk);
  83. struct aead_ctx *ctx = ask->private;
  84. struct aead_sg_list *sgl = &ctx->tsgl;
  85. struct scatterlist *sg = sgl->sg;
  86. unsigned int i;
  87. for (i = 0; i < sgl->cur; i++) {
  88. if (!sg_page(sg + i))
  89. continue;
  90. put_page(sg_page(sg + i));
  91. sg_assign_page(sg + i, NULL);
  92. }
  93. aead_reset_ctx(ctx);
  94. }
  95. static void aead_wmem_wakeup(struct sock *sk)
  96. {
  97. struct socket_wq *wq;
  98. if (!aead_writable(sk))
  99. return;
  100. rcu_read_lock();
  101. wq = rcu_dereference(sk->sk_wq);
  102. if (skwq_has_sleeper(wq))
  103. wake_up_interruptible_sync_poll(&wq->wait, POLLIN |
  104. POLLRDNORM |
  105. POLLRDBAND);
  106. sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
  107. rcu_read_unlock();
  108. }
  109. static int aead_wait_for_data(struct sock *sk, unsigned flags)
  110. {
  111. struct alg_sock *ask = alg_sk(sk);
  112. struct aead_ctx *ctx = ask->private;
  113. long timeout;
  114. DEFINE_WAIT(wait);
  115. int err = -ERESTARTSYS;
  116. if (flags & MSG_DONTWAIT)
  117. return -EAGAIN;
  118. sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
  119. for (;;) {
  120. if (signal_pending(current))
  121. break;
  122. prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
  123. timeout = MAX_SCHEDULE_TIMEOUT;
  124. if (sk_wait_event(sk, &timeout, !ctx->more)) {
  125. err = 0;
  126. break;
  127. }
  128. }
  129. finish_wait(sk_sleep(sk), &wait);
  130. sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
  131. return err;
  132. }
  133. static void aead_data_wakeup(struct sock *sk)
  134. {
  135. struct alg_sock *ask = alg_sk(sk);
  136. struct aead_ctx *ctx = ask->private;
  137. struct socket_wq *wq;
  138. if (ctx->more)
  139. return;
  140. if (!ctx->used)
  141. return;
  142. rcu_read_lock();
  143. wq = rcu_dereference(sk->sk_wq);
  144. if (skwq_has_sleeper(wq))
  145. wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
  146. POLLRDNORM |
  147. POLLRDBAND);
  148. sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
  149. rcu_read_unlock();
  150. }
  151. static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
  152. {
  153. struct sock *sk = sock->sk;
  154. struct alg_sock *ask = alg_sk(sk);
  155. struct aead_ctx *ctx = ask->private;
  156. unsigned ivsize =
  157. crypto_aead_ivsize(crypto_aead_reqtfm(&ctx->aead_req));
  158. struct aead_sg_list *sgl = &ctx->tsgl;
  159. struct af_alg_control con = {};
  160. long copied = 0;
  161. bool enc = 0;
  162. bool init = 0;
  163. int err = -EINVAL;
  164. if (msg->msg_controllen) {
  165. err = af_alg_cmsg_send(msg, &con);
  166. if (err)
  167. return err;
  168. init = 1;
  169. switch (con.op) {
  170. case ALG_OP_ENCRYPT:
  171. enc = 1;
  172. break;
  173. case ALG_OP_DECRYPT:
  174. enc = 0;
  175. break;
  176. default:
  177. return -EINVAL;
  178. }
  179. if (con.iv && con.iv->ivlen != ivsize)
  180. return -EINVAL;
  181. }
  182. lock_sock(sk);
  183. if (!ctx->more && ctx->used)
  184. goto unlock;
  185. if (init) {
  186. ctx->enc = enc;
  187. if (con.iv)
  188. memcpy(ctx->iv, con.iv->iv, ivsize);
  189. ctx->aead_assoclen = con.aead_assoclen;
  190. }
  191. while (size) {
  192. size_t len = size;
  193. struct scatterlist *sg = NULL;
  194. /* use the existing memory in an allocated page */
  195. if (ctx->merge) {
  196. sg = sgl->sg + sgl->cur - 1;
  197. len = min_t(unsigned long, len,
  198. PAGE_SIZE - sg->offset - sg->length);
  199. err = memcpy_from_msg(page_address(sg_page(sg)) +
  200. sg->offset + sg->length,
  201. msg, len);
  202. if (err)
  203. goto unlock;
  204. sg->length += len;
  205. ctx->merge = (sg->offset + sg->length) &
  206. (PAGE_SIZE - 1);
  207. ctx->used += len;
  208. copied += len;
  209. size -= len;
  210. continue;
  211. }
  212. if (!aead_writable(sk)) {
  213. /* user space sent too much data */
  214. aead_put_sgl(sk);
  215. err = -EMSGSIZE;
  216. goto unlock;
  217. }
  218. /* allocate a new page */
  219. len = min_t(unsigned long, size, aead_sndbuf(sk));
  220. while (len) {
  221. size_t plen = 0;
  222. if (sgl->cur >= ALG_MAX_PAGES) {
  223. aead_put_sgl(sk);
  224. err = -E2BIG;
  225. goto unlock;
  226. }
  227. sg = sgl->sg + sgl->cur;
  228. plen = min_t(size_t, len, PAGE_SIZE);
  229. sg_assign_page(sg, alloc_page(GFP_KERNEL));
  230. err = -ENOMEM;
  231. if (!sg_page(sg))
  232. goto unlock;
  233. err = memcpy_from_msg(page_address(sg_page(sg)),
  234. msg, plen);
  235. if (err) {
  236. __free_page(sg_page(sg));
  237. sg_assign_page(sg, NULL);
  238. goto unlock;
  239. }
  240. sg->offset = 0;
  241. sg->length = plen;
  242. len -= plen;
  243. ctx->used += plen;
  244. copied += plen;
  245. sgl->cur++;
  246. size -= plen;
  247. ctx->merge = plen & (PAGE_SIZE - 1);
  248. }
  249. }
  250. err = 0;
  251. ctx->more = msg->msg_flags & MSG_MORE;
  252. if (!ctx->more && !aead_sufficient_data(ctx)) {
  253. aead_put_sgl(sk);
  254. err = -EMSGSIZE;
  255. }
  256. unlock:
  257. aead_data_wakeup(sk);
  258. release_sock(sk);
  259. return err ?: copied;
  260. }
  261. static ssize_t aead_sendpage(struct socket *sock, struct page *page,
  262. int offset, size_t size, int flags)
  263. {
  264. struct sock *sk = sock->sk;
  265. struct alg_sock *ask = alg_sk(sk);
  266. struct aead_ctx *ctx = ask->private;
  267. struct aead_sg_list *sgl = &ctx->tsgl;
  268. int err = -EINVAL;
  269. if (flags & MSG_SENDPAGE_NOTLAST)
  270. flags |= MSG_MORE;
  271. if (sgl->cur >= ALG_MAX_PAGES)
  272. return -E2BIG;
  273. lock_sock(sk);
  274. if (!ctx->more && ctx->used)
  275. goto unlock;
  276. if (!size)
  277. goto done;
  278. if (!aead_writable(sk)) {
  279. /* user space sent too much data */
  280. aead_put_sgl(sk);
  281. err = -EMSGSIZE;
  282. goto unlock;
  283. }
  284. ctx->merge = 0;
  285. get_page(page);
  286. sg_set_page(sgl->sg + sgl->cur, page, size, offset);
  287. sgl->cur++;
  288. ctx->used += size;
  289. err = 0;
  290. done:
  291. ctx->more = flags & MSG_MORE;
  292. if (!ctx->more && !aead_sufficient_data(ctx)) {
  293. aead_put_sgl(sk);
  294. err = -EMSGSIZE;
  295. }
  296. unlock:
  297. aead_data_wakeup(sk);
  298. release_sock(sk);
  299. return err ?: size;
  300. }
  301. #define GET_ASYM_REQ(req, tfm) (struct aead_async_req *) \
  302. ((char *)req + sizeof(struct aead_request) + \
  303. crypto_aead_reqsize(tfm))
  304. #define GET_REQ_SIZE(tfm) sizeof(struct aead_async_req) + \
  305. crypto_aead_reqsize(tfm) + crypto_aead_ivsize(tfm) + \
  306. sizeof(struct aead_request)
  307. static void aead_async_cb(struct crypto_async_request *_req, int err)
  308. {
  309. struct sock *sk = _req->data;
  310. struct alg_sock *ask = alg_sk(sk);
  311. struct aead_ctx *ctx = ask->private;
  312. struct crypto_aead *tfm = crypto_aead_reqtfm(&ctx->aead_req);
  313. struct aead_request *req = aead_request_cast(_req);
  314. struct aead_async_req *areq = GET_ASYM_REQ(req, tfm);
  315. struct scatterlist *sg = areq->tsgl;
  316. struct aead_async_rsgl *rsgl;
  317. struct kiocb *iocb = areq->iocb;
  318. unsigned int i, reqlen = GET_REQ_SIZE(tfm);
  319. list_for_each_entry(rsgl, &areq->list, list) {
  320. af_alg_free_sg(&rsgl->sgl);
  321. if (rsgl != &areq->first_rsgl)
  322. sock_kfree_s(sk, rsgl, sizeof(*rsgl));
  323. }
  324. for (i = 0; i < areq->tsgls; i++)
  325. put_page(sg_page(sg + i));
  326. sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls);
  327. sock_kfree_s(sk, req, reqlen);
  328. __sock_put(sk);
  329. iocb->ki_complete(iocb, err, err);
  330. }
  331. static int aead_recvmsg_async(struct socket *sock, struct msghdr *msg,
  332. int flags)
  333. {
  334. struct sock *sk = sock->sk;
  335. struct alg_sock *ask = alg_sk(sk);
  336. struct aead_ctx *ctx = ask->private;
  337. struct crypto_aead *tfm = crypto_aead_reqtfm(&ctx->aead_req);
  338. struct aead_async_req *areq;
  339. struct aead_request *req = NULL;
  340. struct aead_sg_list *sgl = &ctx->tsgl;
  341. struct aead_async_rsgl *last_rsgl = NULL, *rsgl;
  342. unsigned int as = crypto_aead_authsize(tfm);
  343. unsigned int i, reqlen = GET_REQ_SIZE(tfm);
  344. int err = -ENOMEM;
  345. unsigned long used;
  346. size_t outlen;
  347. size_t usedpages = 0;
  348. lock_sock(sk);
  349. if (ctx->more) {
  350. err = aead_wait_for_data(sk, flags);
  351. if (err)
  352. goto unlock;
  353. }
  354. used = ctx->used;
  355. outlen = used;
  356. if (!aead_sufficient_data(ctx))
  357. goto unlock;
  358. req = sock_kmalloc(sk, reqlen, GFP_KERNEL);
  359. if (unlikely(!req))
  360. goto unlock;
  361. areq = GET_ASYM_REQ(req, tfm);
  362. memset(&areq->first_rsgl, '\0', sizeof(areq->first_rsgl));
  363. INIT_LIST_HEAD(&areq->list);
  364. areq->iocb = msg->msg_iocb;
  365. memcpy(areq->iv, ctx->iv, crypto_aead_ivsize(tfm));
  366. aead_request_set_tfm(req, tfm);
  367. aead_request_set_ad(req, ctx->aead_assoclen);
  368. aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
  369. aead_async_cb, sk);
  370. used -= ctx->aead_assoclen + (ctx->enc ? as : 0);
  371. /* take over all tx sgls from ctx */
  372. areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * sgl->cur,
  373. GFP_KERNEL);
  374. if (unlikely(!areq->tsgl))
  375. goto free;
  376. sg_init_table(areq->tsgl, sgl->cur);
  377. for (i = 0; i < sgl->cur; i++)
  378. sg_set_page(&areq->tsgl[i], sg_page(&sgl->sg[i]),
  379. sgl->sg[i].length, sgl->sg[i].offset);
  380. areq->tsgls = sgl->cur;
  381. /* create rx sgls */
  382. while (iov_iter_count(&msg->msg_iter)) {
  383. size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter),
  384. (outlen - usedpages));
  385. if (list_empty(&areq->list)) {
  386. rsgl = &areq->first_rsgl;
  387. } else {
  388. rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
  389. if (unlikely(!rsgl)) {
  390. err = -ENOMEM;
  391. goto free;
  392. }
  393. }
  394. rsgl->sgl.npages = 0;
  395. list_add_tail(&rsgl->list, &areq->list);
  396. /* make one iovec available as scatterlist */
  397. err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
  398. if (err < 0)
  399. goto free;
  400. usedpages += err;
  401. /* chain the new scatterlist with previous one */
  402. if (last_rsgl)
  403. af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
  404. last_rsgl = rsgl;
  405. /* we do not need more iovecs as we have sufficient memory */
  406. if (outlen <= usedpages)
  407. break;
  408. iov_iter_advance(&msg->msg_iter, err);
  409. }
  410. err = -EINVAL;
  411. /* ensure output buffer is sufficiently large */
  412. if (usedpages < outlen)
  413. goto free;
  414. aead_request_set_crypt(req, areq->tsgl, areq->first_rsgl.sgl.sg, used,
  415. areq->iv);
  416. err = ctx->enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
  417. if (err) {
  418. if (err == -EINPROGRESS) {
  419. sock_hold(sk);
  420. err = -EIOCBQUEUED;
  421. aead_reset_ctx(ctx);
  422. goto unlock;
  423. } else if (err == -EBADMSG) {
  424. aead_put_sgl(sk);
  425. }
  426. goto free;
  427. }
  428. aead_put_sgl(sk);
  429. free:
  430. list_for_each_entry(rsgl, &areq->list, list) {
  431. af_alg_free_sg(&rsgl->sgl);
  432. if (rsgl != &areq->first_rsgl)
  433. sock_kfree_s(sk, rsgl, sizeof(*rsgl));
  434. }
  435. if (areq->tsgl)
  436. sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls);
  437. if (req)
  438. sock_kfree_s(sk, req, reqlen);
  439. unlock:
  440. aead_wmem_wakeup(sk);
  441. release_sock(sk);
  442. return err ? err : outlen;
  443. }
  444. static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
  445. {
  446. struct sock *sk = sock->sk;
  447. struct alg_sock *ask = alg_sk(sk);
  448. struct aead_ctx *ctx = ask->private;
  449. unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
  450. struct aead_sg_list *sgl = &ctx->tsgl;
  451. struct aead_async_rsgl *last_rsgl = NULL;
  452. struct aead_async_rsgl *rsgl, *tmp;
  453. int err = -EINVAL;
  454. unsigned long used = 0;
  455. size_t outlen = 0;
  456. size_t usedpages = 0;
  457. lock_sock(sk);
  458. /*
  459. * AEAD memory structure: For encryption, the tag is appended to the
  460. * ciphertext which implies that the memory allocated for the ciphertext
  461. * must be increased by the tag length. For decryption, the tag
  462. * is expected to be concatenated to the ciphertext. The plaintext
  463. * therefore has a memory size of the ciphertext minus the tag length.
  464. *
  465. * The memory structure for cipher operation has the following
  466. * structure:
  467. * AEAD encryption input: assoc data || plaintext
  468. * AEAD encryption output: cipherntext || auth tag
  469. * AEAD decryption input: assoc data || ciphertext || auth tag
  470. * AEAD decryption output: plaintext
  471. */
  472. if (ctx->more) {
  473. err = aead_wait_for_data(sk, flags);
  474. if (err)
  475. goto unlock;
  476. }
  477. used = ctx->used;
  478. /*
  479. * Make sure sufficient data is present -- note, the same check is
  480. * is also present in sendmsg/sendpage. The checks in sendpage/sendmsg
  481. * shall provide an information to the data sender that something is
  482. * wrong, but they are irrelevant to maintain the kernel integrity.
  483. * We need this check here too in case user space decides to not honor
  484. * the error message in sendmsg/sendpage and still call recvmsg. This
  485. * check here protects the kernel integrity.
  486. */
  487. if (!aead_sufficient_data(ctx))
  488. goto unlock;
  489. outlen = used;
  490. /*
  491. * The cipher operation input data is reduced by the associated data
  492. * length as this data is processed separately later on.
  493. */
  494. used -= ctx->aead_assoclen + (ctx->enc ? as : 0);
  495. /* convert iovecs of output buffers into scatterlists */
  496. while (iov_iter_count(&msg->msg_iter)) {
  497. size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter),
  498. (outlen - usedpages));
  499. if (list_empty(&ctx->list)) {
  500. rsgl = &ctx->first_rsgl;
  501. } else {
  502. rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
  503. if (unlikely(!rsgl)) {
  504. err = -ENOMEM;
  505. goto unlock;
  506. }
  507. }
  508. rsgl->sgl.npages = 0;
  509. list_add_tail(&rsgl->list, &ctx->list);
  510. /* make one iovec available as scatterlist */
  511. err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
  512. if (err < 0)
  513. goto unlock;
  514. usedpages += err;
  515. /* chain the new scatterlist with previous one */
  516. if (last_rsgl)
  517. af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
  518. last_rsgl = rsgl;
  519. /* we do not need more iovecs as we have sufficient memory */
  520. if (outlen <= usedpages)
  521. break;
  522. iov_iter_advance(&msg->msg_iter, err);
  523. }
  524. err = -EINVAL;
  525. /* ensure output buffer is sufficiently large */
  526. if (usedpages < outlen)
  527. goto unlock;
  528. sg_mark_end(sgl->sg + sgl->cur - 1);
  529. aead_request_set_crypt(&ctx->aead_req, sgl->sg, ctx->first_rsgl.sgl.sg,
  530. used, ctx->iv);
  531. aead_request_set_ad(&ctx->aead_req, ctx->aead_assoclen);
  532. err = af_alg_wait_for_completion(ctx->enc ?
  533. crypto_aead_encrypt(&ctx->aead_req) :
  534. crypto_aead_decrypt(&ctx->aead_req),
  535. &ctx->completion);
  536. if (err) {
  537. /* EBADMSG implies a valid cipher operation took place */
  538. if (err == -EBADMSG)
  539. aead_put_sgl(sk);
  540. goto unlock;
  541. }
  542. aead_put_sgl(sk);
  543. err = 0;
  544. unlock:
  545. list_for_each_entry_safe(rsgl, tmp, &ctx->list, list) {
  546. af_alg_free_sg(&rsgl->sgl);
  547. if (rsgl != &ctx->first_rsgl)
  548. sock_kfree_s(sk, rsgl, sizeof(*rsgl));
  549. list_del(&rsgl->list);
  550. }
  551. INIT_LIST_HEAD(&ctx->list);
  552. aead_wmem_wakeup(sk);
  553. release_sock(sk);
  554. return err ? err : outlen;
  555. }
  556. static int aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored,
  557. int flags)
  558. {
  559. return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
  560. aead_recvmsg_async(sock, msg, flags) :
  561. aead_recvmsg_sync(sock, msg, flags);
  562. }
  563. static unsigned int aead_poll(struct file *file, struct socket *sock,
  564. poll_table *wait)
  565. {
  566. struct sock *sk = sock->sk;
  567. struct alg_sock *ask = alg_sk(sk);
  568. struct aead_ctx *ctx = ask->private;
  569. unsigned int mask;
  570. sock_poll_wait(file, sk_sleep(sk), wait);
  571. mask = 0;
  572. if (!ctx->more)
  573. mask |= POLLIN | POLLRDNORM;
  574. if (aead_writable(sk))
  575. mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
  576. return mask;
  577. }
  578. static struct proto_ops algif_aead_ops = {
  579. .family = PF_ALG,
  580. .connect = sock_no_connect,
  581. .socketpair = sock_no_socketpair,
  582. .getname = sock_no_getname,
  583. .ioctl = sock_no_ioctl,
  584. .listen = sock_no_listen,
  585. .shutdown = sock_no_shutdown,
  586. .getsockopt = sock_no_getsockopt,
  587. .mmap = sock_no_mmap,
  588. .bind = sock_no_bind,
  589. .accept = sock_no_accept,
  590. .setsockopt = sock_no_setsockopt,
  591. .release = af_alg_release,
  592. .sendmsg = aead_sendmsg,
  593. .sendpage = aead_sendpage,
  594. .recvmsg = aead_recvmsg,
  595. .poll = aead_poll,
  596. };
  597. static void *aead_bind(const char *name, u32 type, u32 mask)
  598. {
  599. return crypto_alloc_aead(name, type, mask);
  600. }
  601. static void aead_release(void *private)
  602. {
  603. crypto_free_aead(private);
  604. }
  605. static int aead_setauthsize(void *private, unsigned int authsize)
  606. {
  607. return crypto_aead_setauthsize(private, authsize);
  608. }
  609. static int aead_setkey(void *private, const u8 *key, unsigned int keylen)
  610. {
  611. return crypto_aead_setkey(private, key, keylen);
  612. }
  613. static void aead_sock_destruct(struct sock *sk)
  614. {
  615. struct alg_sock *ask = alg_sk(sk);
  616. struct aead_ctx *ctx = ask->private;
  617. unsigned int ivlen = crypto_aead_ivsize(
  618. crypto_aead_reqtfm(&ctx->aead_req));
  619. WARN_ON(atomic_read(&sk->sk_refcnt) != 0);
  620. aead_put_sgl(sk);
  621. sock_kzfree_s(sk, ctx->iv, ivlen);
  622. sock_kfree_s(sk, ctx, ctx->len);
  623. af_alg_release_parent(sk);
  624. }
  625. static int aead_accept_parent(void *private, struct sock *sk)
  626. {
  627. struct aead_ctx *ctx;
  628. struct alg_sock *ask = alg_sk(sk);
  629. unsigned int len = sizeof(*ctx) + crypto_aead_reqsize(private);
  630. unsigned int ivlen = crypto_aead_ivsize(private);
  631. ctx = sock_kmalloc(sk, len, GFP_KERNEL);
  632. if (!ctx)
  633. return -ENOMEM;
  634. memset(ctx, 0, len);
  635. ctx->iv = sock_kmalloc(sk, ivlen, GFP_KERNEL);
  636. if (!ctx->iv) {
  637. sock_kfree_s(sk, ctx, len);
  638. return -ENOMEM;
  639. }
  640. memset(ctx->iv, 0, ivlen);
  641. ctx->len = len;
  642. ctx->used = 0;
  643. ctx->more = 0;
  644. ctx->merge = 0;
  645. ctx->enc = 0;
  646. ctx->tsgl.cur = 0;
  647. ctx->aead_assoclen = 0;
  648. af_alg_init_completion(&ctx->completion);
  649. sg_init_table(ctx->tsgl.sg, ALG_MAX_PAGES);
  650. INIT_LIST_HEAD(&ctx->list);
  651. ask->private = ctx;
  652. aead_request_set_tfm(&ctx->aead_req, private);
  653. aead_request_set_callback(&ctx->aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
  654. af_alg_complete, &ctx->completion);
  655. sk->sk_destruct = aead_sock_destruct;
  656. return 0;
  657. }
  658. static const struct af_alg_type algif_type_aead = {
  659. .bind = aead_bind,
  660. .release = aead_release,
  661. .setkey = aead_setkey,
  662. .setauthsize = aead_setauthsize,
  663. .accept = aead_accept_parent,
  664. .ops = &algif_aead_ops,
  665. .name = "aead",
  666. .owner = THIS_MODULE
  667. };
  668. static int __init algif_aead_init(void)
  669. {
  670. return af_alg_register_type(&algif_type_aead);
  671. }
  672. static void __exit algif_aead_exit(void)
  673. {
  674. int err = af_alg_unregister_type(&algif_type_aead);
  675. BUG_ON(err);
  676. }
  677. module_init(algif_aead_init);
  678. module_exit(algif_aead_exit);
  679. MODULE_LICENSE("GPL");
  680. MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
  681. MODULE_DESCRIPTION("AEAD kernel crypto API user space interface");