tls_device.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998
  1. /* Copyright (c) 2018, Mellanox Technologies All rights reserved.
  2. *
  3. * This software is available to you under a choice of one of two
  4. * licenses. You may choose to be licensed under the terms of the GNU
  5. * General Public License (GPL) Version 2, available from the file
  6. * COPYING in the main directory of this source tree, or the
  7. * OpenIB.org BSD license below:
  8. *
  9. * Redistribution and use in source and binary forms, with or
  10. * without modification, are permitted provided that the following
  11. * conditions are met:
  12. *
  13. * - Redistributions of source code must retain the above
  14. * copyright notice, this list of conditions and the following
  15. * disclaimer.
  16. *
  17. * - Redistributions in binary form must reproduce the above
  18. * copyright notice, this list of conditions and the following
  19. * disclaimer in the documentation and/or other materials
  20. * provided with the distribution.
  21. *
  22. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  26. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  27. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  28. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  29. * SOFTWARE.
  30. */
  31. #include <crypto/aead.h>
  32. #include <linux/highmem.h>
  33. #include <linux/module.h>
  34. #include <linux/netdevice.h>
  35. #include <net/dst.h>
  36. #include <net/inet_connection_sock.h>
  37. #include <net/tcp.h>
  38. #include <net/tls.h>
  39. /* device_offload_lock is used to synchronize tls_dev_add
  40. * against NETDEV_DOWN notifications.
  41. */
  42. static DECLARE_RWSEM(device_offload_lock);
  43. static void tls_device_gc_task(struct work_struct *work);
  44. static DECLARE_WORK(tls_device_gc_work, tls_device_gc_task);
  45. static LIST_HEAD(tls_device_gc_list);
  46. static LIST_HEAD(tls_device_list);
  47. static DEFINE_SPINLOCK(tls_device_lock);
  48. static void tls_device_free_ctx(struct tls_context *ctx)
  49. {
  50. if (ctx->tx_conf == TLS_HW)
  51. kfree(tls_offload_ctx_tx(ctx));
  52. if (ctx->rx_conf == TLS_HW)
  53. kfree(tls_offload_ctx_rx(ctx));
  54. kfree(ctx);
  55. }
  56. static void tls_device_gc_task(struct work_struct *work)
  57. {
  58. struct tls_context *ctx, *tmp;
  59. unsigned long flags;
  60. LIST_HEAD(gc_list);
  61. spin_lock_irqsave(&tls_device_lock, flags);
  62. list_splice_init(&tls_device_gc_list, &gc_list);
  63. spin_unlock_irqrestore(&tls_device_lock, flags);
  64. list_for_each_entry_safe(ctx, tmp, &gc_list, list) {
  65. struct net_device *netdev = ctx->netdev;
  66. if (netdev && ctx->tx_conf == TLS_HW) {
  67. netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
  68. TLS_OFFLOAD_CTX_DIR_TX);
  69. dev_put(netdev);
  70. ctx->netdev = NULL;
  71. }
  72. list_del(&ctx->list);
  73. tls_device_free_ctx(ctx);
  74. }
  75. }
  76. static void tls_device_attach(struct tls_context *ctx, struct sock *sk,
  77. struct net_device *netdev)
  78. {
  79. if (sk->sk_destruct != tls_device_sk_destruct) {
  80. refcount_set(&ctx->refcount, 1);
  81. dev_hold(netdev);
  82. ctx->netdev = netdev;
  83. spin_lock_irq(&tls_device_lock);
  84. list_add_tail(&ctx->list, &tls_device_list);
  85. spin_unlock_irq(&tls_device_lock);
  86. ctx->sk_destruct = sk->sk_destruct;
  87. sk->sk_destruct = tls_device_sk_destruct;
  88. }
  89. }
  90. static void tls_device_queue_ctx_destruction(struct tls_context *ctx)
  91. {
  92. unsigned long flags;
  93. spin_lock_irqsave(&tls_device_lock, flags);
  94. list_move_tail(&ctx->list, &tls_device_gc_list);
  95. /* schedule_work inside the spinlock
  96. * to make sure tls_device_down waits for that work.
  97. */
  98. schedule_work(&tls_device_gc_work);
  99. spin_unlock_irqrestore(&tls_device_lock, flags);
  100. }
  101. /* We assume that the socket is already connected */
  102. static struct net_device *get_netdev_for_sock(struct sock *sk)
  103. {
  104. struct dst_entry *dst = sk_dst_get(sk);
  105. struct net_device *netdev = NULL;
  106. if (likely(dst)) {
  107. netdev = dst->dev;
  108. dev_hold(netdev);
  109. }
  110. dst_release(dst);
  111. return netdev;
  112. }
  113. static void destroy_record(struct tls_record_info *record)
  114. {
  115. int nr_frags = record->num_frags;
  116. skb_frag_t *frag;
  117. while (nr_frags-- > 0) {
  118. frag = &record->frags[nr_frags];
  119. __skb_frag_unref(frag);
  120. }
  121. kfree(record);
  122. }
  123. static void delete_all_records(struct tls_offload_context_tx *offload_ctx)
  124. {
  125. struct tls_record_info *info, *temp;
  126. list_for_each_entry_safe(info, temp, &offload_ctx->records_list, list) {
  127. list_del(&info->list);
  128. destroy_record(info);
  129. }
  130. offload_ctx->retransmit_hint = NULL;
  131. }
  132. static void tls_icsk_clean_acked(struct sock *sk, u32 acked_seq)
  133. {
  134. struct tls_context *tls_ctx = tls_get_ctx(sk);
  135. struct tls_record_info *info, *temp;
  136. struct tls_offload_context_tx *ctx;
  137. u64 deleted_records = 0;
  138. unsigned long flags;
  139. if (!tls_ctx)
  140. return;
  141. ctx = tls_offload_ctx_tx(tls_ctx);
  142. spin_lock_irqsave(&ctx->lock, flags);
  143. info = ctx->retransmit_hint;
  144. if (info && !before(acked_seq, info->end_seq)) {
  145. ctx->retransmit_hint = NULL;
  146. list_del(&info->list);
  147. destroy_record(info);
  148. deleted_records++;
  149. }
  150. list_for_each_entry_safe(info, temp, &ctx->records_list, list) {
  151. if (before(acked_seq, info->end_seq))
  152. break;
  153. list_del(&info->list);
  154. destroy_record(info);
  155. deleted_records++;
  156. }
  157. ctx->unacked_record_sn += deleted_records;
  158. spin_unlock_irqrestore(&ctx->lock, flags);
  159. }
  160. /* At this point, there should be no references on this
  161. * socket and no in-flight SKBs associated with this
  162. * socket, so it is safe to free all the resources.
  163. */
  164. void tls_device_sk_destruct(struct sock *sk)
  165. {
  166. struct tls_context *tls_ctx = tls_get_ctx(sk);
  167. struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx);
  168. tls_ctx->sk_destruct(sk);
  169. if (tls_ctx->tx_conf == TLS_HW) {
  170. if (ctx->open_record)
  171. destroy_record(ctx->open_record);
  172. delete_all_records(ctx);
  173. crypto_free_aead(ctx->aead_send);
  174. clean_acked_data_disable(inet_csk(sk));
  175. }
  176. if (refcount_dec_and_test(&tls_ctx->refcount))
  177. tls_device_queue_ctx_destruction(tls_ctx);
  178. }
  179. EXPORT_SYMBOL(tls_device_sk_destruct);
  180. static void tls_append_frag(struct tls_record_info *record,
  181. struct page_frag *pfrag,
  182. int size)
  183. {
  184. skb_frag_t *frag;
  185. frag = &record->frags[record->num_frags - 1];
  186. if (frag->page.p == pfrag->page &&
  187. frag->page_offset + frag->size == pfrag->offset) {
  188. frag->size += size;
  189. } else {
  190. ++frag;
  191. frag->page.p = pfrag->page;
  192. frag->page_offset = pfrag->offset;
  193. frag->size = size;
  194. ++record->num_frags;
  195. get_page(pfrag->page);
  196. }
  197. pfrag->offset += size;
  198. record->len += size;
  199. }
  200. static int tls_push_record(struct sock *sk,
  201. struct tls_context *ctx,
  202. struct tls_offload_context_tx *offload_ctx,
  203. struct tls_record_info *record,
  204. struct page_frag *pfrag,
  205. int flags,
  206. unsigned char record_type)
  207. {
  208. struct tcp_sock *tp = tcp_sk(sk);
  209. struct page_frag dummy_tag_frag;
  210. skb_frag_t *frag;
  211. int i;
  212. /* fill prepend */
  213. frag = &record->frags[0];
  214. tls_fill_prepend(ctx,
  215. skb_frag_address(frag),
  216. record->len - ctx->tx.prepend_size,
  217. record_type);
  218. /* HW doesn't care about the data in the tag, because it fills it. */
  219. dummy_tag_frag.page = skb_frag_page(frag);
  220. dummy_tag_frag.offset = 0;
  221. tls_append_frag(record, &dummy_tag_frag, ctx->tx.tag_size);
  222. record->end_seq = tp->write_seq + record->len;
  223. spin_lock_irq(&offload_ctx->lock);
  224. list_add_tail(&record->list, &offload_ctx->records_list);
  225. spin_unlock_irq(&offload_ctx->lock);
  226. offload_ctx->open_record = NULL;
  227. set_bit(TLS_PENDING_CLOSED_RECORD, &ctx->flags);
  228. tls_advance_record_sn(sk, &ctx->tx);
  229. for (i = 0; i < record->num_frags; i++) {
  230. frag = &record->frags[i];
  231. sg_unmark_end(&offload_ctx->sg_tx_data[i]);
  232. sg_set_page(&offload_ctx->sg_tx_data[i], skb_frag_page(frag),
  233. frag->size, frag->page_offset);
  234. sk_mem_charge(sk, frag->size);
  235. get_page(skb_frag_page(frag));
  236. }
  237. sg_mark_end(&offload_ctx->sg_tx_data[record->num_frags - 1]);
  238. /* all ready, send */
  239. return tls_push_sg(sk, ctx, offload_ctx->sg_tx_data, 0, flags);
  240. }
  241. static int tls_create_new_record(struct tls_offload_context_tx *offload_ctx,
  242. struct page_frag *pfrag,
  243. size_t prepend_size)
  244. {
  245. struct tls_record_info *record;
  246. skb_frag_t *frag;
  247. record = kmalloc(sizeof(*record), GFP_KERNEL);
  248. if (!record)
  249. return -ENOMEM;
  250. frag = &record->frags[0];
  251. __skb_frag_set_page(frag, pfrag->page);
  252. frag->page_offset = pfrag->offset;
  253. skb_frag_size_set(frag, prepend_size);
  254. get_page(pfrag->page);
  255. pfrag->offset += prepend_size;
  256. record->num_frags = 1;
  257. record->len = prepend_size;
  258. offload_ctx->open_record = record;
  259. return 0;
  260. }
  261. static int tls_do_allocation(struct sock *sk,
  262. struct tls_offload_context_tx *offload_ctx,
  263. struct page_frag *pfrag,
  264. size_t prepend_size)
  265. {
  266. int ret;
  267. if (!offload_ctx->open_record) {
  268. if (unlikely(!skb_page_frag_refill(prepend_size, pfrag,
  269. sk->sk_allocation))) {
  270. sk->sk_prot->enter_memory_pressure(sk);
  271. sk_stream_moderate_sndbuf(sk);
  272. return -ENOMEM;
  273. }
  274. ret = tls_create_new_record(offload_ctx, pfrag, prepend_size);
  275. if (ret)
  276. return ret;
  277. if (pfrag->size > pfrag->offset)
  278. return 0;
  279. }
  280. if (!sk_page_frag_refill(sk, pfrag))
  281. return -ENOMEM;
  282. return 0;
  283. }
  284. static int tls_push_data(struct sock *sk,
  285. struct iov_iter *msg_iter,
  286. size_t size, int flags,
  287. unsigned char record_type)
  288. {
  289. struct tls_context *tls_ctx = tls_get_ctx(sk);
  290. struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx);
  291. int tls_push_record_flags = flags | MSG_SENDPAGE_NOTLAST;
  292. int more = flags & (MSG_SENDPAGE_NOTLAST | MSG_MORE);
  293. struct tls_record_info *record = ctx->open_record;
  294. struct page_frag *pfrag;
  295. size_t orig_size = size;
  296. u32 max_open_record_len;
  297. int copy, rc = 0;
  298. bool done = false;
  299. long timeo;
  300. if (flags &
  301. ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_SENDPAGE_NOTLAST))
  302. return -ENOTSUPP;
  303. if (sk->sk_err)
  304. return -sk->sk_err;
  305. timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
  306. rc = tls_complete_pending_work(sk, tls_ctx, flags, &timeo);
  307. if (rc < 0)
  308. return rc;
  309. pfrag = sk_page_frag(sk);
  310. /* TLS_HEADER_SIZE is not counted as part of the TLS record, and
  311. * we need to leave room for an authentication tag.
  312. */
  313. max_open_record_len = TLS_MAX_PAYLOAD_SIZE +
  314. tls_ctx->tx.prepend_size;
  315. do {
  316. rc = tls_do_allocation(sk, ctx, pfrag,
  317. tls_ctx->tx.prepend_size);
  318. if (rc) {
  319. rc = sk_stream_wait_memory(sk, &timeo);
  320. if (!rc)
  321. continue;
  322. record = ctx->open_record;
  323. if (!record)
  324. break;
  325. handle_error:
  326. if (record_type != TLS_RECORD_TYPE_DATA) {
  327. /* avoid sending partial
  328. * record with type !=
  329. * application_data
  330. */
  331. size = orig_size;
  332. destroy_record(record);
  333. ctx->open_record = NULL;
  334. } else if (record->len > tls_ctx->tx.prepend_size) {
  335. goto last_record;
  336. }
  337. break;
  338. }
  339. record = ctx->open_record;
  340. copy = min_t(size_t, size, (pfrag->size - pfrag->offset));
  341. copy = min_t(size_t, copy, (max_open_record_len - record->len));
  342. if (copy_from_iter_nocache(page_address(pfrag->page) +
  343. pfrag->offset,
  344. copy, msg_iter) != copy) {
  345. rc = -EFAULT;
  346. goto handle_error;
  347. }
  348. tls_append_frag(record, pfrag, copy);
  349. size -= copy;
  350. if (!size) {
  351. last_record:
  352. tls_push_record_flags = flags;
  353. if (more) {
  354. tls_ctx->pending_open_record_frags =
  355. !!record->num_frags;
  356. break;
  357. }
  358. done = true;
  359. }
  360. if (done || record->len >= max_open_record_len ||
  361. (record->num_frags >= MAX_SKB_FRAGS - 1)) {
  362. rc = tls_push_record(sk,
  363. tls_ctx,
  364. ctx,
  365. record,
  366. pfrag,
  367. tls_push_record_flags,
  368. record_type);
  369. if (rc < 0)
  370. break;
  371. }
  372. } while (!done);
  373. if (orig_size - size > 0)
  374. rc = orig_size - size;
  375. return rc;
  376. }
  377. int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
  378. {
  379. unsigned char record_type = TLS_RECORD_TYPE_DATA;
  380. int rc;
  381. lock_sock(sk);
  382. if (unlikely(msg->msg_controllen)) {
  383. rc = tls_proccess_cmsg(sk, msg, &record_type);
  384. if (rc)
  385. goto out;
  386. }
  387. rc = tls_push_data(sk, &msg->msg_iter, size,
  388. msg->msg_flags, record_type);
  389. out:
  390. release_sock(sk);
  391. return rc;
  392. }
  393. int tls_device_sendpage(struct sock *sk, struct page *page,
  394. int offset, size_t size, int flags)
  395. {
  396. struct iov_iter msg_iter;
  397. char *kaddr = kmap(page);
  398. struct kvec iov;
  399. int rc;
  400. if (flags & MSG_SENDPAGE_NOTLAST)
  401. flags |= MSG_MORE;
  402. lock_sock(sk);
  403. if (flags & MSG_OOB) {
  404. rc = -ENOTSUPP;
  405. goto out;
  406. }
  407. iov.iov_base = kaddr + offset;
  408. iov.iov_len = size;
  409. iov_iter_kvec(&msg_iter, WRITE, &iov, 1, size);
  410. rc = tls_push_data(sk, &msg_iter, size,
  411. flags, TLS_RECORD_TYPE_DATA);
  412. kunmap(page);
  413. out:
  414. release_sock(sk);
  415. return rc;
  416. }
  417. struct tls_record_info *tls_get_record(struct tls_offload_context_tx *context,
  418. u32 seq, u64 *p_record_sn)
  419. {
  420. u64 record_sn = context->hint_record_sn;
  421. struct tls_record_info *info;
  422. info = context->retransmit_hint;
  423. if (!info ||
  424. before(seq, info->end_seq - info->len)) {
  425. /* if retransmit_hint is irrelevant start
  426. * from the beggining of the list
  427. */
  428. info = list_first_entry(&context->records_list,
  429. struct tls_record_info, list);
  430. record_sn = context->unacked_record_sn;
  431. }
  432. list_for_each_entry_from(info, &context->records_list, list) {
  433. if (before(seq, info->end_seq)) {
  434. if (!context->retransmit_hint ||
  435. after(info->end_seq,
  436. context->retransmit_hint->end_seq)) {
  437. context->hint_record_sn = record_sn;
  438. context->retransmit_hint = info;
  439. }
  440. *p_record_sn = record_sn;
  441. return info;
  442. }
  443. record_sn++;
  444. }
  445. return NULL;
  446. }
  447. EXPORT_SYMBOL(tls_get_record);
  448. static int tls_device_push_pending_record(struct sock *sk, int flags)
  449. {
  450. struct iov_iter msg_iter;
  451. iov_iter_kvec(&msg_iter, WRITE, NULL, 0, 0);
  452. return tls_push_data(sk, &msg_iter, 0, flags, TLS_RECORD_TYPE_DATA);
  453. }
  454. void handle_device_resync(struct sock *sk, u32 seq, u64 rcd_sn)
  455. {
  456. struct tls_context *tls_ctx = tls_get_ctx(sk);
  457. struct net_device *netdev = tls_ctx->netdev;
  458. struct tls_offload_context_rx *rx_ctx;
  459. u32 is_req_pending;
  460. s64 resync_req;
  461. u32 req_seq;
  462. if (tls_ctx->rx_conf != TLS_HW)
  463. return;
  464. rx_ctx = tls_offload_ctx_rx(tls_ctx);
  465. resync_req = atomic64_read(&rx_ctx->resync_req);
  466. req_seq = ntohl(resync_req >> 32) - ((u32)TLS_HEADER_SIZE - 1);
  467. is_req_pending = resync_req;
  468. if (unlikely(is_req_pending) && req_seq == seq &&
  469. atomic64_try_cmpxchg(&rx_ctx->resync_req, &resync_req, 0))
  470. netdev->tlsdev_ops->tls_dev_resync_rx(netdev, sk,
  471. seq + TLS_HEADER_SIZE - 1,
  472. rcd_sn);
  473. }
  474. static int tls_device_reencrypt(struct sock *sk, struct sk_buff *skb)
  475. {
  476. struct strp_msg *rxm = strp_msg(skb);
  477. int err = 0, offset = rxm->offset, copy, nsg;
  478. struct sk_buff *skb_iter, *unused;
  479. struct scatterlist sg[1];
  480. char *orig_buf, *buf;
  481. orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE +
  482. TLS_CIPHER_AES_GCM_128_IV_SIZE, sk->sk_allocation);
  483. if (!orig_buf)
  484. return -ENOMEM;
  485. buf = orig_buf;
  486. nsg = skb_cow_data(skb, 0, &unused);
  487. if (unlikely(nsg < 0)) {
  488. err = nsg;
  489. goto free_buf;
  490. }
  491. sg_init_table(sg, 1);
  492. sg_set_buf(&sg[0], buf,
  493. rxm->full_len + TLS_HEADER_SIZE +
  494. TLS_CIPHER_AES_GCM_128_IV_SIZE);
  495. skb_copy_bits(skb, offset, buf,
  496. TLS_HEADER_SIZE + TLS_CIPHER_AES_GCM_128_IV_SIZE);
  497. /* We are interested only in the decrypted data not the auth */
  498. err = decrypt_skb(sk, skb, sg);
  499. if (err != -EBADMSG)
  500. goto free_buf;
  501. else
  502. err = 0;
  503. copy = min_t(int, skb_pagelen(skb) - offset,
  504. rxm->full_len - TLS_CIPHER_AES_GCM_128_TAG_SIZE);
  505. if (skb->decrypted)
  506. skb_store_bits(skb, offset, buf, copy);
  507. offset += copy;
  508. buf += copy;
  509. skb_walk_frags(skb, skb_iter) {
  510. copy = min_t(int, skb_iter->len,
  511. rxm->full_len - offset + rxm->offset -
  512. TLS_CIPHER_AES_GCM_128_TAG_SIZE);
  513. if (skb_iter->decrypted)
  514. skb_store_bits(skb_iter, offset, buf, copy);
  515. offset += copy;
  516. buf += copy;
  517. }
  518. free_buf:
  519. kfree(orig_buf);
  520. return err;
  521. }
  522. int tls_device_decrypted(struct sock *sk, struct sk_buff *skb)
  523. {
  524. struct tls_context *tls_ctx = tls_get_ctx(sk);
  525. struct tls_offload_context_rx *ctx = tls_offload_ctx_rx(tls_ctx);
  526. int is_decrypted = skb->decrypted;
  527. int is_encrypted = !is_decrypted;
  528. struct sk_buff *skb_iter;
  529. /* Skip if it is already decrypted */
  530. if (ctx->sw.decrypted)
  531. return 0;
  532. /* Check if all the data is decrypted already */
  533. skb_walk_frags(skb, skb_iter) {
  534. is_decrypted &= skb_iter->decrypted;
  535. is_encrypted &= !skb_iter->decrypted;
  536. }
  537. ctx->sw.decrypted |= is_decrypted;
  538. /* Return immedeatly if the record is either entirely plaintext or
  539. * entirely ciphertext. Otherwise handle reencrypt partially decrypted
  540. * record.
  541. */
  542. return (is_encrypted || is_decrypted) ? 0 :
  543. tls_device_reencrypt(sk, skb);
  544. }
  545. int tls_set_device_offload(struct sock *sk, struct tls_context *ctx)
  546. {
  547. u16 nonce_size, tag_size, iv_size, rec_seq_size;
  548. struct tls_record_info *start_marker_record;
  549. struct tls_offload_context_tx *offload_ctx;
  550. struct tls_crypto_info *crypto_info;
  551. struct net_device *netdev;
  552. char *iv, *rec_seq;
  553. struct sk_buff *skb;
  554. int rc = -EINVAL;
  555. __be64 rcd_sn;
  556. if (!ctx)
  557. goto out;
  558. if (ctx->priv_ctx_tx) {
  559. rc = -EEXIST;
  560. goto out;
  561. }
  562. start_marker_record = kmalloc(sizeof(*start_marker_record), GFP_KERNEL);
  563. if (!start_marker_record) {
  564. rc = -ENOMEM;
  565. goto out;
  566. }
  567. offload_ctx = kzalloc(TLS_OFFLOAD_CONTEXT_SIZE_TX, GFP_KERNEL);
  568. if (!offload_ctx) {
  569. rc = -ENOMEM;
  570. goto free_marker_record;
  571. }
  572. crypto_info = &ctx->crypto_send.info;
  573. switch (crypto_info->cipher_type) {
  574. case TLS_CIPHER_AES_GCM_128:
  575. nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
  576. tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
  577. iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
  578. iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
  579. rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
  580. rec_seq =
  581. ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
  582. break;
  583. default:
  584. rc = -EINVAL;
  585. goto free_offload_ctx;
  586. }
  587. ctx->tx.prepend_size = TLS_HEADER_SIZE + nonce_size;
  588. ctx->tx.tag_size = tag_size;
  589. ctx->tx.overhead_size = ctx->tx.prepend_size + ctx->tx.tag_size;
  590. ctx->tx.iv_size = iv_size;
  591. ctx->tx.iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
  592. GFP_KERNEL);
  593. if (!ctx->tx.iv) {
  594. rc = -ENOMEM;
  595. goto free_offload_ctx;
  596. }
  597. memcpy(ctx->tx.iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
  598. ctx->tx.rec_seq_size = rec_seq_size;
  599. ctx->tx.rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL);
  600. if (!ctx->tx.rec_seq) {
  601. rc = -ENOMEM;
  602. goto free_iv;
  603. }
  604. rc = tls_sw_fallback_init(sk, offload_ctx, crypto_info);
  605. if (rc)
  606. goto free_rec_seq;
  607. /* start at rec_seq - 1 to account for the start marker record */
  608. memcpy(&rcd_sn, ctx->tx.rec_seq, sizeof(rcd_sn));
  609. offload_ctx->unacked_record_sn = be64_to_cpu(rcd_sn) - 1;
  610. start_marker_record->end_seq = tcp_sk(sk)->write_seq;
  611. start_marker_record->len = 0;
  612. start_marker_record->num_frags = 0;
  613. INIT_LIST_HEAD(&offload_ctx->records_list);
  614. list_add_tail(&start_marker_record->list, &offload_ctx->records_list);
  615. spin_lock_init(&offload_ctx->lock);
  616. sg_init_table(offload_ctx->sg_tx_data,
  617. ARRAY_SIZE(offload_ctx->sg_tx_data));
  618. clean_acked_data_enable(inet_csk(sk), &tls_icsk_clean_acked);
  619. ctx->push_pending_record = tls_device_push_pending_record;
  620. /* TLS offload is greatly simplified if we don't send
  621. * SKBs where only part of the payload needs to be encrypted.
  622. * So mark the last skb in the write queue as end of record.
  623. */
  624. skb = tcp_write_queue_tail(sk);
  625. if (skb)
  626. TCP_SKB_CB(skb)->eor = 1;
  627. /* We support starting offload on multiple sockets
  628. * concurrently, so we only need a read lock here.
  629. * This lock must precede get_netdev_for_sock to prevent races between
  630. * NETDEV_DOWN and setsockopt.
  631. */
  632. down_read(&device_offload_lock);
  633. netdev = get_netdev_for_sock(sk);
  634. if (!netdev) {
  635. pr_err_ratelimited("%s: netdev not found\n", __func__);
  636. rc = -EINVAL;
  637. goto release_lock;
  638. }
  639. if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
  640. rc = -ENOTSUPP;
  641. goto release_netdev;
  642. }
  643. /* Avoid offloading if the device is down
  644. * We don't want to offload new flows after
  645. * the NETDEV_DOWN event
  646. */
  647. if (!(netdev->flags & IFF_UP)) {
  648. rc = -EINVAL;
  649. goto release_netdev;
  650. }
  651. ctx->priv_ctx_tx = offload_ctx;
  652. rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
  653. &ctx->crypto_send.info,
  654. tcp_sk(sk)->write_seq);
  655. if (rc)
  656. goto release_netdev;
  657. tls_device_attach(ctx, sk, netdev);
  658. /* following this assignment tls_is_sk_tx_device_offloaded
  659. * will return true and the context might be accessed
  660. * by the netdev's xmit function.
  661. */
  662. smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
  663. dev_put(netdev);
  664. up_read(&device_offload_lock);
  665. goto out;
  666. release_netdev:
  667. dev_put(netdev);
  668. release_lock:
  669. up_read(&device_offload_lock);
  670. clean_acked_data_disable(inet_csk(sk));
  671. crypto_free_aead(offload_ctx->aead_send);
  672. free_rec_seq:
  673. kfree(ctx->tx.rec_seq);
  674. free_iv:
  675. kfree(ctx->tx.iv);
  676. free_offload_ctx:
  677. kfree(offload_ctx);
  678. ctx->priv_ctx_tx = NULL;
  679. free_marker_record:
  680. kfree(start_marker_record);
  681. out:
  682. return rc;
  683. }
  684. int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
  685. {
  686. struct tls_offload_context_rx *context;
  687. struct net_device *netdev;
  688. int rc = 0;
  689. /* We support starting offload on multiple sockets
  690. * concurrently, so we only need a read lock here.
  691. * This lock must precede get_netdev_for_sock to prevent races between
  692. * NETDEV_DOWN and setsockopt.
  693. */
  694. down_read(&device_offload_lock);
  695. netdev = get_netdev_for_sock(sk);
  696. if (!netdev) {
  697. pr_err_ratelimited("%s: netdev not found\n", __func__);
  698. rc = -EINVAL;
  699. goto release_lock;
  700. }
  701. if (!(netdev->features & NETIF_F_HW_TLS_RX)) {
  702. pr_err_ratelimited("%s: netdev %s with no TLS offload\n",
  703. __func__, netdev->name);
  704. rc = -ENOTSUPP;
  705. goto release_netdev;
  706. }
  707. /* Avoid offloading if the device is down
  708. * We don't want to offload new flows after
  709. * the NETDEV_DOWN event
  710. */
  711. if (!(netdev->flags & IFF_UP)) {
  712. rc = -EINVAL;
  713. goto release_netdev;
  714. }
  715. context = kzalloc(TLS_OFFLOAD_CONTEXT_SIZE_RX, GFP_KERNEL);
  716. if (!context) {
  717. rc = -ENOMEM;
  718. goto release_netdev;
  719. }
  720. ctx->priv_ctx_rx = context;
  721. rc = tls_set_sw_offload(sk, ctx, 0);
  722. if (rc)
  723. goto release_ctx;
  724. rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_RX,
  725. &ctx->crypto_recv.info,
  726. tcp_sk(sk)->copied_seq);
  727. if (rc) {
  728. pr_err_ratelimited("%s: The netdev has refused to offload this socket\n",
  729. __func__);
  730. goto free_sw_resources;
  731. }
  732. tls_device_attach(ctx, sk, netdev);
  733. goto release_netdev;
  734. free_sw_resources:
  735. tls_sw_free_resources_rx(sk);
  736. release_ctx:
  737. ctx->priv_ctx_rx = NULL;
  738. release_netdev:
  739. dev_put(netdev);
  740. release_lock:
  741. up_read(&device_offload_lock);
  742. return rc;
  743. }
  744. void tls_device_offload_cleanup_rx(struct sock *sk)
  745. {
  746. struct tls_context *tls_ctx = tls_get_ctx(sk);
  747. struct net_device *netdev;
  748. down_read(&device_offload_lock);
  749. netdev = tls_ctx->netdev;
  750. if (!netdev)
  751. goto out;
  752. if (!(netdev->features & NETIF_F_HW_TLS_RX)) {
  753. pr_err_ratelimited("%s: device is missing NETIF_F_HW_TLS_RX cap\n",
  754. __func__);
  755. goto out;
  756. }
  757. netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
  758. TLS_OFFLOAD_CTX_DIR_RX);
  759. if (tls_ctx->tx_conf != TLS_HW) {
  760. dev_put(netdev);
  761. tls_ctx->netdev = NULL;
  762. }
  763. out:
  764. up_read(&device_offload_lock);
  765. kfree(tls_ctx->rx.rec_seq);
  766. kfree(tls_ctx->rx.iv);
  767. tls_sw_release_resources_rx(sk);
  768. }
  769. static int tls_device_down(struct net_device *netdev)
  770. {
  771. struct tls_context *ctx, *tmp;
  772. unsigned long flags;
  773. LIST_HEAD(list);
  774. /* Request a write lock to block new offload attempts */
  775. down_write(&device_offload_lock);
  776. spin_lock_irqsave(&tls_device_lock, flags);
  777. list_for_each_entry_safe(ctx, tmp, &tls_device_list, list) {
  778. if (ctx->netdev != netdev ||
  779. !refcount_inc_not_zero(&ctx->refcount))
  780. continue;
  781. list_move(&ctx->list, &list);
  782. }
  783. spin_unlock_irqrestore(&tls_device_lock, flags);
  784. list_for_each_entry_safe(ctx, tmp, &list, list) {
  785. if (ctx->tx_conf == TLS_HW)
  786. netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
  787. TLS_OFFLOAD_CTX_DIR_TX);
  788. if (ctx->rx_conf == TLS_HW)
  789. netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
  790. TLS_OFFLOAD_CTX_DIR_RX);
  791. ctx->netdev = NULL;
  792. dev_put(netdev);
  793. list_del_init(&ctx->list);
  794. if (refcount_dec_and_test(&ctx->refcount))
  795. tls_device_free_ctx(ctx);
  796. }
  797. up_write(&device_offload_lock);
  798. flush_work(&tls_device_gc_work);
  799. return NOTIFY_DONE;
  800. }
  801. static int tls_dev_event(struct notifier_block *this, unsigned long event,
  802. void *ptr)
  803. {
  804. struct net_device *dev = netdev_notifier_info_to_dev(ptr);
  805. if (!(dev->features & (NETIF_F_HW_TLS_RX | NETIF_F_HW_TLS_TX)))
  806. return NOTIFY_DONE;
  807. switch (event) {
  808. case NETDEV_REGISTER:
  809. case NETDEV_FEAT_CHANGE:
  810. if ((dev->features & NETIF_F_HW_TLS_RX) &&
  811. !dev->tlsdev_ops->tls_dev_resync_rx)
  812. return NOTIFY_BAD;
  813. if (dev->tlsdev_ops &&
  814. dev->tlsdev_ops->tls_dev_add &&
  815. dev->tlsdev_ops->tls_dev_del)
  816. return NOTIFY_DONE;
  817. else
  818. return NOTIFY_BAD;
  819. case NETDEV_DOWN:
  820. return tls_device_down(dev);
  821. }
  822. return NOTIFY_DONE;
  823. }
  824. static struct notifier_block tls_dev_notifier = {
  825. .notifier_call = tls_dev_event,
  826. };
  827. void __init tls_device_init(void)
  828. {
  829. register_netdevice_notifier(&tls_dev_notifier);
  830. }
  831. void __exit tls_device_cleanup(void)
  832. {
  833. unregister_netdevice_notifier(&tls_dev_notifier);
  834. flush_work(&tls_device_gc_work);
  835. }