virtio_crypto_core.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542
  1. /* Driver for Virtio crypto device.
  2. *
  3. * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, see <http://www.gnu.org/licenses/>.
  17. */
  18. #include <linux/err.h>
  19. #include <linux/module.h>
  20. #include <linux/virtio_config.h>
  21. #include <linux/cpu.h>
  22. #include <uapi/linux/virtio_crypto.h>
  23. #include "virtio_crypto_common.h"
  24. void
  25. virtcrypto_clear_request(struct virtio_crypto_request *vc_req)
  26. {
  27. if (vc_req) {
  28. kzfree(vc_req->req_data);
  29. kfree(vc_req->sgs);
  30. }
  31. }
  32. static void virtcrypto_dataq_callback(struct virtqueue *vq)
  33. {
  34. struct virtio_crypto *vcrypto = vq->vdev->priv;
  35. struct virtio_crypto_request *vc_req;
  36. unsigned long flags;
  37. unsigned int len;
  38. unsigned int qid = vq->index;
  39. spin_lock_irqsave(&vcrypto->data_vq[qid].lock, flags);
  40. do {
  41. virtqueue_disable_cb(vq);
  42. while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) {
  43. spin_unlock_irqrestore(
  44. &vcrypto->data_vq[qid].lock, flags);
  45. if (vc_req->alg_cb)
  46. vc_req->alg_cb(vc_req, len);
  47. spin_lock_irqsave(
  48. &vcrypto->data_vq[qid].lock, flags);
  49. }
  50. } while (!virtqueue_enable_cb(vq));
  51. spin_unlock_irqrestore(&vcrypto->data_vq[qid].lock, flags);
  52. }
  53. static int virtcrypto_find_vqs(struct virtio_crypto *vi)
  54. {
  55. vq_callback_t **callbacks;
  56. struct virtqueue **vqs;
  57. int ret = -ENOMEM;
  58. int i, total_vqs;
  59. const char **names;
  60. struct device *dev = &vi->vdev->dev;
  61. /*
  62. * We expect 1 data virtqueue, followed by
  63. * possible N-1 data queues used in multiqueue mode,
  64. * followed by control vq.
  65. */
  66. total_vqs = vi->max_data_queues + 1;
  67. /* Allocate space for find_vqs parameters */
  68. vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
  69. if (!vqs)
  70. goto err_vq;
  71. callbacks = kcalloc(total_vqs, sizeof(*callbacks), GFP_KERNEL);
  72. if (!callbacks)
  73. goto err_callback;
  74. names = kcalloc(total_vqs, sizeof(*names), GFP_KERNEL);
  75. if (!names)
  76. goto err_names;
  77. /* Parameters for control virtqueue */
  78. callbacks[total_vqs - 1] = NULL;
  79. names[total_vqs - 1] = "controlq";
  80. /* Allocate/initialize parameters for data virtqueues */
  81. for (i = 0; i < vi->max_data_queues; i++) {
  82. callbacks[i] = virtcrypto_dataq_callback;
  83. snprintf(vi->data_vq[i].name, sizeof(vi->data_vq[i].name),
  84. "dataq.%d", i);
  85. names[i] = vi->data_vq[i].name;
  86. }
  87. ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, callbacks, names, NULL);
  88. if (ret)
  89. goto err_find;
  90. vi->ctrl_vq = vqs[total_vqs - 1];
  91. for (i = 0; i < vi->max_data_queues; i++) {
  92. spin_lock_init(&vi->data_vq[i].lock);
  93. vi->data_vq[i].vq = vqs[i];
  94. /* Initialize crypto engine */
  95. vi->data_vq[i].engine = crypto_engine_alloc_init(dev, 1);
  96. if (!vi->data_vq[i].engine) {
  97. ret = -ENOMEM;
  98. goto err_engine;
  99. }
  100. }
  101. kfree(names);
  102. kfree(callbacks);
  103. kfree(vqs);
  104. return 0;
  105. err_engine:
  106. err_find:
  107. kfree(names);
  108. err_names:
  109. kfree(callbacks);
  110. err_callback:
  111. kfree(vqs);
  112. err_vq:
  113. return ret;
  114. }
  115. static int virtcrypto_alloc_queues(struct virtio_crypto *vi)
  116. {
  117. vi->data_vq = kcalloc(vi->max_data_queues, sizeof(*vi->data_vq),
  118. GFP_KERNEL);
  119. if (!vi->data_vq)
  120. return -ENOMEM;
  121. return 0;
  122. }
  123. static void virtcrypto_clean_affinity(struct virtio_crypto *vi, long hcpu)
  124. {
  125. int i;
  126. if (vi->affinity_hint_set) {
  127. for (i = 0; i < vi->max_data_queues; i++)
  128. virtqueue_set_affinity(vi->data_vq[i].vq, NULL);
  129. vi->affinity_hint_set = false;
  130. }
  131. }
  132. static void virtcrypto_set_affinity(struct virtio_crypto *vcrypto)
  133. {
  134. int i = 0;
  135. int cpu;
  136. /*
  137. * In single queue mode, we don't set the cpu affinity.
  138. */
  139. if (vcrypto->curr_queue == 1 || vcrypto->max_data_queues == 1) {
  140. virtcrypto_clean_affinity(vcrypto, -1);
  141. return;
  142. }
  143. /*
  144. * In multiqueue mode, we let the queue to be private to one cpu
  145. * by setting the affinity hint to eliminate the contention.
  146. *
  147. * TODO: adds cpu hotplug support by register cpu notifier.
  148. *
  149. */
  150. for_each_online_cpu(cpu) {
  151. virtqueue_set_affinity(vcrypto->data_vq[i].vq, cpumask_of(cpu));
  152. if (++i >= vcrypto->max_data_queues)
  153. break;
  154. }
  155. vcrypto->affinity_hint_set = true;
  156. }
  157. static void virtcrypto_free_queues(struct virtio_crypto *vi)
  158. {
  159. kfree(vi->data_vq);
  160. }
  161. static int virtcrypto_init_vqs(struct virtio_crypto *vi)
  162. {
  163. int ret;
  164. /* Allocate send & receive queues */
  165. ret = virtcrypto_alloc_queues(vi);
  166. if (ret)
  167. goto err;
  168. ret = virtcrypto_find_vqs(vi);
  169. if (ret)
  170. goto err_free;
  171. get_online_cpus();
  172. virtcrypto_set_affinity(vi);
  173. put_online_cpus();
  174. return 0;
  175. err_free:
  176. virtcrypto_free_queues(vi);
  177. err:
  178. return ret;
  179. }
  180. static int virtcrypto_update_status(struct virtio_crypto *vcrypto)
  181. {
  182. u32 status;
  183. int err;
  184. virtio_cread(vcrypto->vdev,
  185. struct virtio_crypto_config, status, &status);
  186. /*
  187. * Unknown status bits would be a host error and the driver
  188. * should consider the device to be broken.
  189. */
  190. if (status & (~VIRTIO_CRYPTO_S_HW_READY)) {
  191. dev_warn(&vcrypto->vdev->dev,
  192. "Unknown status bits: 0x%x\n", status);
  193. virtio_break_device(vcrypto->vdev);
  194. return -EPERM;
  195. }
  196. if (vcrypto->status == status)
  197. return 0;
  198. vcrypto->status = status;
  199. if (vcrypto->status & VIRTIO_CRYPTO_S_HW_READY) {
  200. err = virtcrypto_dev_start(vcrypto);
  201. if (err) {
  202. dev_err(&vcrypto->vdev->dev,
  203. "Failed to start virtio crypto device.\n");
  204. return -EPERM;
  205. }
  206. dev_info(&vcrypto->vdev->dev, "Accelerator device is ready\n");
  207. } else {
  208. virtcrypto_dev_stop(vcrypto);
  209. dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n");
  210. }
  211. return 0;
  212. }
  213. static int virtcrypto_start_crypto_engines(struct virtio_crypto *vcrypto)
  214. {
  215. int32_t i;
  216. int ret;
  217. for (i = 0; i < vcrypto->max_data_queues; i++) {
  218. if (vcrypto->data_vq[i].engine) {
  219. ret = crypto_engine_start(vcrypto->data_vq[i].engine);
  220. if (ret)
  221. goto err;
  222. }
  223. }
  224. return 0;
  225. err:
  226. while (--i >= 0)
  227. if (vcrypto->data_vq[i].engine)
  228. crypto_engine_exit(vcrypto->data_vq[i].engine);
  229. return ret;
  230. }
  231. static void virtcrypto_clear_crypto_engines(struct virtio_crypto *vcrypto)
  232. {
  233. u32 i;
  234. for (i = 0; i < vcrypto->max_data_queues; i++)
  235. if (vcrypto->data_vq[i].engine)
  236. crypto_engine_exit(vcrypto->data_vq[i].engine);
  237. }
  238. static void virtcrypto_del_vqs(struct virtio_crypto *vcrypto)
  239. {
  240. struct virtio_device *vdev = vcrypto->vdev;
  241. virtcrypto_clean_affinity(vcrypto, -1);
  242. vdev->config->del_vqs(vdev);
  243. virtcrypto_free_queues(vcrypto);
  244. }
  245. static int virtcrypto_probe(struct virtio_device *vdev)
  246. {
  247. int err = -EFAULT;
  248. struct virtio_crypto *vcrypto;
  249. u32 max_data_queues = 0, max_cipher_key_len = 0;
  250. u32 max_auth_key_len = 0;
  251. u64 max_size = 0;
  252. u32 cipher_algo_l = 0;
  253. u32 cipher_algo_h = 0;
  254. u32 hash_algo = 0;
  255. u32 mac_algo_l = 0;
  256. u32 mac_algo_h = 0;
  257. u32 aead_algo = 0;
  258. u32 crypto_services = 0;
  259. if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
  260. return -ENODEV;
  261. if (!vdev->config->get) {
  262. dev_err(&vdev->dev, "%s failure: config access disabled\n",
  263. __func__);
  264. return -EINVAL;
  265. }
  266. if (num_possible_nodes() > 1 && dev_to_node(&vdev->dev) < 0) {
  267. /*
  268. * If the accelerator is connected to a node with no memory
  269. * there is no point in using the accelerator since the remote
  270. * memory transaction will be very slow.
  271. */
  272. dev_err(&vdev->dev, "Invalid NUMA configuration.\n");
  273. return -EINVAL;
  274. }
  275. vcrypto = kzalloc_node(sizeof(*vcrypto), GFP_KERNEL,
  276. dev_to_node(&vdev->dev));
  277. if (!vcrypto)
  278. return -ENOMEM;
  279. virtio_cread(vdev, struct virtio_crypto_config,
  280. max_dataqueues, &max_data_queues);
  281. if (max_data_queues < 1)
  282. max_data_queues = 1;
  283. virtio_cread(vdev, struct virtio_crypto_config,
  284. max_cipher_key_len, &max_cipher_key_len);
  285. virtio_cread(vdev, struct virtio_crypto_config,
  286. max_auth_key_len, &max_auth_key_len);
  287. virtio_cread(vdev, struct virtio_crypto_config,
  288. max_size, &max_size);
  289. virtio_cread(vdev, struct virtio_crypto_config,
  290. crypto_services, &crypto_services);
  291. virtio_cread(vdev, struct virtio_crypto_config,
  292. cipher_algo_l, &cipher_algo_l);
  293. virtio_cread(vdev, struct virtio_crypto_config,
  294. cipher_algo_h, &cipher_algo_h);
  295. virtio_cread(vdev, struct virtio_crypto_config,
  296. hash_algo, &hash_algo);
  297. virtio_cread(vdev, struct virtio_crypto_config,
  298. mac_algo_l, &mac_algo_l);
  299. virtio_cread(vdev, struct virtio_crypto_config,
  300. mac_algo_h, &mac_algo_h);
  301. virtio_cread(vdev, struct virtio_crypto_config,
  302. aead_algo, &aead_algo);
  303. /* Add virtio crypto device to global table */
  304. err = virtcrypto_devmgr_add_dev(vcrypto);
  305. if (err) {
  306. dev_err(&vdev->dev, "Failed to add new virtio crypto device.\n");
  307. goto free;
  308. }
  309. vcrypto->owner = THIS_MODULE;
  310. vcrypto = vdev->priv = vcrypto;
  311. vcrypto->vdev = vdev;
  312. spin_lock_init(&vcrypto->ctrl_lock);
  313. /* Use single data queue as default */
  314. vcrypto->curr_queue = 1;
  315. vcrypto->max_data_queues = max_data_queues;
  316. vcrypto->max_cipher_key_len = max_cipher_key_len;
  317. vcrypto->max_auth_key_len = max_auth_key_len;
  318. vcrypto->max_size = max_size;
  319. vcrypto->crypto_services = crypto_services;
  320. vcrypto->cipher_algo_l = cipher_algo_l;
  321. vcrypto->cipher_algo_h = cipher_algo_h;
  322. vcrypto->mac_algo_l = mac_algo_l;
  323. vcrypto->mac_algo_h = mac_algo_h;
  324. vcrypto->hash_algo = hash_algo;
  325. vcrypto->aead_algo = aead_algo;
  326. dev_info(&vdev->dev,
  327. "max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",
  328. vcrypto->max_data_queues,
  329. vcrypto->max_cipher_key_len,
  330. vcrypto->max_auth_key_len,
  331. vcrypto->max_size);
  332. err = virtcrypto_init_vqs(vcrypto);
  333. if (err) {
  334. dev_err(&vdev->dev, "Failed to initialize vqs.\n");
  335. goto free_dev;
  336. }
  337. err = virtcrypto_start_crypto_engines(vcrypto);
  338. if (err)
  339. goto free_vqs;
  340. virtio_device_ready(vdev);
  341. err = virtcrypto_update_status(vcrypto);
  342. if (err)
  343. goto free_engines;
  344. return 0;
  345. free_engines:
  346. virtcrypto_clear_crypto_engines(vcrypto);
  347. free_vqs:
  348. vcrypto->vdev->config->reset(vdev);
  349. virtcrypto_del_vqs(vcrypto);
  350. free_dev:
  351. virtcrypto_devmgr_rm_dev(vcrypto);
  352. free:
  353. kfree(vcrypto);
  354. return err;
  355. }
  356. static void virtcrypto_free_unused_reqs(struct virtio_crypto *vcrypto)
  357. {
  358. struct virtio_crypto_request *vc_req;
  359. int i;
  360. struct virtqueue *vq;
  361. for (i = 0; i < vcrypto->max_data_queues; i++) {
  362. vq = vcrypto->data_vq[i].vq;
  363. while ((vc_req = virtqueue_detach_unused_buf(vq)) != NULL) {
  364. kfree(vc_req->req_data);
  365. kfree(vc_req->sgs);
  366. }
  367. }
  368. }
  369. static void virtcrypto_remove(struct virtio_device *vdev)
  370. {
  371. struct virtio_crypto *vcrypto = vdev->priv;
  372. dev_info(&vdev->dev, "Start virtcrypto_remove.\n");
  373. if (virtcrypto_dev_started(vcrypto))
  374. virtcrypto_dev_stop(vcrypto);
  375. vdev->config->reset(vdev);
  376. virtcrypto_free_unused_reqs(vcrypto);
  377. virtcrypto_clear_crypto_engines(vcrypto);
  378. virtcrypto_del_vqs(vcrypto);
  379. virtcrypto_devmgr_rm_dev(vcrypto);
  380. kfree(vcrypto);
  381. }
  382. static void virtcrypto_config_changed(struct virtio_device *vdev)
  383. {
  384. struct virtio_crypto *vcrypto = vdev->priv;
  385. virtcrypto_update_status(vcrypto);
  386. }
  387. #ifdef CONFIG_PM_SLEEP
  388. static int virtcrypto_freeze(struct virtio_device *vdev)
  389. {
  390. struct virtio_crypto *vcrypto = vdev->priv;
  391. vdev->config->reset(vdev);
  392. virtcrypto_free_unused_reqs(vcrypto);
  393. if (virtcrypto_dev_started(vcrypto))
  394. virtcrypto_dev_stop(vcrypto);
  395. virtcrypto_clear_crypto_engines(vcrypto);
  396. virtcrypto_del_vqs(vcrypto);
  397. return 0;
  398. }
  399. static int virtcrypto_restore(struct virtio_device *vdev)
  400. {
  401. struct virtio_crypto *vcrypto = vdev->priv;
  402. int err;
  403. err = virtcrypto_init_vqs(vcrypto);
  404. if (err)
  405. return err;
  406. err = virtcrypto_start_crypto_engines(vcrypto);
  407. if (err)
  408. goto free_vqs;
  409. virtio_device_ready(vdev);
  410. err = virtcrypto_dev_start(vcrypto);
  411. if (err) {
  412. dev_err(&vdev->dev, "Failed to start virtio crypto device.\n");
  413. goto free_engines;
  414. }
  415. return 0;
  416. free_engines:
  417. virtcrypto_clear_crypto_engines(vcrypto);
  418. free_vqs:
  419. vcrypto->vdev->config->reset(vdev);
  420. virtcrypto_del_vqs(vcrypto);
  421. return err;
  422. }
  423. #endif
  424. static unsigned int features[] = {
  425. /* none */
  426. };
  427. static struct virtio_device_id id_table[] = {
  428. { VIRTIO_ID_CRYPTO, VIRTIO_DEV_ANY_ID },
  429. { 0 },
  430. };
  431. static struct virtio_driver virtio_crypto_driver = {
  432. .driver.name = KBUILD_MODNAME,
  433. .driver.owner = THIS_MODULE,
  434. .feature_table = features,
  435. .feature_table_size = ARRAY_SIZE(features),
  436. .id_table = id_table,
  437. .probe = virtcrypto_probe,
  438. .remove = virtcrypto_remove,
  439. .config_changed = virtcrypto_config_changed,
  440. #ifdef CONFIG_PM_SLEEP
  441. .freeze = virtcrypto_freeze,
  442. .restore = virtcrypto_restore,
  443. #endif
  444. };
  445. module_virtio_driver(virtio_crypto_driver);
  446. MODULE_DEVICE_TABLE(virtio, id_table);
  447. MODULE_DESCRIPTION("virtio crypto device driver");
  448. MODULE_LICENSE("GPL");
  449. MODULE_AUTHOR("Gonglei <arei.gonglei@huawei.com>");