ccp-dmaengine.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727
  1. /*
  2. * AMD Cryptographic Coprocessor (CCP) driver
  3. *
  4. * Copyright (C) 2016 Advanced Micro Devices, Inc.
  5. *
  6. * Author: Gary R Hook <gary.hook@amd.com>
  7. *
  8. * This program is free software; you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License version 2 as
  10. * published by the Free Software Foundation.
  11. */
  12. #include <linux/kernel.h>
  13. #include <linux/dmaengine.h>
  14. #include <linux/spinlock.h>
  15. #include <linux/mutex.h>
  16. #include <linux/ccp.h>
  17. #include "ccp-dev.h"
  18. #include "../../dma/dmaengine.h"
  19. #define CCP_DMA_WIDTH(_mask) \
  20. ({ \
  21. u64 mask = _mask + 1; \
  22. (mask == 0) ? 64 : fls64(mask); \
  23. })
  24. static void ccp_free_cmd_resources(struct ccp_device *ccp,
  25. struct list_head *list)
  26. {
  27. struct ccp_dma_cmd *cmd, *ctmp;
  28. list_for_each_entry_safe(cmd, ctmp, list, entry) {
  29. list_del(&cmd->entry);
  30. kmem_cache_free(ccp->dma_cmd_cache, cmd);
  31. }
  32. }
  33. static void ccp_free_desc_resources(struct ccp_device *ccp,
  34. struct list_head *list)
  35. {
  36. struct ccp_dma_desc *desc, *dtmp;
  37. list_for_each_entry_safe(desc, dtmp, list, entry) {
  38. ccp_free_cmd_resources(ccp, &desc->active);
  39. ccp_free_cmd_resources(ccp, &desc->pending);
  40. list_del(&desc->entry);
  41. kmem_cache_free(ccp->dma_desc_cache, desc);
  42. }
  43. }
  44. static void ccp_free_chan_resources(struct dma_chan *dma_chan)
  45. {
  46. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  47. dma_chan);
  48. unsigned long flags;
  49. dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan);
  50. spin_lock_irqsave(&chan->lock, flags);
  51. ccp_free_desc_resources(chan->ccp, &chan->complete);
  52. ccp_free_desc_resources(chan->ccp, &chan->active);
  53. ccp_free_desc_resources(chan->ccp, &chan->pending);
  54. spin_unlock_irqrestore(&chan->lock, flags);
  55. }
  56. static void ccp_cleanup_desc_resources(struct ccp_device *ccp,
  57. struct list_head *list)
  58. {
  59. struct ccp_dma_desc *desc, *dtmp;
  60. list_for_each_entry_safe_reverse(desc, dtmp, list, entry) {
  61. if (!async_tx_test_ack(&desc->tx_desc))
  62. continue;
  63. dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
  64. ccp_free_cmd_resources(ccp, &desc->active);
  65. ccp_free_cmd_resources(ccp, &desc->pending);
  66. list_del(&desc->entry);
  67. kmem_cache_free(ccp->dma_desc_cache, desc);
  68. }
  69. }
  70. static void ccp_do_cleanup(unsigned long data)
  71. {
  72. struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data;
  73. unsigned long flags;
  74. dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__,
  75. dma_chan_name(&chan->dma_chan));
  76. spin_lock_irqsave(&chan->lock, flags);
  77. ccp_cleanup_desc_resources(chan->ccp, &chan->complete);
  78. spin_unlock_irqrestore(&chan->lock, flags);
  79. }
  80. static int ccp_issue_next_cmd(struct ccp_dma_desc *desc)
  81. {
  82. struct ccp_dma_cmd *cmd;
  83. int ret;
  84. cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry);
  85. list_move(&cmd->entry, &desc->active);
  86. dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__,
  87. desc->tx_desc.cookie, cmd);
  88. ret = ccp_enqueue_cmd(&cmd->ccp_cmd);
  89. if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY))
  90. return 0;
  91. dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__,
  92. ret, desc->tx_desc.cookie, cmd);
  93. return ret;
  94. }
  95. static void ccp_free_active_cmd(struct ccp_dma_desc *desc)
  96. {
  97. struct ccp_dma_cmd *cmd;
  98. cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd,
  99. entry);
  100. if (!cmd)
  101. return;
  102. dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n",
  103. __func__, desc->tx_desc.cookie, cmd);
  104. list_del(&cmd->entry);
  105. kmem_cache_free(desc->ccp->dma_cmd_cache, cmd);
  106. }
  107. static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan,
  108. struct ccp_dma_desc *desc)
  109. {
  110. /* Move current DMA descriptor to the complete list */
  111. if (desc)
  112. list_move(&desc->entry, &chan->complete);
  113. /* Get the next DMA descriptor on the active list */
  114. desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
  115. entry);
  116. return desc;
  117. }
  118. static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan,
  119. struct ccp_dma_desc *desc)
  120. {
  121. struct dma_async_tx_descriptor *tx_desc;
  122. unsigned long flags;
  123. /* Loop over descriptors until one is found with commands */
  124. do {
  125. if (desc) {
  126. /* Remove the DMA command from the list and free it */
  127. ccp_free_active_cmd(desc);
  128. if (!list_empty(&desc->pending)) {
  129. /* No errors, keep going */
  130. if (desc->status != DMA_ERROR)
  131. return desc;
  132. /* Error, free remaining commands and move on */
  133. ccp_free_cmd_resources(desc->ccp,
  134. &desc->pending);
  135. }
  136. tx_desc = &desc->tx_desc;
  137. } else {
  138. tx_desc = NULL;
  139. }
  140. spin_lock_irqsave(&chan->lock, flags);
  141. if (desc) {
  142. if (desc->status != DMA_ERROR)
  143. desc->status = DMA_COMPLETE;
  144. dev_dbg(desc->ccp->dev,
  145. "%s - tx %d complete, status=%u\n", __func__,
  146. desc->tx_desc.cookie, desc->status);
  147. dma_cookie_complete(tx_desc);
  148. }
  149. desc = __ccp_next_dma_desc(chan, desc);
  150. spin_unlock_irqrestore(&chan->lock, flags);
  151. if (tx_desc) {
  152. if (tx_desc->callback &&
  153. (tx_desc->flags & DMA_PREP_INTERRUPT))
  154. tx_desc->callback(tx_desc->callback_param);
  155. dma_run_dependencies(tx_desc);
  156. }
  157. } while (desc);
  158. return NULL;
  159. }
  160. static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan)
  161. {
  162. struct ccp_dma_desc *desc;
  163. if (list_empty(&chan->pending))
  164. return NULL;
  165. desc = list_empty(&chan->active)
  166. ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry)
  167. : NULL;
  168. list_splice_tail_init(&chan->pending, &chan->active);
  169. return desc;
  170. }
  171. static void ccp_cmd_callback(void *data, int err)
  172. {
  173. struct ccp_dma_desc *desc = data;
  174. struct ccp_dma_chan *chan;
  175. int ret;
  176. if (err == -EINPROGRESS)
  177. return;
  178. chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan,
  179. dma_chan);
  180. dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n",
  181. __func__, desc->tx_desc.cookie, err);
  182. if (err)
  183. desc->status = DMA_ERROR;
  184. while (true) {
  185. /* Check for DMA descriptor completion */
  186. desc = ccp_handle_active_desc(chan, desc);
  187. /* Don't submit cmd if no descriptor or DMA is paused */
  188. if (!desc || (chan->status == DMA_PAUSED))
  189. break;
  190. ret = ccp_issue_next_cmd(desc);
  191. if (!ret)
  192. break;
  193. desc->status = DMA_ERROR;
  194. }
  195. tasklet_schedule(&chan->cleanup_tasklet);
  196. }
  197. static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc)
  198. {
  199. struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc,
  200. tx_desc);
  201. struct ccp_dma_chan *chan;
  202. dma_cookie_t cookie;
  203. unsigned long flags;
  204. chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan);
  205. spin_lock_irqsave(&chan->lock, flags);
  206. cookie = dma_cookie_assign(tx_desc);
  207. list_add_tail(&desc->entry, &chan->pending);
  208. spin_unlock_irqrestore(&chan->lock, flags);
  209. dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n",
  210. __func__, cookie);
  211. return cookie;
  212. }
  213. static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan)
  214. {
  215. struct ccp_dma_cmd *cmd;
  216. cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT);
  217. if (cmd)
  218. memset(cmd, 0, sizeof(*cmd));
  219. return cmd;
  220. }
  221. static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan,
  222. unsigned long flags)
  223. {
  224. struct ccp_dma_desc *desc;
  225. desc = kmem_cache_alloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);
  226. if (!desc)
  227. return NULL;
  228. memset(desc, 0, sizeof(*desc));
  229. dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan);
  230. desc->tx_desc.flags = flags;
  231. desc->tx_desc.tx_submit = ccp_tx_submit;
  232. desc->ccp = chan->ccp;
  233. INIT_LIST_HEAD(&desc->pending);
  234. INIT_LIST_HEAD(&desc->active);
  235. desc->status = DMA_IN_PROGRESS;
  236. return desc;
  237. }
  238. static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan,
  239. struct scatterlist *dst_sg,
  240. unsigned int dst_nents,
  241. struct scatterlist *src_sg,
  242. unsigned int src_nents,
  243. unsigned long flags)
  244. {
  245. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  246. dma_chan);
  247. struct ccp_device *ccp = chan->ccp;
  248. struct ccp_dma_desc *desc;
  249. struct ccp_dma_cmd *cmd;
  250. struct ccp_cmd *ccp_cmd;
  251. struct ccp_passthru_nomap_engine *ccp_pt;
  252. unsigned int src_offset, src_len;
  253. unsigned int dst_offset, dst_len;
  254. unsigned int len;
  255. unsigned long sflags;
  256. size_t total_len;
  257. if (!dst_sg || !src_sg)
  258. return NULL;
  259. if (!dst_nents || !src_nents)
  260. return NULL;
  261. desc = ccp_alloc_dma_desc(chan, flags);
  262. if (!desc)
  263. return NULL;
  264. total_len = 0;
  265. src_len = sg_dma_len(src_sg);
  266. src_offset = 0;
  267. dst_len = sg_dma_len(dst_sg);
  268. dst_offset = 0;
  269. while (true) {
  270. if (!src_len) {
  271. src_nents--;
  272. if (!src_nents)
  273. break;
  274. src_sg = sg_next(src_sg);
  275. if (!src_sg)
  276. break;
  277. src_len = sg_dma_len(src_sg);
  278. src_offset = 0;
  279. continue;
  280. }
  281. if (!dst_len) {
  282. dst_nents--;
  283. if (!dst_nents)
  284. break;
  285. dst_sg = sg_next(dst_sg);
  286. if (!dst_sg)
  287. break;
  288. dst_len = sg_dma_len(dst_sg);
  289. dst_offset = 0;
  290. continue;
  291. }
  292. len = min(dst_len, src_len);
  293. cmd = ccp_alloc_dma_cmd(chan);
  294. if (!cmd)
  295. goto err;
  296. ccp_cmd = &cmd->ccp_cmd;
  297. ccp_pt = &ccp_cmd->u.passthru_nomap;
  298. ccp_cmd->flags = CCP_CMD_MAY_BACKLOG;
  299. ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP;
  300. ccp_cmd->engine = CCP_ENGINE_PASSTHRU;
  301. ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
  302. ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
  303. ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset;
  304. ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset;
  305. ccp_pt->src_len = len;
  306. ccp_pt->final = 1;
  307. ccp_cmd->callback = ccp_cmd_callback;
  308. ccp_cmd->data = desc;
  309. list_add_tail(&cmd->entry, &desc->pending);
  310. dev_dbg(ccp->dev,
  311. "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__,
  312. cmd, &ccp_pt->src_dma,
  313. &ccp_pt->dst_dma, ccp_pt->src_len);
  314. total_len += len;
  315. src_len -= len;
  316. src_offset += len;
  317. dst_len -= len;
  318. dst_offset += len;
  319. }
  320. desc->len = total_len;
  321. if (list_empty(&desc->pending))
  322. goto err;
  323. dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
  324. spin_lock_irqsave(&chan->lock, sflags);
  325. list_add_tail(&desc->entry, &chan->pending);
  326. spin_unlock_irqrestore(&chan->lock, sflags);
  327. return desc;
  328. err:
  329. ccp_free_cmd_resources(ccp, &desc->pending);
  330. kmem_cache_free(ccp->dma_desc_cache, desc);
  331. return NULL;
  332. }
  333. static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy(
  334. struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len,
  335. unsigned long flags)
  336. {
  337. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  338. dma_chan);
  339. struct ccp_dma_desc *desc;
  340. struct scatterlist dst_sg, src_sg;
  341. dev_dbg(chan->ccp->dev,
  342. "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n",
  343. __func__, &src, &dst, len, flags);
  344. sg_init_table(&dst_sg, 1);
  345. sg_dma_address(&dst_sg) = dst;
  346. sg_dma_len(&dst_sg) = len;
  347. sg_init_table(&src_sg, 1);
  348. sg_dma_address(&src_sg) = src;
  349. sg_dma_len(&src_sg) = len;
  350. desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags);
  351. if (!desc)
  352. return NULL;
  353. return &desc->tx_desc;
  354. }
  355. static struct dma_async_tx_descriptor *ccp_prep_dma_sg(
  356. struct dma_chan *dma_chan, struct scatterlist *dst_sg,
  357. unsigned int dst_nents, struct scatterlist *src_sg,
  358. unsigned int src_nents, unsigned long flags)
  359. {
  360. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  361. dma_chan);
  362. struct ccp_dma_desc *desc;
  363. dev_dbg(chan->ccp->dev,
  364. "%s - src=%p, src_nents=%u dst=%p, dst_nents=%u, flags=%#lx\n",
  365. __func__, src_sg, src_nents, dst_sg, dst_nents, flags);
  366. desc = ccp_create_desc(dma_chan, dst_sg, dst_nents, src_sg, src_nents,
  367. flags);
  368. if (!desc)
  369. return NULL;
  370. return &desc->tx_desc;
  371. }
  372. static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt(
  373. struct dma_chan *dma_chan, unsigned long flags)
  374. {
  375. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  376. dma_chan);
  377. struct ccp_dma_desc *desc;
  378. desc = ccp_alloc_dma_desc(chan, flags);
  379. if (!desc)
  380. return NULL;
  381. return &desc->tx_desc;
  382. }
  383. static void ccp_issue_pending(struct dma_chan *dma_chan)
  384. {
  385. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  386. dma_chan);
  387. struct ccp_dma_desc *desc;
  388. unsigned long flags;
  389. dev_dbg(chan->ccp->dev, "%s\n", __func__);
  390. spin_lock_irqsave(&chan->lock, flags);
  391. desc = __ccp_pending_to_active(chan);
  392. spin_unlock_irqrestore(&chan->lock, flags);
  393. /* If there was nothing active, start processing */
  394. if (desc)
  395. ccp_cmd_callback(desc, 0);
  396. }
  397. static enum dma_status ccp_tx_status(struct dma_chan *dma_chan,
  398. dma_cookie_t cookie,
  399. struct dma_tx_state *state)
  400. {
  401. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  402. dma_chan);
  403. struct ccp_dma_desc *desc;
  404. enum dma_status ret;
  405. unsigned long flags;
  406. if (chan->status == DMA_PAUSED) {
  407. ret = DMA_PAUSED;
  408. goto out;
  409. }
  410. ret = dma_cookie_status(dma_chan, cookie, state);
  411. if (ret == DMA_COMPLETE) {
  412. spin_lock_irqsave(&chan->lock, flags);
  413. /* Get status from complete chain, if still there */
  414. list_for_each_entry(desc, &chan->complete, entry) {
  415. if (desc->tx_desc.cookie != cookie)
  416. continue;
  417. ret = desc->status;
  418. break;
  419. }
  420. spin_unlock_irqrestore(&chan->lock, flags);
  421. }
  422. out:
  423. dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret);
  424. return ret;
  425. }
  426. static int ccp_pause(struct dma_chan *dma_chan)
  427. {
  428. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  429. dma_chan);
  430. chan->status = DMA_PAUSED;
  431. /*TODO: Wait for active DMA to complete before returning? */
  432. return 0;
  433. }
  434. static int ccp_resume(struct dma_chan *dma_chan)
  435. {
  436. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  437. dma_chan);
  438. struct ccp_dma_desc *desc;
  439. unsigned long flags;
  440. spin_lock_irqsave(&chan->lock, flags);
  441. desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
  442. entry);
  443. spin_unlock_irqrestore(&chan->lock, flags);
  444. /* Indicate the channel is running again */
  445. chan->status = DMA_IN_PROGRESS;
  446. /* If there was something active, re-start */
  447. if (desc)
  448. ccp_cmd_callback(desc, 0);
  449. return 0;
  450. }
  451. static int ccp_terminate_all(struct dma_chan *dma_chan)
  452. {
  453. struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  454. dma_chan);
  455. unsigned long flags;
  456. dev_dbg(chan->ccp->dev, "%s\n", __func__);
  457. /*TODO: Wait for active DMA to complete before continuing */
  458. spin_lock_irqsave(&chan->lock, flags);
  459. /*TODO: Purge the complete list? */
  460. ccp_free_desc_resources(chan->ccp, &chan->active);
  461. ccp_free_desc_resources(chan->ccp, &chan->pending);
  462. spin_unlock_irqrestore(&chan->lock, flags);
  463. return 0;
  464. }
  465. int ccp_dmaengine_register(struct ccp_device *ccp)
  466. {
  467. struct ccp_dma_chan *chan;
  468. struct dma_device *dma_dev = &ccp->dma_dev;
  469. struct dma_chan *dma_chan;
  470. char *dma_cmd_cache_name;
  471. char *dma_desc_cache_name;
  472. unsigned int i;
  473. int ret;
  474. ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count,
  475. sizeof(*(ccp->ccp_dma_chan)),
  476. GFP_KERNEL);
  477. if (!ccp->ccp_dma_chan)
  478. return -ENOMEM;
  479. dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
  480. "%s-dmaengine-cmd-cache",
  481. ccp->name);
  482. if (!dma_cmd_cache_name)
  483. return -ENOMEM;
  484. ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name,
  485. sizeof(struct ccp_dma_cmd),
  486. sizeof(void *),
  487. SLAB_HWCACHE_ALIGN, NULL);
  488. if (!ccp->dma_cmd_cache)
  489. return -ENOMEM;
  490. dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
  491. "%s-dmaengine-desc-cache",
  492. ccp->name);
  493. if (!dma_cmd_cache_name)
  494. return -ENOMEM;
  495. ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name,
  496. sizeof(struct ccp_dma_desc),
  497. sizeof(void *),
  498. SLAB_HWCACHE_ALIGN, NULL);
  499. if (!ccp->dma_desc_cache) {
  500. ret = -ENOMEM;
  501. goto err_cache;
  502. }
  503. dma_dev->dev = ccp->dev;
  504. dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
  505. dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
  506. dma_dev->directions = DMA_MEM_TO_MEM;
  507. dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
  508. dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
  509. dma_cap_set(DMA_SG, dma_dev->cap_mask);
  510. dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
  511. INIT_LIST_HEAD(&dma_dev->channels);
  512. for (i = 0; i < ccp->cmd_q_count; i++) {
  513. chan = ccp->ccp_dma_chan + i;
  514. dma_chan = &chan->dma_chan;
  515. chan->ccp = ccp;
  516. spin_lock_init(&chan->lock);
  517. INIT_LIST_HEAD(&chan->pending);
  518. INIT_LIST_HEAD(&chan->active);
  519. INIT_LIST_HEAD(&chan->complete);
  520. tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup,
  521. (unsigned long)chan);
  522. dma_chan->device = dma_dev;
  523. dma_cookie_init(dma_chan);
  524. list_add_tail(&dma_chan->device_node, &dma_dev->channels);
  525. }
  526. dma_dev->device_free_chan_resources = ccp_free_chan_resources;
  527. dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy;
  528. dma_dev->device_prep_dma_sg = ccp_prep_dma_sg;
  529. dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt;
  530. dma_dev->device_issue_pending = ccp_issue_pending;
  531. dma_dev->device_tx_status = ccp_tx_status;
  532. dma_dev->device_pause = ccp_pause;
  533. dma_dev->device_resume = ccp_resume;
  534. dma_dev->device_terminate_all = ccp_terminate_all;
  535. ret = dma_async_device_register(dma_dev);
  536. if (ret)
  537. goto err_reg;
  538. return 0;
  539. err_reg:
  540. kmem_cache_destroy(ccp->dma_desc_cache);
  541. err_cache:
  542. kmem_cache_destroy(ccp->dma_cmd_cache);
  543. return ret;
  544. }
  545. void ccp_dmaengine_unregister(struct ccp_device *ccp)
  546. {
  547. struct dma_device *dma_dev = &ccp->dma_dev;
  548. dma_async_device_unregister(dma_dev);
  549. kmem_cache_destroy(ccp->dma_desc_cache);
  550. kmem_cache_destroy(ccp->dma_cmd_cache);
  551. }