caamhash.c 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * caam - Freescale FSL CAAM support for ahash functions of crypto API
  4. *
  5. * Copyright 2011 Freescale Semiconductor, Inc.
  6. *
  7. * Based on caamalg.c crypto API driver.
  8. *
  9. * relationship of digest job descriptor or first job descriptor after init to
  10. * shared descriptors:
  11. *
  12. * --------------- ---------------
  13. * | JobDesc #1 |-------------------->| ShareDesc |
  14. * | *(packet 1) | | (hashKey) |
  15. * --------------- | (operation) |
  16. * ---------------
  17. *
  18. * relationship of subsequent job descriptors to shared descriptors:
  19. *
  20. * --------------- ---------------
  21. * | JobDesc #2 |-------------------->| ShareDesc |
  22. * | *(packet 2) | |------------->| (hashKey) |
  23. * --------------- | |-------->| (operation) |
  24. * . | | | (load ctx2) |
  25. * . | | ---------------
  26. * --------------- | |
  27. * | JobDesc #3 |------| |
  28. * | *(packet 3) | |
  29. * --------------- |
  30. * . |
  31. * . |
  32. * --------------- |
  33. * | JobDesc #4 |------------
  34. * | *(packet 4) |
  35. * ---------------
  36. *
  37. * The SharedDesc never changes for a connection unless rekeyed, but
  38. * each packet will likely be in a different place. So all we need
  39. * to know to process the packet is where the input is, where the
  40. * output goes, and what context we want to process with. Context is
  41. * in the SharedDesc, packet references in the JobDesc.
  42. *
  43. * So, a job desc looks like:
  44. *
  45. * ---------------------
  46. * | Header |
  47. * | ShareDesc Pointer |
  48. * | SEQ_OUT_PTR |
  49. * | (output buffer) |
  50. * | (output length) |
  51. * | SEQ_IN_PTR |
  52. * | (input buffer) |
  53. * | (input length) |
  54. * ---------------------
  55. */
  56. #include "compat.h"
  57. #include "regs.h"
  58. #include "intern.h"
  59. #include "desc_constr.h"
  60. #include "jr.h"
  61. #include "error.h"
  62. #include "sg_sw_sec4.h"
  63. #include "key_gen.h"
  64. #include "caamhash_desc.h"
  65. #define CAAM_CRA_PRIORITY 3000
  66. /* max hash key is max split key size */
  67. #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2)
  68. #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
  69. #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
  70. #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \
  71. CAAM_MAX_HASH_KEY_SIZE)
  72. #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
  73. /* caam context sizes for hashes: running digest + 8 */
  74. #define HASH_MSG_LEN 8
  75. #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
  76. #ifdef DEBUG
  77. /* for print_hex_dumps with line references */
  78. #define debug(format, arg...) printk(format, arg)
  79. #else
  80. #define debug(format, arg...)
  81. #endif
  82. static struct list_head hash_list;
  83. /* ahash per-session context */
  84. struct caam_hash_ctx {
  85. u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
  86. u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
  87. u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
  88. u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
  89. dma_addr_t sh_desc_update_dma ____cacheline_aligned;
  90. dma_addr_t sh_desc_update_first_dma;
  91. dma_addr_t sh_desc_fin_dma;
  92. dma_addr_t sh_desc_digest_dma;
  93. enum dma_data_direction dir;
  94. struct device *jrdev;
  95. u8 key[CAAM_MAX_HASH_KEY_SIZE];
  96. int ctx_len;
  97. struct alginfo adata;
  98. };
  99. /* ahash state */
  100. struct caam_hash_state {
  101. dma_addr_t buf_dma;
  102. dma_addr_t ctx_dma;
  103. u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
  104. int buflen_0;
  105. u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
  106. int buflen_1;
  107. u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
  108. int (*update)(struct ahash_request *req);
  109. int (*final)(struct ahash_request *req);
  110. int (*finup)(struct ahash_request *req);
  111. int current_buf;
  112. };
  113. struct caam_export_state {
  114. u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
  115. u8 caam_ctx[MAX_CTX_LEN];
  116. int buflen;
  117. int (*update)(struct ahash_request *req);
  118. int (*final)(struct ahash_request *req);
  119. int (*finup)(struct ahash_request *req);
  120. };
  121. static inline void switch_buf(struct caam_hash_state *state)
  122. {
  123. state->current_buf ^= 1;
  124. }
  125. static inline u8 *current_buf(struct caam_hash_state *state)
  126. {
  127. return state->current_buf ? state->buf_1 : state->buf_0;
  128. }
  129. static inline u8 *alt_buf(struct caam_hash_state *state)
  130. {
  131. return state->current_buf ? state->buf_0 : state->buf_1;
  132. }
  133. static inline int *current_buflen(struct caam_hash_state *state)
  134. {
  135. return state->current_buf ? &state->buflen_1 : &state->buflen_0;
  136. }
  137. static inline int *alt_buflen(struct caam_hash_state *state)
  138. {
  139. return state->current_buf ? &state->buflen_0 : &state->buflen_1;
  140. }
  141. /* Common job descriptor seq in/out ptr routines */
  142. /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
  143. static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
  144. struct caam_hash_state *state,
  145. int ctx_len)
  146. {
  147. state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
  148. ctx_len, DMA_FROM_DEVICE);
  149. if (dma_mapping_error(jrdev, state->ctx_dma)) {
  150. dev_err(jrdev, "unable to map ctx\n");
  151. state->ctx_dma = 0;
  152. return -ENOMEM;
  153. }
  154. append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
  155. return 0;
  156. }
  157. /* Map req->result, and append seq_out_ptr command that points to it */
  158. static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev,
  159. u8 *result, int digestsize)
  160. {
  161. dma_addr_t dst_dma;
  162. dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE);
  163. append_seq_out_ptr(desc, dst_dma, digestsize, 0);
  164. return dst_dma;
  165. }
  166. /* Map current buffer in state (if length > 0) and put it in link table */
  167. static inline int buf_map_to_sec4_sg(struct device *jrdev,
  168. struct sec4_sg_entry *sec4_sg,
  169. struct caam_hash_state *state)
  170. {
  171. int buflen = *current_buflen(state);
  172. if (!buflen)
  173. return 0;
  174. state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen,
  175. DMA_TO_DEVICE);
  176. if (dma_mapping_error(jrdev, state->buf_dma)) {
  177. dev_err(jrdev, "unable to map buf\n");
  178. state->buf_dma = 0;
  179. return -ENOMEM;
  180. }
  181. dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0);
  182. return 0;
  183. }
  184. /* Map state->caam_ctx, and add it to link table */
  185. static inline int ctx_map_to_sec4_sg(struct device *jrdev,
  186. struct caam_hash_state *state, int ctx_len,
  187. struct sec4_sg_entry *sec4_sg, u32 flag)
  188. {
  189. state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
  190. if (dma_mapping_error(jrdev, state->ctx_dma)) {
  191. dev_err(jrdev, "unable to map ctx\n");
  192. state->ctx_dma = 0;
  193. return -ENOMEM;
  194. }
  195. dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
  196. return 0;
  197. }
  198. static int ahash_set_sh_desc(struct crypto_ahash *ahash)
  199. {
  200. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  201. int digestsize = crypto_ahash_digestsize(ahash);
  202. struct device *jrdev = ctx->jrdev;
  203. struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
  204. u32 *desc;
  205. ctx->adata.key_virt = ctx->key;
  206. /* ahash_update shared descriptor */
  207. desc = ctx->sh_desc_update;
  208. cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
  209. ctx->ctx_len, true, ctrlpriv->era);
  210. dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
  211. desc_bytes(desc), ctx->dir);
  212. #ifdef DEBUG
  213. print_hex_dump(KERN_ERR,
  214. "ahash update shdesc@"__stringify(__LINE__)": ",
  215. DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
  216. #endif
  217. /* ahash_update_first shared descriptor */
  218. desc = ctx->sh_desc_update_first;
  219. cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
  220. ctx->ctx_len, false, ctrlpriv->era);
  221. dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
  222. desc_bytes(desc), ctx->dir);
  223. #ifdef DEBUG
  224. print_hex_dump(KERN_ERR,
  225. "ahash update first shdesc@"__stringify(__LINE__)": ",
  226. DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
  227. #endif
  228. /* ahash_final shared descriptor */
  229. desc = ctx->sh_desc_fin;
  230. cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
  231. ctx->ctx_len, true, ctrlpriv->era);
  232. dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
  233. desc_bytes(desc), ctx->dir);
  234. #ifdef DEBUG
  235. print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
  236. DUMP_PREFIX_ADDRESS, 16, 4, desc,
  237. desc_bytes(desc), 1);
  238. #endif
  239. /* ahash_digest shared descriptor */
  240. desc = ctx->sh_desc_digest;
  241. cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
  242. ctx->ctx_len, false, ctrlpriv->era);
  243. dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
  244. desc_bytes(desc), ctx->dir);
  245. #ifdef DEBUG
  246. print_hex_dump(KERN_ERR,
  247. "ahash digest shdesc@"__stringify(__LINE__)": ",
  248. DUMP_PREFIX_ADDRESS, 16, 4, desc,
  249. desc_bytes(desc), 1);
  250. #endif
  251. return 0;
  252. }
  253. /* Digest hash size if it is too large */
  254. static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
  255. u32 *keylen, u8 *key_out, u32 digestsize)
  256. {
  257. struct device *jrdev = ctx->jrdev;
  258. u32 *desc;
  259. struct split_key_result result;
  260. dma_addr_t src_dma, dst_dma;
  261. int ret;
  262. desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
  263. if (!desc) {
  264. dev_err(jrdev, "unable to allocate key input memory\n");
  265. return -ENOMEM;
  266. }
  267. init_job_desc(desc, 0);
  268. src_dma = dma_map_single(jrdev, (void *)key_in, *keylen,
  269. DMA_TO_DEVICE);
  270. if (dma_mapping_error(jrdev, src_dma)) {
  271. dev_err(jrdev, "unable to map key input memory\n");
  272. kfree(desc);
  273. return -ENOMEM;
  274. }
  275. dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize,
  276. DMA_FROM_DEVICE);
  277. if (dma_mapping_error(jrdev, dst_dma)) {
  278. dev_err(jrdev, "unable to map key output memory\n");
  279. dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
  280. kfree(desc);
  281. return -ENOMEM;
  282. }
  283. /* Job descriptor to perform unkeyed hash on key_in */
  284. append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
  285. OP_ALG_AS_INITFINAL);
  286. append_seq_in_ptr(desc, src_dma, *keylen, 0);
  287. append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
  288. FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
  289. append_seq_out_ptr(desc, dst_dma, digestsize, 0);
  290. append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
  291. LDST_SRCDST_BYTE_CONTEXT);
  292. #ifdef DEBUG
  293. print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
  294. DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
  295. print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
  296. DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
  297. #endif
  298. result.err = 0;
  299. init_completion(&result.completion);
  300. ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
  301. if (!ret) {
  302. /* in progress */
  303. wait_for_completion(&result.completion);
  304. ret = result.err;
  305. #ifdef DEBUG
  306. print_hex_dump(KERN_ERR,
  307. "digested key@"__stringify(__LINE__)": ",
  308. DUMP_PREFIX_ADDRESS, 16, 4, key_in,
  309. digestsize, 1);
  310. #endif
  311. }
  312. dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
  313. dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE);
  314. *keylen = digestsize;
  315. kfree(desc);
  316. return ret;
  317. }
  318. static int ahash_setkey(struct crypto_ahash *ahash,
  319. const u8 *key, unsigned int keylen)
  320. {
  321. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  322. int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
  323. int digestsize = crypto_ahash_digestsize(ahash);
  324. struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
  325. int ret;
  326. u8 *hashed_key = NULL;
  327. #ifdef DEBUG
  328. printk(KERN_ERR "keylen %d\n", keylen);
  329. #endif
  330. if (keylen > blocksize) {
  331. hashed_key = kmalloc_array(digestsize,
  332. sizeof(*hashed_key),
  333. GFP_KERNEL | GFP_DMA);
  334. if (!hashed_key)
  335. return -ENOMEM;
  336. ret = hash_digest_key(ctx, key, &keylen, hashed_key,
  337. digestsize);
  338. if (ret)
  339. goto bad_free_key;
  340. key = hashed_key;
  341. }
  342. /*
  343. * If DKP is supported, use it in the shared descriptor to generate
  344. * the split key.
  345. */
  346. if (ctrlpriv->era >= 6) {
  347. ctx->adata.key_inline = true;
  348. ctx->adata.keylen = keylen;
  349. ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
  350. OP_ALG_ALGSEL_MASK);
  351. if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
  352. goto bad_free_key;
  353. memcpy(ctx->key, key, keylen);
  354. } else {
  355. ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key,
  356. keylen, CAAM_MAX_HASH_KEY_SIZE);
  357. if (ret)
  358. goto bad_free_key;
  359. }
  360. kfree(hashed_key);
  361. return ahash_set_sh_desc(ahash);
  362. bad_free_key:
  363. kfree(hashed_key);
  364. crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
  365. return -EINVAL;
  366. }
  367. /*
  368. * ahash_edesc - s/w-extended ahash descriptor
  369. * @dst_dma: physical mapped address of req->result
  370. * @sec4_sg_dma: physical mapped address of h/w link table
  371. * @src_nents: number of segments in input scatterlist
  372. * @sec4_sg_bytes: length of dma mapped sec4_sg space
  373. * @hw_desc: the h/w job descriptor followed by any referenced link tables
  374. * @sec4_sg: h/w link table
  375. */
  376. struct ahash_edesc {
  377. dma_addr_t dst_dma;
  378. dma_addr_t sec4_sg_dma;
  379. int src_nents;
  380. int sec4_sg_bytes;
  381. u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
  382. struct sec4_sg_entry sec4_sg[0];
  383. };
  384. static inline void ahash_unmap(struct device *dev,
  385. struct ahash_edesc *edesc,
  386. struct ahash_request *req, int dst_len)
  387. {
  388. struct caam_hash_state *state = ahash_request_ctx(req);
  389. if (edesc->src_nents)
  390. dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
  391. if (edesc->dst_dma)
  392. dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
  393. if (edesc->sec4_sg_bytes)
  394. dma_unmap_single(dev, edesc->sec4_sg_dma,
  395. edesc->sec4_sg_bytes, DMA_TO_DEVICE);
  396. if (state->buf_dma) {
  397. dma_unmap_single(dev, state->buf_dma, *current_buflen(state),
  398. DMA_TO_DEVICE);
  399. state->buf_dma = 0;
  400. }
  401. }
  402. static inline void ahash_unmap_ctx(struct device *dev,
  403. struct ahash_edesc *edesc,
  404. struct ahash_request *req, int dst_len, u32 flag)
  405. {
  406. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  407. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  408. struct caam_hash_state *state = ahash_request_ctx(req);
  409. if (state->ctx_dma) {
  410. dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
  411. state->ctx_dma = 0;
  412. }
  413. ahash_unmap(dev, edesc, req, dst_len);
  414. }
  415. static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
  416. void *context)
  417. {
  418. struct ahash_request *req = context;
  419. struct ahash_edesc *edesc;
  420. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  421. int digestsize = crypto_ahash_digestsize(ahash);
  422. #ifdef DEBUG
  423. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  424. struct caam_hash_state *state = ahash_request_ctx(req);
  425. dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
  426. #endif
  427. edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
  428. if (err)
  429. caam_jr_strstatus(jrdev, err);
  430. ahash_unmap(jrdev, edesc, req, digestsize);
  431. kfree(edesc);
  432. #ifdef DEBUG
  433. print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
  434. DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
  435. ctx->ctx_len, 1);
  436. if (req->result)
  437. print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
  438. DUMP_PREFIX_ADDRESS, 16, 4, req->result,
  439. digestsize, 1);
  440. #endif
  441. req->base.complete(&req->base, err);
  442. }
  443. static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
  444. void *context)
  445. {
  446. struct ahash_request *req = context;
  447. struct ahash_edesc *edesc;
  448. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  449. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  450. struct caam_hash_state *state = ahash_request_ctx(req);
  451. #ifdef DEBUG
  452. int digestsize = crypto_ahash_digestsize(ahash);
  453. dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
  454. #endif
  455. edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
  456. if (err)
  457. caam_jr_strstatus(jrdev, err);
  458. ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
  459. switch_buf(state);
  460. kfree(edesc);
  461. #ifdef DEBUG
  462. print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
  463. DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
  464. ctx->ctx_len, 1);
  465. if (req->result)
  466. print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
  467. DUMP_PREFIX_ADDRESS, 16, 4, req->result,
  468. digestsize, 1);
  469. #endif
  470. req->base.complete(&req->base, err);
  471. }
  472. static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
  473. void *context)
  474. {
  475. struct ahash_request *req = context;
  476. struct ahash_edesc *edesc;
  477. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  478. int digestsize = crypto_ahash_digestsize(ahash);
  479. #ifdef DEBUG
  480. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  481. struct caam_hash_state *state = ahash_request_ctx(req);
  482. dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
  483. #endif
  484. edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
  485. if (err)
  486. caam_jr_strstatus(jrdev, err);
  487. ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);
  488. kfree(edesc);
  489. #ifdef DEBUG
  490. print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
  491. DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
  492. ctx->ctx_len, 1);
  493. if (req->result)
  494. print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
  495. DUMP_PREFIX_ADDRESS, 16, 4, req->result,
  496. digestsize, 1);
  497. #endif
  498. req->base.complete(&req->base, err);
  499. }
  500. static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
  501. void *context)
  502. {
  503. struct ahash_request *req = context;
  504. struct ahash_edesc *edesc;
  505. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  506. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  507. struct caam_hash_state *state = ahash_request_ctx(req);
  508. #ifdef DEBUG
  509. int digestsize = crypto_ahash_digestsize(ahash);
  510. dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
  511. #endif
  512. edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
  513. if (err)
  514. caam_jr_strstatus(jrdev, err);
  515. ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
  516. switch_buf(state);
  517. kfree(edesc);
  518. #ifdef DEBUG
  519. print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
  520. DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
  521. ctx->ctx_len, 1);
  522. if (req->result)
  523. print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
  524. DUMP_PREFIX_ADDRESS, 16, 4, req->result,
  525. digestsize, 1);
  526. #endif
  527. req->base.complete(&req->base, err);
  528. }
  529. /*
  530. * Allocate an enhanced descriptor, which contains the hardware descriptor
  531. * and space for hardware scatter table containing sg_num entries.
  532. */
  533. static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx,
  534. int sg_num, u32 *sh_desc,
  535. dma_addr_t sh_desc_dma,
  536. gfp_t flags)
  537. {
  538. struct ahash_edesc *edesc;
  539. unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
  540. edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
  541. if (!edesc) {
  542. dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
  543. return NULL;
  544. }
  545. init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
  546. HDR_SHARE_DEFER | HDR_REVERSE);
  547. return edesc;
  548. }
  549. static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
  550. struct ahash_edesc *edesc,
  551. struct ahash_request *req, int nents,
  552. unsigned int first_sg,
  553. unsigned int first_bytes, size_t to_hash)
  554. {
  555. dma_addr_t src_dma;
  556. u32 options;
  557. if (nents > 1 || first_sg) {
  558. struct sec4_sg_entry *sg = edesc->sec4_sg;
  559. unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
  560. sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
  561. src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
  562. if (dma_mapping_error(ctx->jrdev, src_dma)) {
  563. dev_err(ctx->jrdev, "unable to map S/G table\n");
  564. return -ENOMEM;
  565. }
  566. edesc->sec4_sg_bytes = sgsize;
  567. edesc->sec4_sg_dma = src_dma;
  568. options = LDST_SGF;
  569. } else {
  570. src_dma = sg_dma_address(req->src);
  571. options = 0;
  572. }
  573. append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
  574. options);
  575. return 0;
  576. }
  577. /* submit update job descriptor */
  578. static int ahash_update_ctx(struct ahash_request *req)
  579. {
  580. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  581. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  582. struct caam_hash_state *state = ahash_request_ctx(req);
  583. struct device *jrdev = ctx->jrdev;
  584. gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
  585. GFP_KERNEL : GFP_ATOMIC;
  586. u8 *buf = current_buf(state);
  587. int *buflen = current_buflen(state);
  588. u8 *next_buf = alt_buf(state);
  589. int *next_buflen = alt_buflen(state), last_buflen;
  590. int in_len = *buflen + req->nbytes, to_hash;
  591. u32 *desc;
  592. int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
  593. struct ahash_edesc *edesc;
  594. int ret = 0;
  595. last_buflen = *next_buflen;
  596. *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
  597. to_hash = in_len - *next_buflen;
  598. if (to_hash) {
  599. src_nents = sg_nents_for_len(req->src,
  600. req->nbytes - (*next_buflen));
  601. if (src_nents < 0) {
  602. dev_err(jrdev, "Invalid number of src SG.\n");
  603. return src_nents;
  604. }
  605. if (src_nents) {
  606. mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
  607. DMA_TO_DEVICE);
  608. if (!mapped_nents) {
  609. dev_err(jrdev, "unable to DMA map source\n");
  610. return -ENOMEM;
  611. }
  612. } else {
  613. mapped_nents = 0;
  614. }
  615. sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
  616. sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
  617. sizeof(struct sec4_sg_entry);
  618. /*
  619. * allocate space for base edesc and hw desc commands,
  620. * link tables
  621. */
  622. edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
  623. ctx->sh_desc_update,
  624. ctx->sh_desc_update_dma, flags);
  625. if (!edesc) {
  626. dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
  627. return -ENOMEM;
  628. }
  629. edesc->src_nents = src_nents;
  630. edesc->sec4_sg_bytes = sec4_sg_bytes;
  631. ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
  632. edesc->sec4_sg, DMA_BIDIRECTIONAL);
  633. if (ret)
  634. goto unmap_ctx;
  635. ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
  636. if (ret)
  637. goto unmap_ctx;
  638. if (mapped_nents) {
  639. sg_to_sec4_sg_last(req->src, mapped_nents,
  640. edesc->sec4_sg + sec4_sg_src_index,
  641. 0);
  642. if (*next_buflen)
  643. scatterwalk_map_and_copy(next_buf, req->src,
  644. to_hash - *buflen,
  645. *next_buflen, 0);
  646. } else {
  647. sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index -
  648. 1);
  649. }
  650. desc = edesc->hw_desc;
  651. edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
  652. sec4_sg_bytes,
  653. DMA_TO_DEVICE);
  654. if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
  655. dev_err(jrdev, "unable to map S/G table\n");
  656. ret = -ENOMEM;
  657. goto unmap_ctx;
  658. }
  659. append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
  660. to_hash, LDST_SGF);
  661. append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
  662. #ifdef DEBUG
  663. print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
  664. DUMP_PREFIX_ADDRESS, 16, 4, desc,
  665. desc_bytes(desc), 1);
  666. #endif
  667. ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
  668. if (ret)
  669. goto unmap_ctx;
  670. ret = -EINPROGRESS;
  671. } else if (*next_buflen) {
  672. scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
  673. req->nbytes, 0);
  674. *buflen = *next_buflen;
  675. *next_buflen = last_buflen;
  676. }
  677. #ifdef DEBUG
  678. print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
  679. DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
  680. print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
  681. DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
  682. *next_buflen, 1);
  683. #endif
  684. return ret;
  685. unmap_ctx:
  686. ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
  687. kfree(edesc);
  688. return ret;
  689. }
  690. static int ahash_final_ctx(struct ahash_request *req)
  691. {
  692. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  693. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  694. struct caam_hash_state *state = ahash_request_ctx(req);
  695. struct device *jrdev = ctx->jrdev;
  696. gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
  697. GFP_KERNEL : GFP_ATOMIC;
  698. int buflen = *current_buflen(state);
  699. u32 *desc;
  700. int sec4_sg_bytes, sec4_sg_src_index;
  701. int digestsize = crypto_ahash_digestsize(ahash);
  702. struct ahash_edesc *edesc;
  703. int ret;
  704. sec4_sg_src_index = 1 + (buflen ? 1 : 0);
  705. sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
  706. /* allocate space for base edesc and hw desc commands, link tables */
  707. edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
  708. ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
  709. flags);
  710. if (!edesc)
  711. return -ENOMEM;
  712. desc = edesc->hw_desc;
  713. edesc->sec4_sg_bytes = sec4_sg_bytes;
  714. ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
  715. edesc->sec4_sg, DMA_TO_DEVICE);
  716. if (ret)
  717. goto unmap_ctx;
  718. ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
  719. if (ret)
  720. goto unmap_ctx;
  721. sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1);
  722. edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
  723. sec4_sg_bytes, DMA_TO_DEVICE);
  724. if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
  725. dev_err(jrdev, "unable to map S/G table\n");
  726. ret = -ENOMEM;
  727. goto unmap_ctx;
  728. }
  729. append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
  730. LDST_SGF);
  731. edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
  732. digestsize);
  733. if (dma_mapping_error(jrdev, edesc->dst_dma)) {
  734. dev_err(jrdev, "unable to map dst\n");
  735. ret = -ENOMEM;
  736. goto unmap_ctx;
  737. }
  738. #ifdef DEBUG
  739. print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
  740. DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
  741. #endif
  742. ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
  743. if (ret)
  744. goto unmap_ctx;
  745. return -EINPROGRESS;
  746. unmap_ctx:
  747. ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
  748. kfree(edesc);
  749. return ret;
  750. }
  751. static int ahash_finup_ctx(struct ahash_request *req)
  752. {
  753. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  754. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  755. struct caam_hash_state *state = ahash_request_ctx(req);
  756. struct device *jrdev = ctx->jrdev;
  757. gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
  758. GFP_KERNEL : GFP_ATOMIC;
  759. int buflen = *current_buflen(state);
  760. u32 *desc;
  761. int sec4_sg_src_index;
  762. int src_nents, mapped_nents;
  763. int digestsize = crypto_ahash_digestsize(ahash);
  764. struct ahash_edesc *edesc;
  765. int ret;
  766. src_nents = sg_nents_for_len(req->src, req->nbytes);
  767. if (src_nents < 0) {
  768. dev_err(jrdev, "Invalid number of src SG.\n");
  769. return src_nents;
  770. }
  771. if (src_nents) {
  772. mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
  773. DMA_TO_DEVICE);
  774. if (!mapped_nents) {
  775. dev_err(jrdev, "unable to DMA map source\n");
  776. return -ENOMEM;
  777. }
  778. } else {
  779. mapped_nents = 0;
  780. }
  781. sec4_sg_src_index = 1 + (buflen ? 1 : 0);
  782. /* allocate space for base edesc and hw desc commands, link tables */
  783. edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
  784. ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
  785. flags);
  786. if (!edesc) {
  787. dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
  788. return -ENOMEM;
  789. }
  790. desc = edesc->hw_desc;
  791. edesc->src_nents = src_nents;
  792. ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
  793. edesc->sec4_sg, DMA_TO_DEVICE);
  794. if (ret)
  795. goto unmap_ctx;
  796. ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
  797. if (ret)
  798. goto unmap_ctx;
  799. ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
  800. sec4_sg_src_index, ctx->ctx_len + buflen,
  801. req->nbytes);
  802. if (ret)
  803. goto unmap_ctx;
  804. edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
  805. digestsize);
  806. if (dma_mapping_error(jrdev, edesc->dst_dma)) {
  807. dev_err(jrdev, "unable to map dst\n");
  808. ret = -ENOMEM;
  809. goto unmap_ctx;
  810. }
  811. #ifdef DEBUG
  812. print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
  813. DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
  814. #endif
  815. ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
  816. if (ret)
  817. goto unmap_ctx;
  818. return -EINPROGRESS;
  819. unmap_ctx:
  820. ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
  821. kfree(edesc);
  822. return ret;
  823. }
  824. static int ahash_digest(struct ahash_request *req)
  825. {
  826. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  827. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  828. struct caam_hash_state *state = ahash_request_ctx(req);
  829. struct device *jrdev = ctx->jrdev;
  830. gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
  831. GFP_KERNEL : GFP_ATOMIC;
  832. u32 *desc;
  833. int digestsize = crypto_ahash_digestsize(ahash);
  834. int src_nents, mapped_nents;
  835. struct ahash_edesc *edesc;
  836. int ret;
  837. state->buf_dma = 0;
  838. src_nents = sg_nents_for_len(req->src, req->nbytes);
  839. if (src_nents < 0) {
  840. dev_err(jrdev, "Invalid number of src SG.\n");
  841. return src_nents;
  842. }
  843. if (src_nents) {
  844. mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
  845. DMA_TO_DEVICE);
  846. if (!mapped_nents) {
  847. dev_err(jrdev, "unable to map source for DMA\n");
  848. return -ENOMEM;
  849. }
  850. } else {
  851. mapped_nents = 0;
  852. }
  853. /* allocate space for base edesc and hw desc commands, link tables */
  854. edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0,
  855. ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
  856. flags);
  857. if (!edesc) {
  858. dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
  859. return -ENOMEM;
  860. }
  861. edesc->src_nents = src_nents;
  862. ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
  863. req->nbytes);
  864. if (ret) {
  865. ahash_unmap(jrdev, edesc, req, digestsize);
  866. kfree(edesc);
  867. return ret;
  868. }
  869. desc = edesc->hw_desc;
  870. edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
  871. digestsize);
  872. if (dma_mapping_error(jrdev, edesc->dst_dma)) {
  873. dev_err(jrdev, "unable to map dst\n");
  874. ahash_unmap(jrdev, edesc, req, digestsize);
  875. kfree(edesc);
  876. return -ENOMEM;
  877. }
  878. #ifdef DEBUG
  879. print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
  880. DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
  881. #endif
  882. ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
  883. if (!ret) {
  884. ret = -EINPROGRESS;
  885. } else {
  886. ahash_unmap(jrdev, edesc, req, digestsize);
  887. kfree(edesc);
  888. }
  889. return ret;
  890. }
  891. /* submit ahash final if it the first job descriptor */
  892. static int ahash_final_no_ctx(struct ahash_request *req)
  893. {
  894. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  895. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  896. struct caam_hash_state *state = ahash_request_ctx(req);
  897. struct device *jrdev = ctx->jrdev;
  898. gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
  899. GFP_KERNEL : GFP_ATOMIC;
  900. u8 *buf = current_buf(state);
  901. int buflen = *current_buflen(state);
  902. u32 *desc;
  903. int digestsize = crypto_ahash_digestsize(ahash);
  904. struct ahash_edesc *edesc;
  905. int ret;
  906. /* allocate space for base edesc and hw desc commands, link tables */
  907. edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest,
  908. ctx->sh_desc_digest_dma, flags);
  909. if (!edesc)
  910. return -ENOMEM;
  911. desc = edesc->hw_desc;
  912. state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
  913. if (dma_mapping_error(jrdev, state->buf_dma)) {
  914. dev_err(jrdev, "unable to map src\n");
  915. goto unmap;
  916. }
  917. append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
  918. edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
  919. digestsize);
  920. if (dma_mapping_error(jrdev, edesc->dst_dma)) {
  921. dev_err(jrdev, "unable to map dst\n");
  922. goto unmap;
  923. }
  924. #ifdef DEBUG
  925. print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
  926. DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
  927. #endif
  928. ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
  929. if (!ret) {
  930. ret = -EINPROGRESS;
  931. } else {
  932. ahash_unmap(jrdev, edesc, req, digestsize);
  933. kfree(edesc);
  934. }
  935. return ret;
  936. unmap:
  937. ahash_unmap(jrdev, edesc, req, digestsize);
  938. kfree(edesc);
  939. return -ENOMEM;
  940. }
  941. /* submit ahash update if it the first job descriptor after update */
  942. static int ahash_update_no_ctx(struct ahash_request *req)
  943. {
  944. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  945. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  946. struct caam_hash_state *state = ahash_request_ctx(req);
  947. struct device *jrdev = ctx->jrdev;
  948. gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
  949. GFP_KERNEL : GFP_ATOMIC;
  950. u8 *buf = current_buf(state);
  951. int *buflen = current_buflen(state);
  952. u8 *next_buf = alt_buf(state);
  953. int *next_buflen = alt_buflen(state);
  954. int in_len = *buflen + req->nbytes, to_hash;
  955. int sec4_sg_bytes, src_nents, mapped_nents;
  956. struct ahash_edesc *edesc;
  957. u32 *desc;
  958. int ret = 0;
  959. *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
  960. to_hash = in_len - *next_buflen;
  961. if (to_hash) {
  962. src_nents = sg_nents_for_len(req->src,
  963. req->nbytes - *next_buflen);
  964. if (src_nents < 0) {
  965. dev_err(jrdev, "Invalid number of src SG.\n");
  966. return src_nents;
  967. }
  968. if (src_nents) {
  969. mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
  970. DMA_TO_DEVICE);
  971. if (!mapped_nents) {
  972. dev_err(jrdev, "unable to DMA map source\n");
  973. return -ENOMEM;
  974. }
  975. } else {
  976. mapped_nents = 0;
  977. }
  978. sec4_sg_bytes = (1 + mapped_nents) *
  979. sizeof(struct sec4_sg_entry);
  980. /*
  981. * allocate space for base edesc and hw desc commands,
  982. * link tables
  983. */
  984. edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
  985. ctx->sh_desc_update_first,
  986. ctx->sh_desc_update_first_dma,
  987. flags);
  988. if (!edesc) {
  989. dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
  990. return -ENOMEM;
  991. }
  992. edesc->src_nents = src_nents;
  993. edesc->sec4_sg_bytes = sec4_sg_bytes;
  994. ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
  995. if (ret)
  996. goto unmap_ctx;
  997. sg_to_sec4_sg_last(req->src, mapped_nents,
  998. edesc->sec4_sg + 1, 0);
  999. if (*next_buflen) {
  1000. scatterwalk_map_and_copy(next_buf, req->src,
  1001. to_hash - *buflen,
  1002. *next_buflen, 0);
  1003. }
  1004. desc = edesc->hw_desc;
  1005. edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
  1006. sec4_sg_bytes,
  1007. DMA_TO_DEVICE);
  1008. if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
  1009. dev_err(jrdev, "unable to map S/G table\n");
  1010. ret = -ENOMEM;
  1011. goto unmap_ctx;
  1012. }
  1013. append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
  1014. ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
  1015. if (ret)
  1016. goto unmap_ctx;
  1017. #ifdef DEBUG
  1018. print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
  1019. DUMP_PREFIX_ADDRESS, 16, 4, desc,
  1020. desc_bytes(desc), 1);
  1021. #endif
  1022. ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
  1023. if (ret)
  1024. goto unmap_ctx;
  1025. ret = -EINPROGRESS;
  1026. state->update = ahash_update_ctx;
  1027. state->finup = ahash_finup_ctx;
  1028. state->final = ahash_final_ctx;
  1029. } else if (*next_buflen) {
  1030. scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
  1031. req->nbytes, 0);
  1032. *buflen = *next_buflen;
  1033. *next_buflen = 0;
  1034. }
  1035. #ifdef DEBUG
  1036. print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
  1037. DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
  1038. print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
  1039. DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
  1040. *next_buflen, 1);
  1041. #endif
  1042. return ret;
  1043. unmap_ctx:
  1044. ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
  1045. kfree(edesc);
  1046. return ret;
  1047. }
  1048. /* submit ahash finup if it the first job descriptor after update */
  1049. static int ahash_finup_no_ctx(struct ahash_request *req)
  1050. {
  1051. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  1052. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  1053. struct caam_hash_state *state = ahash_request_ctx(req);
  1054. struct device *jrdev = ctx->jrdev;
  1055. gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
  1056. GFP_KERNEL : GFP_ATOMIC;
  1057. int buflen = *current_buflen(state);
  1058. u32 *desc;
  1059. int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
  1060. int digestsize = crypto_ahash_digestsize(ahash);
  1061. struct ahash_edesc *edesc;
  1062. int ret;
  1063. src_nents = sg_nents_for_len(req->src, req->nbytes);
  1064. if (src_nents < 0) {
  1065. dev_err(jrdev, "Invalid number of src SG.\n");
  1066. return src_nents;
  1067. }
  1068. if (src_nents) {
  1069. mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
  1070. DMA_TO_DEVICE);
  1071. if (!mapped_nents) {
  1072. dev_err(jrdev, "unable to DMA map source\n");
  1073. return -ENOMEM;
  1074. }
  1075. } else {
  1076. mapped_nents = 0;
  1077. }
  1078. sec4_sg_src_index = 2;
  1079. sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
  1080. sizeof(struct sec4_sg_entry);
  1081. /* allocate space for base edesc and hw desc commands, link tables */
  1082. edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
  1083. ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
  1084. flags);
  1085. if (!edesc) {
  1086. dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
  1087. return -ENOMEM;
  1088. }
  1089. desc = edesc->hw_desc;
  1090. edesc->src_nents = src_nents;
  1091. edesc->sec4_sg_bytes = sec4_sg_bytes;
  1092. ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
  1093. if (ret)
  1094. goto unmap;
  1095. ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
  1096. req->nbytes);
  1097. if (ret) {
  1098. dev_err(jrdev, "unable to map S/G table\n");
  1099. goto unmap;
  1100. }
  1101. edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
  1102. digestsize);
  1103. if (dma_mapping_error(jrdev, edesc->dst_dma)) {
  1104. dev_err(jrdev, "unable to map dst\n");
  1105. goto unmap;
  1106. }
  1107. #ifdef DEBUG
  1108. print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
  1109. DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
  1110. #endif
  1111. ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
  1112. if (!ret) {
  1113. ret = -EINPROGRESS;
  1114. } else {
  1115. ahash_unmap(jrdev, edesc, req, digestsize);
  1116. kfree(edesc);
  1117. }
  1118. return ret;
  1119. unmap:
  1120. ahash_unmap(jrdev, edesc, req, digestsize);
  1121. kfree(edesc);
  1122. return -ENOMEM;
  1123. }
  1124. /* submit first update job descriptor after init */
  1125. static int ahash_update_first(struct ahash_request *req)
  1126. {
  1127. struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
  1128. struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
  1129. struct caam_hash_state *state = ahash_request_ctx(req);
  1130. struct device *jrdev = ctx->jrdev;
  1131. gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
  1132. GFP_KERNEL : GFP_ATOMIC;
  1133. u8 *next_buf = alt_buf(state);
  1134. int *next_buflen = alt_buflen(state);
  1135. int to_hash;
  1136. u32 *desc;
  1137. int src_nents, mapped_nents;
  1138. struct ahash_edesc *edesc;
  1139. int ret = 0;
  1140. *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
  1141. 1);
  1142. to_hash = req->nbytes - *next_buflen;
  1143. if (to_hash) {
  1144. src_nents = sg_nents_for_len(req->src,
  1145. req->nbytes - *next_buflen);
  1146. if (src_nents < 0) {
  1147. dev_err(jrdev, "Invalid number of src SG.\n");
  1148. return src_nents;
  1149. }
  1150. if (src_nents) {
  1151. mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
  1152. DMA_TO_DEVICE);
  1153. if (!mapped_nents) {
  1154. dev_err(jrdev, "unable to map source for DMA\n");
  1155. return -ENOMEM;
  1156. }
  1157. } else {
  1158. mapped_nents = 0;
  1159. }
  1160. /*
  1161. * allocate space for base edesc and hw desc commands,
  1162. * link tables
  1163. */
  1164. edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ?
  1165. mapped_nents : 0,
  1166. ctx->sh_desc_update_first,
  1167. ctx->sh_desc_update_first_dma,
  1168. flags);
  1169. if (!edesc) {
  1170. dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
  1171. return -ENOMEM;
  1172. }
  1173. edesc->src_nents = src_nents;
  1174. ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
  1175. to_hash);
  1176. if (ret)
  1177. goto unmap_ctx;
  1178. if (*next_buflen)
  1179. scatterwalk_map_and_copy(next_buf, req->src, to_hash,
  1180. *next_buflen, 0);
  1181. desc = edesc->hw_desc;
  1182. ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
  1183. if (ret)
  1184. goto unmap_ctx;
  1185. #ifdef DEBUG
  1186. print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
  1187. DUMP_PREFIX_ADDRESS, 16, 4, desc,
  1188. desc_bytes(desc), 1);
  1189. #endif
  1190. ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
  1191. if (ret)
  1192. goto unmap_ctx;
  1193. ret = -EINPROGRESS;
  1194. state->update = ahash_update_ctx;
  1195. state->finup = ahash_finup_ctx;
  1196. state->final = ahash_final_ctx;
  1197. } else if (*next_buflen) {
  1198. state->update = ahash_update_no_ctx;
  1199. state->finup = ahash_finup_no_ctx;
  1200. state->final = ahash_final_no_ctx;
  1201. scatterwalk_map_and_copy(next_buf, req->src, 0,
  1202. req->nbytes, 0);
  1203. switch_buf(state);
  1204. }
  1205. #ifdef DEBUG
  1206. print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
  1207. DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
  1208. *next_buflen, 1);
  1209. #endif
  1210. return ret;
  1211. unmap_ctx:
  1212. ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
  1213. kfree(edesc);
  1214. return ret;
  1215. }
  1216. static int ahash_finup_first(struct ahash_request *req)
  1217. {
  1218. return ahash_digest(req);
  1219. }
  1220. static int ahash_init(struct ahash_request *req)
  1221. {
  1222. struct caam_hash_state *state = ahash_request_ctx(req);
  1223. state->update = ahash_update_first;
  1224. state->finup = ahash_finup_first;
  1225. state->final = ahash_final_no_ctx;
  1226. state->ctx_dma = 0;
  1227. state->current_buf = 0;
  1228. state->buf_dma = 0;
  1229. state->buflen_0 = 0;
  1230. state->buflen_1 = 0;
  1231. return 0;
  1232. }
  1233. static int ahash_update(struct ahash_request *req)
  1234. {
  1235. struct caam_hash_state *state = ahash_request_ctx(req);
  1236. return state->update(req);
  1237. }
  1238. static int ahash_finup(struct ahash_request *req)
  1239. {
  1240. struct caam_hash_state *state = ahash_request_ctx(req);
  1241. return state->finup(req);
  1242. }
  1243. static int ahash_final(struct ahash_request *req)
  1244. {
  1245. struct caam_hash_state *state = ahash_request_ctx(req);
  1246. return state->final(req);
  1247. }
  1248. static int ahash_export(struct ahash_request *req, void *out)
  1249. {
  1250. struct caam_hash_state *state = ahash_request_ctx(req);
  1251. struct caam_export_state *export = out;
  1252. int len;
  1253. u8 *buf;
  1254. if (state->current_buf) {
  1255. buf = state->buf_1;
  1256. len = state->buflen_1;
  1257. } else {
  1258. buf = state->buf_0;
  1259. len = state->buflen_0;
  1260. }
  1261. memcpy(export->buf, buf, len);
  1262. memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
  1263. export->buflen = len;
  1264. export->update = state->update;
  1265. export->final = state->final;
  1266. export->finup = state->finup;
  1267. return 0;
  1268. }
  1269. static int ahash_import(struct ahash_request *req, const void *in)
  1270. {
  1271. struct caam_hash_state *state = ahash_request_ctx(req);
  1272. const struct caam_export_state *export = in;
  1273. memset(state, 0, sizeof(*state));
  1274. memcpy(state->buf_0, export->buf, export->buflen);
  1275. memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
  1276. state->buflen_0 = export->buflen;
  1277. state->update = export->update;
  1278. state->final = export->final;
  1279. state->finup = export->finup;
  1280. return 0;
  1281. }
  1282. struct caam_hash_template {
  1283. char name[CRYPTO_MAX_ALG_NAME];
  1284. char driver_name[CRYPTO_MAX_ALG_NAME];
  1285. char hmac_name[CRYPTO_MAX_ALG_NAME];
  1286. char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
  1287. unsigned int blocksize;
  1288. struct ahash_alg template_ahash;
  1289. u32 alg_type;
  1290. };
  1291. /* ahash descriptors */
  1292. static struct caam_hash_template driver_hash[] = {
  1293. {
  1294. .name = "sha1",
  1295. .driver_name = "sha1-caam",
  1296. .hmac_name = "hmac(sha1)",
  1297. .hmac_driver_name = "hmac-sha1-caam",
  1298. .blocksize = SHA1_BLOCK_SIZE,
  1299. .template_ahash = {
  1300. .init = ahash_init,
  1301. .update = ahash_update,
  1302. .final = ahash_final,
  1303. .finup = ahash_finup,
  1304. .digest = ahash_digest,
  1305. .export = ahash_export,
  1306. .import = ahash_import,
  1307. .setkey = ahash_setkey,
  1308. .halg = {
  1309. .digestsize = SHA1_DIGEST_SIZE,
  1310. .statesize = sizeof(struct caam_export_state),
  1311. },
  1312. },
  1313. .alg_type = OP_ALG_ALGSEL_SHA1,
  1314. }, {
  1315. .name = "sha224",
  1316. .driver_name = "sha224-caam",
  1317. .hmac_name = "hmac(sha224)",
  1318. .hmac_driver_name = "hmac-sha224-caam",
  1319. .blocksize = SHA224_BLOCK_SIZE,
  1320. .template_ahash = {
  1321. .init = ahash_init,
  1322. .update = ahash_update,
  1323. .final = ahash_final,
  1324. .finup = ahash_finup,
  1325. .digest = ahash_digest,
  1326. .export = ahash_export,
  1327. .import = ahash_import,
  1328. .setkey = ahash_setkey,
  1329. .halg = {
  1330. .digestsize = SHA224_DIGEST_SIZE,
  1331. .statesize = sizeof(struct caam_export_state),
  1332. },
  1333. },
  1334. .alg_type = OP_ALG_ALGSEL_SHA224,
  1335. }, {
  1336. .name = "sha256",
  1337. .driver_name = "sha256-caam",
  1338. .hmac_name = "hmac(sha256)",
  1339. .hmac_driver_name = "hmac-sha256-caam",
  1340. .blocksize = SHA256_BLOCK_SIZE,
  1341. .template_ahash = {
  1342. .init = ahash_init,
  1343. .update = ahash_update,
  1344. .final = ahash_final,
  1345. .finup = ahash_finup,
  1346. .digest = ahash_digest,
  1347. .export = ahash_export,
  1348. .import = ahash_import,
  1349. .setkey = ahash_setkey,
  1350. .halg = {
  1351. .digestsize = SHA256_DIGEST_SIZE,
  1352. .statesize = sizeof(struct caam_export_state),
  1353. },
  1354. },
  1355. .alg_type = OP_ALG_ALGSEL_SHA256,
  1356. }, {
  1357. .name = "sha384",
  1358. .driver_name = "sha384-caam",
  1359. .hmac_name = "hmac(sha384)",
  1360. .hmac_driver_name = "hmac-sha384-caam",
  1361. .blocksize = SHA384_BLOCK_SIZE,
  1362. .template_ahash = {
  1363. .init = ahash_init,
  1364. .update = ahash_update,
  1365. .final = ahash_final,
  1366. .finup = ahash_finup,
  1367. .digest = ahash_digest,
  1368. .export = ahash_export,
  1369. .import = ahash_import,
  1370. .setkey = ahash_setkey,
  1371. .halg = {
  1372. .digestsize = SHA384_DIGEST_SIZE,
  1373. .statesize = sizeof(struct caam_export_state),
  1374. },
  1375. },
  1376. .alg_type = OP_ALG_ALGSEL_SHA384,
  1377. }, {
  1378. .name = "sha512",
  1379. .driver_name = "sha512-caam",
  1380. .hmac_name = "hmac(sha512)",
  1381. .hmac_driver_name = "hmac-sha512-caam",
  1382. .blocksize = SHA512_BLOCK_SIZE,
  1383. .template_ahash = {
  1384. .init = ahash_init,
  1385. .update = ahash_update,
  1386. .final = ahash_final,
  1387. .finup = ahash_finup,
  1388. .digest = ahash_digest,
  1389. .export = ahash_export,
  1390. .import = ahash_import,
  1391. .setkey = ahash_setkey,
  1392. .halg = {
  1393. .digestsize = SHA512_DIGEST_SIZE,
  1394. .statesize = sizeof(struct caam_export_state),
  1395. },
  1396. },
  1397. .alg_type = OP_ALG_ALGSEL_SHA512,
  1398. }, {
  1399. .name = "md5",
  1400. .driver_name = "md5-caam",
  1401. .hmac_name = "hmac(md5)",
  1402. .hmac_driver_name = "hmac-md5-caam",
  1403. .blocksize = MD5_BLOCK_WORDS * 4,
  1404. .template_ahash = {
  1405. .init = ahash_init,
  1406. .update = ahash_update,
  1407. .final = ahash_final,
  1408. .finup = ahash_finup,
  1409. .digest = ahash_digest,
  1410. .export = ahash_export,
  1411. .import = ahash_import,
  1412. .setkey = ahash_setkey,
  1413. .halg = {
  1414. .digestsize = MD5_DIGEST_SIZE,
  1415. .statesize = sizeof(struct caam_export_state),
  1416. },
  1417. },
  1418. .alg_type = OP_ALG_ALGSEL_MD5,
  1419. },
  1420. };
  1421. struct caam_hash_alg {
  1422. struct list_head entry;
  1423. int alg_type;
  1424. struct ahash_alg ahash_alg;
  1425. };
  1426. static int caam_hash_cra_init(struct crypto_tfm *tfm)
  1427. {
  1428. struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
  1429. struct crypto_alg *base = tfm->__crt_alg;
  1430. struct hash_alg_common *halg =
  1431. container_of(base, struct hash_alg_common, base);
  1432. struct ahash_alg *alg =
  1433. container_of(halg, struct ahash_alg, halg);
  1434. struct caam_hash_alg *caam_hash =
  1435. container_of(alg, struct caam_hash_alg, ahash_alg);
  1436. struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
  1437. /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
  1438. static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
  1439. HASH_MSG_LEN + SHA1_DIGEST_SIZE,
  1440. HASH_MSG_LEN + 32,
  1441. HASH_MSG_LEN + SHA256_DIGEST_SIZE,
  1442. HASH_MSG_LEN + 64,
  1443. HASH_MSG_LEN + SHA512_DIGEST_SIZE };
  1444. dma_addr_t dma_addr;
  1445. struct caam_drv_private *priv;
  1446. /*
  1447. * Get a Job ring from Job Ring driver to ensure in-order
  1448. * crypto request processing per tfm
  1449. */
  1450. ctx->jrdev = caam_jr_alloc();
  1451. if (IS_ERR(ctx->jrdev)) {
  1452. pr_err("Job Ring Device allocation for transform failed\n");
  1453. return PTR_ERR(ctx->jrdev);
  1454. }
  1455. priv = dev_get_drvdata(ctx->jrdev->parent);
  1456. ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
  1457. dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
  1458. offsetof(struct caam_hash_ctx,
  1459. sh_desc_update_dma),
  1460. ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
  1461. if (dma_mapping_error(ctx->jrdev, dma_addr)) {
  1462. dev_err(ctx->jrdev, "unable to map shared descriptors\n");
  1463. caam_jr_free(ctx->jrdev);
  1464. return -ENOMEM;
  1465. }
  1466. ctx->sh_desc_update_dma = dma_addr;
  1467. ctx->sh_desc_update_first_dma = dma_addr +
  1468. offsetof(struct caam_hash_ctx,
  1469. sh_desc_update_first);
  1470. ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx,
  1471. sh_desc_fin);
  1472. ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx,
  1473. sh_desc_digest);
  1474. /* copy descriptor header template value */
  1475. ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
  1476. ctx->ctx_len = runninglen[(ctx->adata.algtype &
  1477. OP_ALG_ALGSEL_SUBMASK) >>
  1478. OP_ALG_ALGSEL_SHIFT];
  1479. crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
  1480. sizeof(struct caam_hash_state));
  1481. return ahash_set_sh_desc(ahash);
  1482. }
  1483. static void caam_hash_cra_exit(struct crypto_tfm *tfm)
  1484. {
  1485. struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
  1486. dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
  1487. offsetof(struct caam_hash_ctx,
  1488. sh_desc_update_dma),
  1489. ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
  1490. caam_jr_free(ctx->jrdev);
  1491. }
  1492. static void __exit caam_algapi_hash_exit(void)
  1493. {
  1494. struct caam_hash_alg *t_alg, *n;
  1495. if (!hash_list.next)
  1496. return;
  1497. list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
  1498. crypto_unregister_ahash(&t_alg->ahash_alg);
  1499. list_del(&t_alg->entry);
  1500. kfree(t_alg);
  1501. }
  1502. }
  1503. static struct caam_hash_alg *
  1504. caam_hash_alloc(struct caam_hash_template *template,
  1505. bool keyed)
  1506. {
  1507. struct caam_hash_alg *t_alg;
  1508. struct ahash_alg *halg;
  1509. struct crypto_alg *alg;
  1510. t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
  1511. if (!t_alg) {
  1512. pr_err("failed to allocate t_alg\n");
  1513. return ERR_PTR(-ENOMEM);
  1514. }
  1515. t_alg->ahash_alg = template->template_ahash;
  1516. halg = &t_alg->ahash_alg;
  1517. alg = &halg->halg.base;
  1518. if (keyed) {
  1519. snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
  1520. template->hmac_name);
  1521. snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
  1522. template->hmac_driver_name);
  1523. } else {
  1524. snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
  1525. template->name);
  1526. snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
  1527. template->driver_name);
  1528. t_alg->ahash_alg.setkey = NULL;
  1529. }
  1530. alg->cra_module = THIS_MODULE;
  1531. alg->cra_init = caam_hash_cra_init;
  1532. alg->cra_exit = caam_hash_cra_exit;
  1533. alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
  1534. alg->cra_priority = CAAM_CRA_PRIORITY;
  1535. alg->cra_blocksize = template->blocksize;
  1536. alg->cra_alignmask = 0;
  1537. alg->cra_flags = CRYPTO_ALG_ASYNC;
  1538. t_alg->alg_type = template->alg_type;
  1539. return t_alg;
  1540. }
  1541. static int __init caam_algapi_hash_init(void)
  1542. {
  1543. struct device_node *dev_node;
  1544. struct platform_device *pdev;
  1545. struct device *ctrldev;
  1546. int i = 0, err = 0;
  1547. struct caam_drv_private *priv;
  1548. unsigned int md_limit = SHA512_DIGEST_SIZE;
  1549. u32 cha_inst, cha_vid;
  1550. dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
  1551. if (!dev_node) {
  1552. dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
  1553. if (!dev_node)
  1554. return -ENODEV;
  1555. }
  1556. pdev = of_find_device_by_node(dev_node);
  1557. if (!pdev) {
  1558. of_node_put(dev_node);
  1559. return -ENODEV;
  1560. }
  1561. ctrldev = &pdev->dev;
  1562. priv = dev_get_drvdata(ctrldev);
  1563. of_node_put(dev_node);
  1564. /*
  1565. * If priv is NULL, it's probably because the caam driver wasn't
  1566. * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
  1567. */
  1568. if (!priv)
  1569. return -ENODEV;
  1570. /*
  1571. * Register crypto algorithms the device supports. First, identify
  1572. * presence and attributes of MD block.
  1573. */
  1574. cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
  1575. cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
  1576. /*
  1577. * Skip registration of any hashing algorithms if MD block
  1578. * is not present.
  1579. */
  1580. if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT))
  1581. return -ENODEV;
  1582. /* Limit digest size based on LP256 */
  1583. if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)
  1584. md_limit = SHA256_DIGEST_SIZE;
  1585. INIT_LIST_HEAD(&hash_list);
  1586. /* register crypto algorithms the device supports */
  1587. for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
  1588. struct caam_hash_alg *t_alg;
  1589. struct caam_hash_template *alg = driver_hash + i;
  1590. /* If MD size is not supported by device, skip registration */
  1591. if (alg->template_ahash.halg.digestsize > md_limit)
  1592. continue;
  1593. /* register hmac version */
  1594. t_alg = caam_hash_alloc(alg, true);
  1595. if (IS_ERR(t_alg)) {
  1596. err = PTR_ERR(t_alg);
  1597. pr_warn("%s alg allocation failed\n", alg->driver_name);
  1598. continue;
  1599. }
  1600. err = crypto_register_ahash(&t_alg->ahash_alg);
  1601. if (err) {
  1602. pr_warn("%s alg registration failed: %d\n",
  1603. t_alg->ahash_alg.halg.base.cra_driver_name,
  1604. err);
  1605. kfree(t_alg);
  1606. } else
  1607. list_add_tail(&t_alg->entry, &hash_list);
  1608. /* register unkeyed version */
  1609. t_alg = caam_hash_alloc(alg, false);
  1610. if (IS_ERR(t_alg)) {
  1611. err = PTR_ERR(t_alg);
  1612. pr_warn("%s alg allocation failed\n", alg->driver_name);
  1613. continue;
  1614. }
  1615. err = crypto_register_ahash(&t_alg->ahash_alg);
  1616. if (err) {
  1617. pr_warn("%s alg registration failed: %d\n",
  1618. t_alg->ahash_alg.halg.base.cra_driver_name,
  1619. err);
  1620. kfree(t_alg);
  1621. } else
  1622. list_add_tail(&t_alg->entry, &hash_list);
  1623. }
  1624. return err;
  1625. }
  1626. module_init(caam_algapi_hash_init);
  1627. module_exit(caam_algapi_hash_exit);
  1628. MODULE_LICENSE("GPL");
  1629. MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
  1630. MODULE_AUTHOR("Freescale Semiconductor - NMG");