crypto.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569
  1. /*
  2. * This contains encryption functions for per-file encryption.
  3. *
  4. * Copyright (C) 2015, Google, Inc.
  5. * Copyright (C) 2015, Motorola Mobility
  6. *
  7. * Written by Michael Halcrow, 2014.
  8. *
  9. * Filename encryption additions
  10. * Uday Savagaonkar, 2014
  11. * Encryption policy handling additions
  12. * Ildar Muslukhov, 2014
  13. * Add fscrypt_pullback_bio_page()
  14. * Jaegeuk Kim, 2015.
  15. *
  16. * This has not yet undergone a rigorous security audit.
  17. *
  18. * The usage of AES-XTS should conform to recommendations in NIST
  19. * Special Publication 800-38E and IEEE P1619/D16.
  20. */
  21. #include <linux/pagemap.h>
  22. #include <linux/mempool.h>
  23. #include <linux/module.h>
  24. #include <linux/scatterlist.h>
  25. #include <linux/ratelimit.h>
  26. #include <linux/bio.h>
  27. #include <linux/dcache.h>
  28. #include <linux/namei.h>
  29. #include <linux/fscrypto.h>
  30. #include <linux/ecryptfs.h>
  31. static unsigned int num_prealloc_crypto_pages = 32;
  32. static unsigned int num_prealloc_crypto_ctxs = 128;
  33. module_param(num_prealloc_crypto_pages, uint, 0444);
  34. MODULE_PARM_DESC(num_prealloc_crypto_pages,
  35. "Number of crypto pages to preallocate");
  36. module_param(num_prealloc_crypto_ctxs, uint, 0444);
  37. MODULE_PARM_DESC(num_prealloc_crypto_ctxs,
  38. "Number of crypto contexts to preallocate");
  39. static mempool_t *fscrypt_bounce_page_pool = NULL;
  40. static LIST_HEAD(fscrypt_free_ctxs);
  41. static DEFINE_SPINLOCK(fscrypt_ctx_lock);
  42. static struct workqueue_struct *fscrypt_read_workqueue;
  43. static DEFINE_MUTEX(fscrypt_init_mutex);
  44. static struct kmem_cache *fscrypt_ctx_cachep;
  45. struct kmem_cache *fscrypt_info_cachep;
  46. /**
  47. * fscrypt_release_ctx() - Releases an encryption context
  48. * @ctx: The encryption context to release.
  49. *
  50. * If the encryption context was allocated from the pre-allocated pool, returns
  51. * it to that pool. Else, frees it.
  52. *
  53. * If there's a bounce page in the context, this frees that.
  54. */
  55. void fscrypt_release_ctx(struct fscrypt_ctx *ctx)
  56. {
  57. unsigned long flags;
  58. if (ctx->flags & FS_WRITE_PATH_FL && ctx->w.bounce_page) {
  59. mempool_free(ctx->w.bounce_page, fscrypt_bounce_page_pool);
  60. ctx->w.bounce_page = NULL;
  61. }
  62. ctx->w.control_page = NULL;
  63. if (ctx->flags & FS_CTX_REQUIRES_FREE_ENCRYPT_FL) {
  64. kmem_cache_free(fscrypt_ctx_cachep, ctx);
  65. } else {
  66. spin_lock_irqsave(&fscrypt_ctx_lock, flags);
  67. list_add(&ctx->free_list, &fscrypt_free_ctxs);
  68. spin_unlock_irqrestore(&fscrypt_ctx_lock, flags);
  69. }
  70. }
  71. EXPORT_SYMBOL(fscrypt_release_ctx);
  72. /**
  73. * fscrypt_get_ctx() - Gets an encryption context
  74. * @inode: The inode for which we are doing the crypto
  75. * @gfp_flags: The gfp flag for memory allocation
  76. *
  77. * Allocates and initializes an encryption context.
  78. *
  79. * Return: An allocated and initialized encryption context on success; error
  80. * value or NULL otherwise.
  81. */
  82. struct fscrypt_ctx *fscrypt_get_ctx(struct inode *inode, gfp_t gfp_flags)
  83. {
  84. struct fscrypt_ctx *ctx = NULL;
  85. struct fscrypt_info *ci = inode->i_crypt_info;
  86. unsigned long flags;
  87. if (ci == NULL)
  88. return ERR_PTR(-ENOKEY);
  89. /*
  90. * We first try getting the ctx from a free list because in
  91. * the common case the ctx will have an allocated and
  92. * initialized crypto tfm, so it's probably a worthwhile
  93. * optimization. For the bounce page, we first try getting it
  94. * from the kernel allocator because that's just about as fast
  95. * as getting it from a list and because a cache of free pages
  96. * should generally be a "last resort" option for a filesystem
  97. * to be able to do its job.
  98. */
  99. spin_lock_irqsave(&fscrypt_ctx_lock, flags);
  100. ctx = list_first_entry_or_null(&fscrypt_free_ctxs,
  101. struct fscrypt_ctx, free_list);
  102. if (ctx)
  103. list_del(&ctx->free_list);
  104. spin_unlock_irqrestore(&fscrypt_ctx_lock, flags);
  105. if (!ctx) {
  106. ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, gfp_flags);
  107. if (!ctx)
  108. return ERR_PTR(-ENOMEM);
  109. ctx->flags |= FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
  110. } else {
  111. ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
  112. }
  113. ctx->flags &= ~FS_WRITE_PATH_FL;
  114. return ctx;
  115. }
  116. EXPORT_SYMBOL(fscrypt_get_ctx);
  117. /**
  118. * fscrypt_complete() - The completion callback for page encryption
  119. * @req: The asynchronous encryption request context
  120. * @res: The result of the encryption operation
  121. */
  122. static void fscrypt_complete(struct crypto_async_request *req, int res)
  123. {
  124. struct fscrypt_completion_result *ecr = req->data;
  125. if (res == -EINPROGRESS)
  126. return;
  127. ecr->res = res;
  128. complete(&ecr->completion);
  129. }
  130. typedef enum {
  131. FS_DECRYPT = 0,
  132. FS_ENCRYPT,
  133. } fscrypt_direction_t;
  134. static int do_page_crypto(struct inode *inode,
  135. fscrypt_direction_t rw, pgoff_t index,
  136. struct page *src_page, struct page *dest_page,
  137. gfp_t gfp_flags)
  138. {
  139. u8 xts_tweak[FS_XTS_TWEAK_SIZE];
  140. struct skcipher_request *req = NULL;
  141. DECLARE_FS_COMPLETION_RESULT(ecr);
  142. struct scatterlist dst, src;
  143. struct fscrypt_info *ci = inode->i_crypt_info;
  144. struct crypto_skcipher *tfm = ci->ci_ctfm;
  145. int res = 0;
  146. req = skcipher_request_alloc(tfm, gfp_flags);
  147. if (!req) {
  148. printk_ratelimited(KERN_ERR
  149. "%s: crypto_request_alloc() failed\n",
  150. __func__);
  151. return -ENOMEM;
  152. }
  153. skcipher_request_set_callback(
  154. req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
  155. fscrypt_complete, &ecr);
  156. BUILD_BUG_ON(FS_XTS_TWEAK_SIZE < sizeof(index));
  157. memcpy(xts_tweak, &index, sizeof(index));
  158. memset(&xts_tweak[sizeof(index)], 0,
  159. FS_XTS_TWEAK_SIZE - sizeof(index));
  160. sg_init_table(&dst, 1);
  161. sg_set_page(&dst, dest_page, PAGE_SIZE, 0);
  162. sg_init_table(&src, 1);
  163. sg_set_page(&src, src_page, PAGE_SIZE, 0);
  164. skcipher_request_set_crypt(req, &src, &dst, PAGE_SIZE,
  165. xts_tweak);
  166. if (rw == FS_DECRYPT)
  167. res = crypto_skcipher_decrypt(req);
  168. else
  169. res = crypto_skcipher_encrypt(req);
  170. if (res == -EINPROGRESS || res == -EBUSY) {
  171. BUG_ON(req->base.data != &ecr);
  172. wait_for_completion(&ecr.completion);
  173. res = ecr.res;
  174. }
  175. skcipher_request_free(req);
  176. if (res) {
  177. printk_ratelimited(KERN_ERR
  178. "%s: crypto_skcipher_encrypt() returned %d\n",
  179. __func__, res);
  180. return res;
  181. }
  182. return 0;
  183. }
  184. static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags)
  185. {
  186. ctx->w.bounce_page = mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);
  187. if (ctx->w.bounce_page == NULL)
  188. return ERR_PTR(-ENOMEM);
  189. ctx->flags |= FS_WRITE_PATH_FL;
  190. return ctx->w.bounce_page;
  191. }
  192. /**
  193. * fscypt_encrypt_page() - Encrypts a page
  194. * @inode: The inode for which the encryption should take place
  195. * @plaintext_page: The page to encrypt. Must be locked.
  196. * @gfp_flags: The gfp flag for memory allocation
  197. *
  198. * Allocates a ciphertext page and encrypts plaintext_page into it using the ctx
  199. * encryption context.
  200. *
  201. * Called on the page write path. The caller must call
  202. * fscrypt_restore_control_page() on the returned ciphertext page to
  203. * release the bounce buffer and the encryption context.
  204. *
  205. * Return: An allocated page with the encrypted content on success. Else, an
  206. * error value or NULL.
  207. */
  208. struct page *fscrypt_encrypt_page(struct inode *inode,
  209. struct page *plaintext_page, gfp_t gfp_flags)
  210. {
  211. struct fscrypt_ctx *ctx;
  212. struct page *ciphertext_page = NULL;
  213. int err;
  214. BUG_ON(!PageLocked(plaintext_page));
  215. ctx = fscrypt_get_ctx(inode, gfp_flags);
  216. if (IS_ERR(ctx))
  217. return (struct page *)ctx;
  218. /* The encryption operation will require a bounce page. */
  219. ciphertext_page = alloc_bounce_page(ctx, gfp_flags);
  220. if (IS_ERR(ciphertext_page))
  221. goto errout;
  222. ctx->w.control_page = plaintext_page;
  223. err = do_page_crypto(inode, FS_ENCRYPT, plaintext_page->index,
  224. plaintext_page, ciphertext_page,
  225. gfp_flags);
  226. if (err) {
  227. ciphertext_page = ERR_PTR(err);
  228. goto errout;
  229. }
  230. SetPagePrivate(ciphertext_page);
  231. set_page_private(ciphertext_page, (unsigned long)ctx);
  232. lock_page(ciphertext_page);
  233. return ciphertext_page;
  234. errout:
  235. fscrypt_release_ctx(ctx);
  236. return ciphertext_page;
  237. }
  238. EXPORT_SYMBOL(fscrypt_encrypt_page);
  239. /**
  240. * f2crypt_decrypt_page() - Decrypts a page in-place
  241. * @page: The page to decrypt. Must be locked.
  242. *
  243. * Decrypts page in-place using the ctx encryption context.
  244. *
  245. * Called from the read completion callback.
  246. *
  247. * Return: Zero on success, non-zero otherwise.
  248. */
  249. int fscrypt_decrypt_page(struct page *page)
  250. {
  251. BUG_ON(!PageLocked(page));
  252. return do_page_crypto(page->mapping->host,
  253. FS_DECRYPT, page->index, page, page, GFP_NOFS);
  254. }
  255. EXPORT_SYMBOL(fscrypt_decrypt_page);
  256. int fscrypt_zeroout_range(struct inode *inode, pgoff_t lblk,
  257. sector_t pblk, unsigned int len)
  258. {
  259. struct fscrypt_ctx *ctx;
  260. struct page *ciphertext_page = NULL;
  261. struct bio *bio;
  262. int ret, err = 0;
  263. BUG_ON(inode->i_sb->s_blocksize != PAGE_SIZE);
  264. ctx = fscrypt_get_ctx(inode, GFP_NOFS);
  265. if (IS_ERR(ctx))
  266. return PTR_ERR(ctx);
  267. ciphertext_page = alloc_bounce_page(ctx, GFP_NOWAIT);
  268. if (IS_ERR(ciphertext_page)) {
  269. err = PTR_ERR(ciphertext_page);
  270. goto errout;
  271. }
  272. while (len--) {
  273. err = do_page_crypto(inode, FS_ENCRYPT, lblk,
  274. ZERO_PAGE(0), ciphertext_page,
  275. GFP_NOFS);
  276. if (err)
  277. goto errout;
  278. bio = bio_alloc(GFP_NOWAIT, 1);
  279. if (!bio) {
  280. err = -ENOMEM;
  281. goto errout;
  282. }
  283. bio->bi_bdev = inode->i_sb->s_bdev;
  284. bio->bi_iter.bi_sector =
  285. pblk << (inode->i_sb->s_blocksize_bits - 9);
  286. bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
  287. ret = bio_add_page(bio, ciphertext_page,
  288. inode->i_sb->s_blocksize, 0);
  289. if (ret != inode->i_sb->s_blocksize) {
  290. /* should never happen! */
  291. WARN_ON(1);
  292. bio_put(bio);
  293. err = -EIO;
  294. goto errout;
  295. }
  296. err = submit_bio_wait(bio);
  297. if ((err == 0) && bio->bi_error)
  298. err = -EIO;
  299. bio_put(bio);
  300. if (err)
  301. goto errout;
  302. lblk++;
  303. pblk++;
  304. }
  305. err = 0;
  306. errout:
  307. fscrypt_release_ctx(ctx);
  308. return err;
  309. }
  310. EXPORT_SYMBOL(fscrypt_zeroout_range);
  311. /*
  312. * Validate dentries for encrypted directories to make sure we aren't
  313. * potentially caching stale data after a key has been added or
  314. * removed.
  315. */
  316. static int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags)
  317. {
  318. struct dentry *dir;
  319. struct fscrypt_info *ci;
  320. int dir_has_key, cached_with_key;
  321. if (flags & LOOKUP_RCU)
  322. return -ECHILD;
  323. dir = dget_parent(dentry);
  324. if (!d_inode(dir)->i_sb->s_cop->is_encrypted(d_inode(dir))) {
  325. dput(dir);
  326. return 0;
  327. }
  328. ci = d_inode(dir)->i_crypt_info;
  329. if (ci && ci->ci_keyring_key &&
  330. (ci->ci_keyring_key->flags & ((1 << KEY_FLAG_INVALIDATED) |
  331. (1 << KEY_FLAG_REVOKED) |
  332. (1 << KEY_FLAG_DEAD))))
  333. ci = NULL;
  334. /* this should eventually be an flag in d_flags */
  335. spin_lock(&dentry->d_lock);
  336. cached_with_key = dentry->d_flags & DCACHE_ENCRYPTED_WITH_KEY;
  337. spin_unlock(&dentry->d_lock);
  338. dir_has_key = (ci != NULL);
  339. dput(dir);
  340. /*
  341. * If the dentry was cached without the key, and it is a
  342. * negative dentry, it might be a valid name. We can't check
  343. * if the key has since been made available due to locking
  344. * reasons, so we fail the validation so ext4_lookup() can do
  345. * this check.
  346. *
  347. * We also fail the validation if the dentry was created with
  348. * the key present, but we no longer have the key, or vice versa.
  349. */
  350. if ((!cached_with_key && d_is_negative(dentry)) ||
  351. (!cached_with_key && dir_has_key) ||
  352. (cached_with_key && !dir_has_key))
  353. return 0;
  354. return 1;
  355. }
  356. const struct dentry_operations fscrypt_d_ops = {
  357. .d_revalidate = fscrypt_d_revalidate,
  358. };
  359. EXPORT_SYMBOL(fscrypt_d_ops);
  360. /*
  361. * Call fscrypt_decrypt_page on every single page, reusing the encryption
  362. * context.
  363. */
  364. static void completion_pages(struct work_struct *work)
  365. {
  366. struct fscrypt_ctx *ctx =
  367. container_of(work, struct fscrypt_ctx, r.work);
  368. struct bio *bio = ctx->r.bio;
  369. struct bio_vec *bv;
  370. int i;
  371. bio_for_each_segment_all(bv, bio, i) {
  372. struct page *page = bv->bv_page;
  373. int ret = fscrypt_decrypt_page(page);
  374. if (ret) {
  375. WARN_ON_ONCE(1);
  376. SetPageError(page);
  377. } else {
  378. SetPageUptodate(page);
  379. }
  380. unlock_page(page);
  381. }
  382. fscrypt_release_ctx(ctx);
  383. bio_put(bio);
  384. }
  385. void fscrypt_decrypt_bio_pages(struct fscrypt_ctx *ctx, struct bio *bio)
  386. {
  387. INIT_WORK(&ctx->r.work, completion_pages);
  388. ctx->r.bio = bio;
  389. queue_work(fscrypt_read_workqueue, &ctx->r.work);
  390. }
  391. EXPORT_SYMBOL(fscrypt_decrypt_bio_pages);
  392. void fscrypt_pullback_bio_page(struct page **page, bool restore)
  393. {
  394. struct fscrypt_ctx *ctx;
  395. struct page *bounce_page;
  396. /* The bounce data pages are unmapped. */
  397. if ((*page)->mapping)
  398. return;
  399. /* The bounce data page is unmapped. */
  400. bounce_page = *page;
  401. ctx = (struct fscrypt_ctx *)page_private(bounce_page);
  402. /* restore control page */
  403. *page = ctx->w.control_page;
  404. if (restore)
  405. fscrypt_restore_control_page(bounce_page);
  406. }
  407. EXPORT_SYMBOL(fscrypt_pullback_bio_page);
  408. void fscrypt_restore_control_page(struct page *page)
  409. {
  410. struct fscrypt_ctx *ctx;
  411. ctx = (struct fscrypt_ctx *)page_private(page);
  412. set_page_private(page, (unsigned long)NULL);
  413. ClearPagePrivate(page);
  414. unlock_page(page);
  415. fscrypt_release_ctx(ctx);
  416. }
  417. EXPORT_SYMBOL(fscrypt_restore_control_page);
  418. static void fscrypt_destroy(void)
  419. {
  420. struct fscrypt_ctx *pos, *n;
  421. list_for_each_entry_safe(pos, n, &fscrypt_free_ctxs, free_list)
  422. kmem_cache_free(fscrypt_ctx_cachep, pos);
  423. INIT_LIST_HEAD(&fscrypt_free_ctxs);
  424. mempool_destroy(fscrypt_bounce_page_pool);
  425. fscrypt_bounce_page_pool = NULL;
  426. }
  427. /**
  428. * fscrypt_initialize() - allocate major buffers for fs encryption.
  429. *
  430. * We only call this when we start accessing encrypted files, since it
  431. * results in memory getting allocated that wouldn't otherwise be used.
  432. *
  433. * Return: Zero on success, non-zero otherwise.
  434. */
  435. int fscrypt_initialize(void)
  436. {
  437. int i, res = -ENOMEM;
  438. if (fscrypt_bounce_page_pool)
  439. return 0;
  440. mutex_lock(&fscrypt_init_mutex);
  441. if (fscrypt_bounce_page_pool)
  442. goto already_initialized;
  443. for (i = 0; i < num_prealloc_crypto_ctxs; i++) {
  444. struct fscrypt_ctx *ctx;
  445. ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, GFP_NOFS);
  446. if (!ctx)
  447. goto fail;
  448. list_add(&ctx->free_list, &fscrypt_free_ctxs);
  449. }
  450. fscrypt_bounce_page_pool =
  451. mempool_create_page_pool(num_prealloc_crypto_pages, 0);
  452. if (!fscrypt_bounce_page_pool)
  453. goto fail;
  454. already_initialized:
  455. mutex_unlock(&fscrypt_init_mutex);
  456. return 0;
  457. fail:
  458. fscrypt_destroy();
  459. mutex_unlock(&fscrypt_init_mutex);
  460. return res;
  461. }
  462. EXPORT_SYMBOL(fscrypt_initialize);
  463. /**
  464. * fscrypt_init() - Set up for fs encryption.
  465. */
  466. static int __init fscrypt_init(void)
  467. {
  468. fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue",
  469. WQ_HIGHPRI, 0);
  470. if (!fscrypt_read_workqueue)
  471. goto fail;
  472. fscrypt_ctx_cachep = KMEM_CACHE(fscrypt_ctx, SLAB_RECLAIM_ACCOUNT);
  473. if (!fscrypt_ctx_cachep)
  474. goto fail_free_queue;
  475. fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT);
  476. if (!fscrypt_info_cachep)
  477. goto fail_free_ctx;
  478. return 0;
  479. fail_free_ctx:
  480. kmem_cache_destroy(fscrypt_ctx_cachep);
  481. fail_free_queue:
  482. destroy_workqueue(fscrypt_read_workqueue);
  483. fail:
  484. return -ENOMEM;
  485. }
  486. module_init(fscrypt_init)
  487. /**
  488. * fscrypt_exit() - Shutdown the fs encryption system
  489. */
  490. static void __exit fscrypt_exit(void)
  491. {
  492. fscrypt_destroy();
  493. if (fscrypt_read_workqueue)
  494. destroy_workqueue(fscrypt_read_workqueue);
  495. kmem_cache_destroy(fscrypt_ctx_cachep);
  496. kmem_cache_destroy(fscrypt_info_cachep);
  497. }
  498. module_exit(fscrypt_exit);
  499. MODULE_LICENSE("GPL");