loop.c 53 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168
  1. /*
  2. * linux/drivers/block/loop.c
  3. *
  4. * Written by Theodore Ts'o, 3/29/93
  5. *
  6. * Copyright 1993 by Theodore Ts'o. Redistribution of this file is
  7. * permitted under the GNU General Public License.
  8. *
  9. * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993
  10. * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996
  11. *
  12. * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994
  13. * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996
  14. *
  15. * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997
  16. *
  17. * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998
  18. *
  19. * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998
  20. *
  21. * Loadable modules and other fixes by AK, 1998
  22. *
  23. * Make real block number available to downstream transfer functions, enables
  24. * CBC (and relatives) mode encryption requiring unique IVs per data block.
  25. * Reed H. Petty, rhp@draper.net
  26. *
  27. * Maximum number of loop devices now dynamic via max_loop module parameter.
  28. * Russell Kroll <rkroll@exploits.org> 19990701
  29. *
  30. * Maximum number of loop devices when compiled-in now selectable by passing
  31. * max_loop=<1-255> to the kernel on boot.
  32. * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999
  33. *
  34. * Completely rewrite request handling to be make_request_fn style and
  35. * non blocking, pushing work to a helper thread. Lots of fixes from
  36. * Al Viro too.
  37. * Jens Axboe <axboe@suse.de>, Nov 2000
  38. *
  39. * Support up to 256 loop devices
  40. * Heinz Mauelshagen <mge@sistina.com>, Feb 2002
  41. *
  42. * Support for falling back on the write file operation when the address space
  43. * operations write_begin is not available on the backing filesystem.
  44. * Anton Altaparmakov, 16 Feb 2005
  45. *
  46. * Still To Fix:
  47. * - Advisory locking is ignored here.
  48. * - Should use an own CAP_* category instead of CAP_SYS_ADMIN
  49. *
  50. */
  51. #include <linux/module.h>
  52. #include <linux/moduleparam.h>
  53. #include <linux/sched.h>
  54. #include <linux/fs.h>
  55. #include <linux/file.h>
  56. #include <linux/stat.h>
  57. #include <linux/errno.h>
  58. #include <linux/major.h>
  59. #include <linux/wait.h>
  60. #include <linux/blkdev.h>
  61. #include <linux/blkpg.h>
  62. #include <linux/init.h>
  63. #include <linux/swap.h>
  64. #include <linux/slab.h>
  65. #include <linux/compat.h>
  66. #include <linux/suspend.h>
  67. #include <linux/freezer.h>
  68. #include <linux/mutex.h>
  69. #include <linux/writeback.h>
  70. #include <linux/completion.h>
  71. #include <linux/highmem.h>
  72. #include <linux/kthread.h>
  73. #include <linux/splice.h>
  74. #include <linux/sysfs.h>
  75. #include <linux/miscdevice.h>
  76. #include <linux/falloc.h>
  77. #include <linux/uio.h>
  78. #include <linux/ioprio.h>
  79. #include "loop.h"
  80. #include <linux/uaccess.h>
  81. static DEFINE_IDR(loop_index_idr);
  82. static DEFINE_MUTEX(loop_index_mutex);
  83. static int max_part;
  84. static int part_shift;
  85. static int transfer_xor(struct loop_device *lo, int cmd,
  86. struct page *raw_page, unsigned raw_off,
  87. struct page *loop_page, unsigned loop_off,
  88. int size, sector_t real_block)
  89. {
  90. char *raw_buf = kmap_atomic(raw_page) + raw_off;
  91. char *loop_buf = kmap_atomic(loop_page) + loop_off;
  92. char *in, *out, *key;
  93. int i, keysize;
  94. if (cmd == READ) {
  95. in = raw_buf;
  96. out = loop_buf;
  97. } else {
  98. in = loop_buf;
  99. out = raw_buf;
  100. }
  101. key = lo->lo_encrypt_key;
  102. keysize = lo->lo_encrypt_key_size;
  103. for (i = 0; i < size; i++)
  104. *out++ = *in++ ^ key[(i & 511) % keysize];
  105. kunmap_atomic(loop_buf);
  106. kunmap_atomic(raw_buf);
  107. cond_resched();
  108. return 0;
  109. }
  110. static int xor_init(struct loop_device *lo, const struct loop_info64 *info)
  111. {
  112. if (unlikely(info->lo_encrypt_key_size <= 0))
  113. return -EINVAL;
  114. return 0;
  115. }
  116. static struct loop_func_table none_funcs = {
  117. .number = LO_CRYPT_NONE,
  118. };
  119. static struct loop_func_table xor_funcs = {
  120. .number = LO_CRYPT_XOR,
  121. .transfer = transfer_xor,
  122. .init = xor_init
  123. };
  124. /* xfer_funcs[0] is special - its release function is never called */
  125. static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
  126. &none_funcs,
  127. &xor_funcs
  128. };
  129. static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file)
  130. {
  131. loff_t loopsize;
  132. /* Compute loopsize in bytes */
  133. loopsize = i_size_read(file->f_mapping->host);
  134. if (offset > 0)
  135. loopsize -= offset;
  136. /* offset is beyond i_size, weird but possible */
  137. if (loopsize < 0)
  138. return 0;
  139. if (sizelimit > 0 && sizelimit < loopsize)
  140. loopsize = sizelimit;
  141. /*
  142. * Unfortunately, if we want to do I/O on the device,
  143. * the number of 512-byte sectors has to fit into a sector_t.
  144. */
  145. return loopsize >> 9;
  146. }
  147. static loff_t get_loop_size(struct loop_device *lo, struct file *file)
  148. {
  149. return get_size(lo->lo_offset, lo->lo_sizelimit, file);
  150. }
  151. static void __loop_update_dio(struct loop_device *lo, bool dio)
  152. {
  153. struct file *file = lo->lo_backing_file;
  154. struct address_space *mapping = file->f_mapping;
  155. struct inode *inode = mapping->host;
  156. unsigned short sb_bsize = 0;
  157. unsigned dio_align = 0;
  158. bool use_dio;
  159. if (inode->i_sb->s_bdev) {
  160. sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev);
  161. dio_align = sb_bsize - 1;
  162. }
  163. /*
  164. * We support direct I/O only if lo_offset is aligned with the
  165. * logical I/O size of backing device, and the logical block
  166. * size of loop is bigger than the backing device's and the loop
  167. * needn't transform transfer.
  168. *
  169. * TODO: the above condition may be loosed in the future, and
  170. * direct I/O may be switched runtime at that time because most
  171. * of requests in sane applications should be PAGE_SIZE aligned
  172. */
  173. if (dio) {
  174. if (queue_logical_block_size(lo->lo_queue) >= sb_bsize &&
  175. !(lo->lo_offset & dio_align) &&
  176. mapping->a_ops->direct_IO &&
  177. !lo->transfer)
  178. use_dio = true;
  179. else
  180. use_dio = false;
  181. } else {
  182. use_dio = false;
  183. }
  184. if (lo->use_dio == use_dio)
  185. return;
  186. /* flush dirty pages before changing direct IO */
  187. vfs_fsync(file, 0);
  188. /*
  189. * The flag of LO_FLAGS_DIRECT_IO is handled similarly with
  190. * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup
  191. * will get updated by ioctl(LOOP_GET_STATUS)
  192. */
  193. blk_mq_freeze_queue(lo->lo_queue);
  194. lo->use_dio = use_dio;
  195. if (use_dio) {
  196. blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, lo->lo_queue);
  197. lo->lo_flags |= LO_FLAGS_DIRECT_IO;
  198. } else {
  199. blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue);
  200. lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
  201. }
  202. blk_mq_unfreeze_queue(lo->lo_queue);
  203. }
  204. static int
  205. figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit)
  206. {
  207. loff_t size = get_size(offset, sizelimit, lo->lo_backing_file);
  208. sector_t x = (sector_t)size;
  209. struct block_device *bdev = lo->lo_device;
  210. if (unlikely((loff_t)x != size))
  211. return -EFBIG;
  212. if (lo->lo_offset != offset)
  213. lo->lo_offset = offset;
  214. if (lo->lo_sizelimit != sizelimit)
  215. lo->lo_sizelimit = sizelimit;
  216. set_capacity(lo->lo_disk, x);
  217. bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9);
  218. /* let user-space know about the new size */
  219. kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
  220. return 0;
  221. }
  222. static inline int
  223. lo_do_transfer(struct loop_device *lo, int cmd,
  224. struct page *rpage, unsigned roffs,
  225. struct page *lpage, unsigned loffs,
  226. int size, sector_t rblock)
  227. {
  228. int ret;
  229. ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock);
  230. if (likely(!ret))
  231. return 0;
  232. printk_ratelimited(KERN_ERR
  233. "loop: Transfer error at byte offset %llu, length %i.\n",
  234. (unsigned long long)rblock << 9, size);
  235. return ret;
  236. }
  237. static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
  238. {
  239. struct iov_iter i;
  240. ssize_t bw;
  241. iov_iter_bvec(&i, ITER_BVEC | WRITE, bvec, 1, bvec->bv_len);
  242. file_start_write(file);
  243. bw = vfs_iter_write(file, &i, ppos, 0);
  244. file_end_write(file);
  245. if (likely(bw == bvec->bv_len))
  246. return 0;
  247. printk_ratelimited(KERN_ERR
  248. "loop: Write error at byte offset %llu, length %i.\n",
  249. (unsigned long long)*ppos, bvec->bv_len);
  250. if (bw >= 0)
  251. bw = -EIO;
  252. return bw;
  253. }
  254. static int lo_write_simple(struct loop_device *lo, struct request *rq,
  255. loff_t pos)
  256. {
  257. struct bio_vec bvec;
  258. struct req_iterator iter;
  259. int ret = 0;
  260. rq_for_each_segment(bvec, rq, iter) {
  261. ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos);
  262. if (ret < 0)
  263. break;
  264. cond_resched();
  265. }
  266. return ret;
  267. }
  268. /*
  269. * This is the slow, transforming version that needs to double buffer the
  270. * data as it cannot do the transformations in place without having direct
  271. * access to the destination pages of the backing file.
  272. */
  273. static int lo_write_transfer(struct loop_device *lo, struct request *rq,
  274. loff_t pos)
  275. {
  276. struct bio_vec bvec, b;
  277. struct req_iterator iter;
  278. struct page *page;
  279. int ret = 0;
  280. page = alloc_page(GFP_NOIO);
  281. if (unlikely(!page))
  282. return -ENOMEM;
  283. rq_for_each_segment(bvec, rq, iter) {
  284. ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page,
  285. bvec.bv_offset, bvec.bv_len, pos >> 9);
  286. if (unlikely(ret))
  287. break;
  288. b.bv_page = page;
  289. b.bv_offset = 0;
  290. b.bv_len = bvec.bv_len;
  291. ret = lo_write_bvec(lo->lo_backing_file, &b, &pos);
  292. if (ret < 0)
  293. break;
  294. }
  295. __free_page(page);
  296. return ret;
  297. }
  298. static int lo_read_simple(struct loop_device *lo, struct request *rq,
  299. loff_t pos)
  300. {
  301. struct bio_vec bvec;
  302. struct req_iterator iter;
  303. struct iov_iter i;
  304. ssize_t len;
  305. rq_for_each_segment(bvec, rq, iter) {
  306. iov_iter_bvec(&i, ITER_BVEC, &bvec, 1, bvec.bv_len);
  307. len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
  308. if (len < 0)
  309. return len;
  310. flush_dcache_page(bvec.bv_page);
  311. if (len != bvec.bv_len) {
  312. struct bio *bio;
  313. __rq_for_each_bio(bio, rq)
  314. zero_fill_bio(bio);
  315. break;
  316. }
  317. cond_resched();
  318. }
  319. return 0;
  320. }
  321. static int lo_read_transfer(struct loop_device *lo, struct request *rq,
  322. loff_t pos)
  323. {
  324. struct bio_vec bvec, b;
  325. struct req_iterator iter;
  326. struct iov_iter i;
  327. struct page *page;
  328. ssize_t len;
  329. int ret = 0;
  330. page = alloc_page(GFP_NOIO);
  331. if (unlikely(!page))
  332. return -ENOMEM;
  333. rq_for_each_segment(bvec, rq, iter) {
  334. loff_t offset = pos;
  335. b.bv_page = page;
  336. b.bv_offset = 0;
  337. b.bv_len = bvec.bv_len;
  338. iov_iter_bvec(&i, ITER_BVEC, &b, 1, b.bv_len);
  339. len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
  340. if (len < 0) {
  341. ret = len;
  342. goto out_free_page;
  343. }
  344. ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page,
  345. bvec.bv_offset, len, offset >> 9);
  346. if (ret)
  347. goto out_free_page;
  348. flush_dcache_page(bvec.bv_page);
  349. if (len != bvec.bv_len) {
  350. struct bio *bio;
  351. __rq_for_each_bio(bio, rq)
  352. zero_fill_bio(bio);
  353. break;
  354. }
  355. }
  356. ret = 0;
  357. out_free_page:
  358. __free_page(page);
  359. return ret;
  360. }
  361. static int lo_discard(struct loop_device *lo, struct request *rq, loff_t pos)
  362. {
  363. /*
  364. * We use punch hole to reclaim the free space used by the
  365. * image a.k.a. discard. However we do not support discard if
  366. * encryption is enabled, because it may give an attacker
  367. * useful information.
  368. */
  369. struct file *file = lo->lo_backing_file;
  370. int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
  371. int ret;
  372. if ((!file->f_op->fallocate) || lo->lo_encrypt_key_size) {
  373. ret = -EOPNOTSUPP;
  374. goto out;
  375. }
  376. ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
  377. if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
  378. ret = -EIO;
  379. out:
  380. return ret;
  381. }
  382. static int lo_req_flush(struct loop_device *lo, struct request *rq)
  383. {
  384. struct file *file = lo->lo_backing_file;
  385. int ret = vfs_fsync(file, 0);
  386. if (unlikely(ret && ret != -EINVAL))
  387. ret = -EIO;
  388. return ret;
  389. }
  390. static void lo_complete_rq(struct request *rq)
  391. {
  392. struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
  393. blk_status_t ret = BLK_STS_OK;
  394. if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) ||
  395. req_op(rq) != REQ_OP_READ) {
  396. if (cmd->ret < 0)
  397. ret = BLK_STS_IOERR;
  398. goto end_io;
  399. }
  400. /*
  401. * Short READ - if we got some data, advance our request and
  402. * retry it. If we got no data, end the rest with EIO.
  403. */
  404. if (cmd->ret) {
  405. blk_update_request(rq, BLK_STS_OK, cmd->ret);
  406. cmd->ret = 0;
  407. blk_mq_requeue_request(rq, true);
  408. } else {
  409. if (cmd->use_aio) {
  410. struct bio *bio = rq->bio;
  411. while (bio) {
  412. zero_fill_bio(bio);
  413. bio = bio->bi_next;
  414. }
  415. }
  416. ret = BLK_STS_IOERR;
  417. end_io:
  418. blk_mq_end_request(rq, ret);
  419. }
  420. }
  421. static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
  422. {
  423. struct request *rq = blk_mq_rq_from_pdu(cmd);
  424. if (!atomic_dec_and_test(&cmd->ref))
  425. return;
  426. kfree(cmd->bvec);
  427. cmd->bvec = NULL;
  428. blk_mq_complete_request(rq);
  429. }
  430. static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
  431. {
  432. struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
  433. if (cmd->css)
  434. css_put(cmd->css);
  435. cmd->ret = ret;
  436. lo_rw_aio_do_completion(cmd);
  437. }
  438. static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
  439. loff_t pos, bool rw)
  440. {
  441. struct iov_iter iter;
  442. struct bio_vec *bvec;
  443. struct request *rq = blk_mq_rq_from_pdu(cmd);
  444. struct bio *bio = rq->bio;
  445. struct file *file = lo->lo_backing_file;
  446. unsigned int offset;
  447. int segments = 0;
  448. int ret;
  449. if (rq->bio != rq->biotail) {
  450. struct req_iterator iter;
  451. struct bio_vec tmp;
  452. __rq_for_each_bio(bio, rq)
  453. segments += bio_segments(bio);
  454. bvec = kmalloc_array(segments, sizeof(struct bio_vec),
  455. GFP_NOIO);
  456. if (!bvec)
  457. return -EIO;
  458. cmd->bvec = bvec;
  459. /*
  460. * The bios of the request may be started from the middle of
  461. * the 'bvec' because of bio splitting, so we can't directly
  462. * copy bio->bi_iov_vec to new bvec. The rq_for_each_segment
  463. * API will take care of all details for us.
  464. */
  465. rq_for_each_segment(tmp, rq, iter) {
  466. *bvec = tmp;
  467. bvec++;
  468. }
  469. bvec = cmd->bvec;
  470. offset = 0;
  471. } else {
  472. /*
  473. * Same here, this bio may be started from the middle of the
  474. * 'bvec' because of bio splitting, so offset from the bvec
  475. * must be passed to iov iterator
  476. */
  477. offset = bio->bi_iter.bi_bvec_done;
  478. bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
  479. segments = bio_segments(bio);
  480. }
  481. atomic_set(&cmd->ref, 2);
  482. iov_iter_bvec(&iter, ITER_BVEC | rw, bvec,
  483. segments, blk_rq_bytes(rq));
  484. iter.iov_offset = offset;
  485. cmd->iocb.ki_pos = pos;
  486. cmd->iocb.ki_filp = file;
  487. cmd->iocb.ki_complete = lo_rw_aio_complete;
  488. cmd->iocb.ki_flags = IOCB_DIRECT;
  489. cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
  490. if (cmd->css)
  491. kthread_associate_blkcg(cmd->css);
  492. if (rw == WRITE)
  493. ret = call_write_iter(file, &cmd->iocb, &iter);
  494. else
  495. ret = call_read_iter(file, &cmd->iocb, &iter);
  496. lo_rw_aio_do_completion(cmd);
  497. kthread_associate_blkcg(NULL);
  498. if (ret != -EIOCBQUEUED)
  499. cmd->iocb.ki_complete(&cmd->iocb, ret, 0);
  500. return 0;
  501. }
  502. static int do_req_filebacked(struct loop_device *lo, struct request *rq)
  503. {
  504. struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
  505. loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
  506. /*
  507. * lo_write_simple and lo_read_simple should have been covered
  508. * by io submit style function like lo_rw_aio(), one blocker
  509. * is that lo_read_simple() need to call flush_dcache_page after
  510. * the page is written from kernel, and it isn't easy to handle
  511. * this in io submit style function which submits all segments
  512. * of the req at one time. And direct read IO doesn't need to
  513. * run flush_dcache_page().
  514. */
  515. switch (req_op(rq)) {
  516. case REQ_OP_FLUSH:
  517. return lo_req_flush(lo, rq);
  518. case REQ_OP_DISCARD:
  519. case REQ_OP_WRITE_ZEROES:
  520. return lo_discard(lo, rq, pos);
  521. case REQ_OP_WRITE:
  522. if (lo->transfer)
  523. return lo_write_transfer(lo, rq, pos);
  524. else if (cmd->use_aio)
  525. return lo_rw_aio(lo, cmd, pos, WRITE);
  526. else
  527. return lo_write_simple(lo, rq, pos);
  528. case REQ_OP_READ:
  529. if (lo->transfer)
  530. return lo_read_transfer(lo, rq, pos);
  531. else if (cmd->use_aio)
  532. return lo_rw_aio(lo, cmd, pos, READ);
  533. else
  534. return lo_read_simple(lo, rq, pos);
  535. default:
  536. WARN_ON_ONCE(1);
  537. return -EIO;
  538. break;
  539. }
  540. }
  541. static inline void loop_update_dio(struct loop_device *lo)
  542. {
  543. __loop_update_dio(lo, io_is_direct(lo->lo_backing_file) |
  544. lo->use_dio);
  545. }
  546. static void loop_reread_partitions(struct loop_device *lo,
  547. struct block_device *bdev)
  548. {
  549. int rc;
  550. /*
  551. * bd_mutex has been held already in release path, so don't
  552. * acquire it if this function is called in such case.
  553. *
  554. * If the reread partition isn't from release path, lo_refcnt
  555. * must be at least one and it can only become zero when the
  556. * current holder is released.
  557. */
  558. if (!atomic_read(&lo->lo_refcnt))
  559. rc = __blkdev_reread_part(bdev);
  560. else
  561. rc = blkdev_reread_part(bdev);
  562. if (rc)
  563. pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n",
  564. __func__, lo->lo_number, lo->lo_file_name, rc);
  565. }
  566. static inline int is_loop_device(struct file *file)
  567. {
  568. struct inode *i = file->f_mapping->host;
  569. return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR;
  570. }
  571. static int loop_validate_file(struct file *file, struct block_device *bdev)
  572. {
  573. struct inode *inode = file->f_mapping->host;
  574. struct file *f = file;
  575. /* Avoid recursion */
  576. while (is_loop_device(f)) {
  577. struct loop_device *l;
  578. if (f->f_mapping->host->i_bdev == bdev)
  579. return -EBADF;
  580. l = f->f_mapping->host->i_bdev->bd_disk->private_data;
  581. if (l->lo_state == Lo_unbound) {
  582. return -EINVAL;
  583. }
  584. f = l->lo_backing_file;
  585. }
  586. if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
  587. return -EINVAL;
  588. return 0;
  589. }
  590. /*
  591. * loop_change_fd switched the backing store of a loopback device to
  592. * a new file. This is useful for operating system installers to free up
  593. * the original file and in High Availability environments to switch to
  594. * an alternative location for the content in case of server meltdown.
  595. * This can only work if the loop device is used read-only, and if the
  596. * new backing store is the same size and type as the old backing store.
  597. */
  598. static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
  599. unsigned int arg)
  600. {
  601. struct file *file, *old_file;
  602. struct inode *inode;
  603. int error;
  604. error = -ENXIO;
  605. if (lo->lo_state != Lo_bound)
  606. goto out;
  607. /* the loop device has to be read-only */
  608. error = -EINVAL;
  609. if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
  610. goto out;
  611. error = -EBADF;
  612. file = fget(arg);
  613. if (!file)
  614. goto out;
  615. error = loop_validate_file(file, bdev);
  616. if (error)
  617. goto out_putf;
  618. inode = file->f_mapping->host;
  619. old_file = lo->lo_backing_file;
  620. error = -EINVAL;
  621. /* size of the new backing store needs to be the same */
  622. if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
  623. goto out_putf;
  624. /* and ... switch */
  625. blk_mq_freeze_queue(lo->lo_queue);
  626. mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
  627. lo->lo_backing_file = file;
  628. lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping);
  629. mapping_set_gfp_mask(file->f_mapping,
  630. lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
  631. loop_update_dio(lo);
  632. blk_mq_unfreeze_queue(lo->lo_queue);
  633. fput(old_file);
  634. if (lo->lo_flags & LO_FLAGS_PARTSCAN)
  635. loop_reread_partitions(lo, bdev);
  636. return 0;
  637. out_putf:
  638. fput(file);
  639. out:
  640. return error;
  641. }
  642. /* loop sysfs attributes */
  643. static ssize_t loop_attr_show(struct device *dev, char *page,
  644. ssize_t (*callback)(struct loop_device *, char *))
  645. {
  646. struct gendisk *disk = dev_to_disk(dev);
  647. struct loop_device *lo = disk->private_data;
  648. return callback(lo, page);
  649. }
  650. #define LOOP_ATTR_RO(_name) \
  651. static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \
  652. static ssize_t loop_attr_do_show_##_name(struct device *d, \
  653. struct device_attribute *attr, char *b) \
  654. { \
  655. return loop_attr_show(d, b, loop_attr_##_name##_show); \
  656. } \
  657. static struct device_attribute loop_attr_##_name = \
  658. __ATTR(_name, 0444, loop_attr_do_show_##_name, NULL);
  659. static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
  660. {
  661. ssize_t ret;
  662. char *p = NULL;
  663. spin_lock_irq(&lo->lo_lock);
  664. if (lo->lo_backing_file)
  665. p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
  666. spin_unlock_irq(&lo->lo_lock);
  667. if (IS_ERR_OR_NULL(p))
  668. ret = PTR_ERR(p);
  669. else {
  670. ret = strlen(p);
  671. memmove(buf, p, ret);
  672. buf[ret++] = '\n';
  673. buf[ret] = 0;
  674. }
  675. return ret;
  676. }
  677. static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
  678. {
  679. return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset);
  680. }
  681. static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
  682. {
  683. return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
  684. }
  685. static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
  686. {
  687. int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);
  688. return sprintf(buf, "%s\n", autoclear ? "1" : "0");
  689. }
  690. static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
  691. {
  692. int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);
  693. return sprintf(buf, "%s\n", partscan ? "1" : "0");
  694. }
  695. static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
  696. {
  697. int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);
  698. return sprintf(buf, "%s\n", dio ? "1" : "0");
  699. }
  700. LOOP_ATTR_RO(backing_file);
  701. LOOP_ATTR_RO(offset);
  702. LOOP_ATTR_RO(sizelimit);
  703. LOOP_ATTR_RO(autoclear);
  704. LOOP_ATTR_RO(partscan);
  705. LOOP_ATTR_RO(dio);
  706. static struct attribute *loop_attrs[] = {
  707. &loop_attr_backing_file.attr,
  708. &loop_attr_offset.attr,
  709. &loop_attr_sizelimit.attr,
  710. &loop_attr_autoclear.attr,
  711. &loop_attr_partscan.attr,
  712. &loop_attr_dio.attr,
  713. NULL,
  714. };
  715. static struct attribute_group loop_attribute_group = {
  716. .name = "loop",
  717. .attrs= loop_attrs,
  718. };
  719. static void loop_sysfs_init(struct loop_device *lo)
  720. {
  721. lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
  722. &loop_attribute_group);
  723. }
  724. static void loop_sysfs_exit(struct loop_device *lo)
  725. {
  726. if (lo->sysfs_inited)
  727. sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
  728. &loop_attribute_group);
  729. }
  730. static void loop_config_discard(struct loop_device *lo)
  731. {
  732. struct file *file = lo->lo_backing_file;
  733. struct inode *inode = file->f_mapping->host;
  734. struct request_queue *q = lo->lo_queue;
  735. /*
  736. * We use punch hole to reclaim the free space used by the
  737. * image a.k.a. discard. However we do not support discard if
  738. * encryption is enabled, because it may give an attacker
  739. * useful information.
  740. */
  741. if ((!file->f_op->fallocate) ||
  742. lo->lo_encrypt_key_size) {
  743. q->limits.discard_granularity = 0;
  744. q->limits.discard_alignment = 0;
  745. blk_queue_max_discard_sectors(q, 0);
  746. blk_queue_max_write_zeroes_sectors(q, 0);
  747. blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q);
  748. return;
  749. }
  750. q->limits.discard_granularity = inode->i_sb->s_blocksize;
  751. q->limits.discard_alignment = 0;
  752. blk_queue_max_discard_sectors(q, UINT_MAX >> 9);
  753. blk_queue_max_write_zeroes_sectors(q, UINT_MAX >> 9);
  754. blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
  755. }
  756. static void loop_unprepare_queue(struct loop_device *lo)
  757. {
  758. kthread_flush_worker(&lo->worker);
  759. kthread_stop(lo->worker_task);
  760. }
  761. static int loop_kthread_worker_fn(void *worker_ptr)
  762. {
  763. current->flags |= PF_LESS_THROTTLE;
  764. return kthread_worker_fn(worker_ptr);
  765. }
  766. static int loop_prepare_queue(struct loop_device *lo)
  767. {
  768. kthread_init_worker(&lo->worker);
  769. lo->worker_task = kthread_run(loop_kthread_worker_fn,
  770. &lo->worker, "loop%d", lo->lo_number);
  771. if (IS_ERR(lo->worker_task))
  772. return -ENOMEM;
  773. set_user_nice(lo->worker_task, MIN_NICE);
  774. return 0;
  775. }
  776. static int loop_set_fd(struct loop_device *lo, fmode_t mode,
  777. struct block_device *bdev, unsigned int arg)
  778. {
  779. struct file *file;
  780. struct inode *inode;
  781. struct address_space *mapping;
  782. int lo_flags = 0;
  783. int error;
  784. loff_t size;
  785. /* This is safe, since we have a reference from open(). */
  786. __module_get(THIS_MODULE);
  787. error = -EBADF;
  788. file = fget(arg);
  789. if (!file)
  790. goto out;
  791. error = -EBUSY;
  792. if (lo->lo_state != Lo_unbound)
  793. goto out_putf;
  794. error = loop_validate_file(file, bdev);
  795. if (error)
  796. goto out_putf;
  797. mapping = file->f_mapping;
  798. inode = mapping->host;
  799. if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) ||
  800. !file->f_op->write_iter)
  801. lo_flags |= LO_FLAGS_READ_ONLY;
  802. error = -EFBIG;
  803. size = get_loop_size(lo, file);
  804. if ((loff_t)(sector_t)size != size)
  805. goto out_putf;
  806. error = loop_prepare_queue(lo);
  807. if (error)
  808. goto out_putf;
  809. error = 0;
  810. set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);
  811. lo->use_dio = false;
  812. lo->lo_device = bdev;
  813. lo->lo_flags = lo_flags;
  814. lo->lo_backing_file = file;
  815. lo->transfer = NULL;
  816. lo->ioctl = NULL;
  817. lo->lo_sizelimit = 0;
  818. lo->old_gfp_mask = mapping_gfp_mask(mapping);
  819. mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
  820. if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
  821. blk_queue_write_cache(lo->lo_queue, true, false);
  822. loop_update_dio(lo);
  823. set_capacity(lo->lo_disk, size);
  824. bd_set_size(bdev, size << 9);
  825. loop_sysfs_init(lo);
  826. /* let user-space know about the new size */
  827. kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
  828. set_blocksize(bdev, S_ISBLK(inode->i_mode) ?
  829. block_size(inode->i_bdev) : PAGE_SIZE);
  830. lo->lo_state = Lo_bound;
  831. if (part_shift)
  832. lo->lo_flags |= LO_FLAGS_PARTSCAN;
  833. if (lo->lo_flags & LO_FLAGS_PARTSCAN)
  834. loop_reread_partitions(lo, bdev);
  835. /* Grab the block_device to prevent its destruction after we
  836. * put /dev/loopXX inode. Later in loop_clr_fd() we bdput(bdev).
  837. */
  838. bdgrab(bdev);
  839. return 0;
  840. out_putf:
  841. fput(file);
  842. out:
  843. /* This is safe: open() is still holding a reference. */
  844. module_put(THIS_MODULE);
  845. return error;
  846. }
  847. static int
  848. loop_release_xfer(struct loop_device *lo)
  849. {
  850. int err = 0;
  851. struct loop_func_table *xfer = lo->lo_encryption;
  852. if (xfer) {
  853. if (xfer->release)
  854. err = xfer->release(lo);
  855. lo->transfer = NULL;
  856. lo->lo_encryption = NULL;
  857. module_put(xfer->owner);
  858. }
  859. return err;
  860. }
  861. static int
  862. loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer,
  863. const struct loop_info64 *i)
  864. {
  865. int err = 0;
  866. if (xfer) {
  867. struct module *owner = xfer->owner;
  868. if (!try_module_get(owner))
  869. return -EINVAL;
  870. if (xfer->init)
  871. err = xfer->init(lo, i);
  872. if (err)
  873. module_put(owner);
  874. else
  875. lo->lo_encryption = xfer;
  876. }
  877. return err;
  878. }
  879. static int loop_clr_fd(struct loop_device *lo)
  880. {
  881. struct file *filp = lo->lo_backing_file;
  882. gfp_t gfp = lo->old_gfp_mask;
  883. struct block_device *bdev = lo->lo_device;
  884. if (lo->lo_state != Lo_bound)
  885. return -ENXIO;
  886. /*
  887. * If we've explicitly asked to tear down the loop device,
  888. * and it has an elevated reference count, set it for auto-teardown when
  889. * the last reference goes away. This stops $!~#$@ udev from
  890. * preventing teardown because it decided that it needs to run blkid on
  891. * the loopback device whenever they appear. xfstests is notorious for
  892. * failing tests because blkid via udev races with a losetup
  893. * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d
  894. * command to fail with EBUSY.
  895. */
  896. if (atomic_read(&lo->lo_refcnt) > 1) {
  897. lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
  898. mutex_unlock(&lo->lo_ctl_mutex);
  899. return 0;
  900. }
  901. if (filp == NULL)
  902. return -EINVAL;
  903. /* freeze request queue during the transition */
  904. blk_mq_freeze_queue(lo->lo_queue);
  905. spin_lock_irq(&lo->lo_lock);
  906. lo->lo_state = Lo_rundown;
  907. lo->lo_backing_file = NULL;
  908. spin_unlock_irq(&lo->lo_lock);
  909. loop_release_xfer(lo);
  910. lo->transfer = NULL;
  911. lo->ioctl = NULL;
  912. lo->lo_device = NULL;
  913. lo->lo_encryption = NULL;
  914. lo->lo_offset = 0;
  915. lo->lo_sizelimit = 0;
  916. lo->lo_encrypt_key_size = 0;
  917. memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
  918. memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
  919. memset(lo->lo_file_name, 0, LO_NAME_SIZE);
  920. blk_queue_logical_block_size(lo->lo_queue, 512);
  921. blk_queue_physical_block_size(lo->lo_queue, 512);
  922. blk_queue_io_min(lo->lo_queue, 512);
  923. if (bdev) {
  924. bdput(bdev);
  925. invalidate_bdev(bdev);
  926. bdev->bd_inode->i_mapping->wb_err = 0;
  927. }
  928. set_capacity(lo->lo_disk, 0);
  929. loop_sysfs_exit(lo);
  930. if (bdev) {
  931. bd_set_size(bdev, 0);
  932. /* let user-space know about this change */
  933. kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
  934. }
  935. mapping_set_gfp_mask(filp->f_mapping, gfp);
  936. lo->lo_state = Lo_unbound;
  937. /* This is safe: open() is still holding a reference. */
  938. module_put(THIS_MODULE);
  939. blk_mq_unfreeze_queue(lo->lo_queue);
  940. if (lo->lo_flags & LO_FLAGS_PARTSCAN && bdev)
  941. loop_reread_partitions(lo, bdev);
  942. lo->lo_flags = 0;
  943. if (!part_shift)
  944. lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN;
  945. loop_unprepare_queue(lo);
  946. mutex_unlock(&lo->lo_ctl_mutex);
  947. /*
  948. * Need not hold lo_ctl_mutex to fput backing file.
  949. * Calling fput holding lo_ctl_mutex triggers a circular
  950. * lock dependency possibility warning as fput can take
  951. * bd_mutex which is usually taken before lo_ctl_mutex.
  952. */
  953. fput(filp);
  954. return 0;
  955. }
  956. static int
  957. loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
  958. {
  959. int err;
  960. struct loop_func_table *xfer;
  961. kuid_t uid = current_uid();
  962. if (lo->lo_encrypt_key_size &&
  963. !uid_eq(lo->lo_key_owner, uid) &&
  964. !capable(CAP_SYS_ADMIN))
  965. return -EPERM;
  966. if (lo->lo_state != Lo_bound)
  967. return -ENXIO;
  968. if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
  969. return -EINVAL;
  970. /* I/O need to be drained during transfer transition */
  971. blk_mq_freeze_queue(lo->lo_queue);
  972. err = loop_release_xfer(lo);
  973. if (err)
  974. goto exit;
  975. if (info->lo_encrypt_type) {
  976. unsigned int type = info->lo_encrypt_type;
  977. if (type >= MAX_LO_CRYPT) {
  978. err = -EINVAL;
  979. goto exit;
  980. }
  981. xfer = xfer_funcs[type];
  982. if (xfer == NULL) {
  983. err = -EINVAL;
  984. goto exit;
  985. }
  986. } else
  987. xfer = NULL;
  988. err = loop_init_xfer(lo, xfer, info);
  989. if (err)
  990. goto exit;
  991. if (lo->lo_offset != info->lo_offset ||
  992. lo->lo_sizelimit != info->lo_sizelimit) {
  993. if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit)) {
  994. err = -EFBIG;
  995. goto exit;
  996. }
  997. }
  998. loop_config_discard(lo);
  999. memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
  1000. memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE);
  1001. lo->lo_file_name[LO_NAME_SIZE-1] = 0;
  1002. lo->lo_crypt_name[LO_NAME_SIZE-1] = 0;
  1003. if (!xfer)
  1004. xfer = &none_funcs;
  1005. lo->transfer = xfer->transfer;
  1006. lo->ioctl = xfer->ioctl;
  1007. if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) !=
  1008. (info->lo_flags & LO_FLAGS_AUTOCLEAR))
  1009. lo->lo_flags ^= LO_FLAGS_AUTOCLEAR;
  1010. lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
  1011. lo->lo_init[0] = info->lo_init[0];
  1012. lo->lo_init[1] = info->lo_init[1];
  1013. if (info->lo_encrypt_key_size) {
  1014. memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
  1015. info->lo_encrypt_key_size);
  1016. lo->lo_key_owner = uid;
  1017. }
  1018. /* update dio if lo_offset or transfer is changed */
  1019. __loop_update_dio(lo, lo->use_dio);
  1020. exit:
  1021. blk_mq_unfreeze_queue(lo->lo_queue);
  1022. if (!err && (info->lo_flags & LO_FLAGS_PARTSCAN) &&
  1023. !(lo->lo_flags & LO_FLAGS_PARTSCAN)) {
  1024. lo->lo_flags |= LO_FLAGS_PARTSCAN;
  1025. lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN;
  1026. loop_reread_partitions(lo, lo->lo_device);
  1027. }
  1028. return err;
  1029. }
  1030. static int
  1031. loop_get_status(struct loop_device *lo, struct loop_info64 *info)
  1032. {
  1033. struct file *file;
  1034. struct kstat stat;
  1035. int ret;
  1036. if (lo->lo_state != Lo_bound) {
  1037. mutex_unlock(&lo->lo_ctl_mutex);
  1038. return -ENXIO;
  1039. }
  1040. memset(info, 0, sizeof(*info));
  1041. info->lo_number = lo->lo_number;
  1042. info->lo_offset = lo->lo_offset;
  1043. info->lo_sizelimit = lo->lo_sizelimit;
  1044. info->lo_flags = lo->lo_flags;
  1045. memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
  1046. memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE);
  1047. info->lo_encrypt_type =
  1048. lo->lo_encryption ? lo->lo_encryption->number : 0;
  1049. if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) {
  1050. info->lo_encrypt_key_size = lo->lo_encrypt_key_size;
  1051. memcpy(info->lo_encrypt_key, lo->lo_encrypt_key,
  1052. lo->lo_encrypt_key_size);
  1053. }
  1054. /* Drop lo_ctl_mutex while we call into the filesystem. */
  1055. file = get_file(lo->lo_backing_file);
  1056. mutex_unlock(&lo->lo_ctl_mutex);
  1057. ret = vfs_getattr(&file->f_path, &stat, STATX_INO,
  1058. AT_STATX_SYNC_AS_STAT);
  1059. if (!ret) {
  1060. info->lo_device = huge_encode_dev(stat.dev);
  1061. info->lo_inode = stat.ino;
  1062. info->lo_rdevice = huge_encode_dev(stat.rdev);
  1063. }
  1064. fput(file);
  1065. return ret;
  1066. }
  1067. static void
  1068. loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
  1069. {
  1070. memset(info64, 0, sizeof(*info64));
  1071. info64->lo_number = info->lo_number;
  1072. info64->lo_device = info->lo_device;
  1073. info64->lo_inode = info->lo_inode;
  1074. info64->lo_rdevice = info->lo_rdevice;
  1075. info64->lo_offset = info->lo_offset;
  1076. info64->lo_sizelimit = 0;
  1077. info64->lo_encrypt_type = info->lo_encrypt_type;
  1078. info64->lo_encrypt_key_size = info->lo_encrypt_key_size;
  1079. info64->lo_flags = info->lo_flags;
  1080. info64->lo_init[0] = info->lo_init[0];
  1081. info64->lo_init[1] = info->lo_init[1];
  1082. if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
  1083. memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE);
  1084. else
  1085. memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
  1086. memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE);
  1087. }
  1088. static int
  1089. loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
  1090. {
  1091. memset(info, 0, sizeof(*info));
  1092. info->lo_number = info64->lo_number;
  1093. info->lo_device = info64->lo_device;
  1094. info->lo_inode = info64->lo_inode;
  1095. info->lo_rdevice = info64->lo_rdevice;
  1096. info->lo_offset = info64->lo_offset;
  1097. info->lo_encrypt_type = info64->lo_encrypt_type;
  1098. info->lo_encrypt_key_size = info64->lo_encrypt_key_size;
  1099. info->lo_flags = info64->lo_flags;
  1100. info->lo_init[0] = info64->lo_init[0];
  1101. info->lo_init[1] = info64->lo_init[1];
  1102. if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
  1103. memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
  1104. else
  1105. memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
  1106. memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
  1107. /* error in case values were truncated */
  1108. if (info->lo_device != info64->lo_device ||
  1109. info->lo_rdevice != info64->lo_rdevice ||
  1110. info->lo_inode != info64->lo_inode ||
  1111. info->lo_offset != info64->lo_offset)
  1112. return -EOVERFLOW;
  1113. return 0;
  1114. }
  1115. static int
  1116. loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
  1117. {
  1118. struct loop_info info;
  1119. struct loop_info64 info64;
  1120. if (copy_from_user(&info, arg, sizeof (struct loop_info)))
  1121. return -EFAULT;
  1122. loop_info64_from_old(&info, &info64);
  1123. return loop_set_status(lo, &info64);
  1124. }
  1125. static int
  1126. loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
  1127. {
  1128. struct loop_info64 info64;
  1129. if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
  1130. return -EFAULT;
  1131. return loop_set_status(lo, &info64);
  1132. }
  1133. static int
  1134. loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
  1135. struct loop_info info;
  1136. struct loop_info64 info64;
  1137. int err;
  1138. if (!arg) {
  1139. mutex_unlock(&lo->lo_ctl_mutex);
  1140. return -EINVAL;
  1141. }
  1142. err = loop_get_status(lo, &info64);
  1143. if (!err)
  1144. err = loop_info64_to_old(&info64, &info);
  1145. if (!err && copy_to_user(arg, &info, sizeof(info)))
  1146. err = -EFAULT;
  1147. return err;
  1148. }
  1149. static int
  1150. loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
  1151. struct loop_info64 info64;
  1152. int err;
  1153. if (!arg) {
  1154. mutex_unlock(&lo->lo_ctl_mutex);
  1155. return -EINVAL;
  1156. }
  1157. err = loop_get_status(lo, &info64);
  1158. if (!err && copy_to_user(arg, &info64, sizeof(info64)))
  1159. err = -EFAULT;
  1160. return err;
  1161. }
  1162. static int loop_set_capacity(struct loop_device *lo)
  1163. {
  1164. if (unlikely(lo->lo_state != Lo_bound))
  1165. return -ENXIO;
  1166. return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit);
  1167. }
  1168. static int loop_set_dio(struct loop_device *lo, unsigned long arg)
  1169. {
  1170. int error = -ENXIO;
  1171. if (lo->lo_state != Lo_bound)
  1172. goto out;
  1173. __loop_update_dio(lo, !!arg);
  1174. if (lo->use_dio == !!arg)
  1175. return 0;
  1176. error = -EINVAL;
  1177. out:
  1178. return error;
  1179. }
  1180. static int loop_set_block_size(struct loop_device *lo, unsigned long arg)
  1181. {
  1182. if (lo->lo_state != Lo_bound)
  1183. return -ENXIO;
  1184. if (arg < 512 || arg > PAGE_SIZE || !is_power_of_2(arg))
  1185. return -EINVAL;
  1186. blk_mq_freeze_queue(lo->lo_queue);
  1187. blk_queue_logical_block_size(lo->lo_queue, arg);
  1188. blk_queue_physical_block_size(lo->lo_queue, arg);
  1189. blk_queue_io_min(lo->lo_queue, arg);
  1190. loop_update_dio(lo);
  1191. blk_mq_unfreeze_queue(lo->lo_queue);
  1192. return 0;
  1193. }
  1194. static int lo_ioctl(struct block_device *bdev, fmode_t mode,
  1195. unsigned int cmd, unsigned long arg)
  1196. {
  1197. struct loop_device *lo = bdev->bd_disk->private_data;
  1198. int err;
  1199. err = mutex_lock_killable_nested(&lo->lo_ctl_mutex, 1);
  1200. if (err)
  1201. goto out_unlocked;
  1202. switch (cmd) {
  1203. case LOOP_SET_FD:
  1204. err = loop_set_fd(lo, mode, bdev, arg);
  1205. break;
  1206. case LOOP_CHANGE_FD:
  1207. err = loop_change_fd(lo, bdev, arg);
  1208. break;
  1209. case LOOP_CLR_FD:
  1210. /* loop_clr_fd would have unlocked lo_ctl_mutex on success */
  1211. err = loop_clr_fd(lo);
  1212. if (!err)
  1213. goto out_unlocked;
  1214. break;
  1215. case LOOP_SET_STATUS:
  1216. err = -EPERM;
  1217. if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
  1218. err = loop_set_status_old(lo,
  1219. (struct loop_info __user *)arg);
  1220. break;
  1221. case LOOP_GET_STATUS:
  1222. err = loop_get_status_old(lo, (struct loop_info __user *) arg);
  1223. /* loop_get_status() unlocks lo_ctl_mutex */
  1224. goto out_unlocked;
  1225. case LOOP_SET_STATUS64:
  1226. err = -EPERM;
  1227. if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
  1228. err = loop_set_status64(lo,
  1229. (struct loop_info64 __user *) arg);
  1230. break;
  1231. case LOOP_GET_STATUS64:
  1232. err = loop_get_status64(lo, (struct loop_info64 __user *) arg);
  1233. /* loop_get_status() unlocks lo_ctl_mutex */
  1234. goto out_unlocked;
  1235. case LOOP_SET_CAPACITY:
  1236. err = -EPERM;
  1237. if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
  1238. err = loop_set_capacity(lo);
  1239. break;
  1240. case LOOP_SET_DIRECT_IO:
  1241. err = -EPERM;
  1242. if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
  1243. err = loop_set_dio(lo, arg);
  1244. break;
  1245. case LOOP_SET_BLOCK_SIZE:
  1246. err = -EPERM;
  1247. if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
  1248. err = loop_set_block_size(lo, arg);
  1249. break;
  1250. default:
  1251. err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
  1252. }
  1253. mutex_unlock(&lo->lo_ctl_mutex);
  1254. out_unlocked:
  1255. return err;
  1256. }
  1257. #ifdef CONFIG_COMPAT
  1258. struct compat_loop_info {
  1259. compat_int_t lo_number; /* ioctl r/o */
  1260. compat_dev_t lo_device; /* ioctl r/o */
  1261. compat_ulong_t lo_inode; /* ioctl r/o */
  1262. compat_dev_t lo_rdevice; /* ioctl r/o */
  1263. compat_int_t lo_offset;
  1264. compat_int_t lo_encrypt_type;
  1265. compat_int_t lo_encrypt_key_size; /* ioctl w/o */
  1266. compat_int_t lo_flags; /* ioctl r/o */
  1267. char lo_name[LO_NAME_SIZE];
  1268. unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
  1269. compat_ulong_t lo_init[2];
  1270. char reserved[4];
  1271. };
  1272. /*
  1273. * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
  1274. * - noinlined to reduce stack space usage in main part of driver
  1275. */
  1276. static noinline int
  1277. loop_info64_from_compat(const struct compat_loop_info __user *arg,
  1278. struct loop_info64 *info64)
  1279. {
  1280. struct compat_loop_info info;
  1281. if (copy_from_user(&info, arg, sizeof(info)))
  1282. return -EFAULT;
  1283. memset(info64, 0, sizeof(*info64));
  1284. info64->lo_number = info.lo_number;
  1285. info64->lo_device = info.lo_device;
  1286. info64->lo_inode = info.lo_inode;
  1287. info64->lo_rdevice = info.lo_rdevice;
  1288. info64->lo_offset = info.lo_offset;
  1289. info64->lo_sizelimit = 0;
  1290. info64->lo_encrypt_type = info.lo_encrypt_type;
  1291. info64->lo_encrypt_key_size = info.lo_encrypt_key_size;
  1292. info64->lo_flags = info.lo_flags;
  1293. info64->lo_init[0] = info.lo_init[0];
  1294. info64->lo_init[1] = info.lo_init[1];
  1295. if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
  1296. memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE);
  1297. else
  1298. memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
  1299. memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE);
  1300. return 0;
  1301. }
  1302. /*
  1303. * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
  1304. * - noinlined to reduce stack space usage in main part of driver
  1305. */
  1306. static noinline int
  1307. loop_info64_to_compat(const struct loop_info64 *info64,
  1308. struct compat_loop_info __user *arg)
  1309. {
  1310. struct compat_loop_info info;
  1311. memset(&info, 0, sizeof(info));
  1312. info.lo_number = info64->lo_number;
  1313. info.lo_device = info64->lo_device;
  1314. info.lo_inode = info64->lo_inode;
  1315. info.lo_rdevice = info64->lo_rdevice;
  1316. info.lo_offset = info64->lo_offset;
  1317. info.lo_encrypt_type = info64->lo_encrypt_type;
  1318. info.lo_encrypt_key_size = info64->lo_encrypt_key_size;
  1319. info.lo_flags = info64->lo_flags;
  1320. info.lo_init[0] = info64->lo_init[0];
  1321. info.lo_init[1] = info64->lo_init[1];
  1322. if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
  1323. memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
  1324. else
  1325. memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
  1326. memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
  1327. /* error in case values were truncated */
  1328. if (info.lo_device != info64->lo_device ||
  1329. info.lo_rdevice != info64->lo_rdevice ||
  1330. info.lo_inode != info64->lo_inode ||
  1331. info.lo_offset != info64->lo_offset ||
  1332. info.lo_init[0] != info64->lo_init[0] ||
  1333. info.lo_init[1] != info64->lo_init[1])
  1334. return -EOVERFLOW;
  1335. if (copy_to_user(arg, &info, sizeof(info)))
  1336. return -EFAULT;
  1337. return 0;
  1338. }
  1339. static int
  1340. loop_set_status_compat(struct loop_device *lo,
  1341. const struct compat_loop_info __user *arg)
  1342. {
  1343. struct loop_info64 info64;
  1344. int ret;
  1345. ret = loop_info64_from_compat(arg, &info64);
  1346. if (ret < 0)
  1347. return ret;
  1348. return loop_set_status(lo, &info64);
  1349. }
  1350. static int
  1351. loop_get_status_compat(struct loop_device *lo,
  1352. struct compat_loop_info __user *arg)
  1353. {
  1354. struct loop_info64 info64;
  1355. int err;
  1356. if (!arg) {
  1357. mutex_unlock(&lo->lo_ctl_mutex);
  1358. return -EINVAL;
  1359. }
  1360. err = loop_get_status(lo, &info64);
  1361. if (!err)
  1362. err = loop_info64_to_compat(&info64, arg);
  1363. return err;
  1364. }
  1365. static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
  1366. unsigned int cmd, unsigned long arg)
  1367. {
  1368. struct loop_device *lo = bdev->bd_disk->private_data;
  1369. int err;
  1370. switch(cmd) {
  1371. case LOOP_SET_STATUS:
  1372. err = mutex_lock_killable(&lo->lo_ctl_mutex);
  1373. if (!err) {
  1374. err = loop_set_status_compat(lo,
  1375. (const struct compat_loop_info __user *)arg);
  1376. mutex_unlock(&lo->lo_ctl_mutex);
  1377. }
  1378. break;
  1379. case LOOP_GET_STATUS:
  1380. err = mutex_lock_killable(&lo->lo_ctl_mutex);
  1381. if (!err) {
  1382. err = loop_get_status_compat(lo,
  1383. (struct compat_loop_info __user *)arg);
  1384. /* loop_get_status() unlocks lo_ctl_mutex */
  1385. }
  1386. break;
  1387. case LOOP_SET_CAPACITY:
  1388. case LOOP_CLR_FD:
  1389. case LOOP_GET_STATUS64:
  1390. case LOOP_SET_STATUS64:
  1391. arg = (unsigned long) compat_ptr(arg);
  1392. case LOOP_SET_FD:
  1393. case LOOP_CHANGE_FD:
  1394. err = lo_ioctl(bdev, mode, cmd, arg);
  1395. break;
  1396. default:
  1397. err = -ENOIOCTLCMD;
  1398. break;
  1399. }
  1400. return err;
  1401. }
  1402. #endif
  1403. static int lo_open(struct block_device *bdev, fmode_t mode)
  1404. {
  1405. struct loop_device *lo;
  1406. int err = 0;
  1407. mutex_lock(&loop_index_mutex);
  1408. lo = bdev->bd_disk->private_data;
  1409. if (!lo) {
  1410. err = -ENXIO;
  1411. goto out;
  1412. }
  1413. atomic_inc(&lo->lo_refcnt);
  1414. out:
  1415. mutex_unlock(&loop_index_mutex);
  1416. return err;
  1417. }
  1418. static void __lo_release(struct loop_device *lo)
  1419. {
  1420. int err;
  1421. if (atomic_dec_return(&lo->lo_refcnt))
  1422. return;
  1423. mutex_lock(&lo->lo_ctl_mutex);
  1424. if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) {
  1425. /*
  1426. * In autoclear mode, stop the loop thread
  1427. * and remove configuration after last close.
  1428. */
  1429. err = loop_clr_fd(lo);
  1430. if (!err)
  1431. return;
  1432. } else if (lo->lo_state == Lo_bound) {
  1433. /*
  1434. * Otherwise keep thread (if running) and config,
  1435. * but flush possible ongoing bios in thread.
  1436. */
  1437. blk_mq_freeze_queue(lo->lo_queue);
  1438. blk_mq_unfreeze_queue(lo->lo_queue);
  1439. }
  1440. mutex_unlock(&lo->lo_ctl_mutex);
  1441. }
  1442. static void lo_release(struct gendisk *disk, fmode_t mode)
  1443. {
  1444. mutex_lock(&loop_index_mutex);
  1445. __lo_release(disk->private_data);
  1446. mutex_unlock(&loop_index_mutex);
  1447. }
  1448. static const struct block_device_operations lo_fops = {
  1449. .owner = THIS_MODULE,
  1450. .open = lo_open,
  1451. .release = lo_release,
  1452. .ioctl = lo_ioctl,
  1453. #ifdef CONFIG_COMPAT
  1454. .compat_ioctl = lo_compat_ioctl,
  1455. #endif
  1456. };
  1457. /*
  1458. * And now the modules code and kernel interface.
  1459. */
  1460. static int max_loop;
  1461. module_param(max_loop, int, 0444);
  1462. MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
  1463. module_param(max_part, int, 0444);
  1464. MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
  1465. MODULE_LICENSE("GPL");
  1466. MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);
  1467. int loop_register_transfer(struct loop_func_table *funcs)
  1468. {
  1469. unsigned int n = funcs->number;
  1470. if (n >= MAX_LO_CRYPT || xfer_funcs[n])
  1471. return -EINVAL;
  1472. xfer_funcs[n] = funcs;
  1473. return 0;
  1474. }
  1475. static int unregister_transfer_cb(int id, void *ptr, void *data)
  1476. {
  1477. struct loop_device *lo = ptr;
  1478. struct loop_func_table *xfer = data;
  1479. mutex_lock(&lo->lo_ctl_mutex);
  1480. if (lo->lo_encryption == xfer)
  1481. loop_release_xfer(lo);
  1482. mutex_unlock(&lo->lo_ctl_mutex);
  1483. return 0;
  1484. }
  1485. int loop_unregister_transfer(int number)
  1486. {
  1487. unsigned int n = number;
  1488. struct loop_func_table *xfer;
  1489. if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL)
  1490. return -EINVAL;
  1491. xfer_funcs[n] = NULL;
  1492. idr_for_each(&loop_index_idr, &unregister_transfer_cb, xfer);
  1493. return 0;
  1494. }
  1495. EXPORT_SYMBOL(loop_register_transfer);
  1496. EXPORT_SYMBOL(loop_unregister_transfer);
  1497. static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,
  1498. const struct blk_mq_queue_data *bd)
  1499. {
  1500. struct request *rq = bd->rq;
  1501. struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
  1502. struct loop_device *lo = rq->q->queuedata;
  1503. blk_mq_start_request(rq);
  1504. if (lo->lo_state != Lo_bound)
  1505. return BLK_STS_IOERR;
  1506. switch (req_op(rq)) {
  1507. case REQ_OP_FLUSH:
  1508. case REQ_OP_DISCARD:
  1509. case REQ_OP_WRITE_ZEROES:
  1510. cmd->use_aio = false;
  1511. break;
  1512. default:
  1513. cmd->use_aio = lo->use_dio;
  1514. break;
  1515. }
  1516. /* always use the first bio's css */
  1517. #ifdef CONFIG_BLK_CGROUP
  1518. if (cmd->use_aio && rq->bio && rq->bio->bi_css) {
  1519. cmd->css = rq->bio->bi_css;
  1520. css_get(cmd->css);
  1521. } else
  1522. #endif
  1523. cmd->css = NULL;
  1524. kthread_queue_work(&lo->worker, &cmd->work);
  1525. return BLK_STS_OK;
  1526. }
  1527. static void loop_handle_cmd(struct loop_cmd *cmd)
  1528. {
  1529. struct request *rq = blk_mq_rq_from_pdu(cmd);
  1530. const bool write = op_is_write(req_op(rq));
  1531. struct loop_device *lo = rq->q->queuedata;
  1532. int ret = 0;
  1533. if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
  1534. ret = -EIO;
  1535. goto failed;
  1536. }
  1537. ret = do_req_filebacked(lo, rq);
  1538. failed:
  1539. /* complete non-aio request */
  1540. if (!cmd->use_aio || ret) {
  1541. cmd->ret = ret ? -EIO : 0;
  1542. blk_mq_complete_request(rq);
  1543. }
  1544. }
  1545. static void loop_queue_work(struct kthread_work *work)
  1546. {
  1547. struct loop_cmd *cmd =
  1548. container_of(work, struct loop_cmd, work);
  1549. loop_handle_cmd(cmd);
  1550. }
  1551. static int loop_init_request(struct blk_mq_tag_set *set, struct request *rq,
  1552. unsigned int hctx_idx, unsigned int numa_node)
  1553. {
  1554. struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
  1555. kthread_init_work(&cmd->work, loop_queue_work);
  1556. return 0;
  1557. }
  1558. static const struct blk_mq_ops loop_mq_ops = {
  1559. .queue_rq = loop_queue_rq,
  1560. .init_request = loop_init_request,
  1561. .complete = lo_complete_rq,
  1562. };
  1563. static int loop_add(struct loop_device **l, int i)
  1564. {
  1565. struct loop_device *lo;
  1566. struct gendisk *disk;
  1567. int err;
  1568. err = -ENOMEM;
  1569. lo = kzalloc(sizeof(*lo), GFP_KERNEL);
  1570. if (!lo)
  1571. goto out;
  1572. lo->lo_state = Lo_unbound;
  1573. /* allocate id, if @id >= 0, we're requesting that specific id */
  1574. if (i >= 0) {
  1575. err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
  1576. if (err == -ENOSPC)
  1577. err = -EEXIST;
  1578. } else {
  1579. err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL);
  1580. }
  1581. if (err < 0)
  1582. goto out_free_dev;
  1583. i = err;
  1584. err = -ENOMEM;
  1585. lo->tag_set.ops = &loop_mq_ops;
  1586. lo->tag_set.nr_hw_queues = 1;
  1587. lo->tag_set.queue_depth = 128;
  1588. lo->tag_set.numa_node = NUMA_NO_NODE;
  1589. lo->tag_set.cmd_size = sizeof(struct loop_cmd);
  1590. lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
  1591. lo->tag_set.driver_data = lo;
  1592. err = blk_mq_alloc_tag_set(&lo->tag_set);
  1593. if (err)
  1594. goto out_free_idr;
  1595. lo->lo_queue = blk_mq_init_queue(&lo->tag_set);
  1596. if (IS_ERR_OR_NULL(lo->lo_queue)) {
  1597. err = PTR_ERR(lo->lo_queue);
  1598. goto out_cleanup_tags;
  1599. }
  1600. lo->lo_queue->queuedata = lo;
  1601. blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS);
  1602. /*
  1603. * By default, we do buffer IO, so it doesn't make sense to enable
  1604. * merge because the I/O submitted to backing file is handled page by
  1605. * page. For directio mode, merge does help to dispatch bigger request
  1606. * to underlayer disk. We will enable merge once directio is enabled.
  1607. */
  1608. blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue);
  1609. err = -ENOMEM;
  1610. disk = lo->lo_disk = alloc_disk(1 << part_shift);
  1611. if (!disk)
  1612. goto out_free_queue;
  1613. /*
  1614. * Disable partition scanning by default. The in-kernel partition
  1615. * scanning can be requested individually per-device during its
  1616. * setup. Userspace can always add and remove partitions from all
  1617. * devices. The needed partition minors are allocated from the
  1618. * extended minor space, the main loop device numbers will continue
  1619. * to match the loop minors, regardless of the number of partitions
  1620. * used.
  1621. *
  1622. * If max_part is given, partition scanning is globally enabled for
  1623. * all loop devices. The minors for the main loop devices will be
  1624. * multiples of max_part.
  1625. *
  1626. * Note: Global-for-all-devices, set-only-at-init, read-only module
  1627. * parameteters like 'max_loop' and 'max_part' make things needlessly
  1628. * complicated, are too static, inflexible and may surprise
  1629. * userspace tools. Parameters like this in general should be avoided.
  1630. */
  1631. if (!part_shift)
  1632. disk->flags |= GENHD_FL_NO_PART_SCAN;
  1633. disk->flags |= GENHD_FL_EXT_DEVT;
  1634. mutex_init(&lo->lo_ctl_mutex);
  1635. atomic_set(&lo->lo_refcnt, 0);
  1636. lo->lo_number = i;
  1637. spin_lock_init(&lo->lo_lock);
  1638. disk->major = LOOP_MAJOR;
  1639. disk->first_minor = i << part_shift;
  1640. disk->fops = &lo_fops;
  1641. disk->private_data = lo;
  1642. disk->queue = lo->lo_queue;
  1643. sprintf(disk->disk_name, "loop%d", i);
  1644. add_disk(disk);
  1645. *l = lo;
  1646. return lo->lo_number;
  1647. out_free_queue:
  1648. blk_cleanup_queue(lo->lo_queue);
  1649. out_cleanup_tags:
  1650. blk_mq_free_tag_set(&lo->tag_set);
  1651. out_free_idr:
  1652. idr_remove(&loop_index_idr, i);
  1653. out_free_dev:
  1654. kfree(lo);
  1655. out:
  1656. return err;
  1657. }
  1658. static void loop_remove(struct loop_device *lo)
  1659. {
  1660. del_gendisk(lo->lo_disk);
  1661. blk_cleanup_queue(lo->lo_queue);
  1662. blk_mq_free_tag_set(&lo->tag_set);
  1663. put_disk(lo->lo_disk);
  1664. kfree(lo);
  1665. }
  1666. static int find_free_cb(int id, void *ptr, void *data)
  1667. {
  1668. struct loop_device *lo = ptr;
  1669. struct loop_device **l = data;
  1670. if (lo->lo_state == Lo_unbound) {
  1671. *l = lo;
  1672. return 1;
  1673. }
  1674. return 0;
  1675. }
  1676. static int loop_lookup(struct loop_device **l, int i)
  1677. {
  1678. struct loop_device *lo;
  1679. int ret = -ENODEV;
  1680. if (i < 0) {
  1681. int err;
  1682. err = idr_for_each(&loop_index_idr, &find_free_cb, &lo);
  1683. if (err == 1) {
  1684. *l = lo;
  1685. ret = lo->lo_number;
  1686. }
  1687. goto out;
  1688. }
  1689. /* lookup and return a specific i */
  1690. lo = idr_find(&loop_index_idr, i);
  1691. if (lo) {
  1692. *l = lo;
  1693. ret = lo->lo_number;
  1694. }
  1695. out:
  1696. return ret;
  1697. }
  1698. static struct kobject *loop_probe(dev_t dev, int *part, void *data)
  1699. {
  1700. struct loop_device *lo;
  1701. struct kobject *kobj;
  1702. int err;
  1703. mutex_lock(&loop_index_mutex);
  1704. err = loop_lookup(&lo, MINOR(dev) >> part_shift);
  1705. if (err < 0)
  1706. err = loop_add(&lo, MINOR(dev) >> part_shift);
  1707. if (err < 0)
  1708. kobj = NULL;
  1709. else
  1710. kobj = get_disk_and_module(lo->lo_disk);
  1711. mutex_unlock(&loop_index_mutex);
  1712. *part = 0;
  1713. return kobj;
  1714. }
  1715. static long loop_control_ioctl(struct file *file, unsigned int cmd,
  1716. unsigned long parm)
  1717. {
  1718. struct loop_device *lo;
  1719. int ret = -ENOSYS;
  1720. mutex_lock(&loop_index_mutex);
  1721. switch (cmd) {
  1722. case LOOP_CTL_ADD:
  1723. ret = loop_lookup(&lo, parm);
  1724. if (ret >= 0) {
  1725. ret = -EEXIST;
  1726. break;
  1727. }
  1728. ret = loop_add(&lo, parm);
  1729. break;
  1730. case LOOP_CTL_REMOVE:
  1731. ret = loop_lookup(&lo, parm);
  1732. if (ret < 0)
  1733. break;
  1734. ret = mutex_lock_killable(&lo->lo_ctl_mutex);
  1735. if (ret)
  1736. break;
  1737. if (lo->lo_state != Lo_unbound) {
  1738. ret = -EBUSY;
  1739. mutex_unlock(&lo->lo_ctl_mutex);
  1740. break;
  1741. }
  1742. if (atomic_read(&lo->lo_refcnt) > 0) {
  1743. ret = -EBUSY;
  1744. mutex_unlock(&lo->lo_ctl_mutex);
  1745. break;
  1746. }
  1747. lo->lo_disk->private_data = NULL;
  1748. mutex_unlock(&lo->lo_ctl_mutex);
  1749. idr_remove(&loop_index_idr, lo->lo_number);
  1750. loop_remove(lo);
  1751. break;
  1752. case LOOP_CTL_GET_FREE:
  1753. ret = loop_lookup(&lo, -1);
  1754. if (ret >= 0)
  1755. break;
  1756. ret = loop_add(&lo, -1);
  1757. }
  1758. mutex_unlock(&loop_index_mutex);
  1759. return ret;
  1760. }
  1761. static const struct file_operations loop_ctl_fops = {
  1762. .open = nonseekable_open,
  1763. .unlocked_ioctl = loop_control_ioctl,
  1764. .compat_ioctl = loop_control_ioctl,
  1765. .owner = THIS_MODULE,
  1766. .llseek = noop_llseek,
  1767. };
  1768. static struct miscdevice loop_misc = {
  1769. .minor = LOOP_CTRL_MINOR,
  1770. .name = "loop-control",
  1771. .fops = &loop_ctl_fops,
  1772. };
  1773. MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR);
  1774. MODULE_ALIAS("devname:loop-control");
  1775. static int __init loop_init(void)
  1776. {
  1777. int i, nr;
  1778. unsigned long range;
  1779. struct loop_device *lo;
  1780. int err;
  1781. part_shift = 0;
  1782. if (max_part > 0) {
  1783. part_shift = fls(max_part);
  1784. /*
  1785. * Adjust max_part according to part_shift as it is exported
  1786. * to user space so that user can decide correct minor number
  1787. * if [s]he want to create more devices.
  1788. *
  1789. * Note that -1 is required because partition 0 is reserved
  1790. * for the whole disk.
  1791. */
  1792. max_part = (1UL << part_shift) - 1;
  1793. }
  1794. if ((1UL << part_shift) > DISK_MAX_PARTS) {
  1795. err = -EINVAL;
  1796. goto err_out;
  1797. }
  1798. if (max_loop > 1UL << (MINORBITS - part_shift)) {
  1799. err = -EINVAL;
  1800. goto err_out;
  1801. }
  1802. /*
  1803. * If max_loop is specified, create that many devices upfront.
  1804. * This also becomes a hard limit. If max_loop is not specified,
  1805. * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module
  1806. * init time. Loop devices can be requested on-demand with the
  1807. * /dev/loop-control interface, or be instantiated by accessing
  1808. * a 'dead' device node.
  1809. */
  1810. if (max_loop) {
  1811. nr = max_loop;
  1812. range = max_loop << part_shift;
  1813. } else {
  1814. nr = CONFIG_BLK_DEV_LOOP_MIN_COUNT;
  1815. range = 1UL << MINORBITS;
  1816. }
  1817. err = misc_register(&loop_misc);
  1818. if (err < 0)
  1819. goto err_out;
  1820. if (register_blkdev(LOOP_MAJOR, "loop")) {
  1821. err = -EIO;
  1822. goto misc_out;
  1823. }
  1824. blk_register_region(MKDEV(LOOP_MAJOR, 0), range,
  1825. THIS_MODULE, loop_probe, NULL, NULL);
  1826. /* pre-create number of devices given by config or max_loop */
  1827. mutex_lock(&loop_index_mutex);
  1828. for (i = 0; i < nr; i++)
  1829. loop_add(&lo, i);
  1830. mutex_unlock(&loop_index_mutex);
  1831. printk(KERN_INFO "loop: module loaded\n");
  1832. return 0;
  1833. misc_out:
  1834. misc_deregister(&loop_misc);
  1835. err_out:
  1836. return err;
  1837. }
  1838. static int loop_exit_cb(int id, void *ptr, void *data)
  1839. {
  1840. struct loop_device *lo = ptr;
  1841. loop_remove(lo);
  1842. return 0;
  1843. }
  1844. static void __exit loop_exit(void)
  1845. {
  1846. unsigned long range;
  1847. range = max_loop ? max_loop << part_shift : 1UL << MINORBITS;
  1848. idr_for_each(&loop_index_idr, &loop_exit_cb, NULL);
  1849. idr_destroy(&loop_index_idr);
  1850. blk_unregister_region(MKDEV(LOOP_MAJOR, 0), range);
  1851. unregister_blkdev(LOOP_MAJOR, "loop");
  1852. misc_deregister(&loop_misc);
  1853. }
  1854. module_init(loop_init);
  1855. module_exit(loop_exit);
  1856. #ifndef MODULE
  1857. static int __init max_loop_setup(char *str)
  1858. {
  1859. max_loop = simple_strtol(str, NULL, 0);
  1860. return 1;
  1861. }
  1862. __setup("max_loop=", max_loop_setup);
  1863. #endif