xfs_file.c 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692
  1. /*
  2. * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  3. * All Rights Reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it would be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write the Free Software Foundation,
  16. * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  17. */
  18. #include "xfs.h"
  19. #include "xfs_fs.h"
  20. #include "xfs_shared.h"
  21. #include "xfs_format.h"
  22. #include "xfs_log_format.h"
  23. #include "xfs_trans_resv.h"
  24. #include "xfs_mount.h"
  25. #include "xfs_da_format.h"
  26. #include "xfs_da_btree.h"
  27. #include "xfs_inode.h"
  28. #include "xfs_trans.h"
  29. #include "xfs_inode_item.h"
  30. #include "xfs_bmap.h"
  31. #include "xfs_bmap_util.h"
  32. #include "xfs_error.h"
  33. #include "xfs_dir2.h"
  34. #include "xfs_dir2_priv.h"
  35. #include "xfs_ioctl.h"
  36. #include "xfs_trace.h"
  37. #include "xfs_log.h"
  38. #include "xfs_icache.h"
  39. #include "xfs_pnfs.h"
  40. #include <linux/dcache.h>
  41. #include <linux/falloc.h>
  42. #include <linux/pagevec.h>
  43. #include <linux/backing-dev.h>
  44. static const struct vm_operations_struct xfs_file_vm_ops;
  45. /*
  46. * Locking primitives for read and write IO paths to ensure we consistently use
  47. * and order the inode->i_mutex, ip->i_lock and ip->i_iolock.
  48. */
  49. static inline void
  50. xfs_rw_ilock(
  51. struct xfs_inode *ip,
  52. int type)
  53. {
  54. if (type & XFS_IOLOCK_EXCL)
  55. inode_lock(VFS_I(ip));
  56. xfs_ilock(ip, type);
  57. }
  58. static inline void
  59. xfs_rw_iunlock(
  60. struct xfs_inode *ip,
  61. int type)
  62. {
  63. xfs_iunlock(ip, type);
  64. if (type & XFS_IOLOCK_EXCL)
  65. inode_unlock(VFS_I(ip));
  66. }
  67. static inline void
  68. xfs_rw_ilock_demote(
  69. struct xfs_inode *ip,
  70. int type)
  71. {
  72. xfs_ilock_demote(ip, type);
  73. if (type & XFS_IOLOCK_EXCL)
  74. inode_unlock(VFS_I(ip));
  75. }
  76. /*
  77. * xfs_iozero clears the specified range supplied via the page cache (except in
  78. * the DAX case). Writes through the page cache will allocate blocks over holes,
  79. * though the callers usually map the holes first and avoid them. If a block is
  80. * not completely zeroed, then it will be read from disk before being partially
  81. * zeroed.
  82. *
  83. * In the DAX case, we can just directly write to the underlying pages. This
  84. * will not allocate blocks, but will avoid holes and unwritten extents and so
  85. * not do unnecessary work.
  86. */
  87. int
  88. xfs_iozero(
  89. struct xfs_inode *ip, /* inode */
  90. loff_t pos, /* offset in file */
  91. size_t count) /* size of data to zero */
  92. {
  93. struct page *page;
  94. struct address_space *mapping;
  95. int status = 0;
  96. mapping = VFS_I(ip)->i_mapping;
  97. do {
  98. unsigned offset, bytes;
  99. void *fsdata;
  100. offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
  101. bytes = PAGE_CACHE_SIZE - offset;
  102. if (bytes > count)
  103. bytes = count;
  104. if (IS_DAX(VFS_I(ip))) {
  105. status = dax_zero_page_range(VFS_I(ip), pos, bytes,
  106. xfs_get_blocks_direct);
  107. if (status)
  108. break;
  109. } else {
  110. status = pagecache_write_begin(NULL, mapping, pos, bytes,
  111. AOP_FLAG_UNINTERRUPTIBLE,
  112. &page, &fsdata);
  113. if (status)
  114. break;
  115. zero_user(page, offset, bytes);
  116. status = pagecache_write_end(NULL, mapping, pos, bytes,
  117. bytes, page, fsdata);
  118. WARN_ON(status <= 0); /* can't return less than zero! */
  119. status = 0;
  120. }
  121. pos += bytes;
  122. count -= bytes;
  123. } while (count);
  124. return status;
  125. }
  126. int
  127. xfs_update_prealloc_flags(
  128. struct xfs_inode *ip,
  129. enum xfs_prealloc_flags flags)
  130. {
  131. struct xfs_trans *tp;
  132. int error;
  133. tp = xfs_trans_alloc(ip->i_mount, XFS_TRANS_WRITEID);
  134. error = xfs_trans_reserve(tp, &M_RES(ip->i_mount)->tr_writeid, 0, 0);
  135. if (error) {
  136. xfs_trans_cancel(tp);
  137. return error;
  138. }
  139. xfs_ilock(ip, XFS_ILOCK_EXCL);
  140. xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
  141. if (!(flags & XFS_PREALLOC_INVISIBLE)) {
  142. ip->i_d.di_mode &= ~S_ISUID;
  143. if (ip->i_d.di_mode & S_IXGRP)
  144. ip->i_d.di_mode &= ~S_ISGID;
  145. xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG);
  146. }
  147. if (flags & XFS_PREALLOC_SET)
  148. ip->i_d.di_flags |= XFS_DIFLAG_PREALLOC;
  149. if (flags & XFS_PREALLOC_CLEAR)
  150. ip->i_d.di_flags &= ~XFS_DIFLAG_PREALLOC;
  151. xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
  152. if (flags & XFS_PREALLOC_SYNC)
  153. xfs_trans_set_sync(tp);
  154. return xfs_trans_commit(tp);
  155. }
  156. /*
  157. * Fsync operations on directories are much simpler than on regular files,
  158. * as there is no file data to flush, and thus also no need for explicit
  159. * cache flush operations, and there are no non-transaction metadata updates
  160. * on directories either.
  161. */
  162. STATIC int
  163. xfs_dir_fsync(
  164. struct file *file,
  165. loff_t start,
  166. loff_t end,
  167. int datasync)
  168. {
  169. struct xfs_inode *ip = XFS_I(file->f_mapping->host);
  170. struct xfs_mount *mp = ip->i_mount;
  171. xfs_lsn_t lsn = 0;
  172. trace_xfs_dir_fsync(ip);
  173. xfs_ilock(ip, XFS_ILOCK_SHARED);
  174. if (xfs_ipincount(ip))
  175. lsn = ip->i_itemp->ili_last_lsn;
  176. xfs_iunlock(ip, XFS_ILOCK_SHARED);
  177. if (!lsn)
  178. return 0;
  179. return _xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, NULL);
  180. }
  181. STATIC int
  182. xfs_file_fsync(
  183. struct file *file,
  184. loff_t start,
  185. loff_t end,
  186. int datasync)
  187. {
  188. struct inode *inode = file->f_mapping->host;
  189. struct xfs_inode *ip = XFS_I(inode);
  190. struct xfs_mount *mp = ip->i_mount;
  191. int error = 0;
  192. int log_flushed = 0;
  193. xfs_lsn_t lsn = 0;
  194. trace_xfs_file_fsync(ip);
  195. error = filemap_write_and_wait_range(inode->i_mapping, start, end);
  196. if (error)
  197. return error;
  198. if (XFS_FORCED_SHUTDOWN(mp))
  199. return -EIO;
  200. xfs_iflags_clear(ip, XFS_ITRUNCATED);
  201. if (mp->m_flags & XFS_MOUNT_BARRIER) {
  202. /*
  203. * If we have an RT and/or log subvolume we need to make sure
  204. * to flush the write cache the device used for file data
  205. * first. This is to ensure newly written file data make
  206. * it to disk before logging the new inode size in case of
  207. * an extending write.
  208. */
  209. if (XFS_IS_REALTIME_INODE(ip))
  210. xfs_blkdev_issue_flush(mp->m_rtdev_targp);
  211. else if (mp->m_logdev_targp != mp->m_ddev_targp)
  212. xfs_blkdev_issue_flush(mp->m_ddev_targp);
  213. }
  214. /*
  215. * All metadata updates are logged, which means that we just have to
  216. * flush the log up to the latest LSN that touched the inode. If we have
  217. * concurrent fsync/fdatasync() calls, we need them to all block on the
  218. * log force before we clear the ili_fsync_fields field. This ensures
  219. * that we don't get a racing sync operation that does not wait for the
  220. * metadata to hit the journal before returning. If we race with
  221. * clearing the ili_fsync_fields, then all that will happen is the log
  222. * force will do nothing as the lsn will already be on disk. We can't
  223. * race with setting ili_fsync_fields because that is done under
  224. * XFS_ILOCK_EXCL, and that can't happen because we hold the lock shared
  225. * until after the ili_fsync_fields is cleared.
  226. */
  227. xfs_ilock(ip, XFS_ILOCK_SHARED);
  228. if (xfs_ipincount(ip)) {
  229. if (!datasync ||
  230. (ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP))
  231. lsn = ip->i_itemp->ili_last_lsn;
  232. }
  233. if (lsn) {
  234. error = _xfs_log_force_lsn(mp, lsn, XFS_LOG_SYNC, &log_flushed);
  235. ip->i_itemp->ili_fsync_fields = 0;
  236. }
  237. xfs_iunlock(ip, XFS_ILOCK_SHARED);
  238. /*
  239. * If we only have a single device, and the log force about was
  240. * a no-op we might have to flush the data device cache here.
  241. * This can only happen for fdatasync/O_DSYNC if we were overwriting
  242. * an already allocated file and thus do not have any metadata to
  243. * commit.
  244. */
  245. if ((mp->m_flags & XFS_MOUNT_BARRIER) &&
  246. mp->m_logdev_targp == mp->m_ddev_targp &&
  247. !XFS_IS_REALTIME_INODE(ip) &&
  248. !log_flushed)
  249. xfs_blkdev_issue_flush(mp->m_ddev_targp);
  250. return error;
  251. }
  252. STATIC ssize_t
  253. xfs_file_read_iter(
  254. struct kiocb *iocb,
  255. struct iov_iter *to)
  256. {
  257. struct file *file = iocb->ki_filp;
  258. struct inode *inode = file->f_mapping->host;
  259. struct xfs_inode *ip = XFS_I(inode);
  260. struct xfs_mount *mp = ip->i_mount;
  261. size_t size = iov_iter_count(to);
  262. ssize_t ret = 0;
  263. int ioflags = 0;
  264. xfs_fsize_t n;
  265. loff_t pos = iocb->ki_pos;
  266. XFS_STATS_INC(mp, xs_read_calls);
  267. if (unlikely(iocb->ki_flags & IOCB_DIRECT))
  268. ioflags |= XFS_IO_ISDIRECT;
  269. if (file->f_mode & FMODE_NOCMTIME)
  270. ioflags |= XFS_IO_INVIS;
  271. if ((ioflags & XFS_IO_ISDIRECT) && !IS_DAX(inode)) {
  272. xfs_buftarg_t *target =
  273. XFS_IS_REALTIME_INODE(ip) ?
  274. mp->m_rtdev_targp : mp->m_ddev_targp;
  275. /* DIO must be aligned to device logical sector size */
  276. if ((pos | size) & target->bt_logical_sectormask) {
  277. if (pos == i_size_read(inode))
  278. return 0;
  279. return -EINVAL;
  280. }
  281. }
  282. n = mp->m_super->s_maxbytes - pos;
  283. if (n <= 0 || size == 0)
  284. return 0;
  285. if (n < size)
  286. size = n;
  287. if (XFS_FORCED_SHUTDOWN(mp))
  288. return -EIO;
  289. /*
  290. * Locking is a bit tricky here. If we take an exclusive lock for direct
  291. * IO, we effectively serialise all new concurrent read IO to this file
  292. * and block it behind IO that is currently in progress because IO in
  293. * progress holds the IO lock shared. We only need to hold the lock
  294. * exclusive to blow away the page cache, so only take lock exclusively
  295. * if the page cache needs invalidation. This allows the normal direct
  296. * IO case of no page cache pages to proceeed concurrently without
  297. * serialisation.
  298. */
  299. xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
  300. if ((ioflags & XFS_IO_ISDIRECT) && inode->i_mapping->nrpages) {
  301. xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
  302. xfs_rw_ilock(ip, XFS_IOLOCK_EXCL);
  303. /*
  304. * The generic dio code only flushes the range of the particular
  305. * I/O. Because we take an exclusive lock here, this whole
  306. * sequence is considerably more expensive for us. This has a
  307. * noticeable performance impact for any file with cached pages,
  308. * even when outside of the range of the particular I/O.
  309. *
  310. * Hence, amortize the cost of the lock against a full file
  311. * flush and reduce the chances of repeated iolock cycles going
  312. * forward.
  313. */
  314. if (inode->i_mapping->nrpages) {
  315. ret = filemap_write_and_wait(VFS_I(ip)->i_mapping);
  316. if (ret) {
  317. xfs_rw_iunlock(ip, XFS_IOLOCK_EXCL);
  318. return ret;
  319. }
  320. /*
  321. * Invalidate whole pages. This can return an error if
  322. * we fail to invalidate a page, but this should never
  323. * happen on XFS. Warn if it does fail.
  324. */
  325. ret = invalidate_inode_pages2(VFS_I(ip)->i_mapping);
  326. WARN_ON_ONCE(ret);
  327. ret = 0;
  328. }
  329. xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL);
  330. }
  331. trace_xfs_file_read(ip, size, pos, ioflags);
  332. ret = generic_file_read_iter(iocb, to);
  333. if (ret > 0)
  334. XFS_STATS_ADD(mp, xs_read_bytes, ret);
  335. xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
  336. return ret;
  337. }
  338. STATIC ssize_t
  339. xfs_file_splice_read(
  340. struct file *infilp,
  341. loff_t *ppos,
  342. struct pipe_inode_info *pipe,
  343. size_t count,
  344. unsigned int flags)
  345. {
  346. struct xfs_inode *ip = XFS_I(infilp->f_mapping->host);
  347. int ioflags = 0;
  348. ssize_t ret;
  349. XFS_STATS_INC(ip->i_mount, xs_read_calls);
  350. if (infilp->f_mode & FMODE_NOCMTIME)
  351. ioflags |= XFS_IO_INVIS;
  352. if (XFS_FORCED_SHUTDOWN(ip->i_mount))
  353. return -EIO;
  354. trace_xfs_file_splice_read(ip, count, *ppos, ioflags);
  355. /*
  356. * DAX inodes cannot ues the page cache for splice, so we have to push
  357. * them through the VFS IO path. This means it goes through
  358. * ->read_iter, which for us takes the XFS_IOLOCK_SHARED. Hence we
  359. * cannot lock the splice operation at this level for DAX inodes.
  360. */
  361. if (IS_DAX(VFS_I(ip))) {
  362. ret = default_file_splice_read(infilp, ppos, pipe, count,
  363. flags);
  364. goto out;
  365. }
  366. xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
  367. ret = generic_file_splice_read(infilp, ppos, pipe, count, flags);
  368. xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
  369. out:
  370. if (ret > 0)
  371. XFS_STATS_ADD(ip->i_mount, xs_read_bytes, ret);
  372. return ret;
  373. }
  374. /*
  375. * This routine is called to handle zeroing any space in the last block of the
  376. * file that is beyond the EOF. We do this since the size is being increased
  377. * without writing anything to that block and we don't want to read the
  378. * garbage on the disk.
  379. */
  380. STATIC int /* error (positive) */
  381. xfs_zero_last_block(
  382. struct xfs_inode *ip,
  383. xfs_fsize_t offset,
  384. xfs_fsize_t isize,
  385. bool *did_zeroing)
  386. {
  387. struct xfs_mount *mp = ip->i_mount;
  388. xfs_fileoff_t last_fsb = XFS_B_TO_FSBT(mp, isize);
  389. int zero_offset = XFS_B_FSB_OFFSET(mp, isize);
  390. int zero_len;
  391. int nimaps = 1;
  392. int error = 0;
  393. struct xfs_bmbt_irec imap;
  394. xfs_ilock(ip, XFS_ILOCK_EXCL);
  395. error = xfs_bmapi_read(ip, last_fsb, 1, &imap, &nimaps, 0);
  396. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  397. if (error)
  398. return error;
  399. ASSERT(nimaps > 0);
  400. /*
  401. * If the block underlying isize is just a hole, then there
  402. * is nothing to zero.
  403. */
  404. if (imap.br_startblock == HOLESTARTBLOCK)
  405. return 0;
  406. zero_len = mp->m_sb.sb_blocksize - zero_offset;
  407. if (isize + zero_len > offset)
  408. zero_len = offset - isize;
  409. *did_zeroing = true;
  410. return xfs_iozero(ip, isize, zero_len);
  411. }
  412. /*
  413. * Zero any on disk space between the current EOF and the new, larger EOF.
  414. *
  415. * This handles the normal case of zeroing the remainder of the last block in
  416. * the file and the unusual case of zeroing blocks out beyond the size of the
  417. * file. This second case only happens with fixed size extents and when the
  418. * system crashes before the inode size was updated but after blocks were
  419. * allocated.
  420. *
  421. * Expects the iolock to be held exclusive, and will take the ilock internally.
  422. */
  423. int /* error (positive) */
  424. xfs_zero_eof(
  425. struct xfs_inode *ip,
  426. xfs_off_t offset, /* starting I/O offset */
  427. xfs_fsize_t isize, /* current inode size */
  428. bool *did_zeroing)
  429. {
  430. struct xfs_mount *mp = ip->i_mount;
  431. xfs_fileoff_t start_zero_fsb;
  432. xfs_fileoff_t end_zero_fsb;
  433. xfs_fileoff_t zero_count_fsb;
  434. xfs_fileoff_t last_fsb;
  435. xfs_fileoff_t zero_off;
  436. xfs_fsize_t zero_len;
  437. int nimaps;
  438. int error = 0;
  439. struct xfs_bmbt_irec imap;
  440. ASSERT(xfs_isilocked(ip, XFS_IOLOCK_EXCL));
  441. ASSERT(offset > isize);
  442. trace_xfs_zero_eof(ip, isize, offset - isize);
  443. /*
  444. * First handle zeroing the block on which isize resides.
  445. *
  446. * We only zero a part of that block so it is handled specially.
  447. */
  448. if (XFS_B_FSB_OFFSET(mp, isize) != 0) {
  449. error = xfs_zero_last_block(ip, offset, isize, did_zeroing);
  450. if (error)
  451. return error;
  452. }
  453. /*
  454. * Calculate the range between the new size and the old where blocks
  455. * needing to be zeroed may exist.
  456. *
  457. * To get the block where the last byte in the file currently resides,
  458. * we need to subtract one from the size and truncate back to a block
  459. * boundary. We subtract 1 in case the size is exactly on a block
  460. * boundary.
  461. */
  462. last_fsb = isize ? XFS_B_TO_FSBT(mp, isize - 1) : (xfs_fileoff_t)-1;
  463. start_zero_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)isize);
  464. end_zero_fsb = XFS_B_TO_FSBT(mp, offset - 1);
  465. ASSERT((xfs_sfiloff_t)last_fsb < (xfs_sfiloff_t)start_zero_fsb);
  466. if (last_fsb == end_zero_fsb) {
  467. /*
  468. * The size was only incremented on its last block.
  469. * We took care of that above, so just return.
  470. */
  471. return 0;
  472. }
  473. ASSERT(start_zero_fsb <= end_zero_fsb);
  474. while (start_zero_fsb <= end_zero_fsb) {
  475. nimaps = 1;
  476. zero_count_fsb = end_zero_fsb - start_zero_fsb + 1;
  477. xfs_ilock(ip, XFS_ILOCK_EXCL);
  478. error = xfs_bmapi_read(ip, start_zero_fsb, zero_count_fsb,
  479. &imap, &nimaps, 0);
  480. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  481. if (error)
  482. return error;
  483. ASSERT(nimaps > 0);
  484. if (imap.br_state == XFS_EXT_UNWRITTEN ||
  485. imap.br_startblock == HOLESTARTBLOCK) {
  486. start_zero_fsb = imap.br_startoff + imap.br_blockcount;
  487. ASSERT(start_zero_fsb <= (end_zero_fsb + 1));
  488. continue;
  489. }
  490. /*
  491. * There are blocks we need to zero.
  492. */
  493. zero_off = XFS_FSB_TO_B(mp, start_zero_fsb);
  494. zero_len = XFS_FSB_TO_B(mp, imap.br_blockcount);
  495. if ((zero_off + zero_len) > offset)
  496. zero_len = offset - zero_off;
  497. error = xfs_iozero(ip, zero_off, zero_len);
  498. if (error)
  499. return error;
  500. *did_zeroing = true;
  501. start_zero_fsb = imap.br_startoff + imap.br_blockcount;
  502. ASSERT(start_zero_fsb <= (end_zero_fsb + 1));
  503. }
  504. return 0;
  505. }
  506. /*
  507. * Common pre-write limit and setup checks.
  508. *
  509. * Called with the iolocked held either shared and exclusive according to
  510. * @iolock, and returns with it held. Might upgrade the iolock to exclusive
  511. * if called for a direct write beyond i_size.
  512. */
  513. STATIC ssize_t
  514. xfs_file_aio_write_checks(
  515. struct kiocb *iocb,
  516. struct iov_iter *from,
  517. int *iolock)
  518. {
  519. struct file *file = iocb->ki_filp;
  520. struct inode *inode = file->f_mapping->host;
  521. struct xfs_inode *ip = XFS_I(inode);
  522. ssize_t error = 0;
  523. size_t count = iov_iter_count(from);
  524. bool drained_dio = false;
  525. restart:
  526. error = generic_write_checks(iocb, from);
  527. if (error <= 0)
  528. return error;
  529. error = xfs_break_layouts(inode, iolock, true);
  530. if (error)
  531. return error;
  532. /* For changing security info in file_remove_privs() we need i_mutex */
  533. if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
  534. xfs_rw_iunlock(ip, *iolock);
  535. *iolock = XFS_IOLOCK_EXCL;
  536. xfs_rw_ilock(ip, *iolock);
  537. goto restart;
  538. }
  539. /*
  540. * If the offset is beyond the size of the file, we need to zero any
  541. * blocks that fall between the existing EOF and the start of this
  542. * write. If zeroing is needed and we are currently holding the
  543. * iolock shared, we need to update it to exclusive which implies
  544. * having to redo all checks before.
  545. *
  546. * We need to serialise against EOF updates that occur in IO
  547. * completions here. We want to make sure that nobody is changing the
  548. * size while we do this check until we have placed an IO barrier (i.e.
  549. * hold the XFS_IOLOCK_EXCL) that prevents new IO from being dispatched.
  550. * The spinlock effectively forms a memory barrier once we have the
  551. * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value
  552. * and hence be able to correctly determine if we need to run zeroing.
  553. */
  554. spin_lock(&ip->i_flags_lock);
  555. if (iocb->ki_pos > i_size_read(inode)) {
  556. bool zero = false;
  557. spin_unlock(&ip->i_flags_lock);
  558. if (!drained_dio) {
  559. if (*iolock == XFS_IOLOCK_SHARED) {
  560. xfs_rw_iunlock(ip, *iolock);
  561. *iolock = XFS_IOLOCK_EXCL;
  562. xfs_rw_ilock(ip, *iolock);
  563. iov_iter_reexpand(from, count);
  564. }
  565. /*
  566. * We now have an IO submission barrier in place, but
  567. * AIO can do EOF updates during IO completion and hence
  568. * we now need to wait for all of them to drain. Non-AIO
  569. * DIO will have drained before we are given the
  570. * XFS_IOLOCK_EXCL, and so for most cases this wait is a
  571. * no-op.
  572. */
  573. inode_dio_wait(inode);
  574. drained_dio = true;
  575. goto restart;
  576. }
  577. error = xfs_zero_eof(ip, iocb->ki_pos, i_size_read(inode), &zero);
  578. if (error)
  579. return error;
  580. } else
  581. spin_unlock(&ip->i_flags_lock);
  582. /*
  583. * Updating the timestamps will grab the ilock again from
  584. * xfs_fs_dirty_inode, so we have to call it after dropping the
  585. * lock above. Eventually we should look into a way to avoid
  586. * the pointless lock roundtrip.
  587. */
  588. if (likely(!(file->f_mode & FMODE_NOCMTIME))) {
  589. error = file_update_time(file);
  590. if (error)
  591. return error;
  592. }
  593. /*
  594. * If we're writing the file then make sure to clear the setuid and
  595. * setgid bits if the process is not being run by root. This keeps
  596. * people from modifying setuid and setgid binaries.
  597. */
  598. if (!IS_NOSEC(inode))
  599. return file_remove_privs(file);
  600. return 0;
  601. }
  602. /*
  603. * xfs_file_dio_aio_write - handle direct IO writes
  604. *
  605. * Lock the inode appropriately to prepare for and issue a direct IO write.
  606. * By separating it from the buffered write path we remove all the tricky to
  607. * follow locking changes and looping.
  608. *
  609. * If there are cached pages or we're extending the file, we need IOLOCK_EXCL
  610. * until we're sure the bytes at the new EOF have been zeroed and/or the cached
  611. * pages are flushed out.
  612. *
  613. * In most cases the direct IO writes will be done holding IOLOCK_SHARED
  614. * allowing them to be done in parallel with reads and other direct IO writes.
  615. * However, if the IO is not aligned to filesystem blocks, the direct IO layer
  616. * needs to do sub-block zeroing and that requires serialisation against other
  617. * direct IOs to the same block. In this case we need to serialise the
  618. * submission of the unaligned IOs so that we don't get racing block zeroing in
  619. * the dio layer. To avoid the problem with aio, we also need to wait for
  620. * outstanding IOs to complete so that unwritten extent conversion is completed
  621. * before we try to map the overlapping block. This is currently implemented by
  622. * hitting it with a big hammer (i.e. inode_dio_wait()).
  623. *
  624. * Returns with locks held indicated by @iolock and errors indicated by
  625. * negative return values.
  626. */
  627. STATIC ssize_t
  628. xfs_file_dio_aio_write(
  629. struct kiocb *iocb,
  630. struct iov_iter *from)
  631. {
  632. struct file *file = iocb->ki_filp;
  633. struct address_space *mapping = file->f_mapping;
  634. struct inode *inode = mapping->host;
  635. struct xfs_inode *ip = XFS_I(inode);
  636. struct xfs_mount *mp = ip->i_mount;
  637. ssize_t ret = 0;
  638. int unaligned_io = 0;
  639. int iolock;
  640. size_t count = iov_iter_count(from);
  641. loff_t pos = iocb->ki_pos;
  642. loff_t end;
  643. struct iov_iter data;
  644. struct xfs_buftarg *target = XFS_IS_REALTIME_INODE(ip) ?
  645. mp->m_rtdev_targp : mp->m_ddev_targp;
  646. /* DIO must be aligned to device logical sector size */
  647. if (!IS_DAX(inode) && ((pos | count) & target->bt_logical_sectormask))
  648. return -EINVAL;
  649. /* "unaligned" here means not aligned to a filesystem block */
  650. if ((pos & mp->m_blockmask) || ((pos + count) & mp->m_blockmask))
  651. unaligned_io = 1;
  652. /*
  653. * We don't need to take an exclusive lock unless there page cache needs
  654. * to be invalidated or unaligned IO is being executed. We don't need to
  655. * consider the EOF extension case here because
  656. * xfs_file_aio_write_checks() will relock the inode as necessary for
  657. * EOF zeroing cases and fill out the new inode size as appropriate.
  658. */
  659. if (unaligned_io || mapping->nrpages)
  660. iolock = XFS_IOLOCK_EXCL;
  661. else
  662. iolock = XFS_IOLOCK_SHARED;
  663. xfs_rw_ilock(ip, iolock);
  664. /*
  665. * Recheck if there are cached pages that need invalidate after we got
  666. * the iolock to protect against other threads adding new pages while
  667. * we were waiting for the iolock.
  668. */
  669. if (mapping->nrpages && iolock == XFS_IOLOCK_SHARED) {
  670. xfs_rw_iunlock(ip, iolock);
  671. iolock = XFS_IOLOCK_EXCL;
  672. xfs_rw_ilock(ip, iolock);
  673. }
  674. ret = xfs_file_aio_write_checks(iocb, from, &iolock);
  675. if (ret)
  676. goto out;
  677. count = iov_iter_count(from);
  678. pos = iocb->ki_pos;
  679. end = pos + count - 1;
  680. /*
  681. * See xfs_file_read_iter() for why we do a full-file flush here.
  682. */
  683. if (mapping->nrpages) {
  684. ret = filemap_write_and_wait(VFS_I(ip)->i_mapping);
  685. if (ret)
  686. goto out;
  687. /*
  688. * Invalidate whole pages. This can return an error if we fail
  689. * to invalidate a page, but this should never happen on XFS.
  690. * Warn if it does fail.
  691. */
  692. ret = invalidate_inode_pages2(VFS_I(ip)->i_mapping);
  693. WARN_ON_ONCE(ret);
  694. ret = 0;
  695. }
  696. /*
  697. * If we are doing unaligned IO, wait for all other IO to drain,
  698. * otherwise demote the lock if we had to flush cached pages
  699. */
  700. if (unaligned_io)
  701. inode_dio_wait(inode);
  702. else if (iolock == XFS_IOLOCK_EXCL) {
  703. xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL);
  704. iolock = XFS_IOLOCK_SHARED;
  705. }
  706. trace_xfs_file_direct_write(ip, count, iocb->ki_pos, 0);
  707. data = *from;
  708. ret = mapping->a_ops->direct_IO(iocb, &data, pos);
  709. /* see generic_file_direct_write() for why this is necessary */
  710. if (mapping->nrpages) {
  711. invalidate_inode_pages2_range(mapping,
  712. pos >> PAGE_CACHE_SHIFT,
  713. end >> PAGE_CACHE_SHIFT);
  714. }
  715. if (ret > 0) {
  716. pos += ret;
  717. iov_iter_advance(from, ret);
  718. iocb->ki_pos = pos;
  719. }
  720. out:
  721. xfs_rw_iunlock(ip, iolock);
  722. /*
  723. * No fallback to buffered IO on errors for XFS. DAX can result in
  724. * partial writes, but direct IO will either complete fully or fail.
  725. */
  726. ASSERT(ret < 0 || ret == count || IS_DAX(VFS_I(ip)));
  727. return ret;
  728. }
  729. STATIC ssize_t
  730. xfs_file_buffered_aio_write(
  731. struct kiocb *iocb,
  732. struct iov_iter *from)
  733. {
  734. struct file *file = iocb->ki_filp;
  735. struct address_space *mapping = file->f_mapping;
  736. struct inode *inode = mapping->host;
  737. struct xfs_inode *ip = XFS_I(inode);
  738. ssize_t ret;
  739. int enospc = 0;
  740. int iolock = XFS_IOLOCK_EXCL;
  741. xfs_rw_ilock(ip, iolock);
  742. ret = xfs_file_aio_write_checks(iocb, from, &iolock);
  743. if (ret)
  744. goto out;
  745. /* We can write back this queue in page reclaim */
  746. current->backing_dev_info = inode_to_bdi(inode);
  747. write_retry:
  748. trace_xfs_file_buffered_write(ip, iov_iter_count(from),
  749. iocb->ki_pos, 0);
  750. ret = generic_perform_write(file, from, iocb->ki_pos);
  751. if (likely(ret >= 0))
  752. iocb->ki_pos += ret;
  753. /*
  754. * If we hit a space limit, try to free up some lingering preallocated
  755. * space before returning an error. In the case of ENOSPC, first try to
  756. * write back all dirty inodes to free up some of the excess reserved
  757. * metadata space. This reduces the chances that the eofblocks scan
  758. * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this
  759. * also behaves as a filter to prevent too many eofblocks scans from
  760. * running at the same time.
  761. */
  762. if (ret == -EDQUOT && !enospc) {
  763. enospc = xfs_inode_free_quota_eofblocks(ip);
  764. if (enospc)
  765. goto write_retry;
  766. } else if (ret == -ENOSPC && !enospc) {
  767. struct xfs_eofblocks eofb = {0};
  768. enospc = 1;
  769. xfs_flush_inodes(ip->i_mount);
  770. eofb.eof_scan_owner = ip->i_ino; /* for locking */
  771. eofb.eof_flags = XFS_EOF_FLAGS_SYNC;
  772. xfs_icache_free_eofblocks(ip->i_mount, &eofb);
  773. goto write_retry;
  774. }
  775. current->backing_dev_info = NULL;
  776. out:
  777. xfs_rw_iunlock(ip, iolock);
  778. return ret;
  779. }
  780. STATIC ssize_t
  781. xfs_file_write_iter(
  782. struct kiocb *iocb,
  783. struct iov_iter *from)
  784. {
  785. struct file *file = iocb->ki_filp;
  786. struct address_space *mapping = file->f_mapping;
  787. struct inode *inode = mapping->host;
  788. struct xfs_inode *ip = XFS_I(inode);
  789. ssize_t ret;
  790. size_t ocount = iov_iter_count(from);
  791. XFS_STATS_INC(ip->i_mount, xs_write_calls);
  792. if (ocount == 0)
  793. return 0;
  794. if (XFS_FORCED_SHUTDOWN(ip->i_mount))
  795. return -EIO;
  796. if ((iocb->ki_flags & IOCB_DIRECT) || IS_DAX(inode))
  797. ret = xfs_file_dio_aio_write(iocb, from);
  798. else
  799. ret = xfs_file_buffered_aio_write(iocb, from);
  800. if (ret > 0) {
  801. ssize_t err;
  802. XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret);
  803. /* Handle various SYNC-type writes */
  804. err = generic_write_sync(file, iocb->ki_pos - ret, ret);
  805. if (err < 0)
  806. ret = err;
  807. }
  808. return ret;
  809. }
  810. #define XFS_FALLOC_FL_SUPPORTED \
  811. (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
  812. FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | \
  813. FALLOC_FL_INSERT_RANGE)
  814. STATIC long
  815. xfs_file_fallocate(
  816. struct file *file,
  817. int mode,
  818. loff_t offset,
  819. loff_t len)
  820. {
  821. struct inode *inode = file_inode(file);
  822. struct xfs_inode *ip = XFS_I(inode);
  823. long error;
  824. enum xfs_prealloc_flags flags = 0;
  825. uint iolock = XFS_IOLOCK_EXCL;
  826. loff_t new_size = 0;
  827. bool do_file_insert = 0;
  828. if (!S_ISREG(inode->i_mode))
  829. return -EINVAL;
  830. if (mode & ~XFS_FALLOC_FL_SUPPORTED)
  831. return -EOPNOTSUPP;
  832. xfs_ilock(ip, iolock);
  833. error = xfs_break_layouts(inode, &iolock, false);
  834. if (error)
  835. goto out_unlock;
  836. xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
  837. iolock |= XFS_MMAPLOCK_EXCL;
  838. if (mode & FALLOC_FL_PUNCH_HOLE) {
  839. error = xfs_free_file_space(ip, offset, len);
  840. if (error)
  841. goto out_unlock;
  842. } else if (mode & FALLOC_FL_COLLAPSE_RANGE) {
  843. unsigned blksize_mask = (1 << inode->i_blkbits) - 1;
  844. if (offset & blksize_mask || len & blksize_mask) {
  845. error = -EINVAL;
  846. goto out_unlock;
  847. }
  848. /*
  849. * There is no need to overlap collapse range with EOF,
  850. * in which case it is effectively a truncate operation
  851. */
  852. if (offset + len >= i_size_read(inode)) {
  853. error = -EINVAL;
  854. goto out_unlock;
  855. }
  856. new_size = i_size_read(inode) - len;
  857. error = xfs_collapse_file_space(ip, offset, len);
  858. if (error)
  859. goto out_unlock;
  860. } else if (mode & FALLOC_FL_INSERT_RANGE) {
  861. unsigned blksize_mask = (1 << inode->i_blkbits) - 1;
  862. new_size = i_size_read(inode) + len;
  863. if (offset & blksize_mask || len & blksize_mask) {
  864. error = -EINVAL;
  865. goto out_unlock;
  866. }
  867. /* check the new inode size does not wrap through zero */
  868. if (new_size > inode->i_sb->s_maxbytes) {
  869. error = -EFBIG;
  870. goto out_unlock;
  871. }
  872. /* Offset should be less than i_size */
  873. if (offset >= i_size_read(inode)) {
  874. error = -EINVAL;
  875. goto out_unlock;
  876. }
  877. do_file_insert = 1;
  878. } else {
  879. flags |= XFS_PREALLOC_SET;
  880. if (!(mode & FALLOC_FL_KEEP_SIZE) &&
  881. offset + len > i_size_read(inode)) {
  882. new_size = offset + len;
  883. error = inode_newsize_ok(inode, new_size);
  884. if (error)
  885. goto out_unlock;
  886. }
  887. if (mode & FALLOC_FL_ZERO_RANGE)
  888. error = xfs_zero_file_space(ip, offset, len);
  889. else
  890. error = xfs_alloc_file_space(ip, offset, len,
  891. XFS_BMAPI_PREALLOC);
  892. if (error)
  893. goto out_unlock;
  894. }
  895. if (file->f_flags & O_DSYNC)
  896. flags |= XFS_PREALLOC_SYNC;
  897. error = xfs_update_prealloc_flags(ip, flags);
  898. if (error)
  899. goto out_unlock;
  900. /* Change file size if needed */
  901. if (new_size) {
  902. struct iattr iattr;
  903. iattr.ia_valid = ATTR_SIZE;
  904. iattr.ia_size = new_size;
  905. error = xfs_setattr_size(ip, &iattr);
  906. if (error)
  907. goto out_unlock;
  908. }
  909. /*
  910. * Perform hole insertion now that the file size has been
  911. * updated so that if we crash during the operation we don't
  912. * leave shifted extents past EOF and hence losing access to
  913. * the data that is contained within them.
  914. */
  915. if (do_file_insert)
  916. error = xfs_insert_file_space(ip, offset, len);
  917. out_unlock:
  918. xfs_iunlock(ip, iolock);
  919. return error;
  920. }
  921. STATIC int
  922. xfs_file_open(
  923. struct inode *inode,
  924. struct file *file)
  925. {
  926. if (!(file->f_flags & O_LARGEFILE) && i_size_read(inode) > MAX_NON_LFS)
  927. return -EFBIG;
  928. if (XFS_FORCED_SHUTDOWN(XFS_M(inode->i_sb)))
  929. return -EIO;
  930. return 0;
  931. }
  932. STATIC int
  933. xfs_dir_open(
  934. struct inode *inode,
  935. struct file *file)
  936. {
  937. struct xfs_inode *ip = XFS_I(inode);
  938. int mode;
  939. int error;
  940. error = xfs_file_open(inode, file);
  941. if (error)
  942. return error;
  943. /*
  944. * If there are any blocks, read-ahead block 0 as we're almost
  945. * certain to have the next operation be a read there.
  946. */
  947. mode = xfs_ilock_data_map_shared(ip);
  948. if (ip->i_d.di_nextents > 0)
  949. xfs_dir3_data_readahead(ip, 0, -1);
  950. xfs_iunlock(ip, mode);
  951. return 0;
  952. }
  953. STATIC int
  954. xfs_file_release(
  955. struct inode *inode,
  956. struct file *filp)
  957. {
  958. return xfs_release(XFS_I(inode));
  959. }
  960. STATIC int
  961. xfs_file_readdir(
  962. struct file *file,
  963. struct dir_context *ctx)
  964. {
  965. struct inode *inode = file_inode(file);
  966. xfs_inode_t *ip = XFS_I(inode);
  967. size_t bufsize;
  968. /*
  969. * The Linux API doesn't pass down the total size of the buffer
  970. * we read into down to the filesystem. With the filldir concept
  971. * it's not needed for correct information, but the XFS dir2 leaf
  972. * code wants an estimate of the buffer size to calculate it's
  973. * readahead window and size the buffers used for mapping to
  974. * physical blocks.
  975. *
  976. * Try to give it an estimate that's good enough, maybe at some
  977. * point we can change the ->readdir prototype to include the
  978. * buffer size. For now we use the current glibc buffer size.
  979. */
  980. bufsize = (size_t)min_t(loff_t, 32768, ip->i_d.di_size);
  981. return xfs_readdir(ip, ctx, bufsize);
  982. }
  983. /*
  984. * This type is designed to indicate the type of offset we would like
  985. * to search from page cache for xfs_seek_hole_data().
  986. */
  987. enum {
  988. HOLE_OFF = 0,
  989. DATA_OFF,
  990. };
  991. /*
  992. * Lookup the desired type of offset from the given page.
  993. *
  994. * On success, return true and the offset argument will point to the
  995. * start of the region that was found. Otherwise this function will
  996. * return false and keep the offset argument unchanged.
  997. */
  998. STATIC bool
  999. xfs_lookup_buffer_offset(
  1000. struct page *page,
  1001. loff_t *offset,
  1002. unsigned int type)
  1003. {
  1004. loff_t lastoff = page_offset(page);
  1005. bool found = false;
  1006. struct buffer_head *bh, *head;
  1007. bh = head = page_buffers(page);
  1008. do {
  1009. /*
  1010. * Unwritten extents that have data in the page
  1011. * cache covering them can be identified by the
  1012. * BH_Unwritten state flag. Pages with multiple
  1013. * buffers might have a mix of holes, data and
  1014. * unwritten extents - any buffer with valid
  1015. * data in it should have BH_Uptodate flag set
  1016. * on it.
  1017. */
  1018. if (buffer_unwritten(bh) ||
  1019. buffer_uptodate(bh)) {
  1020. if (type == DATA_OFF)
  1021. found = true;
  1022. } else {
  1023. if (type == HOLE_OFF)
  1024. found = true;
  1025. }
  1026. if (found) {
  1027. *offset = lastoff;
  1028. break;
  1029. }
  1030. lastoff += bh->b_size;
  1031. } while ((bh = bh->b_this_page) != head);
  1032. return found;
  1033. }
  1034. /*
  1035. * This routine is called to find out and return a data or hole offset
  1036. * from the page cache for unwritten extents according to the desired
  1037. * type for xfs_seek_hole_data().
  1038. *
  1039. * The argument offset is used to tell where we start to search from the
  1040. * page cache. Map is used to figure out the end points of the range to
  1041. * lookup pages.
  1042. *
  1043. * Return true if the desired type of offset was found, and the argument
  1044. * offset is filled with that address. Otherwise, return false and keep
  1045. * offset unchanged.
  1046. */
  1047. STATIC bool
  1048. xfs_find_get_desired_pgoff(
  1049. struct inode *inode,
  1050. struct xfs_bmbt_irec *map,
  1051. unsigned int type,
  1052. loff_t *offset)
  1053. {
  1054. struct xfs_inode *ip = XFS_I(inode);
  1055. struct xfs_mount *mp = ip->i_mount;
  1056. struct pagevec pvec;
  1057. pgoff_t index;
  1058. pgoff_t end;
  1059. loff_t endoff;
  1060. loff_t startoff = *offset;
  1061. loff_t lastoff = startoff;
  1062. bool found = false;
  1063. pagevec_init(&pvec, 0);
  1064. index = startoff >> PAGE_CACHE_SHIFT;
  1065. endoff = XFS_FSB_TO_B(mp, map->br_startoff + map->br_blockcount);
  1066. end = endoff >> PAGE_CACHE_SHIFT;
  1067. do {
  1068. int want;
  1069. unsigned nr_pages;
  1070. unsigned int i;
  1071. want = min_t(pgoff_t, end - index, PAGEVEC_SIZE);
  1072. nr_pages = pagevec_lookup(&pvec, inode->i_mapping, index,
  1073. want);
  1074. /*
  1075. * No page mapped into given range. If we are searching holes
  1076. * and if this is the first time we got into the loop, it means
  1077. * that the given offset is landed in a hole, return it.
  1078. *
  1079. * If we have already stepped through some block buffers to find
  1080. * holes but they all contains data. In this case, the last
  1081. * offset is already updated and pointed to the end of the last
  1082. * mapped page, if it does not reach the endpoint to search,
  1083. * that means there should be a hole between them.
  1084. */
  1085. if (nr_pages == 0) {
  1086. /* Data search found nothing */
  1087. if (type == DATA_OFF)
  1088. break;
  1089. ASSERT(type == HOLE_OFF);
  1090. if (lastoff == startoff || lastoff < endoff) {
  1091. found = true;
  1092. *offset = lastoff;
  1093. }
  1094. break;
  1095. }
  1096. /*
  1097. * At lease we found one page. If this is the first time we
  1098. * step into the loop, and if the first page index offset is
  1099. * greater than the given search offset, a hole was found.
  1100. */
  1101. if (type == HOLE_OFF && lastoff == startoff &&
  1102. lastoff < page_offset(pvec.pages[0])) {
  1103. found = true;
  1104. break;
  1105. }
  1106. for (i = 0; i < nr_pages; i++) {
  1107. struct page *page = pvec.pages[i];
  1108. loff_t b_offset;
  1109. /*
  1110. * At this point, the page may be truncated or
  1111. * invalidated (changing page->mapping to NULL),
  1112. * or even swizzled back from swapper_space to tmpfs
  1113. * file mapping. However, page->index will not change
  1114. * because we have a reference on the page.
  1115. *
  1116. * Searching done if the page index is out of range.
  1117. * If the current offset is not reaches the end of
  1118. * the specified search range, there should be a hole
  1119. * between them.
  1120. */
  1121. if (page->index > end) {
  1122. if (type == HOLE_OFF && lastoff < endoff) {
  1123. *offset = lastoff;
  1124. found = true;
  1125. }
  1126. goto out;
  1127. }
  1128. lock_page(page);
  1129. /*
  1130. * Page truncated or invalidated(page->mapping == NULL).
  1131. * We can freely skip it and proceed to check the next
  1132. * page.
  1133. */
  1134. if (unlikely(page->mapping != inode->i_mapping)) {
  1135. unlock_page(page);
  1136. continue;
  1137. }
  1138. if (!page_has_buffers(page)) {
  1139. unlock_page(page);
  1140. continue;
  1141. }
  1142. found = xfs_lookup_buffer_offset(page, &b_offset, type);
  1143. if (found) {
  1144. /*
  1145. * The found offset may be less than the start
  1146. * point to search if this is the first time to
  1147. * come here.
  1148. */
  1149. *offset = max_t(loff_t, startoff, b_offset);
  1150. unlock_page(page);
  1151. goto out;
  1152. }
  1153. /*
  1154. * We either searching data but nothing was found, or
  1155. * searching hole but found a data buffer. In either
  1156. * case, probably the next page contains the desired
  1157. * things, update the last offset to it so.
  1158. */
  1159. lastoff = page_offset(page) + PAGE_SIZE;
  1160. unlock_page(page);
  1161. }
  1162. /*
  1163. * The number of returned pages less than our desired, search
  1164. * done. In this case, nothing was found for searching data,
  1165. * but we found a hole behind the last offset.
  1166. */
  1167. if (nr_pages < want) {
  1168. if (type == HOLE_OFF) {
  1169. *offset = lastoff;
  1170. found = true;
  1171. }
  1172. break;
  1173. }
  1174. index = pvec.pages[i - 1]->index + 1;
  1175. pagevec_release(&pvec);
  1176. } while (index <= end);
  1177. out:
  1178. pagevec_release(&pvec);
  1179. return found;
  1180. }
  1181. STATIC loff_t
  1182. xfs_seek_hole_data(
  1183. struct file *file,
  1184. loff_t start,
  1185. int whence)
  1186. {
  1187. struct inode *inode = file->f_mapping->host;
  1188. struct xfs_inode *ip = XFS_I(inode);
  1189. struct xfs_mount *mp = ip->i_mount;
  1190. loff_t uninitialized_var(offset);
  1191. xfs_fsize_t isize;
  1192. xfs_fileoff_t fsbno;
  1193. xfs_filblks_t end;
  1194. uint lock;
  1195. int error;
  1196. if (XFS_FORCED_SHUTDOWN(mp))
  1197. return -EIO;
  1198. lock = xfs_ilock_data_map_shared(ip);
  1199. isize = i_size_read(inode);
  1200. if (start >= isize) {
  1201. error = -ENXIO;
  1202. goto out_unlock;
  1203. }
  1204. /*
  1205. * Try to read extents from the first block indicated
  1206. * by fsbno to the end block of the file.
  1207. */
  1208. fsbno = XFS_B_TO_FSBT(mp, start);
  1209. end = XFS_B_TO_FSB(mp, isize);
  1210. for (;;) {
  1211. struct xfs_bmbt_irec map[2];
  1212. int nmap = 2;
  1213. unsigned int i;
  1214. error = xfs_bmapi_read(ip, fsbno, end - fsbno, map, &nmap,
  1215. XFS_BMAPI_ENTIRE);
  1216. if (error)
  1217. goto out_unlock;
  1218. /* No extents at given offset, must be beyond EOF */
  1219. if (nmap == 0) {
  1220. error = -ENXIO;
  1221. goto out_unlock;
  1222. }
  1223. for (i = 0; i < nmap; i++) {
  1224. offset = max_t(loff_t, start,
  1225. XFS_FSB_TO_B(mp, map[i].br_startoff));
  1226. /* Landed in the hole we wanted? */
  1227. if (whence == SEEK_HOLE &&
  1228. map[i].br_startblock == HOLESTARTBLOCK)
  1229. goto out;
  1230. /* Landed in the data extent we wanted? */
  1231. if (whence == SEEK_DATA &&
  1232. (map[i].br_startblock == DELAYSTARTBLOCK ||
  1233. (map[i].br_state == XFS_EXT_NORM &&
  1234. !isnullstartblock(map[i].br_startblock))))
  1235. goto out;
  1236. /*
  1237. * Landed in an unwritten extent, try to search
  1238. * for hole or data from page cache.
  1239. */
  1240. if (map[i].br_state == XFS_EXT_UNWRITTEN) {
  1241. if (xfs_find_get_desired_pgoff(inode, &map[i],
  1242. whence == SEEK_HOLE ? HOLE_OFF : DATA_OFF,
  1243. &offset))
  1244. goto out;
  1245. }
  1246. }
  1247. /*
  1248. * We only received one extent out of the two requested. This
  1249. * means we've hit EOF and didn't find what we are looking for.
  1250. */
  1251. if (nmap == 1) {
  1252. /*
  1253. * If we were looking for a hole, set offset to
  1254. * the end of the file (i.e., there is an implicit
  1255. * hole at the end of any file).
  1256. */
  1257. if (whence == SEEK_HOLE) {
  1258. offset = isize;
  1259. break;
  1260. }
  1261. /*
  1262. * If we were looking for data, it's nowhere to be found
  1263. */
  1264. ASSERT(whence == SEEK_DATA);
  1265. error = -ENXIO;
  1266. goto out_unlock;
  1267. }
  1268. ASSERT(i > 1);
  1269. /*
  1270. * Nothing was found, proceed to the next round of search
  1271. * if the next reading offset is not at or beyond EOF.
  1272. */
  1273. fsbno = map[i - 1].br_startoff + map[i - 1].br_blockcount;
  1274. start = XFS_FSB_TO_B(mp, fsbno);
  1275. if (start >= isize) {
  1276. if (whence == SEEK_HOLE) {
  1277. offset = isize;
  1278. break;
  1279. }
  1280. ASSERT(whence == SEEK_DATA);
  1281. error = -ENXIO;
  1282. goto out_unlock;
  1283. }
  1284. }
  1285. out:
  1286. /*
  1287. * If at this point we have found the hole we wanted, the returned
  1288. * offset may be bigger than the file size as it may be aligned to
  1289. * page boundary for unwritten extents. We need to deal with this
  1290. * situation in particular.
  1291. */
  1292. if (whence == SEEK_HOLE)
  1293. offset = min_t(loff_t, offset, isize);
  1294. offset = vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
  1295. out_unlock:
  1296. xfs_iunlock(ip, lock);
  1297. if (error)
  1298. return error;
  1299. return offset;
  1300. }
  1301. STATIC loff_t
  1302. xfs_file_llseek(
  1303. struct file *file,
  1304. loff_t offset,
  1305. int whence)
  1306. {
  1307. switch (whence) {
  1308. case SEEK_END:
  1309. case SEEK_CUR:
  1310. case SEEK_SET:
  1311. return generic_file_llseek(file, offset, whence);
  1312. case SEEK_HOLE:
  1313. case SEEK_DATA:
  1314. return xfs_seek_hole_data(file, offset, whence);
  1315. default:
  1316. return -EINVAL;
  1317. }
  1318. }
  1319. /*
  1320. * Locking for serialisation of IO during page faults. This results in a lock
  1321. * ordering of:
  1322. *
  1323. * mmap_sem (MM)
  1324. * sb_start_pagefault(vfs, freeze)
  1325. * i_mmaplock (XFS - truncate serialisation)
  1326. * page_lock (MM)
  1327. * i_lock (XFS - extent map serialisation)
  1328. */
  1329. /*
  1330. * mmap()d file has taken write protection fault and is being made writable. We
  1331. * can set the page state up correctly for a writable page, which means we can
  1332. * do correct delalloc accounting (ENOSPC checking!) and unwritten extent
  1333. * mapping.
  1334. */
  1335. STATIC int
  1336. xfs_filemap_page_mkwrite(
  1337. struct vm_area_struct *vma,
  1338. struct vm_fault *vmf)
  1339. {
  1340. struct inode *inode = file_inode(vma->vm_file);
  1341. int ret;
  1342. trace_xfs_filemap_page_mkwrite(XFS_I(inode));
  1343. sb_start_pagefault(inode->i_sb);
  1344. file_update_time(vma->vm_file);
  1345. xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1346. if (IS_DAX(inode)) {
  1347. ret = __dax_mkwrite(vma, vmf, xfs_get_blocks_dax_fault, NULL);
  1348. } else {
  1349. ret = block_page_mkwrite(vma, vmf, xfs_get_blocks);
  1350. ret = block_page_mkwrite_return(ret);
  1351. }
  1352. xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1353. sb_end_pagefault(inode->i_sb);
  1354. return ret;
  1355. }
  1356. STATIC int
  1357. xfs_filemap_fault(
  1358. struct vm_area_struct *vma,
  1359. struct vm_fault *vmf)
  1360. {
  1361. struct inode *inode = file_inode(vma->vm_file);
  1362. int ret;
  1363. trace_xfs_filemap_fault(XFS_I(inode));
  1364. /* DAX can shortcut the normal fault path on write faults! */
  1365. if ((vmf->flags & FAULT_FLAG_WRITE) && IS_DAX(inode))
  1366. return xfs_filemap_page_mkwrite(vma, vmf);
  1367. xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1368. if (IS_DAX(inode)) {
  1369. /*
  1370. * we do not want to trigger unwritten extent conversion on read
  1371. * faults - that is unnecessary overhead and would also require
  1372. * changes to xfs_get_blocks_direct() to map unwritten extent
  1373. * ioend for conversion on read-only mappings.
  1374. */
  1375. ret = __dax_fault(vma, vmf, xfs_get_blocks_dax_fault, NULL);
  1376. } else
  1377. ret = filemap_fault(vma, vmf);
  1378. xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1379. return ret;
  1380. }
  1381. /*
  1382. * Similar to xfs_filemap_fault(), the DAX fault path can call into here on
  1383. * both read and write faults. Hence we need to handle both cases. There is no
  1384. * ->pmd_mkwrite callout for huge pages, so we have a single function here to
  1385. * handle both cases here. @flags carries the information on the type of fault
  1386. * occuring.
  1387. */
  1388. STATIC int
  1389. xfs_filemap_pmd_fault(
  1390. struct vm_area_struct *vma,
  1391. unsigned long addr,
  1392. pmd_t *pmd,
  1393. unsigned int flags)
  1394. {
  1395. struct inode *inode = file_inode(vma->vm_file);
  1396. struct xfs_inode *ip = XFS_I(inode);
  1397. int ret;
  1398. if (!IS_DAX(inode))
  1399. return VM_FAULT_FALLBACK;
  1400. trace_xfs_filemap_pmd_fault(ip);
  1401. if (flags & FAULT_FLAG_WRITE) {
  1402. sb_start_pagefault(inode->i_sb);
  1403. file_update_time(vma->vm_file);
  1404. }
  1405. xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1406. ret = __dax_pmd_fault(vma, addr, pmd, flags, xfs_get_blocks_dax_fault,
  1407. NULL);
  1408. xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
  1409. if (flags & FAULT_FLAG_WRITE)
  1410. sb_end_pagefault(inode->i_sb);
  1411. return ret;
  1412. }
  1413. /*
  1414. * pfn_mkwrite was originally inteneded to ensure we capture time stamp
  1415. * updates on write faults. In reality, it's need to serialise against
  1416. * truncate similar to page_mkwrite. Hence we cycle the XFS_MMAPLOCK_SHARED
  1417. * to ensure we serialise the fault barrier in place.
  1418. */
  1419. static int
  1420. xfs_filemap_pfn_mkwrite(
  1421. struct vm_area_struct *vma,
  1422. struct vm_fault *vmf)
  1423. {
  1424. struct inode *inode = file_inode(vma->vm_file);
  1425. struct xfs_inode *ip = XFS_I(inode);
  1426. int ret = VM_FAULT_NOPAGE;
  1427. loff_t size;
  1428. trace_xfs_filemap_pfn_mkwrite(ip);
  1429. sb_start_pagefault(inode->i_sb);
  1430. file_update_time(vma->vm_file);
  1431. /* check if the faulting page hasn't raced with truncate */
  1432. xfs_ilock(ip, XFS_MMAPLOCK_SHARED);
  1433. size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1434. if (vmf->pgoff >= size)
  1435. ret = VM_FAULT_SIGBUS;
  1436. else if (IS_DAX(inode))
  1437. ret = dax_pfn_mkwrite(vma, vmf);
  1438. xfs_iunlock(ip, XFS_MMAPLOCK_SHARED);
  1439. sb_end_pagefault(inode->i_sb);
  1440. return ret;
  1441. }
  1442. static const struct vm_operations_struct xfs_file_vm_ops = {
  1443. .fault = xfs_filemap_fault,
  1444. .pmd_fault = xfs_filemap_pmd_fault,
  1445. .map_pages = filemap_map_pages,
  1446. .page_mkwrite = xfs_filemap_page_mkwrite,
  1447. .pfn_mkwrite = xfs_filemap_pfn_mkwrite,
  1448. };
  1449. STATIC int
  1450. xfs_file_mmap(
  1451. struct file *filp,
  1452. struct vm_area_struct *vma)
  1453. {
  1454. file_accessed(filp);
  1455. vma->vm_ops = &xfs_file_vm_ops;
  1456. if (IS_DAX(file_inode(filp)))
  1457. vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
  1458. return 0;
  1459. }
  1460. const struct file_operations xfs_file_operations = {
  1461. .llseek = xfs_file_llseek,
  1462. .read_iter = xfs_file_read_iter,
  1463. .write_iter = xfs_file_write_iter,
  1464. .splice_read = xfs_file_splice_read,
  1465. .splice_write = iter_file_splice_write,
  1466. .unlocked_ioctl = xfs_file_ioctl,
  1467. #ifdef CONFIG_COMPAT
  1468. .compat_ioctl = xfs_file_compat_ioctl,
  1469. #endif
  1470. .mmap = xfs_file_mmap,
  1471. .open = xfs_file_open,
  1472. .release = xfs_file_release,
  1473. .fsync = xfs_file_fsync,
  1474. .fallocate = xfs_file_fallocate,
  1475. };
  1476. const struct file_operations xfs_dir_file_operations = {
  1477. .open = xfs_dir_open,
  1478. .read = generic_read_dir,
  1479. .iterate = xfs_file_readdir,
  1480. .llseek = generic_file_llseek,
  1481. .unlocked_ioctl = xfs_file_ioctl,
  1482. #ifdef CONFIG_COMPAT
  1483. .compat_ioctl = xfs_file_compat_ioctl,
  1484. #endif
  1485. .fsync = xfs_dir_fsync,
  1486. };