data.c 28 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175
  1. /*
  2. * fs/f2fs/data.c
  3. *
  4. * Copyright (c) 2012 Samsung Electronics Co., Ltd.
  5. * http://www.samsung.com/
  6. *
  7. * This program is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License version 2 as
  9. * published by the Free Software Foundation.
  10. */
  11. #include <linux/fs.h>
  12. #include <linux/f2fs_fs.h>
  13. #include <linux/buffer_head.h>
  14. #include <linux/mpage.h>
  15. #include <linux/aio.h>
  16. #include <linux/writeback.h>
  17. #include <linux/backing-dev.h>
  18. #include <linux/blkdev.h>
  19. #include <linux/bio.h>
  20. #include <linux/prefetch.h>
  21. #include "f2fs.h"
  22. #include "node.h"
  23. #include "segment.h"
  24. #include <trace/events/f2fs.h>
  25. static void f2fs_read_end_io(struct bio *bio, int err)
  26. {
  27. struct bio_vec *bvec;
  28. int i;
  29. bio_for_each_segment_all(bvec, bio, i) {
  30. struct page *page = bvec->bv_page;
  31. if (!err) {
  32. SetPageUptodate(page);
  33. } else {
  34. ClearPageUptodate(page);
  35. SetPageError(page);
  36. }
  37. unlock_page(page);
  38. }
  39. bio_put(bio);
  40. }
  41. static void f2fs_write_end_io(struct bio *bio, int err)
  42. {
  43. struct f2fs_sb_info *sbi = bio->bi_private;
  44. struct bio_vec *bvec;
  45. int i;
  46. bio_for_each_segment_all(bvec, bio, i) {
  47. struct page *page = bvec->bv_page;
  48. if (unlikely(err)) {
  49. set_page_dirty(page);
  50. set_bit(AS_EIO, &page->mapping->flags);
  51. f2fs_stop_checkpoint(sbi);
  52. }
  53. end_page_writeback(page);
  54. dec_page_count(sbi, F2FS_WRITEBACK);
  55. }
  56. if (!get_pages(sbi, F2FS_WRITEBACK) &&
  57. !list_empty(&sbi->cp_wait.task_list))
  58. wake_up(&sbi->cp_wait);
  59. bio_put(bio);
  60. }
  61. /*
  62. * Low-level block read/write IO operations.
  63. */
  64. static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
  65. int npages, bool is_read)
  66. {
  67. struct bio *bio;
  68. /* No failure on bio allocation */
  69. bio = bio_alloc(GFP_NOIO, npages);
  70. bio->bi_bdev = sbi->sb->s_bdev;
  71. bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
  72. bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
  73. bio->bi_private = sbi;
  74. return bio;
  75. }
  76. static void __submit_merged_bio(struct f2fs_bio_info *io)
  77. {
  78. struct f2fs_io_info *fio = &io->fio;
  79. if (!io->bio)
  80. return;
  81. if (is_read_io(fio->rw))
  82. trace_f2fs_submit_read_bio(io->sbi->sb, fio->rw,
  83. fio->type, io->bio);
  84. else
  85. trace_f2fs_submit_write_bio(io->sbi->sb, fio->rw,
  86. fio->type, io->bio);
  87. submit_bio(fio->rw, io->bio);
  88. io->bio = NULL;
  89. }
  90. void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
  91. enum page_type type, int rw)
  92. {
  93. enum page_type btype = PAGE_TYPE_OF_BIO(type);
  94. struct f2fs_bio_info *io;
  95. io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype];
  96. down_write(&io->io_rwsem);
  97. /* change META to META_FLUSH in the checkpoint procedure */
  98. if (type >= META_FLUSH) {
  99. io->fio.type = META_FLUSH;
  100. if (test_opt(sbi, NOBARRIER))
  101. io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO;
  102. else
  103. io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO;
  104. }
  105. __submit_merged_bio(io);
  106. up_write(&io->io_rwsem);
  107. }
  108. /*
  109. * Fill the locked page with data located in the block address.
  110. * Return unlocked page.
  111. */
  112. int f2fs_submit_page_bio(struct f2fs_sb_info *sbi, struct page *page,
  113. block_t blk_addr, int rw)
  114. {
  115. struct bio *bio;
  116. trace_f2fs_submit_page_bio(page, blk_addr, rw);
  117. /* Allocate a new bio */
  118. bio = __bio_alloc(sbi, blk_addr, 1, is_read_io(rw));
  119. if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
  120. bio_put(bio);
  121. f2fs_put_page(page, 1);
  122. return -EFAULT;
  123. }
  124. submit_bio(rw, bio);
  125. return 0;
  126. }
  127. void f2fs_submit_page_mbio(struct f2fs_sb_info *sbi, struct page *page,
  128. block_t blk_addr, struct f2fs_io_info *fio)
  129. {
  130. enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
  131. struct f2fs_bio_info *io;
  132. bool is_read = is_read_io(fio->rw);
  133. io = is_read ? &sbi->read_io : &sbi->write_io[btype];
  134. verify_block_addr(sbi, blk_addr);
  135. down_write(&io->io_rwsem);
  136. if (!is_read)
  137. inc_page_count(sbi, F2FS_WRITEBACK);
  138. if (io->bio && (io->last_block_in_bio != blk_addr - 1 ||
  139. io->fio.rw != fio->rw))
  140. __submit_merged_bio(io);
  141. alloc_new:
  142. if (io->bio == NULL) {
  143. int bio_blocks = MAX_BIO_BLOCKS(sbi);
  144. io->bio = __bio_alloc(sbi, blk_addr, bio_blocks, is_read);
  145. io->fio = *fio;
  146. }
  147. if (bio_add_page(io->bio, page, PAGE_CACHE_SIZE, 0) <
  148. PAGE_CACHE_SIZE) {
  149. __submit_merged_bio(io);
  150. goto alloc_new;
  151. }
  152. io->last_block_in_bio = blk_addr;
  153. up_write(&io->io_rwsem);
  154. trace_f2fs_submit_page_mbio(page, fio->rw, fio->type, blk_addr);
  155. }
  156. /*
  157. * Lock ordering for the change of data block address:
  158. * ->data_page
  159. * ->node_page
  160. * update block addresses in the node page
  161. */
  162. static void __set_data_blkaddr(struct dnode_of_data *dn, block_t new_addr)
  163. {
  164. struct f2fs_node *rn;
  165. __le32 *addr_array;
  166. struct page *node_page = dn->node_page;
  167. unsigned int ofs_in_node = dn->ofs_in_node;
  168. f2fs_wait_on_page_writeback(node_page, NODE);
  169. rn = F2FS_NODE(node_page);
  170. /* Get physical address of data block */
  171. addr_array = blkaddr_in_node(rn);
  172. addr_array[ofs_in_node] = cpu_to_le32(new_addr);
  173. set_page_dirty(node_page);
  174. }
  175. int reserve_new_block(struct dnode_of_data *dn)
  176. {
  177. struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
  178. if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
  179. return -EPERM;
  180. if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
  181. return -ENOSPC;
  182. trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);
  183. __set_data_blkaddr(dn, NEW_ADDR);
  184. dn->data_blkaddr = NEW_ADDR;
  185. mark_inode_dirty(dn->inode);
  186. sync_inode_page(dn);
  187. return 0;
  188. }
  189. int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
  190. {
  191. bool need_put = dn->inode_page ? false : true;
  192. int err;
  193. err = get_dnode_of_data(dn, index, ALLOC_NODE);
  194. if (err)
  195. return err;
  196. if (dn->data_blkaddr == NULL_ADDR)
  197. err = reserve_new_block(dn);
  198. if (err || need_put)
  199. f2fs_put_dnode(dn);
  200. return err;
  201. }
  202. static int check_extent_cache(struct inode *inode, pgoff_t pgofs,
  203. struct buffer_head *bh_result)
  204. {
  205. struct f2fs_inode_info *fi = F2FS_I(inode);
  206. pgoff_t start_fofs, end_fofs;
  207. block_t start_blkaddr;
  208. if (is_inode_flag_set(fi, FI_NO_EXTENT))
  209. return 0;
  210. read_lock(&fi->ext.ext_lock);
  211. if (fi->ext.len == 0) {
  212. read_unlock(&fi->ext.ext_lock);
  213. return 0;
  214. }
  215. stat_inc_total_hit(inode->i_sb);
  216. start_fofs = fi->ext.fofs;
  217. end_fofs = fi->ext.fofs + fi->ext.len - 1;
  218. start_blkaddr = fi->ext.blk_addr;
  219. if (pgofs >= start_fofs && pgofs <= end_fofs) {
  220. unsigned int blkbits = inode->i_sb->s_blocksize_bits;
  221. size_t count;
  222. clear_buffer_new(bh_result);
  223. map_bh(bh_result, inode->i_sb,
  224. start_blkaddr + pgofs - start_fofs);
  225. count = end_fofs - pgofs + 1;
  226. if (count < (UINT_MAX >> blkbits))
  227. bh_result->b_size = (count << blkbits);
  228. else
  229. bh_result->b_size = UINT_MAX;
  230. stat_inc_read_hit(inode->i_sb);
  231. read_unlock(&fi->ext.ext_lock);
  232. return 1;
  233. }
  234. read_unlock(&fi->ext.ext_lock);
  235. return 0;
  236. }
  237. void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn)
  238. {
  239. struct f2fs_inode_info *fi = F2FS_I(dn->inode);
  240. pgoff_t fofs, start_fofs, end_fofs;
  241. block_t start_blkaddr, end_blkaddr;
  242. int need_update = true;
  243. f2fs_bug_on(F2FS_I_SB(dn->inode), blk_addr == NEW_ADDR);
  244. fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
  245. dn->ofs_in_node;
  246. /* Update the page address in the parent node */
  247. __set_data_blkaddr(dn, blk_addr);
  248. if (is_inode_flag_set(fi, FI_NO_EXTENT))
  249. return;
  250. write_lock(&fi->ext.ext_lock);
  251. start_fofs = fi->ext.fofs;
  252. end_fofs = fi->ext.fofs + fi->ext.len - 1;
  253. start_blkaddr = fi->ext.blk_addr;
  254. end_blkaddr = fi->ext.blk_addr + fi->ext.len - 1;
  255. /* Drop and initialize the matched extent */
  256. if (fi->ext.len == 1 && fofs == start_fofs)
  257. fi->ext.len = 0;
  258. /* Initial extent */
  259. if (fi->ext.len == 0) {
  260. if (blk_addr != NULL_ADDR) {
  261. fi->ext.fofs = fofs;
  262. fi->ext.blk_addr = blk_addr;
  263. fi->ext.len = 1;
  264. }
  265. goto end_update;
  266. }
  267. /* Front merge */
  268. if (fofs == start_fofs - 1 && blk_addr == start_blkaddr - 1) {
  269. fi->ext.fofs--;
  270. fi->ext.blk_addr--;
  271. fi->ext.len++;
  272. goto end_update;
  273. }
  274. /* Back merge */
  275. if (fofs == end_fofs + 1 && blk_addr == end_blkaddr + 1) {
  276. fi->ext.len++;
  277. goto end_update;
  278. }
  279. /* Split the existing extent */
  280. if (fi->ext.len > 1 &&
  281. fofs >= start_fofs && fofs <= end_fofs) {
  282. if ((end_fofs - fofs) < (fi->ext.len >> 1)) {
  283. fi->ext.len = fofs - start_fofs;
  284. } else {
  285. fi->ext.fofs = fofs + 1;
  286. fi->ext.blk_addr = start_blkaddr +
  287. fofs - start_fofs + 1;
  288. fi->ext.len -= fofs - start_fofs + 1;
  289. }
  290. } else {
  291. need_update = false;
  292. }
  293. /* Finally, if the extent is very fragmented, let's drop the cache. */
  294. if (fi->ext.len < F2FS_MIN_EXTENT_LEN) {
  295. fi->ext.len = 0;
  296. set_inode_flag(fi, FI_NO_EXTENT);
  297. need_update = true;
  298. }
  299. end_update:
  300. write_unlock(&fi->ext.ext_lock);
  301. if (need_update)
  302. sync_inode_page(dn);
  303. return;
  304. }
  305. struct page *find_data_page(struct inode *inode, pgoff_t index, bool sync)
  306. {
  307. struct address_space *mapping = inode->i_mapping;
  308. struct dnode_of_data dn;
  309. struct page *page;
  310. int err;
  311. page = find_get_page(mapping, index);
  312. if (page && PageUptodate(page))
  313. return page;
  314. f2fs_put_page(page, 0);
  315. set_new_dnode(&dn, inode, NULL, NULL, 0);
  316. err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
  317. if (err)
  318. return ERR_PTR(err);
  319. f2fs_put_dnode(&dn);
  320. if (dn.data_blkaddr == NULL_ADDR)
  321. return ERR_PTR(-ENOENT);
  322. /* By fallocate(), there is no cached page, but with NEW_ADDR */
  323. if (unlikely(dn.data_blkaddr == NEW_ADDR))
  324. return ERR_PTR(-EINVAL);
  325. page = grab_cache_page(mapping, index);
  326. if (!page)
  327. return ERR_PTR(-ENOMEM);
  328. if (PageUptodate(page)) {
  329. unlock_page(page);
  330. return page;
  331. }
  332. err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, dn.data_blkaddr,
  333. sync ? READ_SYNC : READA);
  334. if (err)
  335. return ERR_PTR(err);
  336. if (sync) {
  337. wait_on_page_locked(page);
  338. if (unlikely(!PageUptodate(page))) {
  339. f2fs_put_page(page, 0);
  340. return ERR_PTR(-EIO);
  341. }
  342. }
  343. return page;
  344. }
  345. /*
  346. * If it tries to access a hole, return an error.
  347. * Because, the callers, functions in dir.c and GC, should be able to know
  348. * whether this page exists or not.
  349. */
  350. struct page *get_lock_data_page(struct inode *inode, pgoff_t index)
  351. {
  352. struct address_space *mapping = inode->i_mapping;
  353. struct dnode_of_data dn;
  354. struct page *page;
  355. int err;
  356. repeat:
  357. page = grab_cache_page(mapping, index);
  358. if (!page)
  359. return ERR_PTR(-ENOMEM);
  360. set_new_dnode(&dn, inode, NULL, NULL, 0);
  361. err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
  362. if (err) {
  363. f2fs_put_page(page, 1);
  364. return ERR_PTR(err);
  365. }
  366. f2fs_put_dnode(&dn);
  367. if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
  368. f2fs_put_page(page, 1);
  369. return ERR_PTR(-ENOENT);
  370. }
  371. if (PageUptodate(page))
  372. return page;
  373. /*
  374. * A new dentry page is allocated but not able to be written, since its
  375. * new inode page couldn't be allocated due to -ENOSPC.
  376. * In such the case, its blkaddr can be remained as NEW_ADDR.
  377. * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
  378. */
  379. if (dn.data_blkaddr == NEW_ADDR) {
  380. zero_user_segment(page, 0, PAGE_CACHE_SIZE);
  381. SetPageUptodate(page);
  382. return page;
  383. }
  384. err = f2fs_submit_page_bio(F2FS_I_SB(inode), page,
  385. dn.data_blkaddr, READ_SYNC);
  386. if (err)
  387. return ERR_PTR(err);
  388. lock_page(page);
  389. if (unlikely(!PageUptodate(page))) {
  390. f2fs_put_page(page, 1);
  391. return ERR_PTR(-EIO);
  392. }
  393. if (unlikely(page->mapping != mapping)) {
  394. f2fs_put_page(page, 1);
  395. goto repeat;
  396. }
  397. return page;
  398. }
  399. /*
  400. * Caller ensures that this data page is never allocated.
  401. * A new zero-filled data page is allocated in the page cache.
  402. *
  403. * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
  404. * f2fs_unlock_op().
  405. * Note that, ipage is set only by make_empty_dir.
  406. */
  407. struct page *get_new_data_page(struct inode *inode,
  408. struct page *ipage, pgoff_t index, bool new_i_size)
  409. {
  410. struct address_space *mapping = inode->i_mapping;
  411. struct page *page;
  412. struct dnode_of_data dn;
  413. int err;
  414. set_new_dnode(&dn, inode, ipage, NULL, 0);
  415. err = f2fs_reserve_block(&dn, index);
  416. if (err)
  417. return ERR_PTR(err);
  418. repeat:
  419. page = grab_cache_page(mapping, index);
  420. if (!page) {
  421. err = -ENOMEM;
  422. goto put_err;
  423. }
  424. if (PageUptodate(page))
  425. return page;
  426. if (dn.data_blkaddr == NEW_ADDR) {
  427. zero_user_segment(page, 0, PAGE_CACHE_SIZE);
  428. SetPageUptodate(page);
  429. } else {
  430. err = f2fs_submit_page_bio(F2FS_I_SB(inode), page,
  431. dn.data_blkaddr, READ_SYNC);
  432. if (err)
  433. goto put_err;
  434. lock_page(page);
  435. if (unlikely(!PageUptodate(page))) {
  436. f2fs_put_page(page, 1);
  437. err = -EIO;
  438. goto put_err;
  439. }
  440. if (unlikely(page->mapping != mapping)) {
  441. f2fs_put_page(page, 1);
  442. goto repeat;
  443. }
  444. }
  445. if (new_i_size &&
  446. i_size_read(inode) < ((index + 1) << PAGE_CACHE_SHIFT)) {
  447. i_size_write(inode, ((index + 1) << PAGE_CACHE_SHIFT));
  448. /* Only the directory inode sets new_i_size */
  449. set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
  450. }
  451. return page;
  452. put_err:
  453. f2fs_put_dnode(&dn);
  454. return ERR_PTR(err);
  455. }
  456. static int __allocate_data_block(struct dnode_of_data *dn)
  457. {
  458. struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
  459. struct f2fs_inode_info *fi = F2FS_I(dn->inode);
  460. struct f2fs_summary sum;
  461. block_t new_blkaddr;
  462. struct node_info ni;
  463. pgoff_t fofs;
  464. int type;
  465. if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
  466. return -EPERM;
  467. if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
  468. return -ENOSPC;
  469. __set_data_blkaddr(dn, NEW_ADDR);
  470. dn->data_blkaddr = NEW_ADDR;
  471. get_node_info(sbi, dn->nid, &ni);
  472. set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
  473. type = CURSEG_WARM_DATA;
  474. allocate_data_block(sbi, NULL, NULL_ADDR, &new_blkaddr, &sum, type);
  475. /* direct IO doesn't use extent cache to maximize the performance */
  476. set_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);
  477. update_extent_cache(new_blkaddr, dn);
  478. clear_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);
  479. /* update i_size */
  480. fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
  481. dn->ofs_in_node;
  482. if (i_size_read(dn->inode) < ((fofs + 1) << PAGE_CACHE_SHIFT))
  483. i_size_write(dn->inode, ((fofs + 1) << PAGE_CACHE_SHIFT));
  484. dn->data_blkaddr = new_blkaddr;
  485. return 0;
  486. }
  487. /*
  488. * get_data_block() now supported readahead/bmap/rw direct_IO with mapped bh.
  489. * If original data blocks are allocated, then give them to blockdev.
  490. * Otherwise,
  491. * a. preallocate requested block addresses
  492. * b. do not use extent cache for better performance
  493. * c. give the block addresses to blockdev
  494. */
  495. static int __get_data_block(struct inode *inode, sector_t iblock,
  496. struct buffer_head *bh_result, int create, bool fiemap)
  497. {
  498. unsigned int blkbits = inode->i_sb->s_blocksize_bits;
  499. unsigned maxblocks = bh_result->b_size >> blkbits;
  500. struct dnode_of_data dn;
  501. int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA;
  502. pgoff_t pgofs, end_offset;
  503. int err = 0, ofs = 1;
  504. bool allocated = false;
  505. /* Get the page offset from the block offset(iblock) */
  506. pgofs = (pgoff_t)(iblock >> (PAGE_CACHE_SHIFT - blkbits));
  507. if (check_extent_cache(inode, pgofs, bh_result))
  508. goto out;
  509. if (create) {
  510. f2fs_balance_fs(F2FS_I_SB(inode));
  511. f2fs_lock_op(F2FS_I_SB(inode));
  512. }
  513. /* When reading holes, we need its node page */
  514. set_new_dnode(&dn, inode, NULL, NULL, 0);
  515. err = get_dnode_of_data(&dn, pgofs, mode);
  516. if (err) {
  517. if (err == -ENOENT)
  518. err = 0;
  519. goto unlock_out;
  520. }
  521. if (dn.data_blkaddr == NEW_ADDR && !fiemap)
  522. goto put_out;
  523. if (dn.data_blkaddr != NULL_ADDR) {
  524. map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
  525. } else if (create) {
  526. err = __allocate_data_block(&dn);
  527. if (err)
  528. goto put_out;
  529. allocated = true;
  530. map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
  531. } else {
  532. goto put_out;
  533. }
  534. end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
  535. bh_result->b_size = (((size_t)1) << blkbits);
  536. dn.ofs_in_node++;
  537. pgofs++;
  538. get_next:
  539. if (dn.ofs_in_node >= end_offset) {
  540. if (allocated)
  541. sync_inode_page(&dn);
  542. allocated = false;
  543. f2fs_put_dnode(&dn);
  544. set_new_dnode(&dn, inode, NULL, NULL, 0);
  545. err = get_dnode_of_data(&dn, pgofs, mode);
  546. if (err) {
  547. if (err == -ENOENT)
  548. err = 0;
  549. goto unlock_out;
  550. }
  551. if (dn.data_blkaddr == NEW_ADDR && !fiemap)
  552. goto put_out;
  553. end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
  554. }
  555. if (maxblocks > (bh_result->b_size >> blkbits)) {
  556. block_t blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
  557. if (blkaddr == NULL_ADDR && create) {
  558. err = __allocate_data_block(&dn);
  559. if (err)
  560. goto sync_out;
  561. allocated = true;
  562. blkaddr = dn.data_blkaddr;
  563. }
  564. /* Give more consecutive addresses for the readahead */
  565. if (blkaddr == (bh_result->b_blocknr + ofs)) {
  566. ofs++;
  567. dn.ofs_in_node++;
  568. pgofs++;
  569. bh_result->b_size += (((size_t)1) << blkbits);
  570. goto get_next;
  571. }
  572. }
  573. sync_out:
  574. if (allocated)
  575. sync_inode_page(&dn);
  576. put_out:
  577. f2fs_put_dnode(&dn);
  578. unlock_out:
  579. if (create)
  580. f2fs_unlock_op(F2FS_I_SB(inode));
  581. out:
  582. trace_f2fs_get_data_block(inode, iblock, bh_result, err);
  583. return err;
  584. }
  585. static int get_data_block(struct inode *inode, sector_t iblock,
  586. struct buffer_head *bh_result, int create)
  587. {
  588. return __get_data_block(inode, iblock, bh_result, create, false);
  589. }
  590. static int get_data_block_fiemap(struct inode *inode, sector_t iblock,
  591. struct buffer_head *bh_result, int create)
  592. {
  593. return __get_data_block(inode, iblock, bh_result, create, true);
  594. }
  595. int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
  596. u64 start, u64 len)
  597. {
  598. return generic_block_fiemap(inode, fieinfo,
  599. start, len, get_data_block_fiemap);
  600. }
  601. static int f2fs_read_data_page(struct file *file, struct page *page)
  602. {
  603. struct inode *inode = page->mapping->host;
  604. int ret = -EAGAIN;
  605. trace_f2fs_readpage(page, DATA);
  606. /* If the file has inline data, try to read it directly */
  607. if (f2fs_has_inline_data(inode))
  608. ret = f2fs_read_inline_data(inode, page);
  609. if (ret == -EAGAIN)
  610. ret = mpage_readpage(page, get_data_block);
  611. return ret;
  612. }
  613. static int f2fs_read_data_pages(struct file *file,
  614. struct address_space *mapping,
  615. struct list_head *pages, unsigned nr_pages)
  616. {
  617. struct inode *inode = file->f_mapping->host;
  618. /* If the file has inline data, skip readpages */
  619. if (f2fs_has_inline_data(inode))
  620. return 0;
  621. return mpage_readpages(mapping, pages, nr_pages, get_data_block);
  622. }
  623. int do_write_data_page(struct page *page, struct f2fs_io_info *fio)
  624. {
  625. struct inode *inode = page->mapping->host;
  626. block_t old_blkaddr, new_blkaddr;
  627. struct dnode_of_data dn;
  628. int err = 0;
  629. set_new_dnode(&dn, inode, NULL, NULL, 0);
  630. err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
  631. if (err)
  632. return err;
  633. old_blkaddr = dn.data_blkaddr;
  634. /* This page is already truncated */
  635. if (old_blkaddr == NULL_ADDR)
  636. goto out_writepage;
  637. set_page_writeback(page);
  638. /*
  639. * If current allocation needs SSR,
  640. * it had better in-place writes for updated data.
  641. */
  642. if (unlikely(old_blkaddr != NEW_ADDR &&
  643. !is_cold_data(page) &&
  644. need_inplace_update(inode))) {
  645. rewrite_data_page(page, old_blkaddr, fio);
  646. set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE);
  647. } else {
  648. write_data_page(page, &dn, &new_blkaddr, fio);
  649. update_extent_cache(new_blkaddr, &dn);
  650. set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE);
  651. }
  652. out_writepage:
  653. f2fs_put_dnode(&dn);
  654. return err;
  655. }
  656. static int f2fs_write_data_page(struct page *page,
  657. struct writeback_control *wbc)
  658. {
  659. struct inode *inode = page->mapping->host;
  660. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  661. loff_t i_size = i_size_read(inode);
  662. const pgoff_t end_index = ((unsigned long long) i_size)
  663. >> PAGE_CACHE_SHIFT;
  664. unsigned offset = 0;
  665. bool need_balance_fs = false;
  666. int err = 0;
  667. struct f2fs_io_info fio = {
  668. .type = DATA,
  669. .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE,
  670. };
  671. trace_f2fs_writepage(page, DATA);
  672. if (page->index < end_index)
  673. goto write;
  674. /*
  675. * If the offset is out-of-range of file size,
  676. * this page does not have to be written to disk.
  677. */
  678. offset = i_size & (PAGE_CACHE_SIZE - 1);
  679. if ((page->index >= end_index + 1) || !offset)
  680. goto out;
  681. zero_user_segment(page, offset, PAGE_CACHE_SIZE);
  682. write:
  683. if (unlikely(sbi->por_doing))
  684. goto redirty_out;
  685. /* Dentry blocks are controlled by checkpoint */
  686. if (S_ISDIR(inode->i_mode)) {
  687. if (unlikely(f2fs_cp_error(sbi)))
  688. goto redirty_out;
  689. err = do_write_data_page(page, &fio);
  690. goto done;
  691. }
  692. /* we should bypass data pages to proceed the kworkder jobs */
  693. if (unlikely(f2fs_cp_error(sbi))) {
  694. SetPageError(page);
  695. unlock_page(page);
  696. goto out;
  697. }
  698. if (!wbc->for_reclaim)
  699. need_balance_fs = true;
  700. else if (has_not_enough_free_secs(sbi, 0))
  701. goto redirty_out;
  702. err = -EAGAIN;
  703. f2fs_lock_op(sbi);
  704. if (f2fs_has_inline_data(inode))
  705. err = f2fs_write_inline_data(inode, page);
  706. if (err == -EAGAIN)
  707. err = do_write_data_page(page, &fio);
  708. f2fs_unlock_op(sbi);
  709. done:
  710. if (err && err != -ENOENT)
  711. goto redirty_out;
  712. clear_cold_data(page);
  713. out:
  714. inode_dec_dirty_pages(inode);
  715. unlock_page(page);
  716. if (need_balance_fs)
  717. f2fs_balance_fs(sbi);
  718. if (wbc->for_reclaim)
  719. f2fs_submit_merged_bio(sbi, DATA, WRITE);
  720. return 0;
  721. redirty_out:
  722. redirty_page_for_writepage(wbc, page);
  723. return AOP_WRITEPAGE_ACTIVATE;
  724. }
  725. static int __f2fs_writepage(struct page *page, struct writeback_control *wbc,
  726. void *data)
  727. {
  728. struct address_space *mapping = data;
  729. int ret = mapping->a_ops->writepage(page, wbc);
  730. mapping_set_error(mapping, ret);
  731. return ret;
  732. }
  733. static int f2fs_write_data_pages(struct address_space *mapping,
  734. struct writeback_control *wbc)
  735. {
  736. struct inode *inode = mapping->host;
  737. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  738. bool locked = false;
  739. int ret;
  740. long diff;
  741. trace_f2fs_writepages(mapping->host, wbc, DATA);
  742. /* deal with chardevs and other special file */
  743. if (!mapping->a_ops->writepage)
  744. return 0;
  745. if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
  746. get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
  747. available_free_memory(sbi, DIRTY_DENTS))
  748. goto skip_write;
  749. diff = nr_pages_to_write(sbi, DATA, wbc);
  750. if (!S_ISDIR(inode->i_mode)) {
  751. mutex_lock(&sbi->writepages);
  752. locked = true;
  753. }
  754. ret = write_cache_pages(mapping, wbc, __f2fs_writepage, mapping);
  755. if (locked)
  756. mutex_unlock(&sbi->writepages);
  757. f2fs_submit_merged_bio(sbi, DATA, WRITE);
  758. remove_dirty_dir_inode(inode);
  759. wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
  760. return ret;
  761. skip_write:
  762. wbc->pages_skipped += get_dirty_pages(inode);
  763. return 0;
  764. }
  765. static void f2fs_write_failed(struct address_space *mapping, loff_t to)
  766. {
  767. struct inode *inode = mapping->host;
  768. if (to > inode->i_size) {
  769. truncate_pagecache(inode, inode->i_size);
  770. truncate_blocks(inode, inode->i_size, true);
  771. }
  772. }
  773. static int f2fs_write_begin(struct file *file, struct address_space *mapping,
  774. loff_t pos, unsigned len, unsigned flags,
  775. struct page **pagep, void **fsdata)
  776. {
  777. struct inode *inode = mapping->host;
  778. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  779. struct page *page, *ipage;
  780. pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
  781. struct dnode_of_data dn;
  782. int err = 0;
  783. trace_f2fs_write_begin(inode, pos, len, flags);
  784. f2fs_balance_fs(sbi);
  785. /*
  786. * We should check this at this moment to avoid deadlock on inode page
  787. * and #0 page. The locking rule for inline_data conversion should be:
  788. * lock_page(page #0) -> lock_page(inode_page)
  789. */
  790. if (index != 0) {
  791. err = f2fs_convert_inline_inode(inode);
  792. if (err)
  793. goto fail;
  794. }
  795. repeat:
  796. page = grab_cache_page_write_begin(mapping, index, flags);
  797. if (!page) {
  798. err = -ENOMEM;
  799. goto fail;
  800. }
  801. *pagep = page;
  802. f2fs_lock_op(sbi);
  803. /* check inline_data */
  804. ipage = get_node_page(sbi, inode->i_ino);
  805. if (IS_ERR(ipage)) {
  806. err = PTR_ERR(ipage);
  807. goto unlock_fail;
  808. }
  809. set_new_dnode(&dn, inode, ipage, ipage, 0);
  810. if (f2fs_has_inline_data(inode)) {
  811. if (pos + len <= MAX_INLINE_DATA) {
  812. read_inline_data(page, ipage);
  813. set_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
  814. sync_inode_page(&dn);
  815. goto put_next;
  816. }
  817. err = f2fs_convert_inline_page(&dn, page);
  818. if (err)
  819. goto put_fail;
  820. }
  821. err = f2fs_reserve_block(&dn, index);
  822. if (err)
  823. goto put_fail;
  824. put_next:
  825. f2fs_put_dnode(&dn);
  826. f2fs_unlock_op(sbi);
  827. if ((len == PAGE_CACHE_SIZE) || PageUptodate(page))
  828. return 0;
  829. f2fs_wait_on_page_writeback(page, DATA);
  830. if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
  831. unsigned start = pos & (PAGE_CACHE_SIZE - 1);
  832. unsigned end = start + len;
  833. /* Reading beyond i_size is simple: memset to zero */
  834. zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE);
  835. goto out;
  836. }
  837. if (dn.data_blkaddr == NEW_ADDR) {
  838. zero_user_segment(page, 0, PAGE_CACHE_SIZE);
  839. } else {
  840. err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
  841. READ_SYNC);
  842. if (err)
  843. goto fail;
  844. lock_page(page);
  845. if (unlikely(!PageUptodate(page))) {
  846. f2fs_put_page(page, 1);
  847. err = -EIO;
  848. goto fail;
  849. }
  850. if (unlikely(page->mapping != mapping)) {
  851. f2fs_put_page(page, 1);
  852. goto repeat;
  853. }
  854. }
  855. out:
  856. SetPageUptodate(page);
  857. clear_cold_data(page);
  858. return 0;
  859. put_fail:
  860. f2fs_put_dnode(&dn);
  861. unlock_fail:
  862. f2fs_unlock_op(sbi);
  863. f2fs_put_page(page, 1);
  864. fail:
  865. f2fs_write_failed(mapping, pos + len);
  866. return err;
  867. }
  868. static int f2fs_write_end(struct file *file,
  869. struct address_space *mapping,
  870. loff_t pos, unsigned len, unsigned copied,
  871. struct page *page, void *fsdata)
  872. {
  873. struct inode *inode = page->mapping->host;
  874. trace_f2fs_write_end(inode, pos, len, copied);
  875. set_page_dirty(page);
  876. if (pos + copied > i_size_read(inode)) {
  877. i_size_write(inode, pos + copied);
  878. mark_inode_dirty(inode);
  879. update_inode_page(inode);
  880. }
  881. f2fs_put_page(page, 1);
  882. return copied;
  883. }
  884. static int check_direct_IO(struct inode *inode, int rw,
  885. struct iov_iter *iter, loff_t offset)
  886. {
  887. unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
  888. if (rw == READ)
  889. return 0;
  890. if (offset & blocksize_mask)
  891. return -EINVAL;
  892. if (iov_iter_alignment(iter) & blocksize_mask)
  893. return -EINVAL;
  894. return 0;
  895. }
  896. static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
  897. struct iov_iter *iter, loff_t offset)
  898. {
  899. struct file *file = iocb->ki_filp;
  900. struct address_space *mapping = file->f_mapping;
  901. struct inode *inode = mapping->host;
  902. size_t count = iov_iter_count(iter);
  903. int err;
  904. /* we don't need to use inline_data strictly */
  905. if (f2fs_has_inline_data(inode)) {
  906. err = f2fs_convert_inline_inode(inode);
  907. if (err)
  908. return err;
  909. }
  910. if (check_direct_IO(inode, rw, iter, offset))
  911. return 0;
  912. trace_f2fs_direct_IO_enter(inode, offset, count, rw);
  913. err = blockdev_direct_IO(rw, iocb, inode, iter, offset, get_data_block);
  914. if (err < 0 && (rw & WRITE))
  915. f2fs_write_failed(mapping, offset + count);
  916. trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
  917. return err;
  918. }
  919. static void f2fs_invalidate_data_page(struct page *page, unsigned int offset,
  920. unsigned int length)
  921. {
  922. struct inode *inode = page->mapping->host;
  923. if (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE)
  924. return;
  925. if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode))
  926. invalidate_inmem_page(inode, page);
  927. if (PageDirty(page))
  928. inode_dec_dirty_pages(inode);
  929. ClearPagePrivate(page);
  930. }
  931. static int f2fs_release_data_page(struct page *page, gfp_t wait)
  932. {
  933. ClearPagePrivate(page);
  934. return 1;
  935. }
  936. static int f2fs_set_data_page_dirty(struct page *page)
  937. {
  938. struct address_space *mapping = page->mapping;
  939. struct inode *inode = mapping->host;
  940. trace_f2fs_set_page_dirty(page, DATA);
  941. SetPageUptodate(page);
  942. if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode)) {
  943. register_inmem_page(inode, page);
  944. return 1;
  945. }
  946. mark_inode_dirty(inode);
  947. if (!PageDirty(page)) {
  948. __set_page_dirty_nobuffers(page);
  949. update_dirty_page(inode, page);
  950. return 1;
  951. }
  952. return 0;
  953. }
  954. static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
  955. {
  956. struct inode *inode = mapping->host;
  957. /* we don't need to use inline_data strictly */
  958. if (f2fs_has_inline_data(inode)) {
  959. int err = f2fs_convert_inline_inode(inode);
  960. if (err)
  961. return err;
  962. }
  963. return generic_block_bmap(mapping, block, get_data_block);
  964. }
  965. const struct address_space_operations f2fs_dblock_aops = {
  966. .readpage = f2fs_read_data_page,
  967. .readpages = f2fs_read_data_pages,
  968. .writepage = f2fs_write_data_page,
  969. .writepages = f2fs_write_data_pages,
  970. .write_begin = f2fs_write_begin,
  971. .write_end = f2fs_write_end,
  972. .set_page_dirty = f2fs_set_data_page_dirty,
  973. .invalidatepage = f2fs_invalidate_data_page,
  974. .releasepage = f2fs_release_data_page,
  975. .direct_IO = f2fs_direct_IO,
  976. .bmap = f2fs_bmap,
  977. };