checkpoint.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074
  1. /*
  2. * fs/f2fs/checkpoint.c
  3. *
  4. * Copyright (c) 2012 Samsung Electronics Co., Ltd.
  5. * http://www.samsung.com/
  6. *
  7. * This program is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License version 2 as
  9. * published by the Free Software Foundation.
  10. */
  11. #include <linux/fs.h>
  12. #include <linux/bio.h>
  13. #include <linux/mpage.h>
  14. #include <linux/writeback.h>
  15. #include <linux/blkdev.h>
  16. #include <linux/f2fs_fs.h>
  17. #include <linux/pagevec.h>
  18. #include <linux/swap.h>
  19. #include "f2fs.h"
  20. #include "node.h"
  21. #include "segment.h"
  22. #include <trace/events/f2fs.h>
  23. static struct kmem_cache *ino_entry_slab;
  24. static struct kmem_cache *inode_entry_slab;
  25. /*
  26. * We guarantee no failure on the returned page.
  27. */
  28. struct page *grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
  29. {
  30. struct address_space *mapping = META_MAPPING(sbi);
  31. struct page *page = NULL;
  32. repeat:
  33. page = grab_cache_page(mapping, index);
  34. if (!page) {
  35. cond_resched();
  36. goto repeat;
  37. }
  38. f2fs_wait_on_page_writeback(page, META);
  39. SetPageUptodate(page);
  40. return page;
  41. }
  42. /*
  43. * We guarantee no failure on the returned page.
  44. */
  45. struct page *get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
  46. {
  47. struct address_space *mapping = META_MAPPING(sbi);
  48. struct page *page;
  49. repeat:
  50. page = grab_cache_page(mapping, index);
  51. if (!page) {
  52. cond_resched();
  53. goto repeat;
  54. }
  55. if (PageUptodate(page))
  56. goto out;
  57. if (f2fs_submit_page_bio(sbi, page, index,
  58. READ_SYNC | REQ_META | REQ_PRIO))
  59. goto repeat;
  60. lock_page(page);
  61. if (unlikely(page->mapping != mapping)) {
  62. f2fs_put_page(page, 1);
  63. goto repeat;
  64. }
  65. out:
  66. return page;
  67. }
  68. struct page *get_meta_page_ra(struct f2fs_sb_info *sbi, pgoff_t index)
  69. {
  70. bool readahead = false;
  71. struct page *page;
  72. page = find_get_page(META_MAPPING(sbi), index);
  73. if (!page || (page && !PageUptodate(page)))
  74. readahead = true;
  75. f2fs_put_page(page, 0);
  76. if (readahead)
  77. ra_meta_pages(sbi, index, MAX_BIO_BLOCKS(sbi), META_POR);
  78. return get_meta_page(sbi, index);
  79. }
  80. static inline block_t get_max_meta_blks(struct f2fs_sb_info *sbi, int type)
  81. {
  82. switch (type) {
  83. case META_NAT:
  84. return NM_I(sbi)->max_nid / NAT_ENTRY_PER_BLOCK;
  85. case META_SIT:
  86. return SIT_BLK_CNT(sbi);
  87. case META_SSA:
  88. case META_CP:
  89. return 0;
  90. case META_POR:
  91. return MAX_BLKADDR(sbi);
  92. default:
  93. BUG();
  94. }
  95. }
  96. /*
  97. * Readahead CP/NAT/SIT/SSA pages
  98. */
  99. int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, int type)
  100. {
  101. block_t prev_blk_addr = 0;
  102. struct page *page;
  103. block_t blkno = start;
  104. block_t max_blks = get_max_meta_blks(sbi, type);
  105. struct f2fs_io_info fio = {
  106. .type = META,
  107. .rw = READ_SYNC | REQ_META | REQ_PRIO
  108. };
  109. for (; nrpages-- > 0; blkno++) {
  110. block_t blk_addr;
  111. switch (type) {
  112. case META_NAT:
  113. /* get nat block addr */
  114. if (unlikely(blkno >= max_blks))
  115. blkno = 0;
  116. blk_addr = current_nat_addr(sbi,
  117. blkno * NAT_ENTRY_PER_BLOCK);
  118. break;
  119. case META_SIT:
  120. /* get sit block addr */
  121. if (unlikely(blkno >= max_blks))
  122. goto out;
  123. blk_addr = current_sit_addr(sbi,
  124. blkno * SIT_ENTRY_PER_BLOCK);
  125. if (blkno != start && prev_blk_addr + 1 != blk_addr)
  126. goto out;
  127. prev_blk_addr = blk_addr;
  128. break;
  129. case META_SSA:
  130. case META_CP:
  131. case META_POR:
  132. if (unlikely(blkno >= max_blks))
  133. goto out;
  134. if (unlikely(blkno < SEG0_BLKADDR(sbi)))
  135. goto out;
  136. blk_addr = blkno;
  137. break;
  138. default:
  139. BUG();
  140. }
  141. page = grab_cache_page(META_MAPPING(sbi), blk_addr);
  142. if (!page)
  143. continue;
  144. if (PageUptodate(page)) {
  145. f2fs_put_page(page, 1);
  146. continue;
  147. }
  148. f2fs_submit_page_mbio(sbi, page, blk_addr, &fio);
  149. f2fs_put_page(page, 0);
  150. }
  151. out:
  152. f2fs_submit_merged_bio(sbi, META, READ);
  153. return blkno - start;
  154. }
  155. static int f2fs_write_meta_page(struct page *page,
  156. struct writeback_control *wbc)
  157. {
  158. struct f2fs_sb_info *sbi = F2FS_P_SB(page);
  159. trace_f2fs_writepage(page, META);
  160. if (unlikely(sbi->por_doing))
  161. goto redirty_out;
  162. if (wbc->for_reclaim)
  163. goto redirty_out;
  164. if (unlikely(f2fs_cp_error(sbi)))
  165. goto redirty_out;
  166. f2fs_wait_on_page_writeback(page, META);
  167. write_meta_page(sbi, page);
  168. dec_page_count(sbi, F2FS_DIRTY_META);
  169. unlock_page(page);
  170. return 0;
  171. redirty_out:
  172. redirty_page_for_writepage(wbc, page);
  173. return AOP_WRITEPAGE_ACTIVATE;
  174. }
  175. static int f2fs_write_meta_pages(struct address_space *mapping,
  176. struct writeback_control *wbc)
  177. {
  178. struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
  179. long diff, written;
  180. trace_f2fs_writepages(mapping->host, wbc, META);
  181. /* collect a number of dirty meta pages and write together */
  182. if (wbc->for_kupdate ||
  183. get_pages(sbi, F2FS_DIRTY_META) < nr_pages_to_skip(sbi, META))
  184. goto skip_write;
  185. /* if mounting is failed, skip writing node pages */
  186. mutex_lock(&sbi->cp_mutex);
  187. diff = nr_pages_to_write(sbi, META, wbc);
  188. written = sync_meta_pages(sbi, META, wbc->nr_to_write);
  189. mutex_unlock(&sbi->cp_mutex);
  190. wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff);
  191. return 0;
  192. skip_write:
  193. wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META);
  194. return 0;
  195. }
  196. long sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
  197. long nr_to_write)
  198. {
  199. struct address_space *mapping = META_MAPPING(sbi);
  200. pgoff_t index = 0, end = LONG_MAX;
  201. struct pagevec pvec;
  202. long nwritten = 0;
  203. struct writeback_control wbc = {
  204. .for_reclaim = 0,
  205. };
  206. pagevec_init(&pvec, 0);
  207. while (index <= end) {
  208. int i, nr_pages;
  209. nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
  210. PAGECACHE_TAG_DIRTY,
  211. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
  212. if (unlikely(nr_pages == 0))
  213. break;
  214. for (i = 0; i < nr_pages; i++) {
  215. struct page *page = pvec.pages[i];
  216. lock_page(page);
  217. if (unlikely(page->mapping != mapping)) {
  218. continue_unlock:
  219. unlock_page(page);
  220. continue;
  221. }
  222. if (!PageDirty(page)) {
  223. /* someone wrote it for us */
  224. goto continue_unlock;
  225. }
  226. if (!clear_page_dirty_for_io(page))
  227. goto continue_unlock;
  228. if (f2fs_write_meta_page(page, &wbc)) {
  229. unlock_page(page);
  230. break;
  231. }
  232. nwritten++;
  233. if (unlikely(nwritten >= nr_to_write))
  234. break;
  235. }
  236. pagevec_release(&pvec);
  237. cond_resched();
  238. }
  239. if (nwritten)
  240. f2fs_submit_merged_bio(sbi, type, WRITE);
  241. return nwritten;
  242. }
  243. static int f2fs_set_meta_page_dirty(struct page *page)
  244. {
  245. trace_f2fs_set_page_dirty(page, META);
  246. SetPageUptodate(page);
  247. if (!PageDirty(page)) {
  248. __set_page_dirty_nobuffers(page);
  249. inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_META);
  250. return 1;
  251. }
  252. return 0;
  253. }
  254. const struct address_space_operations f2fs_meta_aops = {
  255. .writepage = f2fs_write_meta_page,
  256. .writepages = f2fs_write_meta_pages,
  257. .set_page_dirty = f2fs_set_meta_page_dirty,
  258. };
  259. static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
  260. {
  261. struct ino_entry *e;
  262. retry:
  263. spin_lock(&sbi->ino_lock[type]);
  264. e = radix_tree_lookup(&sbi->ino_root[type], ino);
  265. if (!e) {
  266. e = kmem_cache_alloc(ino_entry_slab, GFP_ATOMIC);
  267. if (!e) {
  268. spin_unlock(&sbi->ino_lock[type]);
  269. goto retry;
  270. }
  271. if (radix_tree_insert(&sbi->ino_root[type], ino, e)) {
  272. spin_unlock(&sbi->ino_lock[type]);
  273. kmem_cache_free(ino_entry_slab, e);
  274. goto retry;
  275. }
  276. memset(e, 0, sizeof(struct ino_entry));
  277. e->ino = ino;
  278. list_add_tail(&e->list, &sbi->ino_list[type]);
  279. }
  280. spin_unlock(&sbi->ino_lock[type]);
  281. }
  282. static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
  283. {
  284. struct ino_entry *e;
  285. spin_lock(&sbi->ino_lock[type]);
  286. e = radix_tree_lookup(&sbi->ino_root[type], ino);
  287. if (e) {
  288. list_del(&e->list);
  289. radix_tree_delete(&sbi->ino_root[type], ino);
  290. if (type == ORPHAN_INO)
  291. sbi->n_orphans--;
  292. spin_unlock(&sbi->ino_lock[type]);
  293. kmem_cache_free(ino_entry_slab, e);
  294. return;
  295. }
  296. spin_unlock(&sbi->ino_lock[type]);
  297. }
  298. void add_dirty_inode(struct f2fs_sb_info *sbi, nid_t ino, int type)
  299. {
  300. /* add new dirty ino entry into list */
  301. __add_ino_entry(sbi, ino, type);
  302. }
  303. void remove_dirty_inode(struct f2fs_sb_info *sbi, nid_t ino, int type)
  304. {
  305. /* remove dirty ino entry from list */
  306. __remove_ino_entry(sbi, ino, type);
  307. }
  308. /* mode should be APPEND_INO or UPDATE_INO */
  309. bool exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode)
  310. {
  311. struct ino_entry *e;
  312. spin_lock(&sbi->ino_lock[mode]);
  313. e = radix_tree_lookup(&sbi->ino_root[mode], ino);
  314. spin_unlock(&sbi->ino_lock[mode]);
  315. return e ? true : false;
  316. }
  317. void release_dirty_inode(struct f2fs_sb_info *sbi)
  318. {
  319. struct ino_entry *e, *tmp;
  320. int i;
  321. for (i = APPEND_INO; i <= UPDATE_INO; i++) {
  322. spin_lock(&sbi->ino_lock[i]);
  323. list_for_each_entry_safe(e, tmp, &sbi->ino_list[i], list) {
  324. list_del(&e->list);
  325. radix_tree_delete(&sbi->ino_root[i], e->ino);
  326. kmem_cache_free(ino_entry_slab, e);
  327. }
  328. spin_unlock(&sbi->ino_lock[i]);
  329. }
  330. }
  331. int acquire_orphan_inode(struct f2fs_sb_info *sbi)
  332. {
  333. int err = 0;
  334. spin_lock(&sbi->ino_lock[ORPHAN_INO]);
  335. if (unlikely(sbi->n_orphans >= sbi->max_orphans))
  336. err = -ENOSPC;
  337. else
  338. sbi->n_orphans++;
  339. spin_unlock(&sbi->ino_lock[ORPHAN_INO]);
  340. return err;
  341. }
  342. void release_orphan_inode(struct f2fs_sb_info *sbi)
  343. {
  344. spin_lock(&sbi->ino_lock[ORPHAN_INO]);
  345. f2fs_bug_on(sbi, sbi->n_orphans == 0);
  346. sbi->n_orphans--;
  347. spin_unlock(&sbi->ino_lock[ORPHAN_INO]);
  348. }
  349. void add_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
  350. {
  351. /* add new orphan ino entry into list */
  352. __add_ino_entry(sbi, ino, ORPHAN_INO);
  353. }
  354. void remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
  355. {
  356. /* remove orphan entry from orphan list */
  357. __remove_ino_entry(sbi, ino, ORPHAN_INO);
  358. }
  359. static void recover_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
  360. {
  361. struct inode *inode = f2fs_iget(sbi->sb, ino);
  362. f2fs_bug_on(sbi, IS_ERR(inode));
  363. clear_nlink(inode);
  364. /* truncate all the data during iput */
  365. iput(inode);
  366. }
  367. void recover_orphan_inodes(struct f2fs_sb_info *sbi)
  368. {
  369. block_t start_blk, orphan_blkaddr, i, j;
  370. if (!is_set_ckpt_flags(F2FS_CKPT(sbi), CP_ORPHAN_PRESENT_FLAG))
  371. return;
  372. sbi->por_doing = true;
  373. start_blk = __start_cp_addr(sbi) + 1 +
  374. le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload);
  375. orphan_blkaddr = __start_sum_addr(sbi) - 1;
  376. ra_meta_pages(sbi, start_blk, orphan_blkaddr, META_CP);
  377. for (i = 0; i < orphan_blkaddr; i++) {
  378. struct page *page = get_meta_page(sbi, start_blk + i);
  379. struct f2fs_orphan_block *orphan_blk;
  380. orphan_blk = (struct f2fs_orphan_block *)page_address(page);
  381. for (j = 0; j < le32_to_cpu(orphan_blk->entry_count); j++) {
  382. nid_t ino = le32_to_cpu(orphan_blk->ino[j]);
  383. recover_orphan_inode(sbi, ino);
  384. }
  385. f2fs_put_page(page, 1);
  386. }
  387. /* clear Orphan Flag */
  388. clear_ckpt_flags(F2FS_CKPT(sbi), CP_ORPHAN_PRESENT_FLAG);
  389. sbi->por_doing = false;
  390. return;
  391. }
  392. static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk)
  393. {
  394. struct list_head *head;
  395. struct f2fs_orphan_block *orphan_blk = NULL;
  396. unsigned int nentries = 0;
  397. unsigned short index;
  398. unsigned short orphan_blocks =
  399. (unsigned short)GET_ORPHAN_BLOCKS(sbi->n_orphans);
  400. struct page *page = NULL;
  401. struct ino_entry *orphan = NULL;
  402. for (index = 0; index < orphan_blocks; index++)
  403. grab_meta_page(sbi, start_blk + index);
  404. index = 1;
  405. spin_lock(&sbi->ino_lock[ORPHAN_INO]);
  406. head = &sbi->ino_list[ORPHAN_INO];
  407. /* loop for each orphan inode entry and write them in Jornal block */
  408. list_for_each_entry(orphan, head, list) {
  409. if (!page) {
  410. page = find_get_page(META_MAPPING(sbi), start_blk++);
  411. f2fs_bug_on(sbi, !page);
  412. orphan_blk =
  413. (struct f2fs_orphan_block *)page_address(page);
  414. memset(orphan_blk, 0, sizeof(*orphan_blk));
  415. f2fs_put_page(page, 0);
  416. }
  417. orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino);
  418. if (nentries == F2FS_ORPHANS_PER_BLOCK) {
  419. /*
  420. * an orphan block is full of 1020 entries,
  421. * then we need to flush current orphan blocks
  422. * and bring another one in memory
  423. */
  424. orphan_blk->blk_addr = cpu_to_le16(index);
  425. orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
  426. orphan_blk->entry_count = cpu_to_le32(nentries);
  427. set_page_dirty(page);
  428. f2fs_put_page(page, 1);
  429. index++;
  430. nentries = 0;
  431. page = NULL;
  432. }
  433. }
  434. if (page) {
  435. orphan_blk->blk_addr = cpu_to_le16(index);
  436. orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
  437. orphan_blk->entry_count = cpu_to_le32(nentries);
  438. set_page_dirty(page);
  439. f2fs_put_page(page, 1);
  440. }
  441. spin_unlock(&sbi->ino_lock[ORPHAN_INO]);
  442. }
  443. static struct page *validate_checkpoint(struct f2fs_sb_info *sbi,
  444. block_t cp_addr, unsigned long long *version)
  445. {
  446. struct page *cp_page_1, *cp_page_2 = NULL;
  447. unsigned long blk_size = sbi->blocksize;
  448. struct f2fs_checkpoint *cp_block;
  449. unsigned long long cur_version = 0, pre_version = 0;
  450. size_t crc_offset;
  451. __u32 crc = 0;
  452. /* Read the 1st cp block in this CP pack */
  453. cp_page_1 = get_meta_page(sbi, cp_addr);
  454. /* get the version number */
  455. cp_block = (struct f2fs_checkpoint *)page_address(cp_page_1);
  456. crc_offset = le32_to_cpu(cp_block->checksum_offset);
  457. if (crc_offset >= blk_size)
  458. goto invalid_cp1;
  459. crc = le32_to_cpu(*((__u32 *)((unsigned char *)cp_block + crc_offset)));
  460. if (!f2fs_crc_valid(crc, cp_block, crc_offset))
  461. goto invalid_cp1;
  462. pre_version = cur_cp_version(cp_block);
  463. /* Read the 2nd cp block in this CP pack */
  464. cp_addr += le32_to_cpu(cp_block->cp_pack_total_block_count) - 1;
  465. cp_page_2 = get_meta_page(sbi, cp_addr);
  466. cp_block = (struct f2fs_checkpoint *)page_address(cp_page_2);
  467. crc_offset = le32_to_cpu(cp_block->checksum_offset);
  468. if (crc_offset >= blk_size)
  469. goto invalid_cp2;
  470. crc = le32_to_cpu(*((__u32 *)((unsigned char *)cp_block + crc_offset)));
  471. if (!f2fs_crc_valid(crc, cp_block, crc_offset))
  472. goto invalid_cp2;
  473. cur_version = cur_cp_version(cp_block);
  474. if (cur_version == pre_version) {
  475. *version = cur_version;
  476. f2fs_put_page(cp_page_2, 1);
  477. return cp_page_1;
  478. }
  479. invalid_cp2:
  480. f2fs_put_page(cp_page_2, 1);
  481. invalid_cp1:
  482. f2fs_put_page(cp_page_1, 1);
  483. return NULL;
  484. }
  485. int get_valid_checkpoint(struct f2fs_sb_info *sbi)
  486. {
  487. struct f2fs_checkpoint *cp_block;
  488. struct f2fs_super_block *fsb = sbi->raw_super;
  489. struct page *cp1, *cp2, *cur_page;
  490. unsigned long blk_size = sbi->blocksize;
  491. unsigned long long cp1_version = 0, cp2_version = 0;
  492. unsigned long long cp_start_blk_no;
  493. unsigned int cp_blks = 1 + le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload);
  494. block_t cp_blk_no;
  495. int i;
  496. sbi->ckpt = kzalloc(cp_blks * blk_size, GFP_KERNEL);
  497. if (!sbi->ckpt)
  498. return -ENOMEM;
  499. /*
  500. * Finding out valid cp block involves read both
  501. * sets( cp pack1 and cp pack 2)
  502. */
  503. cp_start_blk_no = le32_to_cpu(fsb->cp_blkaddr);
  504. cp1 = validate_checkpoint(sbi, cp_start_blk_no, &cp1_version);
  505. /* The second checkpoint pack should start at the next segment */
  506. cp_start_blk_no += ((unsigned long long)1) <<
  507. le32_to_cpu(fsb->log_blocks_per_seg);
  508. cp2 = validate_checkpoint(sbi, cp_start_blk_no, &cp2_version);
  509. if (cp1 && cp2) {
  510. if (ver_after(cp2_version, cp1_version))
  511. cur_page = cp2;
  512. else
  513. cur_page = cp1;
  514. } else if (cp1) {
  515. cur_page = cp1;
  516. } else if (cp2) {
  517. cur_page = cp2;
  518. } else {
  519. goto fail_no_cp;
  520. }
  521. cp_block = (struct f2fs_checkpoint *)page_address(cur_page);
  522. memcpy(sbi->ckpt, cp_block, blk_size);
  523. if (cp_blks <= 1)
  524. goto done;
  525. cp_blk_no = le32_to_cpu(fsb->cp_blkaddr);
  526. if (cur_page == cp2)
  527. cp_blk_no += 1 << le32_to_cpu(fsb->log_blocks_per_seg);
  528. for (i = 1; i < cp_blks; i++) {
  529. void *sit_bitmap_ptr;
  530. unsigned char *ckpt = (unsigned char *)sbi->ckpt;
  531. cur_page = get_meta_page(sbi, cp_blk_no + i);
  532. sit_bitmap_ptr = page_address(cur_page);
  533. memcpy(ckpt + i * blk_size, sit_bitmap_ptr, blk_size);
  534. f2fs_put_page(cur_page, 1);
  535. }
  536. done:
  537. f2fs_put_page(cp1, 1);
  538. f2fs_put_page(cp2, 1);
  539. return 0;
  540. fail_no_cp:
  541. kfree(sbi->ckpt);
  542. return -EINVAL;
  543. }
  544. static int __add_dirty_inode(struct inode *inode, struct dir_inode_entry *new)
  545. {
  546. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  547. if (is_inode_flag_set(F2FS_I(inode), FI_DIRTY_DIR))
  548. return -EEXIST;
  549. set_inode_flag(F2FS_I(inode), FI_DIRTY_DIR);
  550. F2FS_I(inode)->dirty_dir = new;
  551. list_add_tail(&new->list, &sbi->dir_inode_list);
  552. stat_inc_dirty_dir(sbi);
  553. return 0;
  554. }
  555. void update_dirty_page(struct inode *inode, struct page *page)
  556. {
  557. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  558. struct dir_inode_entry *new;
  559. int ret = 0;
  560. if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode))
  561. return;
  562. if (!S_ISDIR(inode->i_mode)) {
  563. inode_inc_dirty_pages(inode);
  564. goto out;
  565. }
  566. new = f2fs_kmem_cache_alloc(inode_entry_slab, GFP_NOFS);
  567. new->inode = inode;
  568. INIT_LIST_HEAD(&new->list);
  569. spin_lock(&sbi->dir_inode_lock);
  570. ret = __add_dirty_inode(inode, new);
  571. inode_inc_dirty_pages(inode);
  572. spin_unlock(&sbi->dir_inode_lock);
  573. if (ret)
  574. kmem_cache_free(inode_entry_slab, new);
  575. out:
  576. SetPagePrivate(page);
  577. }
  578. void add_dirty_dir_inode(struct inode *inode)
  579. {
  580. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  581. struct dir_inode_entry *new =
  582. f2fs_kmem_cache_alloc(inode_entry_slab, GFP_NOFS);
  583. int ret = 0;
  584. new->inode = inode;
  585. INIT_LIST_HEAD(&new->list);
  586. spin_lock(&sbi->dir_inode_lock);
  587. ret = __add_dirty_inode(inode, new);
  588. spin_unlock(&sbi->dir_inode_lock);
  589. if (ret)
  590. kmem_cache_free(inode_entry_slab, new);
  591. }
  592. void remove_dirty_dir_inode(struct inode *inode)
  593. {
  594. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  595. struct dir_inode_entry *entry;
  596. if (!S_ISDIR(inode->i_mode))
  597. return;
  598. spin_lock(&sbi->dir_inode_lock);
  599. if (get_dirty_pages(inode) ||
  600. !is_inode_flag_set(F2FS_I(inode), FI_DIRTY_DIR)) {
  601. spin_unlock(&sbi->dir_inode_lock);
  602. return;
  603. }
  604. entry = F2FS_I(inode)->dirty_dir;
  605. list_del(&entry->list);
  606. F2FS_I(inode)->dirty_dir = NULL;
  607. clear_inode_flag(F2FS_I(inode), FI_DIRTY_DIR);
  608. stat_dec_dirty_dir(sbi);
  609. spin_unlock(&sbi->dir_inode_lock);
  610. kmem_cache_free(inode_entry_slab, entry);
  611. /* Only from the recovery routine */
  612. if (is_inode_flag_set(F2FS_I(inode), FI_DELAY_IPUT)) {
  613. clear_inode_flag(F2FS_I(inode), FI_DELAY_IPUT);
  614. iput(inode);
  615. }
  616. }
  617. void sync_dirty_dir_inodes(struct f2fs_sb_info *sbi)
  618. {
  619. struct list_head *head;
  620. struct dir_inode_entry *entry;
  621. struct inode *inode;
  622. retry:
  623. spin_lock(&sbi->dir_inode_lock);
  624. head = &sbi->dir_inode_list;
  625. if (list_empty(head)) {
  626. spin_unlock(&sbi->dir_inode_lock);
  627. return;
  628. }
  629. entry = list_entry(head->next, struct dir_inode_entry, list);
  630. inode = igrab(entry->inode);
  631. spin_unlock(&sbi->dir_inode_lock);
  632. if (inode) {
  633. filemap_fdatawrite(inode->i_mapping);
  634. iput(inode);
  635. } else {
  636. /*
  637. * We should submit bio, since it exists several
  638. * wribacking dentry pages in the freeing inode.
  639. */
  640. f2fs_submit_merged_bio(sbi, DATA, WRITE);
  641. }
  642. goto retry;
  643. }
  644. /*
  645. * Freeze all the FS-operations for checkpoint.
  646. */
  647. static int block_operations(struct f2fs_sb_info *sbi)
  648. {
  649. struct writeback_control wbc = {
  650. .sync_mode = WB_SYNC_ALL,
  651. .nr_to_write = LONG_MAX,
  652. .for_reclaim = 0,
  653. };
  654. struct blk_plug plug;
  655. int err = 0;
  656. blk_start_plug(&plug);
  657. retry_flush_dents:
  658. f2fs_lock_all(sbi);
  659. /* write all the dirty dentry pages */
  660. if (get_pages(sbi, F2FS_DIRTY_DENTS)) {
  661. f2fs_unlock_all(sbi);
  662. sync_dirty_dir_inodes(sbi);
  663. if (unlikely(f2fs_cp_error(sbi))) {
  664. err = -EIO;
  665. goto out;
  666. }
  667. goto retry_flush_dents;
  668. }
  669. /*
  670. * POR: we should ensure that there are no dirty node pages
  671. * until finishing nat/sit flush.
  672. */
  673. retry_flush_nodes:
  674. down_write(&sbi->node_write);
  675. if (get_pages(sbi, F2FS_DIRTY_NODES)) {
  676. up_write(&sbi->node_write);
  677. sync_node_pages(sbi, 0, &wbc);
  678. if (unlikely(f2fs_cp_error(sbi))) {
  679. f2fs_unlock_all(sbi);
  680. err = -EIO;
  681. goto out;
  682. }
  683. goto retry_flush_nodes;
  684. }
  685. out:
  686. blk_finish_plug(&plug);
  687. return err;
  688. }
  689. static void unblock_operations(struct f2fs_sb_info *sbi)
  690. {
  691. up_write(&sbi->node_write);
  692. f2fs_unlock_all(sbi);
  693. }
  694. static void wait_on_all_pages_writeback(struct f2fs_sb_info *sbi)
  695. {
  696. DEFINE_WAIT(wait);
  697. for (;;) {
  698. prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE);
  699. if (!get_pages(sbi, F2FS_WRITEBACK))
  700. break;
  701. io_schedule();
  702. }
  703. finish_wait(&sbi->cp_wait, &wait);
  704. }
  705. static void do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
  706. {
  707. struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
  708. struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_WARM_NODE);
  709. struct f2fs_nm_info *nm_i = NM_I(sbi);
  710. nid_t last_nid = nm_i->next_scan_nid;
  711. block_t start_blk;
  712. struct page *cp_page;
  713. unsigned int data_sum_blocks, orphan_blocks;
  714. __u32 crc32 = 0;
  715. void *kaddr;
  716. int i;
  717. int cp_payload_blks = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload);
  718. /*
  719. * This avoids to conduct wrong roll-forward operations and uses
  720. * metapages, so should be called prior to sync_meta_pages below.
  721. */
  722. discard_next_dnode(sbi, NEXT_FREE_BLKADDR(sbi, curseg));
  723. /* Flush all the NAT/SIT pages */
  724. while (get_pages(sbi, F2FS_DIRTY_META)) {
  725. sync_meta_pages(sbi, META, LONG_MAX);
  726. if (unlikely(f2fs_cp_error(sbi)))
  727. return;
  728. }
  729. next_free_nid(sbi, &last_nid);
  730. /*
  731. * modify checkpoint
  732. * version number is already updated
  733. */
  734. ckpt->elapsed_time = cpu_to_le64(get_mtime(sbi));
  735. ckpt->valid_block_count = cpu_to_le64(valid_user_blocks(sbi));
  736. ckpt->free_segment_count = cpu_to_le32(free_segments(sbi));
  737. for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) {
  738. ckpt->cur_node_segno[i] =
  739. cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_NODE));
  740. ckpt->cur_node_blkoff[i] =
  741. cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_NODE));
  742. ckpt->alloc_type[i + CURSEG_HOT_NODE] =
  743. curseg_alloc_type(sbi, i + CURSEG_HOT_NODE);
  744. }
  745. for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) {
  746. ckpt->cur_data_segno[i] =
  747. cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_DATA));
  748. ckpt->cur_data_blkoff[i] =
  749. cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_DATA));
  750. ckpt->alloc_type[i + CURSEG_HOT_DATA] =
  751. curseg_alloc_type(sbi, i + CURSEG_HOT_DATA);
  752. }
  753. ckpt->valid_node_count = cpu_to_le32(valid_node_count(sbi));
  754. ckpt->valid_inode_count = cpu_to_le32(valid_inode_count(sbi));
  755. ckpt->next_free_nid = cpu_to_le32(last_nid);
  756. /* 2 cp + n data seg summary + orphan inode blocks */
  757. data_sum_blocks = npages_for_summary_flush(sbi);
  758. if (data_sum_blocks < NR_CURSEG_DATA_TYPE)
  759. set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
  760. else
  761. clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
  762. orphan_blocks = GET_ORPHAN_BLOCKS(sbi->n_orphans);
  763. ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks +
  764. orphan_blocks);
  765. if (cpc->reason == CP_UMOUNT) {
  766. set_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
  767. ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS+
  768. cp_payload_blks + data_sum_blocks +
  769. orphan_blocks + NR_CURSEG_NODE_TYPE);
  770. } else {
  771. clear_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
  772. ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS +
  773. cp_payload_blks + data_sum_blocks +
  774. orphan_blocks);
  775. }
  776. if (sbi->n_orphans)
  777. set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
  778. else
  779. clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
  780. if (sbi->need_fsck)
  781. set_ckpt_flags(ckpt, CP_FSCK_FLAG);
  782. /* update SIT/NAT bitmap */
  783. get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP));
  784. get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP));
  785. crc32 = f2fs_crc32(ckpt, le32_to_cpu(ckpt->checksum_offset));
  786. *((__le32 *)((unsigned char *)ckpt +
  787. le32_to_cpu(ckpt->checksum_offset)))
  788. = cpu_to_le32(crc32);
  789. start_blk = __start_cp_addr(sbi);
  790. /* write out checkpoint buffer at block 0 */
  791. cp_page = grab_meta_page(sbi, start_blk++);
  792. kaddr = page_address(cp_page);
  793. memcpy(kaddr, ckpt, (1 << sbi->log_blocksize));
  794. set_page_dirty(cp_page);
  795. f2fs_put_page(cp_page, 1);
  796. for (i = 1; i < 1 + cp_payload_blks; i++) {
  797. cp_page = grab_meta_page(sbi, start_blk++);
  798. kaddr = page_address(cp_page);
  799. memcpy(kaddr, (char *)ckpt + i * F2FS_BLKSIZE,
  800. (1 << sbi->log_blocksize));
  801. set_page_dirty(cp_page);
  802. f2fs_put_page(cp_page, 1);
  803. }
  804. if (sbi->n_orphans) {
  805. write_orphan_inodes(sbi, start_blk);
  806. start_blk += orphan_blocks;
  807. }
  808. write_data_summaries(sbi, start_blk);
  809. start_blk += data_sum_blocks;
  810. if (cpc->reason == CP_UMOUNT) {
  811. write_node_summaries(sbi, start_blk);
  812. start_blk += NR_CURSEG_NODE_TYPE;
  813. }
  814. /* writeout checkpoint block */
  815. cp_page = grab_meta_page(sbi, start_blk);
  816. kaddr = page_address(cp_page);
  817. memcpy(kaddr, ckpt, (1 << sbi->log_blocksize));
  818. set_page_dirty(cp_page);
  819. f2fs_put_page(cp_page, 1);
  820. /* wait for previous submitted node/meta pages writeback */
  821. wait_on_all_pages_writeback(sbi);
  822. if (unlikely(f2fs_cp_error(sbi)))
  823. return;
  824. filemap_fdatawait_range(NODE_MAPPING(sbi), 0, LONG_MAX);
  825. filemap_fdatawait_range(META_MAPPING(sbi), 0, LONG_MAX);
  826. /* update user_block_counts */
  827. sbi->last_valid_block_count = sbi->total_valid_block_count;
  828. sbi->alloc_valid_block_count = 0;
  829. /* Here, we only have one bio having CP pack */
  830. sync_meta_pages(sbi, META_FLUSH, LONG_MAX);
  831. release_dirty_inode(sbi);
  832. if (unlikely(f2fs_cp_error(sbi)))
  833. return;
  834. clear_prefree_segments(sbi);
  835. F2FS_RESET_SB_DIRT(sbi);
  836. }
  837. /*
  838. * We guarantee that this checkpoint procedure will not fail.
  839. */
  840. void write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
  841. {
  842. struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
  843. unsigned long long ckpt_ver;
  844. trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "start block_ops");
  845. mutex_lock(&sbi->cp_mutex);
  846. if (!sbi->s_dirty && cpc->reason != CP_DISCARD)
  847. goto out;
  848. if (unlikely(f2fs_cp_error(sbi)))
  849. goto out;
  850. if (block_operations(sbi))
  851. goto out;
  852. trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish block_ops");
  853. f2fs_submit_merged_bio(sbi, DATA, WRITE);
  854. f2fs_submit_merged_bio(sbi, NODE, WRITE);
  855. f2fs_submit_merged_bio(sbi, META, WRITE);
  856. /*
  857. * update checkpoint pack index
  858. * Increase the version number so that
  859. * SIT entries and seg summaries are written at correct place
  860. */
  861. ckpt_ver = cur_cp_version(ckpt);
  862. ckpt->checkpoint_ver = cpu_to_le64(++ckpt_ver);
  863. /* write cached NAT/SIT entries to NAT/SIT area */
  864. flush_nat_entries(sbi);
  865. flush_sit_entries(sbi, cpc);
  866. /* unlock all the fs_lock[] in do_checkpoint() */
  867. do_checkpoint(sbi, cpc);
  868. unblock_operations(sbi);
  869. stat_inc_cp_count(sbi->stat_info);
  870. out:
  871. mutex_unlock(&sbi->cp_mutex);
  872. trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish checkpoint");
  873. }
  874. void init_ino_entry_info(struct f2fs_sb_info *sbi)
  875. {
  876. int i;
  877. for (i = 0; i < MAX_INO_ENTRY; i++) {
  878. INIT_RADIX_TREE(&sbi->ino_root[i], GFP_ATOMIC);
  879. spin_lock_init(&sbi->ino_lock[i]);
  880. INIT_LIST_HEAD(&sbi->ino_list[i]);
  881. }
  882. /*
  883. * considering 512 blocks in a segment 8 blocks are needed for cp
  884. * and log segment summaries. Remaining blocks are used to keep
  885. * orphan entries with the limitation one reserved segment
  886. * for cp pack we can have max 1020*504 orphan entries
  887. */
  888. sbi->n_orphans = 0;
  889. sbi->max_orphans = (sbi->blocks_per_seg - F2FS_CP_PACKS -
  890. NR_CURSEG_TYPE) * F2FS_ORPHANS_PER_BLOCK;
  891. }
  892. int __init create_checkpoint_caches(void)
  893. {
  894. ino_entry_slab = f2fs_kmem_cache_create("f2fs_ino_entry",
  895. sizeof(struct ino_entry));
  896. if (!ino_entry_slab)
  897. return -ENOMEM;
  898. inode_entry_slab = f2fs_kmem_cache_create("f2fs_dirty_dir_entry",
  899. sizeof(struct dir_inode_entry));
  900. if (!inode_entry_slab) {
  901. kmem_cache_destroy(ino_entry_slab);
  902. return -ENOMEM;
  903. }
  904. return 0;
  905. }
  906. void destroy_checkpoint_caches(void)
  907. {
  908. kmem_cache_destroy(ino_entry_slab);
  909. kmem_cache_destroy(inode_entry_slab);
  910. }