xfs_icache.c 44 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739
  1. /*
  2. * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  3. * All Rights Reserved.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License as
  7. * published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope that it would be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. * GNU General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write the Free Software Foundation,
  16. * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  17. */
  18. #include "xfs.h"
  19. #include "xfs_fs.h"
  20. #include "xfs_format.h"
  21. #include "xfs_log_format.h"
  22. #include "xfs_trans_resv.h"
  23. #include "xfs_sb.h"
  24. #include "xfs_mount.h"
  25. #include "xfs_inode.h"
  26. #include "xfs_error.h"
  27. #include "xfs_trans.h"
  28. #include "xfs_trans_priv.h"
  29. #include "xfs_inode_item.h"
  30. #include "xfs_quota.h"
  31. #include "xfs_trace.h"
  32. #include "xfs_icache.h"
  33. #include "xfs_bmap_util.h"
  34. #include "xfs_dquot_item.h"
  35. #include "xfs_dquot.h"
  36. #include "xfs_reflink.h"
  37. #include <linux/kthread.h>
  38. #include <linux/freezer.h>
  39. /*
  40. * Allocate and initialise an xfs_inode.
  41. */
  42. struct xfs_inode *
  43. xfs_inode_alloc(
  44. struct xfs_mount *mp,
  45. xfs_ino_t ino)
  46. {
  47. struct xfs_inode *ip;
  48. /*
  49. * if this didn't occur in transactions, we could use
  50. * KM_MAYFAIL and return NULL here on ENOMEM. Set the
  51. * code up to do this anyway.
  52. */
  53. ip = kmem_zone_alloc(xfs_inode_zone, KM_SLEEP);
  54. if (!ip)
  55. return NULL;
  56. if (inode_init_always(mp->m_super, VFS_I(ip))) {
  57. kmem_zone_free(xfs_inode_zone, ip);
  58. return NULL;
  59. }
  60. /* VFS doesn't initialise i_mode! */
  61. VFS_I(ip)->i_mode = 0;
  62. XFS_STATS_INC(mp, vn_active);
  63. ASSERT(atomic_read(&ip->i_pincount) == 0);
  64. ASSERT(!xfs_isiflocked(ip));
  65. ASSERT(ip->i_ino == 0);
  66. /* initialise the xfs inode */
  67. ip->i_ino = ino;
  68. ip->i_mount = mp;
  69. memset(&ip->i_imap, 0, sizeof(struct xfs_imap));
  70. ip->i_afp = NULL;
  71. ip->i_cowfp = NULL;
  72. ip->i_cnextents = 0;
  73. ip->i_cformat = XFS_DINODE_FMT_EXTENTS;
  74. memset(&ip->i_df, 0, sizeof(xfs_ifork_t));
  75. ip->i_flags = 0;
  76. ip->i_delayed_blks = 0;
  77. memset(&ip->i_d, 0, sizeof(ip->i_d));
  78. return ip;
  79. }
  80. STATIC void
  81. xfs_inode_free_callback(
  82. struct rcu_head *head)
  83. {
  84. struct inode *inode = container_of(head, struct inode, i_rcu);
  85. struct xfs_inode *ip = XFS_I(inode);
  86. switch (VFS_I(ip)->i_mode & S_IFMT) {
  87. case S_IFREG:
  88. case S_IFDIR:
  89. case S_IFLNK:
  90. xfs_idestroy_fork(ip, XFS_DATA_FORK);
  91. break;
  92. }
  93. if (ip->i_afp)
  94. xfs_idestroy_fork(ip, XFS_ATTR_FORK);
  95. if (ip->i_cowfp)
  96. xfs_idestroy_fork(ip, XFS_COW_FORK);
  97. if (ip->i_itemp) {
  98. ASSERT(!(ip->i_itemp->ili_item.li_flags & XFS_LI_IN_AIL));
  99. xfs_inode_item_destroy(ip);
  100. ip->i_itemp = NULL;
  101. }
  102. kmem_zone_free(xfs_inode_zone, ip);
  103. }
  104. static void
  105. __xfs_inode_free(
  106. struct xfs_inode *ip)
  107. {
  108. /* asserts to verify all state is correct here */
  109. ASSERT(atomic_read(&ip->i_pincount) == 0);
  110. XFS_STATS_DEC(ip->i_mount, vn_active);
  111. call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback);
  112. }
  113. void
  114. xfs_inode_free(
  115. struct xfs_inode *ip)
  116. {
  117. ASSERT(!xfs_isiflocked(ip));
  118. /*
  119. * Because we use RCU freeing we need to ensure the inode always
  120. * appears to be reclaimed with an invalid inode number when in the
  121. * free state. The ip->i_flags_lock provides the barrier against lookup
  122. * races.
  123. */
  124. spin_lock(&ip->i_flags_lock);
  125. ip->i_flags = XFS_IRECLAIM;
  126. ip->i_ino = 0;
  127. spin_unlock(&ip->i_flags_lock);
  128. __xfs_inode_free(ip);
  129. }
  130. /*
  131. * Queue a new inode reclaim pass if there are reclaimable inodes and there
  132. * isn't a reclaim pass already in progress. By default it runs every 5s based
  133. * on the xfs periodic sync default of 30s. Perhaps this should have it's own
  134. * tunable, but that can be done if this method proves to be ineffective or too
  135. * aggressive.
  136. */
  137. static void
  138. xfs_reclaim_work_queue(
  139. struct xfs_mount *mp)
  140. {
  141. rcu_read_lock();
  142. if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
  143. queue_delayed_work(mp->m_reclaim_workqueue, &mp->m_reclaim_work,
  144. msecs_to_jiffies(xfs_syncd_centisecs / 6 * 10));
  145. }
  146. rcu_read_unlock();
  147. }
  148. /*
  149. * This is a fast pass over the inode cache to try to get reclaim moving on as
  150. * many inodes as possible in a short period of time. It kicks itself every few
  151. * seconds, as well as being kicked by the inode cache shrinker when memory
  152. * goes low. It scans as quickly as possible avoiding locked inodes or those
  153. * already being flushed, and once done schedules a future pass.
  154. */
  155. void
  156. xfs_reclaim_worker(
  157. struct work_struct *work)
  158. {
  159. struct xfs_mount *mp = container_of(to_delayed_work(work),
  160. struct xfs_mount, m_reclaim_work);
  161. xfs_reclaim_inodes(mp, SYNC_TRYLOCK);
  162. xfs_reclaim_work_queue(mp);
  163. }
  164. static void
  165. xfs_perag_set_reclaim_tag(
  166. struct xfs_perag *pag)
  167. {
  168. struct xfs_mount *mp = pag->pag_mount;
  169. lockdep_assert_held(&pag->pag_ici_lock);
  170. if (pag->pag_ici_reclaimable++)
  171. return;
  172. /* propagate the reclaim tag up into the perag radix tree */
  173. spin_lock(&mp->m_perag_lock);
  174. radix_tree_tag_set(&mp->m_perag_tree, pag->pag_agno,
  175. XFS_ICI_RECLAIM_TAG);
  176. spin_unlock(&mp->m_perag_lock);
  177. /* schedule periodic background inode reclaim */
  178. xfs_reclaim_work_queue(mp);
  179. trace_xfs_perag_set_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
  180. }
  181. static void
  182. xfs_perag_clear_reclaim_tag(
  183. struct xfs_perag *pag)
  184. {
  185. struct xfs_mount *mp = pag->pag_mount;
  186. lockdep_assert_held(&pag->pag_ici_lock);
  187. if (--pag->pag_ici_reclaimable)
  188. return;
  189. /* clear the reclaim tag from the perag radix tree */
  190. spin_lock(&mp->m_perag_lock);
  191. radix_tree_tag_clear(&mp->m_perag_tree, pag->pag_agno,
  192. XFS_ICI_RECLAIM_TAG);
  193. spin_unlock(&mp->m_perag_lock);
  194. trace_xfs_perag_clear_reclaim(mp, pag->pag_agno, -1, _RET_IP_);
  195. }
  196. /*
  197. * We set the inode flag atomically with the radix tree tag.
  198. * Once we get tag lookups on the radix tree, this inode flag
  199. * can go away.
  200. */
  201. void
  202. xfs_inode_set_reclaim_tag(
  203. struct xfs_inode *ip)
  204. {
  205. struct xfs_mount *mp = ip->i_mount;
  206. struct xfs_perag *pag;
  207. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
  208. spin_lock(&pag->pag_ici_lock);
  209. spin_lock(&ip->i_flags_lock);
  210. radix_tree_tag_set(&pag->pag_ici_root, XFS_INO_TO_AGINO(mp, ip->i_ino),
  211. XFS_ICI_RECLAIM_TAG);
  212. xfs_perag_set_reclaim_tag(pag);
  213. __xfs_iflags_set(ip, XFS_IRECLAIMABLE);
  214. spin_unlock(&ip->i_flags_lock);
  215. spin_unlock(&pag->pag_ici_lock);
  216. xfs_perag_put(pag);
  217. }
  218. STATIC void
  219. xfs_inode_clear_reclaim_tag(
  220. struct xfs_perag *pag,
  221. xfs_ino_t ino)
  222. {
  223. radix_tree_tag_clear(&pag->pag_ici_root,
  224. XFS_INO_TO_AGINO(pag->pag_mount, ino),
  225. XFS_ICI_RECLAIM_TAG);
  226. xfs_perag_clear_reclaim_tag(pag);
  227. }
  228. static void
  229. xfs_inew_wait(
  230. struct xfs_inode *ip)
  231. {
  232. wait_queue_head_t *wq = bit_waitqueue(&ip->i_flags, __XFS_INEW_BIT);
  233. DEFINE_WAIT_BIT(wait, &ip->i_flags, __XFS_INEW_BIT);
  234. do {
  235. prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
  236. if (!xfs_iflags_test(ip, XFS_INEW))
  237. break;
  238. schedule();
  239. } while (true);
  240. finish_wait(wq, &wait.wq_entry);
  241. }
  242. /*
  243. * When we recycle a reclaimable inode, we need to re-initialise the VFS inode
  244. * part of the structure. This is made more complex by the fact we store
  245. * information about the on-disk values in the VFS inode and so we can't just
  246. * overwrite the values unconditionally. Hence we save the parameters we
  247. * need to retain across reinitialisation, and rewrite them into the VFS inode
  248. * after reinitialisation even if it fails.
  249. */
  250. static int
  251. xfs_reinit_inode(
  252. struct xfs_mount *mp,
  253. struct inode *inode)
  254. {
  255. int error;
  256. uint32_t nlink = inode->i_nlink;
  257. uint32_t generation = inode->i_generation;
  258. uint64_t version = inode->i_version;
  259. umode_t mode = inode->i_mode;
  260. error = inode_init_always(mp->m_super, inode);
  261. set_nlink(inode, nlink);
  262. inode->i_generation = generation;
  263. inode->i_version = version;
  264. inode->i_mode = mode;
  265. return error;
  266. }
  267. /*
  268. * Check the validity of the inode we just found it the cache
  269. */
  270. static int
  271. xfs_iget_cache_hit(
  272. struct xfs_perag *pag,
  273. struct xfs_inode *ip,
  274. xfs_ino_t ino,
  275. int flags,
  276. int lock_flags) __releases(RCU)
  277. {
  278. struct inode *inode = VFS_I(ip);
  279. struct xfs_mount *mp = ip->i_mount;
  280. int error;
  281. /*
  282. * check for re-use of an inode within an RCU grace period due to the
  283. * radix tree nodes not being updated yet. We monitor for this by
  284. * setting the inode number to zero before freeing the inode structure.
  285. * If the inode has been reallocated and set up, then the inode number
  286. * will not match, so check for that, too.
  287. */
  288. spin_lock(&ip->i_flags_lock);
  289. if (ip->i_ino != ino) {
  290. trace_xfs_iget_skip(ip);
  291. XFS_STATS_INC(mp, xs_ig_frecycle);
  292. error = -EAGAIN;
  293. goto out_error;
  294. }
  295. /*
  296. * If we are racing with another cache hit that is currently
  297. * instantiating this inode or currently recycling it out of
  298. * reclaimabe state, wait for the initialisation to complete
  299. * before continuing.
  300. *
  301. * XXX(hch): eventually we should do something equivalent to
  302. * wait_on_inode to wait for these flags to be cleared
  303. * instead of polling for it.
  304. */
  305. if (ip->i_flags & (XFS_INEW|XFS_IRECLAIM)) {
  306. trace_xfs_iget_skip(ip);
  307. XFS_STATS_INC(mp, xs_ig_frecycle);
  308. error = -EAGAIN;
  309. goto out_error;
  310. }
  311. /*
  312. * If lookup is racing with unlink return an error immediately.
  313. */
  314. if (VFS_I(ip)->i_mode == 0 && !(flags & XFS_IGET_CREATE)) {
  315. error = -ENOENT;
  316. goto out_error;
  317. }
  318. /*
  319. * If IRECLAIMABLE is set, we've torn down the VFS inode already.
  320. * Need to carefully get it back into useable state.
  321. */
  322. if (ip->i_flags & XFS_IRECLAIMABLE) {
  323. trace_xfs_iget_reclaim(ip);
  324. if (flags & XFS_IGET_INCORE) {
  325. error = -EAGAIN;
  326. goto out_error;
  327. }
  328. /*
  329. * We need to set XFS_IRECLAIM to prevent xfs_reclaim_inode
  330. * from stomping over us while we recycle the inode. We can't
  331. * clear the radix tree reclaimable tag yet as it requires
  332. * pag_ici_lock to be held exclusive.
  333. */
  334. ip->i_flags |= XFS_IRECLAIM;
  335. spin_unlock(&ip->i_flags_lock);
  336. rcu_read_unlock();
  337. error = xfs_reinit_inode(mp, inode);
  338. if (error) {
  339. bool wake;
  340. /*
  341. * Re-initializing the inode failed, and we are in deep
  342. * trouble. Try to re-add it to the reclaim list.
  343. */
  344. rcu_read_lock();
  345. spin_lock(&ip->i_flags_lock);
  346. wake = !!__xfs_iflags_test(ip, XFS_INEW);
  347. ip->i_flags &= ~(XFS_INEW | XFS_IRECLAIM);
  348. if (wake)
  349. wake_up_bit(&ip->i_flags, __XFS_INEW_BIT);
  350. ASSERT(ip->i_flags & XFS_IRECLAIMABLE);
  351. trace_xfs_iget_reclaim_fail(ip);
  352. goto out_error;
  353. }
  354. spin_lock(&pag->pag_ici_lock);
  355. spin_lock(&ip->i_flags_lock);
  356. /*
  357. * Clear the per-lifetime state in the inode as we are now
  358. * effectively a new inode and need to return to the initial
  359. * state before reuse occurs.
  360. */
  361. ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS;
  362. ip->i_flags |= XFS_INEW;
  363. xfs_inode_clear_reclaim_tag(pag, ip->i_ino);
  364. inode->i_state = I_NEW;
  365. ASSERT(!rwsem_is_locked(&inode->i_rwsem));
  366. init_rwsem(&inode->i_rwsem);
  367. spin_unlock(&ip->i_flags_lock);
  368. spin_unlock(&pag->pag_ici_lock);
  369. } else {
  370. /* If the VFS inode is being torn down, pause and try again. */
  371. if (!igrab(inode)) {
  372. trace_xfs_iget_skip(ip);
  373. error = -EAGAIN;
  374. goto out_error;
  375. }
  376. /* We've got a live one. */
  377. spin_unlock(&ip->i_flags_lock);
  378. rcu_read_unlock();
  379. trace_xfs_iget_hit(ip);
  380. }
  381. if (lock_flags != 0)
  382. xfs_ilock(ip, lock_flags);
  383. if (!(flags & XFS_IGET_INCORE))
  384. xfs_iflags_clear(ip, XFS_ISTALE | XFS_IDONTCACHE);
  385. XFS_STATS_INC(mp, xs_ig_found);
  386. return 0;
  387. out_error:
  388. spin_unlock(&ip->i_flags_lock);
  389. rcu_read_unlock();
  390. return error;
  391. }
  392. static int
  393. xfs_iget_cache_miss(
  394. struct xfs_mount *mp,
  395. struct xfs_perag *pag,
  396. xfs_trans_t *tp,
  397. xfs_ino_t ino,
  398. struct xfs_inode **ipp,
  399. int flags,
  400. int lock_flags)
  401. {
  402. struct xfs_inode *ip;
  403. int error;
  404. xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ino);
  405. int iflags;
  406. ip = xfs_inode_alloc(mp, ino);
  407. if (!ip)
  408. return -ENOMEM;
  409. error = xfs_iread(mp, tp, ip, flags);
  410. if (error)
  411. goto out_destroy;
  412. trace_xfs_iget_miss(ip);
  413. if ((VFS_I(ip)->i_mode == 0) && !(flags & XFS_IGET_CREATE)) {
  414. error = -ENOENT;
  415. goto out_destroy;
  416. }
  417. /*
  418. * Preload the radix tree so we can insert safely under the
  419. * write spinlock. Note that we cannot sleep inside the preload
  420. * region. Since we can be called from transaction context, don't
  421. * recurse into the file system.
  422. */
  423. if (radix_tree_preload(GFP_NOFS)) {
  424. error = -EAGAIN;
  425. goto out_destroy;
  426. }
  427. /*
  428. * Because the inode hasn't been added to the radix-tree yet it can't
  429. * be found by another thread, so we can do the non-sleeping lock here.
  430. */
  431. if (lock_flags) {
  432. if (!xfs_ilock_nowait(ip, lock_flags))
  433. BUG();
  434. }
  435. /*
  436. * These values must be set before inserting the inode into the radix
  437. * tree as the moment it is inserted a concurrent lookup (allowed by the
  438. * RCU locking mechanism) can find it and that lookup must see that this
  439. * is an inode currently under construction (i.e. that XFS_INEW is set).
  440. * The ip->i_flags_lock that protects the XFS_INEW flag forms the
  441. * memory barrier that ensures this detection works correctly at lookup
  442. * time.
  443. */
  444. iflags = XFS_INEW;
  445. if (flags & XFS_IGET_DONTCACHE)
  446. iflags |= XFS_IDONTCACHE;
  447. ip->i_udquot = NULL;
  448. ip->i_gdquot = NULL;
  449. ip->i_pdquot = NULL;
  450. xfs_iflags_set(ip, iflags);
  451. /* insert the new inode */
  452. spin_lock(&pag->pag_ici_lock);
  453. error = radix_tree_insert(&pag->pag_ici_root, agino, ip);
  454. if (unlikely(error)) {
  455. WARN_ON(error != -EEXIST);
  456. XFS_STATS_INC(mp, xs_ig_dup);
  457. error = -EAGAIN;
  458. goto out_preload_end;
  459. }
  460. spin_unlock(&pag->pag_ici_lock);
  461. radix_tree_preload_end();
  462. *ipp = ip;
  463. return 0;
  464. out_preload_end:
  465. spin_unlock(&pag->pag_ici_lock);
  466. radix_tree_preload_end();
  467. if (lock_flags)
  468. xfs_iunlock(ip, lock_flags);
  469. out_destroy:
  470. __destroy_inode(VFS_I(ip));
  471. xfs_inode_free(ip);
  472. return error;
  473. }
  474. /*
  475. * Look up an inode by number in the given file system.
  476. * The inode is looked up in the cache held in each AG.
  477. * If the inode is found in the cache, initialise the vfs inode
  478. * if necessary.
  479. *
  480. * If it is not in core, read it in from the file system's device,
  481. * add it to the cache and initialise the vfs inode.
  482. *
  483. * The inode is locked according to the value of the lock_flags parameter.
  484. * This flag parameter indicates how and if the inode's IO lock and inode lock
  485. * should be taken.
  486. *
  487. * mp -- the mount point structure for the current file system. It points
  488. * to the inode hash table.
  489. * tp -- a pointer to the current transaction if there is one. This is
  490. * simply passed through to the xfs_iread() call.
  491. * ino -- the number of the inode desired. This is the unique identifier
  492. * within the file system for the inode being requested.
  493. * lock_flags -- flags indicating how to lock the inode. See the comment
  494. * for xfs_ilock() for a list of valid values.
  495. */
  496. int
  497. xfs_iget(
  498. xfs_mount_t *mp,
  499. xfs_trans_t *tp,
  500. xfs_ino_t ino,
  501. uint flags,
  502. uint lock_flags,
  503. xfs_inode_t **ipp)
  504. {
  505. xfs_inode_t *ip;
  506. int error;
  507. xfs_perag_t *pag;
  508. xfs_agino_t agino;
  509. /*
  510. * xfs_reclaim_inode() uses the ILOCK to ensure an inode
  511. * doesn't get freed while it's being referenced during a
  512. * radix tree traversal here. It assumes this function
  513. * aqcuires only the ILOCK (and therefore it has no need to
  514. * involve the IOLOCK in this synchronization).
  515. */
  516. ASSERT((lock_flags & (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED)) == 0);
  517. /* reject inode numbers outside existing AGs */
  518. if (!ino || XFS_INO_TO_AGNO(mp, ino) >= mp->m_sb.sb_agcount)
  519. return -EINVAL;
  520. XFS_STATS_INC(mp, xs_ig_attempts);
  521. /* get the perag structure and ensure that it's inode capable */
  522. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ino));
  523. agino = XFS_INO_TO_AGINO(mp, ino);
  524. again:
  525. error = 0;
  526. rcu_read_lock();
  527. ip = radix_tree_lookup(&pag->pag_ici_root, agino);
  528. if (ip) {
  529. error = xfs_iget_cache_hit(pag, ip, ino, flags, lock_flags);
  530. if (error)
  531. goto out_error_or_again;
  532. } else {
  533. rcu_read_unlock();
  534. if (flags & XFS_IGET_INCORE) {
  535. error = -ENOENT;
  536. goto out_error_or_again;
  537. }
  538. XFS_STATS_INC(mp, xs_ig_missed);
  539. error = xfs_iget_cache_miss(mp, pag, tp, ino, &ip,
  540. flags, lock_flags);
  541. if (error)
  542. goto out_error_or_again;
  543. }
  544. xfs_perag_put(pag);
  545. *ipp = ip;
  546. /*
  547. * If we have a real type for an on-disk inode, we can setup the inode
  548. * now. If it's a new inode being created, xfs_ialloc will handle it.
  549. */
  550. if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0)
  551. xfs_setup_existing_inode(ip);
  552. return 0;
  553. out_error_or_again:
  554. if (!(flags & XFS_IGET_INCORE) && error == -EAGAIN) {
  555. delay(1);
  556. goto again;
  557. }
  558. xfs_perag_put(pag);
  559. return error;
  560. }
  561. /*
  562. * "Is this a cached inode that's also allocated?"
  563. *
  564. * Look up an inode by number in the given file system. If the inode is
  565. * in cache and isn't in purgatory, return 1 if the inode is allocated
  566. * and 0 if it is not. For all other cases (not in cache, being torn
  567. * down, etc.), return a negative error code.
  568. *
  569. * The caller has to prevent inode allocation and freeing activity,
  570. * presumably by locking the AGI buffer. This is to ensure that an
  571. * inode cannot transition from allocated to freed until the caller is
  572. * ready to allow that. If the inode is in an intermediate state (new,
  573. * reclaimable, or being reclaimed), -EAGAIN will be returned; if the
  574. * inode is not in the cache, -ENOENT will be returned. The caller must
  575. * deal with these scenarios appropriately.
  576. *
  577. * This is a specialized use case for the online scrubber; if you're
  578. * reading this, you probably want xfs_iget.
  579. */
  580. int
  581. xfs_icache_inode_is_allocated(
  582. struct xfs_mount *mp,
  583. struct xfs_trans *tp,
  584. xfs_ino_t ino,
  585. bool *inuse)
  586. {
  587. struct xfs_inode *ip;
  588. int error;
  589. error = xfs_iget(mp, tp, ino, XFS_IGET_INCORE, 0, &ip);
  590. if (error)
  591. return error;
  592. *inuse = !!(VFS_I(ip)->i_mode);
  593. IRELE(ip);
  594. return 0;
  595. }
  596. /*
  597. * The inode lookup is done in batches to keep the amount of lock traffic and
  598. * radix tree lookups to a minimum. The batch size is a trade off between
  599. * lookup reduction and stack usage. This is in the reclaim path, so we can't
  600. * be too greedy.
  601. */
  602. #define XFS_LOOKUP_BATCH 32
  603. STATIC int
  604. xfs_inode_ag_walk_grab(
  605. struct xfs_inode *ip,
  606. int flags)
  607. {
  608. struct inode *inode = VFS_I(ip);
  609. bool newinos = !!(flags & XFS_AGITER_INEW_WAIT);
  610. ASSERT(rcu_read_lock_held());
  611. /*
  612. * check for stale RCU freed inode
  613. *
  614. * If the inode has been reallocated, it doesn't matter if it's not in
  615. * the AG we are walking - we are walking for writeback, so if it
  616. * passes all the "valid inode" checks and is dirty, then we'll write
  617. * it back anyway. If it has been reallocated and still being
  618. * initialised, the XFS_INEW check below will catch it.
  619. */
  620. spin_lock(&ip->i_flags_lock);
  621. if (!ip->i_ino)
  622. goto out_unlock_noent;
  623. /* avoid new or reclaimable inodes. Leave for reclaim code to flush */
  624. if ((!newinos && __xfs_iflags_test(ip, XFS_INEW)) ||
  625. __xfs_iflags_test(ip, XFS_IRECLAIMABLE | XFS_IRECLAIM))
  626. goto out_unlock_noent;
  627. spin_unlock(&ip->i_flags_lock);
  628. /* nothing to sync during shutdown */
  629. if (XFS_FORCED_SHUTDOWN(ip->i_mount))
  630. return -EFSCORRUPTED;
  631. /* If we can't grab the inode, it must on it's way to reclaim. */
  632. if (!igrab(inode))
  633. return -ENOENT;
  634. /* inode is valid */
  635. return 0;
  636. out_unlock_noent:
  637. spin_unlock(&ip->i_flags_lock);
  638. return -ENOENT;
  639. }
  640. STATIC int
  641. xfs_inode_ag_walk(
  642. struct xfs_mount *mp,
  643. struct xfs_perag *pag,
  644. int (*execute)(struct xfs_inode *ip, int flags,
  645. void *args),
  646. int flags,
  647. void *args,
  648. int tag,
  649. int iter_flags)
  650. {
  651. uint32_t first_index;
  652. int last_error = 0;
  653. int skipped;
  654. int done;
  655. int nr_found;
  656. restart:
  657. done = 0;
  658. skipped = 0;
  659. first_index = 0;
  660. nr_found = 0;
  661. do {
  662. struct xfs_inode *batch[XFS_LOOKUP_BATCH];
  663. int error = 0;
  664. int i;
  665. rcu_read_lock();
  666. if (tag == -1)
  667. nr_found = radix_tree_gang_lookup(&pag->pag_ici_root,
  668. (void **)batch, first_index,
  669. XFS_LOOKUP_BATCH);
  670. else
  671. nr_found = radix_tree_gang_lookup_tag(
  672. &pag->pag_ici_root,
  673. (void **) batch, first_index,
  674. XFS_LOOKUP_BATCH, tag);
  675. if (!nr_found) {
  676. rcu_read_unlock();
  677. break;
  678. }
  679. /*
  680. * Grab the inodes before we drop the lock. if we found
  681. * nothing, nr == 0 and the loop will be skipped.
  682. */
  683. for (i = 0; i < nr_found; i++) {
  684. struct xfs_inode *ip = batch[i];
  685. if (done || xfs_inode_ag_walk_grab(ip, iter_flags))
  686. batch[i] = NULL;
  687. /*
  688. * Update the index for the next lookup. Catch
  689. * overflows into the next AG range which can occur if
  690. * we have inodes in the last block of the AG and we
  691. * are currently pointing to the last inode.
  692. *
  693. * Because we may see inodes that are from the wrong AG
  694. * due to RCU freeing and reallocation, only update the
  695. * index if it lies in this AG. It was a race that lead
  696. * us to see this inode, so another lookup from the
  697. * same index will not find it again.
  698. */
  699. if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag->pag_agno)
  700. continue;
  701. first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
  702. if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
  703. done = 1;
  704. }
  705. /* unlock now we've grabbed the inodes. */
  706. rcu_read_unlock();
  707. for (i = 0; i < nr_found; i++) {
  708. if (!batch[i])
  709. continue;
  710. if ((iter_flags & XFS_AGITER_INEW_WAIT) &&
  711. xfs_iflags_test(batch[i], XFS_INEW))
  712. xfs_inew_wait(batch[i]);
  713. error = execute(batch[i], flags, args);
  714. IRELE(batch[i]);
  715. if (error == -EAGAIN) {
  716. skipped++;
  717. continue;
  718. }
  719. if (error && last_error != -EFSCORRUPTED)
  720. last_error = error;
  721. }
  722. /* bail out if the filesystem is corrupted. */
  723. if (error == -EFSCORRUPTED)
  724. break;
  725. cond_resched();
  726. } while (nr_found && !done);
  727. if (skipped) {
  728. delay(1);
  729. goto restart;
  730. }
  731. return last_error;
  732. }
  733. /*
  734. * Background scanning to trim post-EOF preallocated space. This is queued
  735. * based on the 'speculative_prealloc_lifetime' tunable (5m by default).
  736. */
  737. void
  738. xfs_queue_eofblocks(
  739. struct xfs_mount *mp)
  740. {
  741. rcu_read_lock();
  742. if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_EOFBLOCKS_TAG))
  743. queue_delayed_work(mp->m_eofblocks_workqueue,
  744. &mp->m_eofblocks_work,
  745. msecs_to_jiffies(xfs_eofb_secs * 1000));
  746. rcu_read_unlock();
  747. }
  748. void
  749. xfs_eofblocks_worker(
  750. struct work_struct *work)
  751. {
  752. struct xfs_mount *mp = container_of(to_delayed_work(work),
  753. struct xfs_mount, m_eofblocks_work);
  754. xfs_icache_free_eofblocks(mp, NULL);
  755. xfs_queue_eofblocks(mp);
  756. }
  757. /*
  758. * Background scanning to trim preallocated CoW space. This is queued
  759. * based on the 'speculative_cow_prealloc_lifetime' tunable (5m by default).
  760. * (We'll just piggyback on the post-EOF prealloc space workqueue.)
  761. */
  762. STATIC void
  763. xfs_queue_cowblocks(
  764. struct xfs_mount *mp)
  765. {
  766. rcu_read_lock();
  767. if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_COWBLOCKS_TAG))
  768. queue_delayed_work(mp->m_eofblocks_workqueue,
  769. &mp->m_cowblocks_work,
  770. msecs_to_jiffies(xfs_cowb_secs * 1000));
  771. rcu_read_unlock();
  772. }
  773. void
  774. xfs_cowblocks_worker(
  775. struct work_struct *work)
  776. {
  777. struct xfs_mount *mp = container_of(to_delayed_work(work),
  778. struct xfs_mount, m_cowblocks_work);
  779. xfs_icache_free_cowblocks(mp, NULL);
  780. xfs_queue_cowblocks(mp);
  781. }
  782. int
  783. xfs_inode_ag_iterator_flags(
  784. struct xfs_mount *mp,
  785. int (*execute)(struct xfs_inode *ip, int flags,
  786. void *args),
  787. int flags,
  788. void *args,
  789. int iter_flags)
  790. {
  791. struct xfs_perag *pag;
  792. int error = 0;
  793. int last_error = 0;
  794. xfs_agnumber_t ag;
  795. ag = 0;
  796. while ((pag = xfs_perag_get(mp, ag))) {
  797. ag = pag->pag_agno + 1;
  798. error = xfs_inode_ag_walk(mp, pag, execute, flags, args, -1,
  799. iter_flags);
  800. xfs_perag_put(pag);
  801. if (error) {
  802. last_error = error;
  803. if (error == -EFSCORRUPTED)
  804. break;
  805. }
  806. }
  807. return last_error;
  808. }
  809. int
  810. xfs_inode_ag_iterator(
  811. struct xfs_mount *mp,
  812. int (*execute)(struct xfs_inode *ip, int flags,
  813. void *args),
  814. int flags,
  815. void *args)
  816. {
  817. return xfs_inode_ag_iterator_flags(mp, execute, flags, args, 0);
  818. }
  819. int
  820. xfs_inode_ag_iterator_tag(
  821. struct xfs_mount *mp,
  822. int (*execute)(struct xfs_inode *ip, int flags,
  823. void *args),
  824. int flags,
  825. void *args,
  826. int tag)
  827. {
  828. struct xfs_perag *pag;
  829. int error = 0;
  830. int last_error = 0;
  831. xfs_agnumber_t ag;
  832. ag = 0;
  833. while ((pag = xfs_perag_get_tag(mp, ag, tag))) {
  834. ag = pag->pag_agno + 1;
  835. error = xfs_inode_ag_walk(mp, pag, execute, flags, args, tag,
  836. 0);
  837. xfs_perag_put(pag);
  838. if (error) {
  839. last_error = error;
  840. if (error == -EFSCORRUPTED)
  841. break;
  842. }
  843. }
  844. return last_error;
  845. }
  846. /*
  847. * Grab the inode for reclaim exclusively.
  848. * Return 0 if we grabbed it, non-zero otherwise.
  849. */
  850. STATIC int
  851. xfs_reclaim_inode_grab(
  852. struct xfs_inode *ip,
  853. int flags)
  854. {
  855. ASSERT(rcu_read_lock_held());
  856. /* quick check for stale RCU freed inode */
  857. if (!ip->i_ino)
  858. return 1;
  859. /*
  860. * If we are asked for non-blocking operation, do unlocked checks to
  861. * see if the inode already is being flushed or in reclaim to avoid
  862. * lock traffic.
  863. */
  864. if ((flags & SYNC_TRYLOCK) &&
  865. __xfs_iflags_test(ip, XFS_IFLOCK | XFS_IRECLAIM))
  866. return 1;
  867. /*
  868. * The radix tree lock here protects a thread in xfs_iget from racing
  869. * with us starting reclaim on the inode. Once we have the
  870. * XFS_IRECLAIM flag set it will not touch us.
  871. *
  872. * Due to RCU lookup, we may find inodes that have been freed and only
  873. * have XFS_IRECLAIM set. Indeed, we may see reallocated inodes that
  874. * aren't candidates for reclaim at all, so we must check the
  875. * XFS_IRECLAIMABLE is set first before proceeding to reclaim.
  876. */
  877. spin_lock(&ip->i_flags_lock);
  878. if (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) ||
  879. __xfs_iflags_test(ip, XFS_IRECLAIM)) {
  880. /* not a reclaim candidate. */
  881. spin_unlock(&ip->i_flags_lock);
  882. return 1;
  883. }
  884. __xfs_iflags_set(ip, XFS_IRECLAIM);
  885. spin_unlock(&ip->i_flags_lock);
  886. return 0;
  887. }
  888. /*
  889. * Inodes in different states need to be treated differently. The following
  890. * table lists the inode states and the reclaim actions necessary:
  891. *
  892. * inode state iflush ret required action
  893. * --------------- ---------- ---------------
  894. * bad - reclaim
  895. * shutdown EIO unpin and reclaim
  896. * clean, unpinned 0 reclaim
  897. * stale, unpinned 0 reclaim
  898. * clean, pinned(*) 0 requeue
  899. * stale, pinned EAGAIN requeue
  900. * dirty, async - requeue
  901. * dirty, sync 0 reclaim
  902. *
  903. * (*) dgc: I don't think the clean, pinned state is possible but it gets
  904. * handled anyway given the order of checks implemented.
  905. *
  906. * Also, because we get the flush lock first, we know that any inode that has
  907. * been flushed delwri has had the flush completed by the time we check that
  908. * the inode is clean.
  909. *
  910. * Note that because the inode is flushed delayed write by AIL pushing, the
  911. * flush lock may already be held here and waiting on it can result in very
  912. * long latencies. Hence for sync reclaims, where we wait on the flush lock,
  913. * the caller should push the AIL first before trying to reclaim inodes to
  914. * minimise the amount of time spent waiting. For background relaim, we only
  915. * bother to reclaim clean inodes anyway.
  916. *
  917. * Hence the order of actions after gaining the locks should be:
  918. * bad => reclaim
  919. * shutdown => unpin and reclaim
  920. * pinned, async => requeue
  921. * pinned, sync => unpin
  922. * stale => reclaim
  923. * clean => reclaim
  924. * dirty, async => requeue
  925. * dirty, sync => flush, wait and reclaim
  926. */
  927. STATIC int
  928. xfs_reclaim_inode(
  929. struct xfs_inode *ip,
  930. struct xfs_perag *pag,
  931. int sync_mode)
  932. {
  933. struct xfs_buf *bp = NULL;
  934. xfs_ino_t ino = ip->i_ino; /* for radix_tree_delete */
  935. int error;
  936. restart:
  937. error = 0;
  938. xfs_ilock(ip, XFS_ILOCK_EXCL);
  939. if (!xfs_iflock_nowait(ip)) {
  940. if (!(sync_mode & SYNC_WAIT))
  941. goto out;
  942. xfs_iflock(ip);
  943. }
  944. if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
  945. xfs_iunpin_wait(ip);
  946. /* xfs_iflush_abort() drops the flush lock */
  947. xfs_iflush_abort(ip, false);
  948. goto reclaim;
  949. }
  950. if (xfs_ipincount(ip)) {
  951. if (!(sync_mode & SYNC_WAIT))
  952. goto out_ifunlock;
  953. xfs_iunpin_wait(ip);
  954. }
  955. if (xfs_iflags_test(ip, XFS_ISTALE) || xfs_inode_clean(ip)) {
  956. xfs_ifunlock(ip);
  957. goto reclaim;
  958. }
  959. /*
  960. * Never flush out dirty data during non-blocking reclaim, as it would
  961. * just contend with AIL pushing trying to do the same job.
  962. */
  963. if (!(sync_mode & SYNC_WAIT))
  964. goto out_ifunlock;
  965. /*
  966. * Now we have an inode that needs flushing.
  967. *
  968. * Note that xfs_iflush will never block on the inode buffer lock, as
  969. * xfs_ifree_cluster() can lock the inode buffer before it locks the
  970. * ip->i_lock, and we are doing the exact opposite here. As a result,
  971. * doing a blocking xfs_imap_to_bp() to get the cluster buffer would
  972. * result in an ABBA deadlock with xfs_ifree_cluster().
  973. *
  974. * As xfs_ifree_cluser() must gather all inodes that are active in the
  975. * cache to mark them stale, if we hit this case we don't actually want
  976. * to do IO here - we want the inode marked stale so we can simply
  977. * reclaim it. Hence if we get an EAGAIN error here, just unlock the
  978. * inode, back off and try again. Hopefully the next pass through will
  979. * see the stale flag set on the inode.
  980. */
  981. error = xfs_iflush(ip, &bp);
  982. if (error == -EAGAIN) {
  983. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  984. /* backoff longer than in xfs_ifree_cluster */
  985. delay(2);
  986. goto restart;
  987. }
  988. if (!error) {
  989. error = xfs_bwrite(bp);
  990. xfs_buf_relse(bp);
  991. }
  992. reclaim:
  993. ASSERT(!xfs_isiflocked(ip));
  994. /*
  995. * Because we use RCU freeing we need to ensure the inode always appears
  996. * to be reclaimed with an invalid inode number when in the free state.
  997. * We do this as early as possible under the ILOCK so that
  998. * xfs_iflush_cluster() and xfs_ifree_cluster() can be guaranteed to
  999. * detect races with us here. By doing this, we guarantee that once
  1000. * xfs_iflush_cluster() or xfs_ifree_cluster() has locked XFS_ILOCK that
  1001. * it will see either a valid inode that will serialise correctly, or it
  1002. * will see an invalid inode that it can skip.
  1003. */
  1004. spin_lock(&ip->i_flags_lock);
  1005. ip->i_flags = XFS_IRECLAIM;
  1006. ip->i_ino = 0;
  1007. spin_unlock(&ip->i_flags_lock);
  1008. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  1009. XFS_STATS_INC(ip->i_mount, xs_ig_reclaims);
  1010. /*
  1011. * Remove the inode from the per-AG radix tree.
  1012. *
  1013. * Because radix_tree_delete won't complain even if the item was never
  1014. * added to the tree assert that it's been there before to catch
  1015. * problems with the inode life time early on.
  1016. */
  1017. spin_lock(&pag->pag_ici_lock);
  1018. if (!radix_tree_delete(&pag->pag_ici_root,
  1019. XFS_INO_TO_AGINO(ip->i_mount, ino)))
  1020. ASSERT(0);
  1021. xfs_perag_clear_reclaim_tag(pag);
  1022. spin_unlock(&pag->pag_ici_lock);
  1023. /*
  1024. * Here we do an (almost) spurious inode lock in order to coordinate
  1025. * with inode cache radix tree lookups. This is because the lookup
  1026. * can reference the inodes in the cache without taking references.
  1027. *
  1028. * We make that OK here by ensuring that we wait until the inode is
  1029. * unlocked after the lookup before we go ahead and free it.
  1030. */
  1031. xfs_ilock(ip, XFS_ILOCK_EXCL);
  1032. xfs_qm_dqdetach(ip);
  1033. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  1034. __xfs_inode_free(ip);
  1035. return error;
  1036. out_ifunlock:
  1037. xfs_ifunlock(ip);
  1038. out:
  1039. xfs_iflags_clear(ip, XFS_IRECLAIM);
  1040. xfs_iunlock(ip, XFS_ILOCK_EXCL);
  1041. /*
  1042. * We could return -EAGAIN here to make reclaim rescan the inode tree in
  1043. * a short while. However, this just burns CPU time scanning the tree
  1044. * waiting for IO to complete and the reclaim work never goes back to
  1045. * the idle state. Instead, return 0 to let the next scheduled
  1046. * background reclaim attempt to reclaim the inode again.
  1047. */
  1048. return 0;
  1049. }
  1050. /*
  1051. * Walk the AGs and reclaim the inodes in them. Even if the filesystem is
  1052. * corrupted, we still want to try to reclaim all the inodes. If we don't,
  1053. * then a shut down during filesystem unmount reclaim walk leak all the
  1054. * unreclaimed inodes.
  1055. */
  1056. STATIC int
  1057. xfs_reclaim_inodes_ag(
  1058. struct xfs_mount *mp,
  1059. int flags,
  1060. int *nr_to_scan)
  1061. {
  1062. struct xfs_perag *pag;
  1063. int error = 0;
  1064. int last_error = 0;
  1065. xfs_agnumber_t ag;
  1066. int trylock = flags & SYNC_TRYLOCK;
  1067. int skipped;
  1068. restart:
  1069. ag = 0;
  1070. skipped = 0;
  1071. while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
  1072. unsigned long first_index = 0;
  1073. int done = 0;
  1074. int nr_found = 0;
  1075. ag = pag->pag_agno + 1;
  1076. if (trylock) {
  1077. if (!mutex_trylock(&pag->pag_ici_reclaim_lock)) {
  1078. skipped++;
  1079. xfs_perag_put(pag);
  1080. continue;
  1081. }
  1082. first_index = pag->pag_ici_reclaim_cursor;
  1083. } else
  1084. mutex_lock(&pag->pag_ici_reclaim_lock);
  1085. do {
  1086. struct xfs_inode *batch[XFS_LOOKUP_BATCH];
  1087. int i;
  1088. rcu_read_lock();
  1089. nr_found = radix_tree_gang_lookup_tag(
  1090. &pag->pag_ici_root,
  1091. (void **)batch, first_index,
  1092. XFS_LOOKUP_BATCH,
  1093. XFS_ICI_RECLAIM_TAG);
  1094. if (!nr_found) {
  1095. done = 1;
  1096. rcu_read_unlock();
  1097. break;
  1098. }
  1099. /*
  1100. * Grab the inodes before we drop the lock. if we found
  1101. * nothing, nr == 0 and the loop will be skipped.
  1102. */
  1103. for (i = 0; i < nr_found; i++) {
  1104. struct xfs_inode *ip = batch[i];
  1105. if (done || xfs_reclaim_inode_grab(ip, flags))
  1106. batch[i] = NULL;
  1107. /*
  1108. * Update the index for the next lookup. Catch
  1109. * overflows into the next AG range which can
  1110. * occur if we have inodes in the last block of
  1111. * the AG and we are currently pointing to the
  1112. * last inode.
  1113. *
  1114. * Because we may see inodes that are from the
  1115. * wrong AG due to RCU freeing and
  1116. * reallocation, only update the index if it
  1117. * lies in this AG. It was a race that lead us
  1118. * to see this inode, so another lookup from
  1119. * the same index will not find it again.
  1120. */
  1121. if (XFS_INO_TO_AGNO(mp, ip->i_ino) !=
  1122. pag->pag_agno)
  1123. continue;
  1124. first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
  1125. if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
  1126. done = 1;
  1127. }
  1128. /* unlock now we've grabbed the inodes. */
  1129. rcu_read_unlock();
  1130. for (i = 0; i < nr_found; i++) {
  1131. if (!batch[i])
  1132. continue;
  1133. error = xfs_reclaim_inode(batch[i], pag, flags);
  1134. if (error && last_error != -EFSCORRUPTED)
  1135. last_error = error;
  1136. }
  1137. *nr_to_scan -= XFS_LOOKUP_BATCH;
  1138. cond_resched();
  1139. } while (nr_found && !done && *nr_to_scan > 0);
  1140. if (trylock && !done)
  1141. pag->pag_ici_reclaim_cursor = first_index;
  1142. else
  1143. pag->pag_ici_reclaim_cursor = 0;
  1144. mutex_unlock(&pag->pag_ici_reclaim_lock);
  1145. xfs_perag_put(pag);
  1146. }
  1147. /*
  1148. * if we skipped any AG, and we still have scan count remaining, do
  1149. * another pass this time using blocking reclaim semantics (i.e
  1150. * waiting on the reclaim locks and ignoring the reclaim cursors). This
  1151. * ensure that when we get more reclaimers than AGs we block rather
  1152. * than spin trying to execute reclaim.
  1153. */
  1154. if (skipped && (flags & SYNC_WAIT) && *nr_to_scan > 0) {
  1155. trylock = 0;
  1156. goto restart;
  1157. }
  1158. return last_error;
  1159. }
  1160. int
  1161. xfs_reclaim_inodes(
  1162. xfs_mount_t *mp,
  1163. int mode)
  1164. {
  1165. int nr_to_scan = INT_MAX;
  1166. return xfs_reclaim_inodes_ag(mp, mode, &nr_to_scan);
  1167. }
  1168. /*
  1169. * Scan a certain number of inodes for reclaim.
  1170. *
  1171. * When called we make sure that there is a background (fast) inode reclaim in
  1172. * progress, while we will throttle the speed of reclaim via doing synchronous
  1173. * reclaim of inodes. That means if we come across dirty inodes, we wait for
  1174. * them to be cleaned, which we hope will not be very long due to the
  1175. * background walker having already kicked the IO off on those dirty inodes.
  1176. */
  1177. long
  1178. xfs_reclaim_inodes_nr(
  1179. struct xfs_mount *mp,
  1180. int nr_to_scan)
  1181. {
  1182. /* kick background reclaimer and push the AIL */
  1183. xfs_reclaim_work_queue(mp);
  1184. xfs_ail_push_all(mp->m_ail);
  1185. return xfs_reclaim_inodes_ag(mp, SYNC_TRYLOCK | SYNC_WAIT, &nr_to_scan);
  1186. }
  1187. /*
  1188. * Return the number of reclaimable inodes in the filesystem for
  1189. * the shrinker to determine how much to reclaim.
  1190. */
  1191. int
  1192. xfs_reclaim_inodes_count(
  1193. struct xfs_mount *mp)
  1194. {
  1195. struct xfs_perag *pag;
  1196. xfs_agnumber_t ag = 0;
  1197. int reclaimable = 0;
  1198. while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
  1199. ag = pag->pag_agno + 1;
  1200. reclaimable += pag->pag_ici_reclaimable;
  1201. xfs_perag_put(pag);
  1202. }
  1203. return reclaimable;
  1204. }
  1205. STATIC int
  1206. xfs_inode_match_id(
  1207. struct xfs_inode *ip,
  1208. struct xfs_eofblocks *eofb)
  1209. {
  1210. if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
  1211. !uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
  1212. return 0;
  1213. if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
  1214. !gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
  1215. return 0;
  1216. if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
  1217. xfs_get_projid(ip) != eofb->eof_prid)
  1218. return 0;
  1219. return 1;
  1220. }
  1221. /*
  1222. * A union-based inode filtering algorithm. Process the inode if any of the
  1223. * criteria match. This is for global/internal scans only.
  1224. */
  1225. STATIC int
  1226. xfs_inode_match_id_union(
  1227. struct xfs_inode *ip,
  1228. struct xfs_eofblocks *eofb)
  1229. {
  1230. if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) &&
  1231. uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid))
  1232. return 1;
  1233. if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) &&
  1234. gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid))
  1235. return 1;
  1236. if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) &&
  1237. xfs_get_projid(ip) == eofb->eof_prid)
  1238. return 1;
  1239. return 0;
  1240. }
  1241. STATIC int
  1242. xfs_inode_free_eofblocks(
  1243. struct xfs_inode *ip,
  1244. int flags,
  1245. void *args)
  1246. {
  1247. int ret = 0;
  1248. struct xfs_eofblocks *eofb = args;
  1249. int match;
  1250. if (!xfs_can_free_eofblocks(ip, false)) {
  1251. /* inode could be preallocated or append-only */
  1252. trace_xfs_inode_free_eofblocks_invalid(ip);
  1253. xfs_inode_clear_eofblocks_tag(ip);
  1254. return 0;
  1255. }
  1256. /*
  1257. * If the mapping is dirty the operation can block and wait for some
  1258. * time. Unless we are waiting, skip it.
  1259. */
  1260. if (!(flags & SYNC_WAIT) &&
  1261. mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY))
  1262. return 0;
  1263. if (eofb) {
  1264. if (eofb->eof_flags & XFS_EOF_FLAGS_UNION)
  1265. match = xfs_inode_match_id_union(ip, eofb);
  1266. else
  1267. match = xfs_inode_match_id(ip, eofb);
  1268. if (!match)
  1269. return 0;
  1270. /* skip the inode if the file size is too small */
  1271. if (eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE &&
  1272. XFS_ISIZE(ip) < eofb->eof_min_file_size)
  1273. return 0;
  1274. }
  1275. /*
  1276. * If the caller is waiting, return -EAGAIN to keep the background
  1277. * scanner moving and revisit the inode in a subsequent pass.
  1278. */
  1279. if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
  1280. if (flags & SYNC_WAIT)
  1281. ret = -EAGAIN;
  1282. return ret;
  1283. }
  1284. ret = xfs_free_eofblocks(ip);
  1285. xfs_iunlock(ip, XFS_IOLOCK_EXCL);
  1286. return ret;
  1287. }
  1288. static int
  1289. __xfs_icache_free_eofblocks(
  1290. struct xfs_mount *mp,
  1291. struct xfs_eofblocks *eofb,
  1292. int (*execute)(struct xfs_inode *ip, int flags,
  1293. void *args),
  1294. int tag)
  1295. {
  1296. int flags = SYNC_TRYLOCK;
  1297. if (eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC))
  1298. flags = SYNC_WAIT;
  1299. return xfs_inode_ag_iterator_tag(mp, execute, flags,
  1300. eofb, tag);
  1301. }
  1302. int
  1303. xfs_icache_free_eofblocks(
  1304. struct xfs_mount *mp,
  1305. struct xfs_eofblocks *eofb)
  1306. {
  1307. return __xfs_icache_free_eofblocks(mp, eofb, xfs_inode_free_eofblocks,
  1308. XFS_ICI_EOFBLOCKS_TAG);
  1309. }
  1310. /*
  1311. * Run eofblocks scans on the quotas applicable to the inode. For inodes with
  1312. * multiple quotas, we don't know exactly which quota caused an allocation
  1313. * failure. We make a best effort by including each quota under low free space
  1314. * conditions (less than 1% free space) in the scan.
  1315. */
  1316. static int
  1317. __xfs_inode_free_quota_eofblocks(
  1318. struct xfs_inode *ip,
  1319. int (*execute)(struct xfs_mount *mp,
  1320. struct xfs_eofblocks *eofb))
  1321. {
  1322. int scan = 0;
  1323. struct xfs_eofblocks eofb = {0};
  1324. struct xfs_dquot *dq;
  1325. /*
  1326. * Run a sync scan to increase effectiveness and use the union filter to
  1327. * cover all applicable quotas in a single scan.
  1328. */
  1329. eofb.eof_flags = XFS_EOF_FLAGS_UNION|XFS_EOF_FLAGS_SYNC;
  1330. if (XFS_IS_UQUOTA_ENFORCED(ip->i_mount)) {
  1331. dq = xfs_inode_dquot(ip, XFS_DQ_USER);
  1332. if (dq && xfs_dquot_lowsp(dq)) {
  1333. eofb.eof_uid = VFS_I(ip)->i_uid;
  1334. eofb.eof_flags |= XFS_EOF_FLAGS_UID;
  1335. scan = 1;
  1336. }
  1337. }
  1338. if (XFS_IS_GQUOTA_ENFORCED(ip->i_mount)) {
  1339. dq = xfs_inode_dquot(ip, XFS_DQ_GROUP);
  1340. if (dq && xfs_dquot_lowsp(dq)) {
  1341. eofb.eof_gid = VFS_I(ip)->i_gid;
  1342. eofb.eof_flags |= XFS_EOF_FLAGS_GID;
  1343. scan = 1;
  1344. }
  1345. }
  1346. if (scan)
  1347. execute(ip->i_mount, &eofb);
  1348. return scan;
  1349. }
  1350. int
  1351. xfs_inode_free_quota_eofblocks(
  1352. struct xfs_inode *ip)
  1353. {
  1354. return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_eofblocks);
  1355. }
  1356. static void
  1357. __xfs_inode_set_eofblocks_tag(
  1358. xfs_inode_t *ip,
  1359. void (*execute)(struct xfs_mount *mp),
  1360. void (*set_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
  1361. int error, unsigned long caller_ip),
  1362. int tag)
  1363. {
  1364. struct xfs_mount *mp = ip->i_mount;
  1365. struct xfs_perag *pag;
  1366. int tagged;
  1367. /*
  1368. * Don't bother locking the AG and looking up in the radix trees
  1369. * if we already know that we have the tag set.
  1370. */
  1371. if (ip->i_flags & XFS_IEOFBLOCKS)
  1372. return;
  1373. spin_lock(&ip->i_flags_lock);
  1374. ip->i_flags |= XFS_IEOFBLOCKS;
  1375. spin_unlock(&ip->i_flags_lock);
  1376. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
  1377. spin_lock(&pag->pag_ici_lock);
  1378. tagged = radix_tree_tagged(&pag->pag_ici_root, tag);
  1379. radix_tree_tag_set(&pag->pag_ici_root,
  1380. XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
  1381. if (!tagged) {
  1382. /* propagate the eofblocks tag up into the perag radix tree */
  1383. spin_lock(&ip->i_mount->m_perag_lock);
  1384. radix_tree_tag_set(&ip->i_mount->m_perag_tree,
  1385. XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
  1386. tag);
  1387. spin_unlock(&ip->i_mount->m_perag_lock);
  1388. /* kick off background trimming */
  1389. execute(ip->i_mount);
  1390. set_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
  1391. }
  1392. spin_unlock(&pag->pag_ici_lock);
  1393. xfs_perag_put(pag);
  1394. }
  1395. void
  1396. xfs_inode_set_eofblocks_tag(
  1397. xfs_inode_t *ip)
  1398. {
  1399. trace_xfs_inode_set_eofblocks_tag(ip);
  1400. return __xfs_inode_set_eofblocks_tag(ip, xfs_queue_eofblocks,
  1401. trace_xfs_perag_set_eofblocks,
  1402. XFS_ICI_EOFBLOCKS_TAG);
  1403. }
  1404. static void
  1405. __xfs_inode_clear_eofblocks_tag(
  1406. xfs_inode_t *ip,
  1407. void (*clear_tp)(struct xfs_mount *mp, xfs_agnumber_t agno,
  1408. int error, unsigned long caller_ip),
  1409. int tag)
  1410. {
  1411. struct xfs_mount *mp = ip->i_mount;
  1412. struct xfs_perag *pag;
  1413. spin_lock(&ip->i_flags_lock);
  1414. ip->i_flags &= ~XFS_IEOFBLOCKS;
  1415. spin_unlock(&ip->i_flags_lock);
  1416. pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
  1417. spin_lock(&pag->pag_ici_lock);
  1418. radix_tree_tag_clear(&pag->pag_ici_root,
  1419. XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino), tag);
  1420. if (!radix_tree_tagged(&pag->pag_ici_root, tag)) {
  1421. /* clear the eofblocks tag from the perag radix tree */
  1422. spin_lock(&ip->i_mount->m_perag_lock);
  1423. radix_tree_tag_clear(&ip->i_mount->m_perag_tree,
  1424. XFS_INO_TO_AGNO(ip->i_mount, ip->i_ino),
  1425. tag);
  1426. spin_unlock(&ip->i_mount->m_perag_lock);
  1427. clear_tp(ip->i_mount, pag->pag_agno, -1, _RET_IP_);
  1428. }
  1429. spin_unlock(&pag->pag_ici_lock);
  1430. xfs_perag_put(pag);
  1431. }
  1432. void
  1433. xfs_inode_clear_eofblocks_tag(
  1434. xfs_inode_t *ip)
  1435. {
  1436. trace_xfs_inode_clear_eofblocks_tag(ip);
  1437. return __xfs_inode_clear_eofblocks_tag(ip,
  1438. trace_xfs_perag_clear_eofblocks, XFS_ICI_EOFBLOCKS_TAG);
  1439. }
  1440. /*
  1441. * Automatic CoW Reservation Freeing
  1442. *
  1443. * These functions automatically garbage collect leftover CoW reservations
  1444. * that were made on behalf of a cowextsize hint when we start to run out
  1445. * of quota or when the reservations sit around for too long. If the file
  1446. * has dirty pages or is undergoing writeback, its CoW reservations will
  1447. * be retained.
  1448. *
  1449. * The actual garbage collection piggybacks off the same code that runs
  1450. * the speculative EOF preallocation garbage collector.
  1451. */
  1452. STATIC int
  1453. xfs_inode_free_cowblocks(
  1454. struct xfs_inode *ip,
  1455. int flags,
  1456. void *args)
  1457. {
  1458. int ret;
  1459. struct xfs_eofblocks *eofb = args;
  1460. int match;
  1461. struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
  1462. /*
  1463. * Just clear the tag if we have an empty cow fork or none at all. It's
  1464. * possible the inode was fully unshared since it was originally tagged.
  1465. */
  1466. if (!xfs_is_reflink_inode(ip) || !ifp->if_bytes) {
  1467. trace_xfs_inode_free_cowblocks_invalid(ip);
  1468. xfs_inode_clear_cowblocks_tag(ip);
  1469. return 0;
  1470. }
  1471. /*
  1472. * If the mapping is dirty or under writeback we cannot touch the
  1473. * CoW fork. Leave it alone if we're in the midst of a directio.
  1474. */
  1475. if ((VFS_I(ip)->i_state & I_DIRTY_PAGES) ||
  1476. mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY) ||
  1477. mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_WRITEBACK) ||
  1478. atomic_read(&VFS_I(ip)->i_dio_count))
  1479. return 0;
  1480. if (eofb) {
  1481. if (eofb->eof_flags & XFS_EOF_FLAGS_UNION)
  1482. match = xfs_inode_match_id_union(ip, eofb);
  1483. else
  1484. match = xfs_inode_match_id(ip, eofb);
  1485. if (!match)
  1486. return 0;
  1487. /* skip the inode if the file size is too small */
  1488. if (eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE &&
  1489. XFS_ISIZE(ip) < eofb->eof_min_file_size)
  1490. return 0;
  1491. }
  1492. /* Free the CoW blocks */
  1493. xfs_ilock(ip, XFS_IOLOCK_EXCL);
  1494. xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
  1495. ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false);
  1496. xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
  1497. xfs_iunlock(ip, XFS_IOLOCK_EXCL);
  1498. return ret;
  1499. }
  1500. int
  1501. xfs_icache_free_cowblocks(
  1502. struct xfs_mount *mp,
  1503. struct xfs_eofblocks *eofb)
  1504. {
  1505. return __xfs_icache_free_eofblocks(mp, eofb, xfs_inode_free_cowblocks,
  1506. XFS_ICI_COWBLOCKS_TAG);
  1507. }
  1508. int
  1509. xfs_inode_free_quota_cowblocks(
  1510. struct xfs_inode *ip)
  1511. {
  1512. return __xfs_inode_free_quota_eofblocks(ip, xfs_icache_free_cowblocks);
  1513. }
  1514. void
  1515. xfs_inode_set_cowblocks_tag(
  1516. xfs_inode_t *ip)
  1517. {
  1518. trace_xfs_inode_set_cowblocks_tag(ip);
  1519. return __xfs_inode_set_eofblocks_tag(ip, xfs_queue_cowblocks,
  1520. trace_xfs_perag_set_cowblocks,
  1521. XFS_ICI_COWBLOCKS_TAG);
  1522. }
  1523. void
  1524. xfs_inode_clear_cowblocks_tag(
  1525. xfs_inode_t *ip)
  1526. {
  1527. trace_xfs_inode_clear_cowblocks_tag(ip);
  1528. return __xfs_inode_clear_eofblocks_tag(ip,
  1529. trace_xfs_perag_clear_cowblocks, XFS_ICI_COWBLOCKS_TAG);
  1530. }