file.c 83 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/fs.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/highmem.h>
  21. #include <linux/time.h>
  22. #include <linux/init.h>
  23. #include <linux/string.h>
  24. #include <linux/backing-dev.h>
  25. #include <linux/mpage.h>
  26. #include <linux/falloc.h>
  27. #include <linux/swap.h>
  28. #include <linux/writeback.h>
  29. #include <linux/compat.h>
  30. #include <linux/slab.h>
  31. #include <linux/btrfs.h>
  32. #include <linux/uio.h>
  33. #include "ctree.h"
  34. #include "disk-io.h"
  35. #include "transaction.h"
  36. #include "btrfs_inode.h"
  37. #include "print-tree.h"
  38. #include "tree-log.h"
  39. #include "locking.h"
  40. #include "volumes.h"
  41. #include "qgroup.h"
  42. #include "compression.h"
  43. static struct kmem_cache *btrfs_inode_defrag_cachep;
  44. /*
  45. * when auto defrag is enabled we
  46. * queue up these defrag structs to remember which
  47. * inodes need defragging passes
  48. */
  49. struct inode_defrag {
  50. struct rb_node rb_node;
  51. /* objectid */
  52. u64 ino;
  53. /*
  54. * transid where the defrag was added, we search for
  55. * extents newer than this
  56. */
  57. u64 transid;
  58. /* root objectid */
  59. u64 root;
  60. /* last offset we were able to defrag */
  61. u64 last_offset;
  62. /* if we've wrapped around back to zero once already */
  63. int cycled;
  64. };
  65. static int __compare_inode_defrag(struct inode_defrag *defrag1,
  66. struct inode_defrag *defrag2)
  67. {
  68. if (defrag1->root > defrag2->root)
  69. return 1;
  70. else if (defrag1->root < defrag2->root)
  71. return -1;
  72. else if (defrag1->ino > defrag2->ino)
  73. return 1;
  74. else if (defrag1->ino < defrag2->ino)
  75. return -1;
  76. else
  77. return 0;
  78. }
  79. /* pop a record for an inode into the defrag tree. The lock
  80. * must be held already
  81. *
  82. * If you're inserting a record for an older transid than an
  83. * existing record, the transid already in the tree is lowered
  84. *
  85. * If an existing record is found the defrag item you
  86. * pass in is freed
  87. */
  88. static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
  89. struct inode_defrag *defrag)
  90. {
  91. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  92. struct inode_defrag *entry;
  93. struct rb_node **p;
  94. struct rb_node *parent = NULL;
  95. int ret;
  96. p = &fs_info->defrag_inodes.rb_node;
  97. while (*p) {
  98. parent = *p;
  99. entry = rb_entry(parent, struct inode_defrag, rb_node);
  100. ret = __compare_inode_defrag(defrag, entry);
  101. if (ret < 0)
  102. p = &parent->rb_left;
  103. else if (ret > 0)
  104. p = &parent->rb_right;
  105. else {
  106. /* if we're reinserting an entry for
  107. * an old defrag run, make sure to
  108. * lower the transid of our existing record
  109. */
  110. if (defrag->transid < entry->transid)
  111. entry->transid = defrag->transid;
  112. if (defrag->last_offset > entry->last_offset)
  113. entry->last_offset = defrag->last_offset;
  114. return -EEXIST;
  115. }
  116. }
  117. set_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags);
  118. rb_link_node(&defrag->rb_node, parent, p);
  119. rb_insert_color(&defrag->rb_node, &fs_info->defrag_inodes);
  120. return 0;
  121. }
  122. static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)
  123. {
  124. if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
  125. return 0;
  126. if (btrfs_fs_closing(fs_info))
  127. return 0;
  128. return 1;
  129. }
  130. /*
  131. * insert a defrag record for this inode if auto defrag is
  132. * enabled
  133. */
  134. int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
  135. struct btrfs_inode *inode)
  136. {
  137. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  138. struct btrfs_root *root = inode->root;
  139. struct inode_defrag *defrag;
  140. u64 transid;
  141. int ret;
  142. if (!__need_auto_defrag(fs_info))
  143. return 0;
  144. if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))
  145. return 0;
  146. if (trans)
  147. transid = trans->transid;
  148. else
  149. transid = inode->root->last_trans;
  150. defrag = kmem_cache_zalloc(btrfs_inode_defrag_cachep, GFP_NOFS);
  151. if (!defrag)
  152. return -ENOMEM;
  153. defrag->ino = btrfs_ino(inode);
  154. defrag->transid = transid;
  155. defrag->root = root->root_key.objectid;
  156. spin_lock(&fs_info->defrag_inodes_lock);
  157. if (!test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags)) {
  158. /*
  159. * If we set IN_DEFRAG flag and evict the inode from memory,
  160. * and then re-read this inode, this new inode doesn't have
  161. * IN_DEFRAG flag. At the case, we may find the existed defrag.
  162. */
  163. ret = __btrfs_add_inode_defrag(inode, defrag);
  164. if (ret)
  165. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  166. } else {
  167. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  168. }
  169. spin_unlock(&fs_info->defrag_inodes_lock);
  170. return 0;
  171. }
  172. /*
  173. * Requeue the defrag object. If there is a defrag object that points to
  174. * the same inode in the tree, we will merge them together (by
  175. * __btrfs_add_inode_defrag()) and free the one that we want to requeue.
  176. */
  177. static void btrfs_requeue_inode_defrag(struct btrfs_inode *inode,
  178. struct inode_defrag *defrag)
  179. {
  180. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  181. int ret;
  182. if (!__need_auto_defrag(fs_info))
  183. goto out;
  184. /*
  185. * Here we don't check the IN_DEFRAG flag, because we need merge
  186. * them together.
  187. */
  188. spin_lock(&fs_info->defrag_inodes_lock);
  189. ret = __btrfs_add_inode_defrag(inode, defrag);
  190. spin_unlock(&fs_info->defrag_inodes_lock);
  191. if (ret)
  192. goto out;
  193. return;
  194. out:
  195. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  196. }
  197. /*
  198. * pick the defragable inode that we want, if it doesn't exist, we will get
  199. * the next one.
  200. */
  201. static struct inode_defrag *
  202. btrfs_pick_defrag_inode(struct btrfs_fs_info *fs_info, u64 root, u64 ino)
  203. {
  204. struct inode_defrag *entry = NULL;
  205. struct inode_defrag tmp;
  206. struct rb_node *p;
  207. struct rb_node *parent = NULL;
  208. int ret;
  209. tmp.ino = ino;
  210. tmp.root = root;
  211. spin_lock(&fs_info->defrag_inodes_lock);
  212. p = fs_info->defrag_inodes.rb_node;
  213. while (p) {
  214. parent = p;
  215. entry = rb_entry(parent, struct inode_defrag, rb_node);
  216. ret = __compare_inode_defrag(&tmp, entry);
  217. if (ret < 0)
  218. p = parent->rb_left;
  219. else if (ret > 0)
  220. p = parent->rb_right;
  221. else
  222. goto out;
  223. }
  224. if (parent && __compare_inode_defrag(&tmp, entry) > 0) {
  225. parent = rb_next(parent);
  226. if (parent)
  227. entry = rb_entry(parent, struct inode_defrag, rb_node);
  228. else
  229. entry = NULL;
  230. }
  231. out:
  232. if (entry)
  233. rb_erase(parent, &fs_info->defrag_inodes);
  234. spin_unlock(&fs_info->defrag_inodes_lock);
  235. return entry;
  236. }
  237. void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info)
  238. {
  239. struct inode_defrag *defrag;
  240. struct rb_node *node;
  241. spin_lock(&fs_info->defrag_inodes_lock);
  242. node = rb_first(&fs_info->defrag_inodes);
  243. while (node) {
  244. rb_erase(node, &fs_info->defrag_inodes);
  245. defrag = rb_entry(node, struct inode_defrag, rb_node);
  246. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  247. cond_resched_lock(&fs_info->defrag_inodes_lock);
  248. node = rb_first(&fs_info->defrag_inodes);
  249. }
  250. spin_unlock(&fs_info->defrag_inodes_lock);
  251. }
  252. #define BTRFS_DEFRAG_BATCH 1024
  253. static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info,
  254. struct inode_defrag *defrag)
  255. {
  256. struct btrfs_root *inode_root;
  257. struct inode *inode;
  258. struct btrfs_key key;
  259. struct btrfs_ioctl_defrag_range_args range;
  260. int num_defrag;
  261. int index;
  262. int ret;
  263. /* get the inode */
  264. key.objectid = defrag->root;
  265. key.type = BTRFS_ROOT_ITEM_KEY;
  266. key.offset = (u64)-1;
  267. index = srcu_read_lock(&fs_info->subvol_srcu);
  268. inode_root = btrfs_read_fs_root_no_name(fs_info, &key);
  269. if (IS_ERR(inode_root)) {
  270. ret = PTR_ERR(inode_root);
  271. goto cleanup;
  272. }
  273. key.objectid = defrag->ino;
  274. key.type = BTRFS_INODE_ITEM_KEY;
  275. key.offset = 0;
  276. inode = btrfs_iget(fs_info->sb, &key, inode_root, NULL);
  277. if (IS_ERR(inode)) {
  278. ret = PTR_ERR(inode);
  279. goto cleanup;
  280. }
  281. srcu_read_unlock(&fs_info->subvol_srcu, index);
  282. /* do a chunk of defrag */
  283. clear_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags);
  284. memset(&range, 0, sizeof(range));
  285. range.len = (u64)-1;
  286. range.start = defrag->last_offset;
  287. sb_start_write(fs_info->sb);
  288. num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
  289. BTRFS_DEFRAG_BATCH);
  290. sb_end_write(fs_info->sb);
  291. /*
  292. * if we filled the whole defrag batch, there
  293. * must be more work to do. Queue this defrag
  294. * again
  295. */
  296. if (num_defrag == BTRFS_DEFRAG_BATCH) {
  297. defrag->last_offset = range.start;
  298. btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
  299. } else if (defrag->last_offset && !defrag->cycled) {
  300. /*
  301. * we didn't fill our defrag batch, but
  302. * we didn't start at zero. Make sure we loop
  303. * around to the start of the file.
  304. */
  305. defrag->last_offset = 0;
  306. defrag->cycled = 1;
  307. btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
  308. } else {
  309. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  310. }
  311. iput(inode);
  312. return 0;
  313. cleanup:
  314. srcu_read_unlock(&fs_info->subvol_srcu, index);
  315. kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
  316. return ret;
  317. }
  318. /*
  319. * run through the list of inodes in the FS that need
  320. * defragging
  321. */
  322. int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
  323. {
  324. struct inode_defrag *defrag;
  325. u64 first_ino = 0;
  326. u64 root_objectid = 0;
  327. atomic_inc(&fs_info->defrag_running);
  328. while (1) {
  329. /* Pause the auto defragger. */
  330. if (test_bit(BTRFS_FS_STATE_REMOUNTING,
  331. &fs_info->fs_state))
  332. break;
  333. if (!__need_auto_defrag(fs_info))
  334. break;
  335. /* find an inode to defrag */
  336. defrag = btrfs_pick_defrag_inode(fs_info, root_objectid,
  337. first_ino);
  338. if (!defrag) {
  339. if (root_objectid || first_ino) {
  340. root_objectid = 0;
  341. first_ino = 0;
  342. continue;
  343. } else {
  344. break;
  345. }
  346. }
  347. first_ino = defrag->ino + 1;
  348. root_objectid = defrag->root;
  349. __btrfs_run_defrag_inode(fs_info, defrag);
  350. }
  351. atomic_dec(&fs_info->defrag_running);
  352. /*
  353. * during unmount, we use the transaction_wait queue to
  354. * wait for the defragger to stop
  355. */
  356. wake_up(&fs_info->transaction_wait);
  357. return 0;
  358. }
  359. /* simple helper to fault in pages and copy. This should go away
  360. * and be replaced with calls into generic code.
  361. */
  362. static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
  363. struct page **prepared_pages,
  364. struct iov_iter *i)
  365. {
  366. size_t copied = 0;
  367. size_t total_copied = 0;
  368. int pg = 0;
  369. int offset = pos & (PAGE_SIZE - 1);
  370. while (write_bytes > 0) {
  371. size_t count = min_t(size_t,
  372. PAGE_SIZE - offset, write_bytes);
  373. struct page *page = prepared_pages[pg];
  374. /*
  375. * Copy data from userspace to the current page
  376. */
  377. copied = iov_iter_copy_from_user_atomic(page, i, offset, count);
  378. /* Flush processor's dcache for this page */
  379. flush_dcache_page(page);
  380. /*
  381. * if we get a partial write, we can end up with
  382. * partially up to date pages. These add
  383. * a lot of complexity, so make sure they don't
  384. * happen by forcing this copy to be retried.
  385. *
  386. * The rest of the btrfs_file_write code will fall
  387. * back to page at a time copies after we return 0.
  388. */
  389. if (!PageUptodate(page) && copied < count)
  390. copied = 0;
  391. iov_iter_advance(i, copied);
  392. write_bytes -= copied;
  393. total_copied += copied;
  394. /* Return to btrfs_file_write_iter to fault page */
  395. if (unlikely(copied == 0))
  396. break;
  397. if (copied < PAGE_SIZE - offset) {
  398. offset += copied;
  399. } else {
  400. pg++;
  401. offset = 0;
  402. }
  403. }
  404. return total_copied;
  405. }
  406. /*
  407. * unlocks pages after btrfs_file_write is done with them
  408. */
  409. static void btrfs_drop_pages(struct page **pages, size_t num_pages)
  410. {
  411. size_t i;
  412. for (i = 0; i < num_pages; i++) {
  413. /* page checked is some magic around finding pages that
  414. * have been modified without going through btrfs_set_page_dirty
  415. * clear it here. There should be no need to mark the pages
  416. * accessed as prepare_pages should have marked them accessed
  417. * in prepare_pages via find_or_create_page()
  418. */
  419. ClearPageChecked(pages[i]);
  420. unlock_page(pages[i]);
  421. put_page(pages[i]);
  422. }
  423. }
  424. /*
  425. * after copy_from_user, pages need to be dirtied and we need to make
  426. * sure holes are created between the current EOF and the start of
  427. * any next extents (if required).
  428. *
  429. * this also makes the decision about creating an inline extent vs
  430. * doing real data extents, marking pages dirty and delalloc as required.
  431. */
  432. int btrfs_dirty_pages(struct inode *inode, struct page **pages,
  433. size_t num_pages, loff_t pos, size_t write_bytes,
  434. struct extent_state **cached)
  435. {
  436. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  437. int err = 0;
  438. int i;
  439. u64 num_bytes;
  440. u64 start_pos;
  441. u64 end_of_last_block;
  442. u64 end_pos = pos + write_bytes;
  443. loff_t isize = i_size_read(inode);
  444. start_pos = pos & ~((u64) fs_info->sectorsize - 1);
  445. num_bytes = round_up(write_bytes + pos - start_pos,
  446. fs_info->sectorsize);
  447. end_of_last_block = start_pos + num_bytes - 1;
  448. err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
  449. cached, 0);
  450. if (err)
  451. return err;
  452. for (i = 0; i < num_pages; i++) {
  453. struct page *p = pages[i];
  454. SetPageUptodate(p);
  455. ClearPageChecked(p);
  456. set_page_dirty(p);
  457. }
  458. /*
  459. * we've only changed i_size in ram, and we haven't updated
  460. * the disk i_size. There is no need to log the inode
  461. * at this time.
  462. */
  463. if (end_pos > isize)
  464. i_size_write(inode, end_pos);
  465. return 0;
  466. }
  467. /*
  468. * this drops all the extents in the cache that intersect the range
  469. * [start, end]. Existing extents are split as required.
  470. */
  471. void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end,
  472. int skip_pinned)
  473. {
  474. struct extent_map *em;
  475. struct extent_map *split = NULL;
  476. struct extent_map *split2 = NULL;
  477. struct extent_map_tree *em_tree = &inode->extent_tree;
  478. u64 len = end - start + 1;
  479. u64 gen;
  480. int ret;
  481. int testend = 1;
  482. unsigned long flags;
  483. int compressed = 0;
  484. bool modified;
  485. WARN_ON(end < start);
  486. if (end == (u64)-1) {
  487. len = (u64)-1;
  488. testend = 0;
  489. }
  490. while (1) {
  491. int no_splits = 0;
  492. modified = false;
  493. if (!split)
  494. split = alloc_extent_map();
  495. if (!split2)
  496. split2 = alloc_extent_map();
  497. if (!split || !split2)
  498. no_splits = 1;
  499. write_lock(&em_tree->lock);
  500. em = lookup_extent_mapping(em_tree, start, len);
  501. if (!em) {
  502. write_unlock(&em_tree->lock);
  503. break;
  504. }
  505. flags = em->flags;
  506. gen = em->generation;
  507. if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
  508. if (testend && em->start + em->len >= start + len) {
  509. free_extent_map(em);
  510. write_unlock(&em_tree->lock);
  511. break;
  512. }
  513. start = em->start + em->len;
  514. if (testend)
  515. len = start + len - (em->start + em->len);
  516. free_extent_map(em);
  517. write_unlock(&em_tree->lock);
  518. continue;
  519. }
  520. compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  521. clear_bit(EXTENT_FLAG_PINNED, &em->flags);
  522. clear_bit(EXTENT_FLAG_LOGGING, &flags);
  523. modified = !list_empty(&em->list);
  524. if (no_splits)
  525. goto next;
  526. if (em->start < start) {
  527. split->start = em->start;
  528. split->len = start - em->start;
  529. if (em->block_start < EXTENT_MAP_LAST_BYTE) {
  530. split->orig_start = em->orig_start;
  531. split->block_start = em->block_start;
  532. if (compressed)
  533. split->block_len = em->block_len;
  534. else
  535. split->block_len = split->len;
  536. split->orig_block_len = max(split->block_len,
  537. em->orig_block_len);
  538. split->ram_bytes = em->ram_bytes;
  539. } else {
  540. split->orig_start = split->start;
  541. split->block_len = 0;
  542. split->block_start = em->block_start;
  543. split->orig_block_len = 0;
  544. split->ram_bytes = split->len;
  545. }
  546. split->generation = gen;
  547. split->bdev = em->bdev;
  548. split->flags = flags;
  549. split->compress_type = em->compress_type;
  550. replace_extent_mapping(em_tree, em, split, modified);
  551. free_extent_map(split);
  552. split = split2;
  553. split2 = NULL;
  554. }
  555. if (testend && em->start + em->len > start + len) {
  556. u64 diff = start + len - em->start;
  557. split->start = start + len;
  558. split->len = em->start + em->len - (start + len);
  559. split->bdev = em->bdev;
  560. split->flags = flags;
  561. split->compress_type = em->compress_type;
  562. split->generation = gen;
  563. if (em->block_start < EXTENT_MAP_LAST_BYTE) {
  564. split->orig_block_len = max(em->block_len,
  565. em->orig_block_len);
  566. split->ram_bytes = em->ram_bytes;
  567. if (compressed) {
  568. split->block_len = em->block_len;
  569. split->block_start = em->block_start;
  570. split->orig_start = em->orig_start;
  571. } else {
  572. split->block_len = split->len;
  573. split->block_start = em->block_start
  574. + diff;
  575. split->orig_start = em->orig_start;
  576. }
  577. } else {
  578. split->ram_bytes = split->len;
  579. split->orig_start = split->start;
  580. split->block_len = 0;
  581. split->block_start = em->block_start;
  582. split->orig_block_len = 0;
  583. }
  584. if (extent_map_in_tree(em)) {
  585. replace_extent_mapping(em_tree, em, split,
  586. modified);
  587. } else {
  588. ret = add_extent_mapping(em_tree, split,
  589. modified);
  590. ASSERT(ret == 0); /* Logic error */
  591. }
  592. free_extent_map(split);
  593. split = NULL;
  594. }
  595. next:
  596. if (extent_map_in_tree(em))
  597. remove_extent_mapping(em_tree, em);
  598. write_unlock(&em_tree->lock);
  599. /* once for us */
  600. free_extent_map(em);
  601. /* once for the tree*/
  602. free_extent_map(em);
  603. }
  604. if (split)
  605. free_extent_map(split);
  606. if (split2)
  607. free_extent_map(split2);
  608. }
  609. /*
  610. * this is very complex, but the basic idea is to drop all extents
  611. * in the range start - end. hint_block is filled in with a block number
  612. * that would be a good hint to the block allocator for this file.
  613. *
  614. * If an extent intersects the range but is not entirely inside the range
  615. * it is either truncated or split. Anything entirely inside the range
  616. * is deleted from the tree.
  617. */
  618. int __btrfs_drop_extents(struct btrfs_trans_handle *trans,
  619. struct btrfs_root *root, struct inode *inode,
  620. struct btrfs_path *path, u64 start, u64 end,
  621. u64 *drop_end, int drop_cache,
  622. int replace_extent,
  623. u32 extent_item_size,
  624. int *key_inserted)
  625. {
  626. struct btrfs_fs_info *fs_info = root->fs_info;
  627. struct extent_buffer *leaf;
  628. struct btrfs_file_extent_item *fi;
  629. struct btrfs_key key;
  630. struct btrfs_key new_key;
  631. u64 ino = btrfs_ino(BTRFS_I(inode));
  632. u64 search_start = start;
  633. u64 disk_bytenr = 0;
  634. u64 num_bytes = 0;
  635. u64 extent_offset = 0;
  636. u64 extent_end = 0;
  637. u64 last_end = start;
  638. int del_nr = 0;
  639. int del_slot = 0;
  640. int extent_type;
  641. int recow;
  642. int ret;
  643. int modify_tree = -1;
  644. int update_refs;
  645. int found = 0;
  646. int leafs_visited = 0;
  647. if (drop_cache)
  648. btrfs_drop_extent_cache(BTRFS_I(inode), start, end - 1, 0);
  649. if (start >= BTRFS_I(inode)->disk_i_size && !replace_extent)
  650. modify_tree = 0;
  651. update_refs = (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
  652. root == fs_info->tree_root);
  653. while (1) {
  654. recow = 0;
  655. ret = btrfs_lookup_file_extent(trans, root, path, ino,
  656. search_start, modify_tree);
  657. if (ret < 0)
  658. break;
  659. if (ret > 0 && path->slots[0] > 0 && search_start == start) {
  660. leaf = path->nodes[0];
  661. btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
  662. if (key.objectid == ino &&
  663. key.type == BTRFS_EXTENT_DATA_KEY)
  664. path->slots[0]--;
  665. }
  666. ret = 0;
  667. leafs_visited++;
  668. next_slot:
  669. leaf = path->nodes[0];
  670. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  671. BUG_ON(del_nr > 0);
  672. ret = btrfs_next_leaf(root, path);
  673. if (ret < 0)
  674. break;
  675. if (ret > 0) {
  676. ret = 0;
  677. break;
  678. }
  679. leafs_visited++;
  680. leaf = path->nodes[0];
  681. recow = 1;
  682. }
  683. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  684. if (key.objectid > ino)
  685. break;
  686. if (WARN_ON_ONCE(key.objectid < ino) ||
  687. key.type < BTRFS_EXTENT_DATA_KEY) {
  688. ASSERT(del_nr == 0);
  689. path->slots[0]++;
  690. goto next_slot;
  691. }
  692. if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= end)
  693. break;
  694. fi = btrfs_item_ptr(leaf, path->slots[0],
  695. struct btrfs_file_extent_item);
  696. extent_type = btrfs_file_extent_type(leaf, fi);
  697. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  698. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  699. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  700. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  701. extent_offset = btrfs_file_extent_offset(leaf, fi);
  702. extent_end = key.offset +
  703. btrfs_file_extent_num_bytes(leaf, fi);
  704. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  705. extent_end = key.offset +
  706. btrfs_file_extent_inline_len(leaf,
  707. path->slots[0], fi);
  708. } else {
  709. /* can't happen */
  710. BUG();
  711. }
  712. /*
  713. * Don't skip extent items representing 0 byte lengths. They
  714. * used to be created (bug) if while punching holes we hit
  715. * -ENOSPC condition. So if we find one here, just ensure we
  716. * delete it, otherwise we would insert a new file extent item
  717. * with the same key (offset) as that 0 bytes length file
  718. * extent item in the call to setup_items_for_insert() later
  719. * in this function.
  720. */
  721. if (extent_end == key.offset && extent_end >= search_start) {
  722. last_end = extent_end;
  723. goto delete_extent_item;
  724. }
  725. if (extent_end <= search_start) {
  726. path->slots[0]++;
  727. goto next_slot;
  728. }
  729. found = 1;
  730. search_start = max(key.offset, start);
  731. if (recow || !modify_tree) {
  732. modify_tree = -1;
  733. btrfs_release_path(path);
  734. continue;
  735. }
  736. /*
  737. * | - range to drop - |
  738. * | -------- extent -------- |
  739. */
  740. if (start > key.offset && end < extent_end) {
  741. BUG_ON(del_nr > 0);
  742. if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  743. ret = -EOPNOTSUPP;
  744. break;
  745. }
  746. memcpy(&new_key, &key, sizeof(new_key));
  747. new_key.offset = start;
  748. ret = btrfs_duplicate_item(trans, root, path,
  749. &new_key);
  750. if (ret == -EAGAIN) {
  751. btrfs_release_path(path);
  752. continue;
  753. }
  754. if (ret < 0)
  755. break;
  756. leaf = path->nodes[0];
  757. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  758. struct btrfs_file_extent_item);
  759. btrfs_set_file_extent_num_bytes(leaf, fi,
  760. start - key.offset);
  761. fi = btrfs_item_ptr(leaf, path->slots[0],
  762. struct btrfs_file_extent_item);
  763. extent_offset += start - key.offset;
  764. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  765. btrfs_set_file_extent_num_bytes(leaf, fi,
  766. extent_end - start);
  767. btrfs_mark_buffer_dirty(leaf);
  768. if (update_refs && disk_bytenr > 0) {
  769. ret = btrfs_inc_extent_ref(trans, fs_info,
  770. disk_bytenr, num_bytes, 0,
  771. root->root_key.objectid,
  772. new_key.objectid,
  773. start - extent_offset);
  774. BUG_ON(ret); /* -ENOMEM */
  775. }
  776. key.offset = start;
  777. }
  778. /*
  779. * From here on out we will have actually dropped something, so
  780. * last_end can be updated.
  781. */
  782. last_end = extent_end;
  783. /*
  784. * | ---- range to drop ----- |
  785. * | -------- extent -------- |
  786. */
  787. if (start <= key.offset && end < extent_end) {
  788. if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  789. ret = -EOPNOTSUPP;
  790. break;
  791. }
  792. memcpy(&new_key, &key, sizeof(new_key));
  793. new_key.offset = end;
  794. btrfs_set_item_key_safe(fs_info, path, &new_key);
  795. extent_offset += end - key.offset;
  796. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  797. btrfs_set_file_extent_num_bytes(leaf, fi,
  798. extent_end - end);
  799. btrfs_mark_buffer_dirty(leaf);
  800. if (update_refs && disk_bytenr > 0)
  801. inode_sub_bytes(inode, end - key.offset);
  802. break;
  803. }
  804. search_start = extent_end;
  805. /*
  806. * | ---- range to drop ----- |
  807. * | -------- extent -------- |
  808. */
  809. if (start > key.offset && end >= extent_end) {
  810. BUG_ON(del_nr > 0);
  811. if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  812. ret = -EOPNOTSUPP;
  813. break;
  814. }
  815. btrfs_set_file_extent_num_bytes(leaf, fi,
  816. start - key.offset);
  817. btrfs_mark_buffer_dirty(leaf);
  818. if (update_refs && disk_bytenr > 0)
  819. inode_sub_bytes(inode, extent_end - start);
  820. if (end == extent_end)
  821. break;
  822. path->slots[0]++;
  823. goto next_slot;
  824. }
  825. /*
  826. * | ---- range to drop ----- |
  827. * | ------ extent ------ |
  828. */
  829. if (start <= key.offset && end >= extent_end) {
  830. delete_extent_item:
  831. if (del_nr == 0) {
  832. del_slot = path->slots[0];
  833. del_nr = 1;
  834. } else {
  835. BUG_ON(del_slot + del_nr != path->slots[0]);
  836. del_nr++;
  837. }
  838. if (update_refs &&
  839. extent_type == BTRFS_FILE_EXTENT_INLINE) {
  840. inode_sub_bytes(inode,
  841. extent_end - key.offset);
  842. extent_end = ALIGN(extent_end,
  843. fs_info->sectorsize);
  844. } else if (update_refs && disk_bytenr > 0) {
  845. ret = btrfs_free_extent(trans, fs_info,
  846. disk_bytenr, num_bytes, 0,
  847. root->root_key.objectid,
  848. key.objectid, key.offset -
  849. extent_offset);
  850. BUG_ON(ret); /* -ENOMEM */
  851. inode_sub_bytes(inode,
  852. extent_end - key.offset);
  853. }
  854. if (end == extent_end)
  855. break;
  856. if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
  857. path->slots[0]++;
  858. goto next_slot;
  859. }
  860. ret = btrfs_del_items(trans, root, path, del_slot,
  861. del_nr);
  862. if (ret) {
  863. btrfs_abort_transaction(trans, ret);
  864. break;
  865. }
  866. del_nr = 0;
  867. del_slot = 0;
  868. btrfs_release_path(path);
  869. continue;
  870. }
  871. BUG_ON(1);
  872. }
  873. if (!ret && del_nr > 0) {
  874. /*
  875. * Set path->slots[0] to first slot, so that after the delete
  876. * if items are move off from our leaf to its immediate left or
  877. * right neighbor leafs, we end up with a correct and adjusted
  878. * path->slots[0] for our insertion (if replace_extent != 0).
  879. */
  880. path->slots[0] = del_slot;
  881. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  882. if (ret)
  883. btrfs_abort_transaction(trans, ret);
  884. }
  885. leaf = path->nodes[0];
  886. /*
  887. * If btrfs_del_items() was called, it might have deleted a leaf, in
  888. * which case it unlocked our path, so check path->locks[0] matches a
  889. * write lock.
  890. */
  891. if (!ret && replace_extent && leafs_visited == 1 &&
  892. (path->locks[0] == BTRFS_WRITE_LOCK_BLOCKING ||
  893. path->locks[0] == BTRFS_WRITE_LOCK) &&
  894. btrfs_leaf_free_space(fs_info, leaf) >=
  895. sizeof(struct btrfs_item) + extent_item_size) {
  896. key.objectid = ino;
  897. key.type = BTRFS_EXTENT_DATA_KEY;
  898. key.offset = start;
  899. if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) {
  900. struct btrfs_key slot_key;
  901. btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]);
  902. if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)
  903. path->slots[0]++;
  904. }
  905. setup_items_for_insert(root, path, &key,
  906. &extent_item_size,
  907. extent_item_size,
  908. sizeof(struct btrfs_item) +
  909. extent_item_size, 1);
  910. *key_inserted = 1;
  911. }
  912. if (!replace_extent || !(*key_inserted))
  913. btrfs_release_path(path);
  914. if (drop_end)
  915. *drop_end = found ? min(end, last_end) : end;
  916. return ret;
  917. }
  918. int btrfs_drop_extents(struct btrfs_trans_handle *trans,
  919. struct btrfs_root *root, struct inode *inode, u64 start,
  920. u64 end, int drop_cache)
  921. {
  922. struct btrfs_path *path;
  923. int ret;
  924. path = btrfs_alloc_path();
  925. if (!path)
  926. return -ENOMEM;
  927. ret = __btrfs_drop_extents(trans, root, inode, path, start, end, NULL,
  928. drop_cache, 0, 0, NULL);
  929. btrfs_free_path(path);
  930. return ret;
  931. }
  932. static int extent_mergeable(struct extent_buffer *leaf, int slot,
  933. u64 objectid, u64 bytenr, u64 orig_offset,
  934. u64 *start, u64 *end)
  935. {
  936. struct btrfs_file_extent_item *fi;
  937. struct btrfs_key key;
  938. u64 extent_end;
  939. if (slot < 0 || slot >= btrfs_header_nritems(leaf))
  940. return 0;
  941. btrfs_item_key_to_cpu(leaf, &key, slot);
  942. if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
  943. return 0;
  944. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  945. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
  946. btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
  947. btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
  948. btrfs_file_extent_compression(leaf, fi) ||
  949. btrfs_file_extent_encryption(leaf, fi) ||
  950. btrfs_file_extent_other_encoding(leaf, fi))
  951. return 0;
  952. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  953. if ((*start && *start != key.offset) || (*end && *end != extent_end))
  954. return 0;
  955. *start = key.offset;
  956. *end = extent_end;
  957. return 1;
  958. }
  959. /*
  960. * Mark extent in the range start - end as written.
  961. *
  962. * This changes extent type from 'pre-allocated' to 'regular'. If only
  963. * part of extent is marked as written, the extent will be split into
  964. * two or three.
  965. */
  966. int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
  967. struct btrfs_inode *inode, u64 start, u64 end)
  968. {
  969. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  970. struct btrfs_root *root = inode->root;
  971. struct extent_buffer *leaf;
  972. struct btrfs_path *path;
  973. struct btrfs_file_extent_item *fi;
  974. struct btrfs_key key;
  975. struct btrfs_key new_key;
  976. u64 bytenr;
  977. u64 num_bytes;
  978. u64 extent_end;
  979. u64 orig_offset;
  980. u64 other_start;
  981. u64 other_end;
  982. u64 split;
  983. int del_nr = 0;
  984. int del_slot = 0;
  985. int recow;
  986. int ret;
  987. u64 ino = btrfs_ino(inode);
  988. path = btrfs_alloc_path();
  989. if (!path)
  990. return -ENOMEM;
  991. again:
  992. recow = 0;
  993. split = start;
  994. key.objectid = ino;
  995. key.type = BTRFS_EXTENT_DATA_KEY;
  996. key.offset = split;
  997. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  998. if (ret < 0)
  999. goto out;
  1000. if (ret > 0 && path->slots[0] > 0)
  1001. path->slots[0]--;
  1002. leaf = path->nodes[0];
  1003. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  1004. if (key.objectid != ino ||
  1005. key.type != BTRFS_EXTENT_DATA_KEY) {
  1006. ret = -EINVAL;
  1007. btrfs_abort_transaction(trans, ret);
  1008. goto out;
  1009. }
  1010. fi = btrfs_item_ptr(leaf, path->slots[0],
  1011. struct btrfs_file_extent_item);
  1012. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC) {
  1013. ret = -EINVAL;
  1014. btrfs_abort_transaction(trans, ret);
  1015. goto out;
  1016. }
  1017. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  1018. if (key.offset > start || extent_end < end) {
  1019. ret = -EINVAL;
  1020. btrfs_abort_transaction(trans, ret);
  1021. goto out;
  1022. }
  1023. bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  1024. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  1025. orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
  1026. memcpy(&new_key, &key, sizeof(new_key));
  1027. if (start == key.offset && end < extent_end) {
  1028. other_start = 0;
  1029. other_end = start;
  1030. if (extent_mergeable(leaf, path->slots[0] - 1,
  1031. ino, bytenr, orig_offset,
  1032. &other_start, &other_end)) {
  1033. new_key.offset = end;
  1034. btrfs_set_item_key_safe(fs_info, path, &new_key);
  1035. fi = btrfs_item_ptr(leaf, path->slots[0],
  1036. struct btrfs_file_extent_item);
  1037. btrfs_set_file_extent_generation(leaf, fi,
  1038. trans->transid);
  1039. btrfs_set_file_extent_num_bytes(leaf, fi,
  1040. extent_end - end);
  1041. btrfs_set_file_extent_offset(leaf, fi,
  1042. end - orig_offset);
  1043. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  1044. struct btrfs_file_extent_item);
  1045. btrfs_set_file_extent_generation(leaf, fi,
  1046. trans->transid);
  1047. btrfs_set_file_extent_num_bytes(leaf, fi,
  1048. end - other_start);
  1049. btrfs_mark_buffer_dirty(leaf);
  1050. goto out;
  1051. }
  1052. }
  1053. if (start > key.offset && end == extent_end) {
  1054. other_start = end;
  1055. other_end = 0;
  1056. if (extent_mergeable(leaf, path->slots[0] + 1,
  1057. ino, bytenr, orig_offset,
  1058. &other_start, &other_end)) {
  1059. fi = btrfs_item_ptr(leaf, path->slots[0],
  1060. struct btrfs_file_extent_item);
  1061. btrfs_set_file_extent_num_bytes(leaf, fi,
  1062. start - key.offset);
  1063. btrfs_set_file_extent_generation(leaf, fi,
  1064. trans->transid);
  1065. path->slots[0]++;
  1066. new_key.offset = start;
  1067. btrfs_set_item_key_safe(fs_info, path, &new_key);
  1068. fi = btrfs_item_ptr(leaf, path->slots[0],
  1069. struct btrfs_file_extent_item);
  1070. btrfs_set_file_extent_generation(leaf, fi,
  1071. trans->transid);
  1072. btrfs_set_file_extent_num_bytes(leaf, fi,
  1073. other_end - start);
  1074. btrfs_set_file_extent_offset(leaf, fi,
  1075. start - orig_offset);
  1076. btrfs_mark_buffer_dirty(leaf);
  1077. goto out;
  1078. }
  1079. }
  1080. while (start > key.offset || end < extent_end) {
  1081. if (key.offset == start)
  1082. split = end;
  1083. new_key.offset = split;
  1084. ret = btrfs_duplicate_item(trans, root, path, &new_key);
  1085. if (ret == -EAGAIN) {
  1086. btrfs_release_path(path);
  1087. goto again;
  1088. }
  1089. if (ret < 0) {
  1090. btrfs_abort_transaction(trans, ret);
  1091. goto out;
  1092. }
  1093. leaf = path->nodes[0];
  1094. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  1095. struct btrfs_file_extent_item);
  1096. btrfs_set_file_extent_generation(leaf, fi, trans->transid);
  1097. btrfs_set_file_extent_num_bytes(leaf, fi,
  1098. split - key.offset);
  1099. fi = btrfs_item_ptr(leaf, path->slots[0],
  1100. struct btrfs_file_extent_item);
  1101. btrfs_set_file_extent_generation(leaf, fi, trans->transid);
  1102. btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
  1103. btrfs_set_file_extent_num_bytes(leaf, fi,
  1104. extent_end - split);
  1105. btrfs_mark_buffer_dirty(leaf);
  1106. ret = btrfs_inc_extent_ref(trans, fs_info, bytenr, num_bytes,
  1107. 0, root->root_key.objectid,
  1108. ino, orig_offset);
  1109. if (ret) {
  1110. btrfs_abort_transaction(trans, ret);
  1111. goto out;
  1112. }
  1113. if (split == start) {
  1114. key.offset = start;
  1115. } else {
  1116. if (start != key.offset) {
  1117. ret = -EINVAL;
  1118. btrfs_abort_transaction(trans, ret);
  1119. goto out;
  1120. }
  1121. path->slots[0]--;
  1122. extent_end = end;
  1123. }
  1124. recow = 1;
  1125. }
  1126. other_start = end;
  1127. other_end = 0;
  1128. if (extent_mergeable(leaf, path->slots[0] + 1,
  1129. ino, bytenr, orig_offset,
  1130. &other_start, &other_end)) {
  1131. if (recow) {
  1132. btrfs_release_path(path);
  1133. goto again;
  1134. }
  1135. extent_end = other_end;
  1136. del_slot = path->slots[0] + 1;
  1137. del_nr++;
  1138. ret = btrfs_free_extent(trans, fs_info, bytenr, num_bytes,
  1139. 0, root->root_key.objectid,
  1140. ino, orig_offset);
  1141. if (ret) {
  1142. btrfs_abort_transaction(trans, ret);
  1143. goto out;
  1144. }
  1145. }
  1146. other_start = 0;
  1147. other_end = start;
  1148. if (extent_mergeable(leaf, path->slots[0] - 1,
  1149. ino, bytenr, orig_offset,
  1150. &other_start, &other_end)) {
  1151. if (recow) {
  1152. btrfs_release_path(path);
  1153. goto again;
  1154. }
  1155. key.offset = other_start;
  1156. del_slot = path->slots[0];
  1157. del_nr++;
  1158. ret = btrfs_free_extent(trans, fs_info, bytenr, num_bytes,
  1159. 0, root->root_key.objectid,
  1160. ino, orig_offset);
  1161. if (ret) {
  1162. btrfs_abort_transaction(trans, ret);
  1163. goto out;
  1164. }
  1165. }
  1166. if (del_nr == 0) {
  1167. fi = btrfs_item_ptr(leaf, path->slots[0],
  1168. struct btrfs_file_extent_item);
  1169. btrfs_set_file_extent_type(leaf, fi,
  1170. BTRFS_FILE_EXTENT_REG);
  1171. btrfs_set_file_extent_generation(leaf, fi, trans->transid);
  1172. btrfs_mark_buffer_dirty(leaf);
  1173. } else {
  1174. fi = btrfs_item_ptr(leaf, del_slot - 1,
  1175. struct btrfs_file_extent_item);
  1176. btrfs_set_file_extent_type(leaf, fi,
  1177. BTRFS_FILE_EXTENT_REG);
  1178. btrfs_set_file_extent_generation(leaf, fi, trans->transid);
  1179. btrfs_set_file_extent_num_bytes(leaf, fi,
  1180. extent_end - key.offset);
  1181. btrfs_mark_buffer_dirty(leaf);
  1182. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  1183. if (ret < 0) {
  1184. btrfs_abort_transaction(trans, ret);
  1185. goto out;
  1186. }
  1187. }
  1188. out:
  1189. btrfs_free_path(path);
  1190. return 0;
  1191. }
  1192. /*
  1193. * on error we return an unlocked page and the error value
  1194. * on success we return a locked page and 0
  1195. */
  1196. static int prepare_uptodate_page(struct inode *inode,
  1197. struct page *page, u64 pos,
  1198. bool force_uptodate)
  1199. {
  1200. int ret = 0;
  1201. if (((pos & (PAGE_SIZE - 1)) || force_uptodate) &&
  1202. !PageUptodate(page)) {
  1203. ret = btrfs_readpage(NULL, page);
  1204. if (ret)
  1205. return ret;
  1206. lock_page(page);
  1207. if (!PageUptodate(page)) {
  1208. unlock_page(page);
  1209. return -EIO;
  1210. }
  1211. if (page->mapping != inode->i_mapping) {
  1212. unlock_page(page);
  1213. return -EAGAIN;
  1214. }
  1215. }
  1216. return 0;
  1217. }
  1218. /*
  1219. * this just gets pages into the page cache and locks them down.
  1220. */
  1221. static noinline int prepare_pages(struct inode *inode, struct page **pages,
  1222. size_t num_pages, loff_t pos,
  1223. size_t write_bytes, bool force_uptodate)
  1224. {
  1225. int i;
  1226. unsigned long index = pos >> PAGE_SHIFT;
  1227. gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
  1228. int err = 0;
  1229. int faili;
  1230. for (i = 0; i < num_pages; i++) {
  1231. again:
  1232. pages[i] = find_or_create_page(inode->i_mapping, index + i,
  1233. mask | __GFP_WRITE);
  1234. if (!pages[i]) {
  1235. faili = i - 1;
  1236. err = -ENOMEM;
  1237. goto fail;
  1238. }
  1239. if (i == 0)
  1240. err = prepare_uptodate_page(inode, pages[i], pos,
  1241. force_uptodate);
  1242. if (!err && i == num_pages - 1)
  1243. err = prepare_uptodate_page(inode, pages[i],
  1244. pos + write_bytes, false);
  1245. if (err) {
  1246. put_page(pages[i]);
  1247. if (err == -EAGAIN) {
  1248. err = 0;
  1249. goto again;
  1250. }
  1251. faili = i - 1;
  1252. goto fail;
  1253. }
  1254. wait_on_page_writeback(pages[i]);
  1255. }
  1256. return 0;
  1257. fail:
  1258. while (faili >= 0) {
  1259. unlock_page(pages[faili]);
  1260. put_page(pages[faili]);
  1261. faili--;
  1262. }
  1263. return err;
  1264. }
  1265. static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode,
  1266. const u64 start,
  1267. const u64 len,
  1268. struct extent_state **cached_state)
  1269. {
  1270. u64 search_start = start;
  1271. const u64 end = start + len - 1;
  1272. while (search_start < end) {
  1273. const u64 search_len = end - search_start + 1;
  1274. struct extent_map *em;
  1275. u64 em_len;
  1276. int ret = 0;
  1277. em = btrfs_get_extent(inode, NULL, 0, search_start,
  1278. search_len, 0);
  1279. if (IS_ERR(em))
  1280. return PTR_ERR(em);
  1281. if (em->block_start != EXTENT_MAP_HOLE)
  1282. goto next;
  1283. em_len = em->len;
  1284. if (em->start < search_start)
  1285. em_len -= search_start - em->start;
  1286. if (em_len > search_len)
  1287. em_len = search_len;
  1288. ret = set_extent_bit(&inode->io_tree, search_start,
  1289. search_start + em_len - 1,
  1290. EXTENT_DELALLOC_NEW,
  1291. NULL, cached_state, GFP_NOFS);
  1292. next:
  1293. search_start = extent_map_end(em);
  1294. free_extent_map(em);
  1295. if (ret)
  1296. return ret;
  1297. }
  1298. return 0;
  1299. }
  1300. /*
  1301. * This function locks the extent and properly waits for data=ordered extents
  1302. * to finish before allowing the pages to be modified if need.
  1303. *
  1304. * The return value:
  1305. * 1 - the extent is locked
  1306. * 0 - the extent is not locked, and everything is OK
  1307. * -EAGAIN - need re-prepare the pages
  1308. * the other < 0 number - Something wrong happens
  1309. */
  1310. static noinline int
  1311. lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
  1312. size_t num_pages, loff_t pos,
  1313. size_t write_bytes,
  1314. u64 *lockstart, u64 *lockend,
  1315. struct extent_state **cached_state)
  1316. {
  1317. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  1318. u64 start_pos;
  1319. u64 last_pos;
  1320. int i;
  1321. int ret = 0;
  1322. start_pos = round_down(pos, fs_info->sectorsize);
  1323. last_pos = start_pos
  1324. + round_up(pos + write_bytes - start_pos,
  1325. fs_info->sectorsize) - 1;
  1326. if (start_pos < inode->vfs_inode.i_size ||
  1327. (inode->flags & BTRFS_INODE_PREALLOC)) {
  1328. struct btrfs_ordered_extent *ordered;
  1329. unsigned int clear_bits;
  1330. lock_extent_bits(&inode->io_tree, start_pos, last_pos,
  1331. cached_state);
  1332. ordered = btrfs_lookup_ordered_range(inode, start_pos,
  1333. last_pos - start_pos + 1);
  1334. if (ordered &&
  1335. ordered->file_offset + ordered->len > start_pos &&
  1336. ordered->file_offset <= last_pos) {
  1337. unlock_extent_cached(&inode->io_tree, start_pos,
  1338. last_pos, cached_state, GFP_NOFS);
  1339. for (i = 0; i < num_pages; i++) {
  1340. unlock_page(pages[i]);
  1341. put_page(pages[i]);
  1342. }
  1343. btrfs_start_ordered_extent(&inode->vfs_inode,
  1344. ordered, 1);
  1345. btrfs_put_ordered_extent(ordered);
  1346. return -EAGAIN;
  1347. }
  1348. if (ordered)
  1349. btrfs_put_ordered_extent(ordered);
  1350. ret = btrfs_find_new_delalloc_bytes(inode, start_pos,
  1351. last_pos - start_pos + 1,
  1352. cached_state);
  1353. clear_bits = EXTENT_DIRTY | EXTENT_DELALLOC |
  1354. EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG;
  1355. if (ret)
  1356. clear_bits |= EXTENT_DELALLOC_NEW | EXTENT_LOCKED;
  1357. clear_extent_bit(&inode->io_tree, start_pos,
  1358. last_pos, clear_bits,
  1359. (clear_bits & EXTENT_LOCKED) ? 1 : 0,
  1360. 0, cached_state, GFP_NOFS);
  1361. if (ret)
  1362. return ret;
  1363. *lockstart = start_pos;
  1364. *lockend = last_pos;
  1365. ret = 1;
  1366. }
  1367. for (i = 0; i < num_pages; i++) {
  1368. if (clear_page_dirty_for_io(pages[i]))
  1369. account_page_redirty(pages[i]);
  1370. set_page_extent_mapped(pages[i]);
  1371. WARN_ON(!PageLocked(pages[i]));
  1372. }
  1373. return ret;
  1374. }
  1375. static noinline int check_can_nocow(struct btrfs_inode *inode, loff_t pos,
  1376. size_t *write_bytes)
  1377. {
  1378. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  1379. struct btrfs_root *root = inode->root;
  1380. struct btrfs_ordered_extent *ordered;
  1381. u64 lockstart, lockend;
  1382. u64 num_bytes;
  1383. int ret;
  1384. ret = btrfs_start_write_no_snapshoting(root);
  1385. if (!ret)
  1386. return -ENOSPC;
  1387. lockstart = round_down(pos, fs_info->sectorsize);
  1388. lockend = round_up(pos + *write_bytes,
  1389. fs_info->sectorsize) - 1;
  1390. while (1) {
  1391. lock_extent(&inode->io_tree, lockstart, lockend);
  1392. ordered = btrfs_lookup_ordered_range(inode, lockstart,
  1393. lockend - lockstart + 1);
  1394. if (!ordered) {
  1395. break;
  1396. }
  1397. unlock_extent(&inode->io_tree, lockstart, lockend);
  1398. btrfs_start_ordered_extent(&inode->vfs_inode, ordered, 1);
  1399. btrfs_put_ordered_extent(ordered);
  1400. }
  1401. num_bytes = lockend - lockstart + 1;
  1402. ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes,
  1403. NULL, NULL, NULL);
  1404. if (ret <= 0) {
  1405. ret = 0;
  1406. btrfs_end_write_no_snapshoting(root);
  1407. } else {
  1408. *write_bytes = min_t(size_t, *write_bytes ,
  1409. num_bytes - pos + lockstart);
  1410. }
  1411. unlock_extent(&inode->io_tree, lockstart, lockend);
  1412. return ret;
  1413. }
  1414. static noinline ssize_t __btrfs_buffered_write(struct file *file,
  1415. struct iov_iter *i,
  1416. loff_t pos)
  1417. {
  1418. struct inode *inode = file_inode(file);
  1419. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1420. struct btrfs_root *root = BTRFS_I(inode)->root;
  1421. struct page **pages = NULL;
  1422. struct extent_state *cached_state = NULL;
  1423. u64 release_bytes = 0;
  1424. u64 lockstart;
  1425. u64 lockend;
  1426. size_t num_written = 0;
  1427. int nrptrs;
  1428. int ret = 0;
  1429. bool only_release_metadata = false;
  1430. bool force_page_uptodate = false;
  1431. bool need_unlock;
  1432. nrptrs = min(DIV_ROUND_UP(iov_iter_count(i), PAGE_SIZE),
  1433. PAGE_SIZE / (sizeof(struct page *)));
  1434. nrptrs = min(nrptrs, current->nr_dirtied_pause - current->nr_dirtied);
  1435. nrptrs = max(nrptrs, 8);
  1436. pages = kmalloc_array(nrptrs, sizeof(struct page *), GFP_KERNEL);
  1437. if (!pages)
  1438. return -ENOMEM;
  1439. while (iov_iter_count(i) > 0) {
  1440. size_t offset = pos & (PAGE_SIZE - 1);
  1441. size_t sector_offset;
  1442. size_t write_bytes = min(iov_iter_count(i),
  1443. nrptrs * (size_t)PAGE_SIZE -
  1444. offset);
  1445. size_t num_pages = DIV_ROUND_UP(write_bytes + offset,
  1446. PAGE_SIZE);
  1447. size_t reserve_bytes;
  1448. size_t dirty_pages;
  1449. size_t copied;
  1450. size_t dirty_sectors;
  1451. size_t num_sectors;
  1452. WARN_ON(num_pages > nrptrs);
  1453. /*
  1454. * Fault pages before locking them in prepare_pages
  1455. * to avoid recursive lock
  1456. */
  1457. if (unlikely(iov_iter_fault_in_readable(i, write_bytes))) {
  1458. ret = -EFAULT;
  1459. break;
  1460. }
  1461. sector_offset = pos & (fs_info->sectorsize - 1);
  1462. reserve_bytes = round_up(write_bytes + sector_offset,
  1463. fs_info->sectorsize);
  1464. ret = btrfs_check_data_free_space(inode, pos, write_bytes);
  1465. if (ret < 0) {
  1466. if ((BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW |
  1467. BTRFS_INODE_PREALLOC)) &&
  1468. check_can_nocow(BTRFS_I(inode), pos,
  1469. &write_bytes) > 0) {
  1470. /*
  1471. * For nodata cow case, no need to reserve
  1472. * data space.
  1473. */
  1474. only_release_metadata = true;
  1475. /*
  1476. * our prealloc extent may be smaller than
  1477. * write_bytes, so scale down.
  1478. */
  1479. num_pages = DIV_ROUND_UP(write_bytes + offset,
  1480. PAGE_SIZE);
  1481. reserve_bytes = round_up(write_bytes +
  1482. sector_offset,
  1483. fs_info->sectorsize);
  1484. } else {
  1485. break;
  1486. }
  1487. }
  1488. ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode),
  1489. reserve_bytes);
  1490. if (ret) {
  1491. if (!only_release_metadata)
  1492. btrfs_free_reserved_data_space(inode, pos,
  1493. write_bytes);
  1494. else
  1495. btrfs_end_write_no_snapshoting(root);
  1496. break;
  1497. }
  1498. release_bytes = reserve_bytes;
  1499. need_unlock = false;
  1500. again:
  1501. /*
  1502. * This is going to setup the pages array with the number of
  1503. * pages we want, so we don't really need to worry about the
  1504. * contents of pages from loop to loop
  1505. */
  1506. ret = prepare_pages(inode, pages, num_pages,
  1507. pos, write_bytes,
  1508. force_page_uptodate);
  1509. if (ret)
  1510. break;
  1511. ret = lock_and_cleanup_extent_if_need(BTRFS_I(inode), pages,
  1512. num_pages, pos, write_bytes, &lockstart,
  1513. &lockend, &cached_state);
  1514. if (ret < 0) {
  1515. if (ret == -EAGAIN)
  1516. goto again;
  1517. break;
  1518. } else if (ret > 0) {
  1519. need_unlock = true;
  1520. ret = 0;
  1521. }
  1522. copied = btrfs_copy_from_user(pos, write_bytes, pages, i);
  1523. num_sectors = BTRFS_BYTES_TO_BLKS(fs_info, reserve_bytes);
  1524. dirty_sectors = round_up(copied + sector_offset,
  1525. fs_info->sectorsize);
  1526. dirty_sectors = BTRFS_BYTES_TO_BLKS(fs_info, dirty_sectors);
  1527. /*
  1528. * if we have trouble faulting in the pages, fall
  1529. * back to one page at a time
  1530. */
  1531. if (copied < write_bytes)
  1532. nrptrs = 1;
  1533. if (copied == 0) {
  1534. force_page_uptodate = true;
  1535. dirty_sectors = 0;
  1536. dirty_pages = 0;
  1537. } else {
  1538. force_page_uptodate = false;
  1539. dirty_pages = DIV_ROUND_UP(copied + offset,
  1540. PAGE_SIZE);
  1541. }
  1542. /*
  1543. * If we had a short copy we need to release the excess delaloc
  1544. * bytes we reserved. We need to increment outstanding_extents
  1545. * because btrfs_delalloc_release_space and
  1546. * btrfs_delalloc_release_metadata will decrement it, but
  1547. * we still have an outstanding extent for the chunk we actually
  1548. * managed to copy.
  1549. */
  1550. if (num_sectors > dirty_sectors) {
  1551. /* release everything except the sectors we dirtied */
  1552. release_bytes -= dirty_sectors <<
  1553. fs_info->sb->s_blocksize_bits;
  1554. if (copied > 0) {
  1555. spin_lock(&BTRFS_I(inode)->lock);
  1556. BTRFS_I(inode)->outstanding_extents++;
  1557. spin_unlock(&BTRFS_I(inode)->lock);
  1558. }
  1559. if (only_release_metadata) {
  1560. btrfs_delalloc_release_metadata(BTRFS_I(inode),
  1561. release_bytes);
  1562. } else {
  1563. u64 __pos;
  1564. __pos = round_down(pos,
  1565. fs_info->sectorsize) +
  1566. (dirty_pages << PAGE_SHIFT);
  1567. btrfs_delalloc_release_space(inode, __pos,
  1568. release_bytes);
  1569. }
  1570. }
  1571. release_bytes = round_up(copied + sector_offset,
  1572. fs_info->sectorsize);
  1573. if (copied > 0)
  1574. ret = btrfs_dirty_pages(inode, pages, dirty_pages,
  1575. pos, copied, NULL);
  1576. if (need_unlock)
  1577. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1578. lockstart, lockend, &cached_state,
  1579. GFP_NOFS);
  1580. if (ret) {
  1581. btrfs_drop_pages(pages, num_pages);
  1582. break;
  1583. }
  1584. release_bytes = 0;
  1585. if (only_release_metadata)
  1586. btrfs_end_write_no_snapshoting(root);
  1587. if (only_release_metadata && copied > 0) {
  1588. lockstart = round_down(pos,
  1589. fs_info->sectorsize);
  1590. lockend = round_up(pos + copied,
  1591. fs_info->sectorsize) - 1;
  1592. set_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
  1593. lockend, EXTENT_NORESERVE, NULL,
  1594. NULL, GFP_NOFS);
  1595. only_release_metadata = false;
  1596. }
  1597. btrfs_drop_pages(pages, num_pages);
  1598. cond_resched();
  1599. balance_dirty_pages_ratelimited(inode->i_mapping);
  1600. if (dirty_pages < (fs_info->nodesize >> PAGE_SHIFT) + 1)
  1601. btrfs_btree_balance_dirty(fs_info);
  1602. pos += copied;
  1603. num_written += copied;
  1604. }
  1605. kfree(pages);
  1606. if (release_bytes) {
  1607. if (only_release_metadata) {
  1608. btrfs_end_write_no_snapshoting(root);
  1609. btrfs_delalloc_release_metadata(BTRFS_I(inode),
  1610. release_bytes);
  1611. } else {
  1612. btrfs_delalloc_release_space(inode,
  1613. round_down(pos, fs_info->sectorsize),
  1614. release_bytes);
  1615. }
  1616. }
  1617. return num_written ? num_written : ret;
  1618. }
  1619. static ssize_t __btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
  1620. {
  1621. struct file *file = iocb->ki_filp;
  1622. struct inode *inode = file_inode(file);
  1623. loff_t pos = iocb->ki_pos;
  1624. ssize_t written;
  1625. ssize_t written_buffered;
  1626. loff_t endbyte;
  1627. int err;
  1628. written = generic_file_direct_write(iocb, from);
  1629. if (written < 0 || !iov_iter_count(from))
  1630. return written;
  1631. pos += written;
  1632. written_buffered = __btrfs_buffered_write(file, from, pos);
  1633. if (written_buffered < 0) {
  1634. err = written_buffered;
  1635. goto out;
  1636. }
  1637. /*
  1638. * Ensure all data is persisted. We want the next direct IO read to be
  1639. * able to read what was just written.
  1640. */
  1641. endbyte = pos + written_buffered - 1;
  1642. err = btrfs_fdatawrite_range(inode, pos, endbyte);
  1643. if (err)
  1644. goto out;
  1645. err = filemap_fdatawait_range(inode->i_mapping, pos, endbyte);
  1646. if (err)
  1647. goto out;
  1648. written += written_buffered;
  1649. iocb->ki_pos = pos + written_buffered;
  1650. invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT,
  1651. endbyte >> PAGE_SHIFT);
  1652. out:
  1653. return written ? written : err;
  1654. }
  1655. static void update_time_for_write(struct inode *inode)
  1656. {
  1657. struct timespec now;
  1658. if (IS_NOCMTIME(inode))
  1659. return;
  1660. now = current_time(inode);
  1661. if (!timespec_equal(&inode->i_mtime, &now))
  1662. inode->i_mtime = now;
  1663. if (!timespec_equal(&inode->i_ctime, &now))
  1664. inode->i_ctime = now;
  1665. if (IS_I_VERSION(inode))
  1666. inode_inc_iversion(inode);
  1667. }
  1668. static ssize_t btrfs_file_write_iter(struct kiocb *iocb,
  1669. struct iov_iter *from)
  1670. {
  1671. struct file *file = iocb->ki_filp;
  1672. struct inode *inode = file_inode(file);
  1673. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1674. struct btrfs_root *root = BTRFS_I(inode)->root;
  1675. u64 start_pos;
  1676. u64 end_pos;
  1677. ssize_t num_written = 0;
  1678. bool sync = (file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host);
  1679. ssize_t err;
  1680. loff_t pos;
  1681. size_t count;
  1682. loff_t oldsize;
  1683. int clean_page = 0;
  1684. inode_lock(inode);
  1685. err = generic_write_checks(iocb, from);
  1686. if (err <= 0) {
  1687. inode_unlock(inode);
  1688. return err;
  1689. }
  1690. current->backing_dev_info = inode_to_bdi(inode);
  1691. err = file_remove_privs(file);
  1692. if (err) {
  1693. inode_unlock(inode);
  1694. goto out;
  1695. }
  1696. /*
  1697. * If BTRFS flips readonly due to some impossible error
  1698. * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
  1699. * although we have opened a file as writable, we have
  1700. * to stop this write operation to ensure FS consistency.
  1701. */
  1702. if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) {
  1703. inode_unlock(inode);
  1704. err = -EROFS;
  1705. goto out;
  1706. }
  1707. /*
  1708. * We reserve space for updating the inode when we reserve space for the
  1709. * extent we are going to write, so we will enospc out there. We don't
  1710. * need to start yet another transaction to update the inode as we will
  1711. * update the inode when we finish writing whatever data we write.
  1712. */
  1713. update_time_for_write(inode);
  1714. pos = iocb->ki_pos;
  1715. count = iov_iter_count(from);
  1716. start_pos = round_down(pos, fs_info->sectorsize);
  1717. oldsize = i_size_read(inode);
  1718. if (start_pos > oldsize) {
  1719. /* Expand hole size to cover write data, preventing empty gap */
  1720. end_pos = round_up(pos + count,
  1721. fs_info->sectorsize);
  1722. err = btrfs_cont_expand(inode, oldsize, end_pos);
  1723. if (err) {
  1724. inode_unlock(inode);
  1725. goto out;
  1726. }
  1727. if (start_pos > round_up(oldsize, fs_info->sectorsize))
  1728. clean_page = 1;
  1729. }
  1730. if (sync)
  1731. atomic_inc(&BTRFS_I(inode)->sync_writers);
  1732. if (iocb->ki_flags & IOCB_DIRECT) {
  1733. num_written = __btrfs_direct_write(iocb, from);
  1734. } else {
  1735. num_written = __btrfs_buffered_write(file, from, pos);
  1736. if (num_written > 0)
  1737. iocb->ki_pos = pos + num_written;
  1738. if (clean_page)
  1739. pagecache_isize_extended(inode, oldsize,
  1740. i_size_read(inode));
  1741. }
  1742. inode_unlock(inode);
  1743. /*
  1744. * We also have to set last_sub_trans to the current log transid,
  1745. * otherwise subsequent syncs to a file that's been synced in this
  1746. * transaction will appear to have already occurred.
  1747. */
  1748. spin_lock(&BTRFS_I(inode)->lock);
  1749. BTRFS_I(inode)->last_sub_trans = root->log_transid;
  1750. spin_unlock(&BTRFS_I(inode)->lock);
  1751. if (num_written > 0)
  1752. num_written = generic_write_sync(iocb, num_written);
  1753. if (sync)
  1754. atomic_dec(&BTRFS_I(inode)->sync_writers);
  1755. out:
  1756. current->backing_dev_info = NULL;
  1757. return num_written ? num_written : err;
  1758. }
  1759. int btrfs_release_file(struct inode *inode, struct file *filp)
  1760. {
  1761. if (filp->private_data)
  1762. btrfs_ioctl_trans_end(filp);
  1763. /*
  1764. * ordered_data_close is set by settattr when we are about to truncate
  1765. * a file from a non-zero size to a zero size. This tries to
  1766. * flush down new bytes that may have been written if the
  1767. * application were using truncate to replace a file in place.
  1768. */
  1769. if (test_and_clear_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
  1770. &BTRFS_I(inode)->runtime_flags))
  1771. filemap_flush(inode->i_mapping);
  1772. return 0;
  1773. }
  1774. static int start_ordered_ops(struct inode *inode, loff_t start, loff_t end)
  1775. {
  1776. int ret;
  1777. atomic_inc(&BTRFS_I(inode)->sync_writers);
  1778. ret = btrfs_fdatawrite_range(inode, start, end);
  1779. atomic_dec(&BTRFS_I(inode)->sync_writers);
  1780. return ret;
  1781. }
  1782. /*
  1783. * fsync call for both files and directories. This logs the inode into
  1784. * the tree log instead of forcing full commits whenever possible.
  1785. *
  1786. * It needs to call filemap_fdatawait so that all ordered extent updates are
  1787. * in the metadata btree are up to date for copying to the log.
  1788. *
  1789. * It drops the inode mutex before doing the tree log commit. This is an
  1790. * important optimization for directories because holding the mutex prevents
  1791. * new operations on the dir while we write to disk.
  1792. */
  1793. int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
  1794. {
  1795. struct dentry *dentry = file_dentry(file);
  1796. struct inode *inode = d_inode(dentry);
  1797. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1798. struct btrfs_root *root = BTRFS_I(inode)->root;
  1799. struct btrfs_trans_handle *trans;
  1800. struct btrfs_log_ctx ctx;
  1801. int ret = 0;
  1802. bool full_sync = 0;
  1803. u64 len;
  1804. /*
  1805. * The range length can be represented by u64, we have to do the typecasts
  1806. * to avoid signed overflow if it's [0, LLONG_MAX] eg. from fsync()
  1807. */
  1808. len = (u64)end - (u64)start + 1;
  1809. trace_btrfs_sync_file(file, datasync);
  1810. /*
  1811. * We write the dirty pages in the range and wait until they complete
  1812. * out of the ->i_mutex. If so, we can flush the dirty pages by
  1813. * multi-task, and make the performance up. See
  1814. * btrfs_wait_ordered_range for an explanation of the ASYNC check.
  1815. */
  1816. ret = start_ordered_ops(inode, start, end);
  1817. if (ret)
  1818. return ret;
  1819. inode_lock(inode);
  1820. atomic_inc(&root->log_batch);
  1821. full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  1822. &BTRFS_I(inode)->runtime_flags);
  1823. /*
  1824. * We might have have had more pages made dirty after calling
  1825. * start_ordered_ops and before acquiring the inode's i_mutex.
  1826. */
  1827. if (full_sync) {
  1828. /*
  1829. * For a full sync, we need to make sure any ordered operations
  1830. * start and finish before we start logging the inode, so that
  1831. * all extents are persisted and the respective file extent
  1832. * items are in the fs/subvol btree.
  1833. */
  1834. ret = btrfs_wait_ordered_range(inode, start, len);
  1835. } else {
  1836. /*
  1837. * Start any new ordered operations before starting to log the
  1838. * inode. We will wait for them to finish in btrfs_sync_log().
  1839. *
  1840. * Right before acquiring the inode's mutex, we might have new
  1841. * writes dirtying pages, which won't immediately start the
  1842. * respective ordered operations - that is done through the
  1843. * fill_delalloc callbacks invoked from the writepage and
  1844. * writepages address space operations. So make sure we start
  1845. * all ordered operations before starting to log our inode. Not
  1846. * doing this means that while logging the inode, writeback
  1847. * could start and invoke writepage/writepages, which would call
  1848. * the fill_delalloc callbacks (cow_file_range,
  1849. * submit_compressed_extents). These callbacks add first an
  1850. * extent map to the modified list of extents and then create
  1851. * the respective ordered operation, which means in
  1852. * tree-log.c:btrfs_log_inode() we might capture all existing
  1853. * ordered operations (with btrfs_get_logged_extents()) before
  1854. * the fill_delalloc callback adds its ordered operation, and by
  1855. * the time we visit the modified list of extent maps (with
  1856. * btrfs_log_changed_extents()), we see and process the extent
  1857. * map they created. We then use the extent map to construct a
  1858. * file extent item for logging without waiting for the
  1859. * respective ordered operation to finish - this file extent
  1860. * item points to a disk location that might not have yet been
  1861. * written to, containing random data - so after a crash a log
  1862. * replay will make our inode have file extent items that point
  1863. * to disk locations containing invalid data, as we returned
  1864. * success to userspace without waiting for the respective
  1865. * ordered operation to finish, because it wasn't captured by
  1866. * btrfs_get_logged_extents().
  1867. */
  1868. ret = start_ordered_ops(inode, start, end);
  1869. }
  1870. if (ret) {
  1871. inode_unlock(inode);
  1872. goto out;
  1873. }
  1874. atomic_inc(&root->log_batch);
  1875. /*
  1876. * If the last transaction that changed this file was before the current
  1877. * transaction and we have the full sync flag set in our inode, we can
  1878. * bail out now without any syncing.
  1879. *
  1880. * Note that we can't bail out if the full sync flag isn't set. This is
  1881. * because when the full sync flag is set we start all ordered extents
  1882. * and wait for them to fully complete - when they complete they update
  1883. * the inode's last_trans field through:
  1884. *
  1885. * btrfs_finish_ordered_io() ->
  1886. * btrfs_update_inode_fallback() ->
  1887. * btrfs_update_inode() ->
  1888. * btrfs_set_inode_last_trans()
  1889. *
  1890. * So we are sure that last_trans is up to date and can do this check to
  1891. * bail out safely. For the fast path, when the full sync flag is not
  1892. * set in our inode, we can not do it because we start only our ordered
  1893. * extents and don't wait for them to complete (that is when
  1894. * btrfs_finish_ordered_io runs), so here at this point their last_trans
  1895. * value might be less than or equals to fs_info->last_trans_committed,
  1896. * and setting a speculative last_trans for an inode when a buffered
  1897. * write is made (such as fs_info->generation + 1 for example) would not
  1898. * be reliable since after setting the value and before fsync is called
  1899. * any number of transactions can start and commit (transaction kthread
  1900. * commits the current transaction periodically), and a transaction
  1901. * commit does not start nor waits for ordered extents to complete.
  1902. */
  1903. smp_mb();
  1904. if (btrfs_inode_in_log(BTRFS_I(inode), fs_info->generation) ||
  1905. (full_sync && BTRFS_I(inode)->last_trans <=
  1906. fs_info->last_trans_committed) ||
  1907. (!btrfs_have_ordered_extents_in_range(inode, start, len) &&
  1908. BTRFS_I(inode)->last_trans
  1909. <= fs_info->last_trans_committed)) {
  1910. /*
  1911. * We've had everything committed since the last time we were
  1912. * modified so clear this flag in case it was set for whatever
  1913. * reason, it's no longer relevant.
  1914. */
  1915. clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  1916. &BTRFS_I(inode)->runtime_flags);
  1917. /*
  1918. * An ordered extent might have started before and completed
  1919. * already with io errors, in which case the inode was not
  1920. * updated and we end up here. So check the inode's mapping
  1921. * flags for any errors that might have happened while doing
  1922. * writeback of file data.
  1923. */
  1924. ret = filemap_check_errors(inode->i_mapping);
  1925. inode_unlock(inode);
  1926. goto out;
  1927. }
  1928. /*
  1929. * ok we haven't committed the transaction yet, lets do a commit
  1930. */
  1931. if (file->private_data)
  1932. btrfs_ioctl_trans_end(file);
  1933. /*
  1934. * We use start here because we will need to wait on the IO to complete
  1935. * in btrfs_sync_log, which could require joining a transaction (for
  1936. * example checking cross references in the nocow path). If we use join
  1937. * here we could get into a situation where we're waiting on IO to
  1938. * happen that is blocked on a transaction trying to commit. With start
  1939. * we inc the extwriter counter, so we wait for all extwriters to exit
  1940. * before we start blocking join'ers. This comment is to keep somebody
  1941. * from thinking they are super smart and changing this to
  1942. * btrfs_join_transaction *cough*Josef*cough*.
  1943. */
  1944. trans = btrfs_start_transaction(root, 0);
  1945. if (IS_ERR(trans)) {
  1946. ret = PTR_ERR(trans);
  1947. inode_unlock(inode);
  1948. goto out;
  1949. }
  1950. trans->sync = true;
  1951. btrfs_init_log_ctx(&ctx, inode);
  1952. ret = btrfs_log_dentry_safe(trans, root, dentry, start, end, &ctx);
  1953. if (ret < 0) {
  1954. /* Fallthrough and commit/free transaction. */
  1955. ret = 1;
  1956. }
  1957. /* we've logged all the items and now have a consistent
  1958. * version of the file in the log. It is possible that
  1959. * someone will come in and modify the file, but that's
  1960. * fine because the log is consistent on disk, and we
  1961. * have references to all of the file's extents
  1962. *
  1963. * It is possible that someone will come in and log the
  1964. * file again, but that will end up using the synchronization
  1965. * inside btrfs_sync_log to keep things safe.
  1966. */
  1967. inode_unlock(inode);
  1968. /*
  1969. * If any of the ordered extents had an error, just return it to user
  1970. * space, so that the application knows some writes didn't succeed and
  1971. * can take proper action (retry for e.g.). Blindly committing the
  1972. * transaction in this case, would fool userspace that everything was
  1973. * successful. And we also want to make sure our log doesn't contain
  1974. * file extent items pointing to extents that weren't fully written to -
  1975. * just like in the non fast fsync path, where we check for the ordered
  1976. * operation's error flag before writing to the log tree and return -EIO
  1977. * if any of them had this flag set (btrfs_wait_ordered_range) -
  1978. * therefore we need to check for errors in the ordered operations,
  1979. * which are indicated by ctx.io_err.
  1980. */
  1981. if (ctx.io_err) {
  1982. btrfs_end_transaction(trans);
  1983. ret = ctx.io_err;
  1984. goto out;
  1985. }
  1986. if (ret != BTRFS_NO_LOG_SYNC) {
  1987. if (!ret) {
  1988. ret = btrfs_sync_log(trans, root, &ctx);
  1989. if (!ret) {
  1990. ret = btrfs_end_transaction(trans);
  1991. goto out;
  1992. }
  1993. }
  1994. if (!full_sync) {
  1995. ret = btrfs_wait_ordered_range(inode, start, len);
  1996. if (ret) {
  1997. btrfs_end_transaction(trans);
  1998. goto out;
  1999. }
  2000. }
  2001. ret = btrfs_commit_transaction(trans);
  2002. } else {
  2003. ret = btrfs_end_transaction(trans);
  2004. }
  2005. out:
  2006. return ret > 0 ? -EIO : ret;
  2007. }
  2008. static const struct vm_operations_struct btrfs_file_vm_ops = {
  2009. .fault = filemap_fault,
  2010. .map_pages = filemap_map_pages,
  2011. .page_mkwrite = btrfs_page_mkwrite,
  2012. };
  2013. static int btrfs_file_mmap(struct file *filp, struct vm_area_struct *vma)
  2014. {
  2015. struct address_space *mapping = filp->f_mapping;
  2016. if (!mapping->a_ops->readpage)
  2017. return -ENOEXEC;
  2018. file_accessed(filp);
  2019. vma->vm_ops = &btrfs_file_vm_ops;
  2020. return 0;
  2021. }
  2022. static int hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf,
  2023. int slot, u64 start, u64 end)
  2024. {
  2025. struct btrfs_file_extent_item *fi;
  2026. struct btrfs_key key;
  2027. if (slot < 0 || slot >= btrfs_header_nritems(leaf))
  2028. return 0;
  2029. btrfs_item_key_to_cpu(leaf, &key, slot);
  2030. if (key.objectid != btrfs_ino(inode) ||
  2031. key.type != BTRFS_EXTENT_DATA_KEY)
  2032. return 0;
  2033. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  2034. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
  2035. return 0;
  2036. if (btrfs_file_extent_disk_bytenr(leaf, fi))
  2037. return 0;
  2038. if (key.offset == end)
  2039. return 1;
  2040. if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start)
  2041. return 1;
  2042. return 0;
  2043. }
  2044. static int fill_holes(struct btrfs_trans_handle *trans,
  2045. struct btrfs_inode *inode,
  2046. struct btrfs_path *path, u64 offset, u64 end)
  2047. {
  2048. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  2049. struct btrfs_root *root = inode->root;
  2050. struct extent_buffer *leaf;
  2051. struct btrfs_file_extent_item *fi;
  2052. struct extent_map *hole_em;
  2053. struct extent_map_tree *em_tree = &inode->extent_tree;
  2054. struct btrfs_key key;
  2055. int ret;
  2056. if (btrfs_fs_incompat(fs_info, NO_HOLES))
  2057. goto out;
  2058. key.objectid = btrfs_ino(inode);
  2059. key.type = BTRFS_EXTENT_DATA_KEY;
  2060. key.offset = offset;
  2061. ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
  2062. if (ret <= 0) {
  2063. /*
  2064. * We should have dropped this offset, so if we find it then
  2065. * something has gone horribly wrong.
  2066. */
  2067. if (ret == 0)
  2068. ret = -EINVAL;
  2069. return ret;
  2070. }
  2071. leaf = path->nodes[0];
  2072. if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) {
  2073. u64 num_bytes;
  2074. path->slots[0]--;
  2075. fi = btrfs_item_ptr(leaf, path->slots[0],
  2076. struct btrfs_file_extent_item);
  2077. num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
  2078. end - offset;
  2079. btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
  2080. btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
  2081. btrfs_set_file_extent_offset(leaf, fi, 0);
  2082. btrfs_mark_buffer_dirty(leaf);
  2083. goto out;
  2084. }
  2085. if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) {
  2086. u64 num_bytes;
  2087. key.offset = offset;
  2088. btrfs_set_item_key_safe(fs_info, path, &key);
  2089. fi = btrfs_item_ptr(leaf, path->slots[0],
  2090. struct btrfs_file_extent_item);
  2091. num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end -
  2092. offset;
  2093. btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
  2094. btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
  2095. btrfs_set_file_extent_offset(leaf, fi, 0);
  2096. btrfs_mark_buffer_dirty(leaf);
  2097. goto out;
  2098. }
  2099. btrfs_release_path(path);
  2100. ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode),
  2101. offset, 0, 0, end - offset, 0, end - offset, 0, 0, 0);
  2102. if (ret)
  2103. return ret;
  2104. out:
  2105. btrfs_release_path(path);
  2106. hole_em = alloc_extent_map();
  2107. if (!hole_em) {
  2108. btrfs_drop_extent_cache(inode, offset, end - 1, 0);
  2109. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
  2110. } else {
  2111. hole_em->start = offset;
  2112. hole_em->len = end - offset;
  2113. hole_em->ram_bytes = hole_em->len;
  2114. hole_em->orig_start = offset;
  2115. hole_em->block_start = EXTENT_MAP_HOLE;
  2116. hole_em->block_len = 0;
  2117. hole_em->orig_block_len = 0;
  2118. hole_em->bdev = fs_info->fs_devices->latest_bdev;
  2119. hole_em->compress_type = BTRFS_COMPRESS_NONE;
  2120. hole_em->generation = trans->transid;
  2121. do {
  2122. btrfs_drop_extent_cache(inode, offset, end - 1, 0);
  2123. write_lock(&em_tree->lock);
  2124. ret = add_extent_mapping(em_tree, hole_em, 1);
  2125. write_unlock(&em_tree->lock);
  2126. } while (ret == -EEXIST);
  2127. free_extent_map(hole_em);
  2128. if (ret)
  2129. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  2130. &inode->runtime_flags);
  2131. }
  2132. return 0;
  2133. }
  2134. /*
  2135. * Find a hole extent on given inode and change start/len to the end of hole
  2136. * extent.(hole/vacuum extent whose em->start <= start &&
  2137. * em->start + em->len > start)
  2138. * When a hole extent is found, return 1 and modify start/len.
  2139. */
  2140. static int find_first_non_hole(struct inode *inode, u64 *start, u64 *len)
  2141. {
  2142. struct extent_map *em;
  2143. int ret = 0;
  2144. em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, *start, *len, 0);
  2145. if (IS_ERR(em))
  2146. return PTR_ERR(em);
  2147. /* Hole or vacuum extent(only exists in no-hole mode) */
  2148. if (em->block_start == EXTENT_MAP_HOLE) {
  2149. ret = 1;
  2150. *len = em->start + em->len > *start + *len ?
  2151. 0 : *start + *len - em->start - em->len;
  2152. *start = em->start + em->len;
  2153. }
  2154. free_extent_map(em);
  2155. return ret;
  2156. }
  2157. static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
  2158. {
  2159. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2160. struct btrfs_root *root = BTRFS_I(inode)->root;
  2161. struct extent_state *cached_state = NULL;
  2162. struct btrfs_path *path;
  2163. struct btrfs_block_rsv *rsv;
  2164. struct btrfs_trans_handle *trans;
  2165. u64 lockstart;
  2166. u64 lockend;
  2167. u64 tail_start;
  2168. u64 tail_len;
  2169. u64 orig_start = offset;
  2170. u64 cur_offset;
  2171. u64 min_size = btrfs_calc_trans_metadata_size(fs_info, 1);
  2172. u64 drop_end;
  2173. int ret = 0;
  2174. int err = 0;
  2175. unsigned int rsv_count;
  2176. bool same_block;
  2177. bool no_holes = btrfs_fs_incompat(fs_info, NO_HOLES);
  2178. u64 ino_size;
  2179. bool truncated_block = false;
  2180. bool updated_inode = false;
  2181. ret = btrfs_wait_ordered_range(inode, offset, len);
  2182. if (ret)
  2183. return ret;
  2184. inode_lock(inode);
  2185. ino_size = round_up(inode->i_size, fs_info->sectorsize);
  2186. ret = find_first_non_hole(inode, &offset, &len);
  2187. if (ret < 0)
  2188. goto out_only_mutex;
  2189. if (ret && !len) {
  2190. /* Already in a large hole */
  2191. ret = 0;
  2192. goto out_only_mutex;
  2193. }
  2194. lockstart = round_up(offset, btrfs_inode_sectorsize(inode));
  2195. lockend = round_down(offset + len,
  2196. btrfs_inode_sectorsize(inode)) - 1;
  2197. same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset))
  2198. == (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1));
  2199. /*
  2200. * We needn't truncate any block which is beyond the end of the file
  2201. * because we are sure there is no data there.
  2202. */
  2203. /*
  2204. * Only do this if we are in the same block and we aren't doing the
  2205. * entire block.
  2206. */
  2207. if (same_block && len < fs_info->sectorsize) {
  2208. if (offset < ino_size) {
  2209. truncated_block = true;
  2210. ret = btrfs_truncate_block(inode, offset, len, 0);
  2211. } else {
  2212. ret = 0;
  2213. }
  2214. goto out_only_mutex;
  2215. }
  2216. /* zero back part of the first block */
  2217. if (offset < ino_size) {
  2218. truncated_block = true;
  2219. ret = btrfs_truncate_block(inode, offset, 0, 0);
  2220. if (ret) {
  2221. inode_unlock(inode);
  2222. return ret;
  2223. }
  2224. }
  2225. /* Check the aligned pages after the first unaligned page,
  2226. * if offset != orig_start, which means the first unaligned page
  2227. * including several following pages are already in holes,
  2228. * the extra check can be skipped */
  2229. if (offset == orig_start) {
  2230. /* after truncate page, check hole again */
  2231. len = offset + len - lockstart;
  2232. offset = lockstart;
  2233. ret = find_first_non_hole(inode, &offset, &len);
  2234. if (ret < 0)
  2235. goto out_only_mutex;
  2236. if (ret && !len) {
  2237. ret = 0;
  2238. goto out_only_mutex;
  2239. }
  2240. lockstart = offset;
  2241. }
  2242. /* Check the tail unaligned part is in a hole */
  2243. tail_start = lockend + 1;
  2244. tail_len = offset + len - tail_start;
  2245. if (tail_len) {
  2246. ret = find_first_non_hole(inode, &tail_start, &tail_len);
  2247. if (unlikely(ret < 0))
  2248. goto out_only_mutex;
  2249. if (!ret) {
  2250. /* zero the front end of the last page */
  2251. if (tail_start + tail_len < ino_size) {
  2252. truncated_block = true;
  2253. ret = btrfs_truncate_block(inode,
  2254. tail_start + tail_len,
  2255. 0, 1);
  2256. if (ret)
  2257. goto out_only_mutex;
  2258. }
  2259. }
  2260. }
  2261. if (lockend < lockstart) {
  2262. ret = 0;
  2263. goto out_only_mutex;
  2264. }
  2265. while (1) {
  2266. struct btrfs_ordered_extent *ordered;
  2267. truncate_pagecache_range(inode, lockstart, lockend);
  2268. lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  2269. &cached_state);
  2270. ordered = btrfs_lookup_first_ordered_extent(inode, lockend);
  2271. /*
  2272. * We need to make sure we have no ordered extents in this range
  2273. * and nobody raced in and read a page in this range, if we did
  2274. * we need to try again.
  2275. */
  2276. if ((!ordered ||
  2277. (ordered->file_offset + ordered->len <= lockstart ||
  2278. ordered->file_offset > lockend)) &&
  2279. !btrfs_page_exists_in_range(inode, lockstart, lockend)) {
  2280. if (ordered)
  2281. btrfs_put_ordered_extent(ordered);
  2282. break;
  2283. }
  2284. if (ordered)
  2285. btrfs_put_ordered_extent(ordered);
  2286. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
  2287. lockend, &cached_state, GFP_NOFS);
  2288. ret = btrfs_wait_ordered_range(inode, lockstart,
  2289. lockend - lockstart + 1);
  2290. if (ret) {
  2291. inode_unlock(inode);
  2292. return ret;
  2293. }
  2294. }
  2295. path = btrfs_alloc_path();
  2296. if (!path) {
  2297. ret = -ENOMEM;
  2298. goto out;
  2299. }
  2300. rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
  2301. if (!rsv) {
  2302. ret = -ENOMEM;
  2303. goto out_free;
  2304. }
  2305. rsv->size = btrfs_calc_trans_metadata_size(fs_info, 1);
  2306. rsv->failfast = 1;
  2307. /*
  2308. * 1 - update the inode
  2309. * 1 - removing the extents in the range
  2310. * 1 - adding the hole extent if no_holes isn't set
  2311. */
  2312. rsv_count = no_holes ? 2 : 3;
  2313. trans = btrfs_start_transaction(root, rsv_count);
  2314. if (IS_ERR(trans)) {
  2315. err = PTR_ERR(trans);
  2316. goto out_free;
  2317. }
  2318. ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
  2319. min_size, 0);
  2320. BUG_ON(ret);
  2321. trans->block_rsv = rsv;
  2322. cur_offset = lockstart;
  2323. len = lockend - cur_offset;
  2324. while (cur_offset < lockend) {
  2325. ret = __btrfs_drop_extents(trans, root, inode, path,
  2326. cur_offset, lockend + 1,
  2327. &drop_end, 1, 0, 0, NULL);
  2328. if (ret != -ENOSPC)
  2329. break;
  2330. trans->block_rsv = &fs_info->trans_block_rsv;
  2331. if (cur_offset < drop_end && cur_offset < ino_size) {
  2332. ret = fill_holes(trans, BTRFS_I(inode), path,
  2333. cur_offset, drop_end);
  2334. if (ret) {
  2335. /*
  2336. * If we failed then we didn't insert our hole
  2337. * entries for the area we dropped, so now the
  2338. * fs is corrupted, so we must abort the
  2339. * transaction.
  2340. */
  2341. btrfs_abort_transaction(trans, ret);
  2342. err = ret;
  2343. break;
  2344. }
  2345. }
  2346. cur_offset = drop_end;
  2347. ret = btrfs_update_inode(trans, root, inode);
  2348. if (ret) {
  2349. err = ret;
  2350. break;
  2351. }
  2352. btrfs_end_transaction(trans);
  2353. btrfs_btree_balance_dirty(fs_info);
  2354. trans = btrfs_start_transaction(root, rsv_count);
  2355. if (IS_ERR(trans)) {
  2356. ret = PTR_ERR(trans);
  2357. trans = NULL;
  2358. break;
  2359. }
  2360. ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
  2361. rsv, min_size, 0);
  2362. BUG_ON(ret); /* shouldn't happen */
  2363. trans->block_rsv = rsv;
  2364. ret = find_first_non_hole(inode, &cur_offset, &len);
  2365. if (unlikely(ret < 0))
  2366. break;
  2367. if (ret && !len) {
  2368. ret = 0;
  2369. break;
  2370. }
  2371. }
  2372. if (ret) {
  2373. err = ret;
  2374. goto out_trans;
  2375. }
  2376. trans->block_rsv = &fs_info->trans_block_rsv;
  2377. /*
  2378. * If we are using the NO_HOLES feature we might have had already an
  2379. * hole that overlaps a part of the region [lockstart, lockend] and
  2380. * ends at (or beyond) lockend. Since we have no file extent items to
  2381. * represent holes, drop_end can be less than lockend and so we must
  2382. * make sure we have an extent map representing the existing hole (the
  2383. * call to __btrfs_drop_extents() might have dropped the existing extent
  2384. * map representing the existing hole), otherwise the fast fsync path
  2385. * will not record the existence of the hole region
  2386. * [existing_hole_start, lockend].
  2387. */
  2388. if (drop_end <= lockend)
  2389. drop_end = lockend + 1;
  2390. /*
  2391. * Don't insert file hole extent item if it's for a range beyond eof
  2392. * (because it's useless) or if it represents a 0 bytes range (when
  2393. * cur_offset == drop_end).
  2394. */
  2395. if (cur_offset < ino_size && cur_offset < drop_end) {
  2396. ret = fill_holes(trans, BTRFS_I(inode), path,
  2397. cur_offset, drop_end);
  2398. if (ret) {
  2399. /* Same comment as above. */
  2400. btrfs_abort_transaction(trans, ret);
  2401. err = ret;
  2402. goto out_trans;
  2403. }
  2404. }
  2405. out_trans:
  2406. if (!trans)
  2407. goto out_free;
  2408. inode_inc_iversion(inode);
  2409. inode->i_mtime = inode->i_ctime = current_time(inode);
  2410. trans->block_rsv = &fs_info->trans_block_rsv;
  2411. ret = btrfs_update_inode(trans, root, inode);
  2412. updated_inode = true;
  2413. btrfs_end_transaction(trans);
  2414. btrfs_btree_balance_dirty(fs_info);
  2415. out_free:
  2416. btrfs_free_path(path);
  2417. btrfs_free_block_rsv(fs_info, rsv);
  2418. out:
  2419. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  2420. &cached_state, GFP_NOFS);
  2421. out_only_mutex:
  2422. if (!updated_inode && truncated_block && !ret && !err) {
  2423. /*
  2424. * If we only end up zeroing part of a page, we still need to
  2425. * update the inode item, so that all the time fields are
  2426. * updated as well as the necessary btrfs inode in memory fields
  2427. * for detecting, at fsync time, if the inode isn't yet in the
  2428. * log tree or it's there but not up to date.
  2429. */
  2430. trans = btrfs_start_transaction(root, 1);
  2431. if (IS_ERR(trans)) {
  2432. err = PTR_ERR(trans);
  2433. } else {
  2434. err = btrfs_update_inode(trans, root, inode);
  2435. ret = btrfs_end_transaction(trans);
  2436. }
  2437. }
  2438. inode_unlock(inode);
  2439. if (ret && !err)
  2440. err = ret;
  2441. return err;
  2442. }
  2443. /* Helper structure to record which range is already reserved */
  2444. struct falloc_range {
  2445. struct list_head list;
  2446. u64 start;
  2447. u64 len;
  2448. };
  2449. /*
  2450. * Helper function to add falloc range
  2451. *
  2452. * Caller should have locked the larger range of extent containing
  2453. * [start, len)
  2454. */
  2455. static int add_falloc_range(struct list_head *head, u64 start, u64 len)
  2456. {
  2457. struct falloc_range *prev = NULL;
  2458. struct falloc_range *range = NULL;
  2459. if (list_empty(head))
  2460. goto insert;
  2461. /*
  2462. * As fallocate iterate by bytenr order, we only need to check
  2463. * the last range.
  2464. */
  2465. prev = list_entry(head->prev, struct falloc_range, list);
  2466. if (prev->start + prev->len == start) {
  2467. prev->len += len;
  2468. return 0;
  2469. }
  2470. insert:
  2471. range = kmalloc(sizeof(*range), GFP_KERNEL);
  2472. if (!range)
  2473. return -ENOMEM;
  2474. range->start = start;
  2475. range->len = len;
  2476. list_add_tail(&range->list, head);
  2477. return 0;
  2478. }
  2479. static long btrfs_fallocate(struct file *file, int mode,
  2480. loff_t offset, loff_t len)
  2481. {
  2482. struct inode *inode = file_inode(file);
  2483. struct extent_state *cached_state = NULL;
  2484. struct falloc_range *range;
  2485. struct falloc_range *tmp;
  2486. struct list_head reserve_list;
  2487. u64 cur_offset;
  2488. u64 last_byte;
  2489. u64 alloc_start;
  2490. u64 alloc_end;
  2491. u64 alloc_hint = 0;
  2492. u64 locked_end;
  2493. u64 actual_end = 0;
  2494. struct extent_map *em;
  2495. int blocksize = btrfs_inode_sectorsize(inode);
  2496. int ret;
  2497. alloc_start = round_down(offset, blocksize);
  2498. alloc_end = round_up(offset + len, blocksize);
  2499. cur_offset = alloc_start;
  2500. /* Make sure we aren't being give some crap mode */
  2501. if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
  2502. return -EOPNOTSUPP;
  2503. if (mode & FALLOC_FL_PUNCH_HOLE)
  2504. return btrfs_punch_hole(inode, offset, len);
  2505. /*
  2506. * Only trigger disk allocation, don't trigger qgroup reserve
  2507. *
  2508. * For qgroup space, it will be checked later.
  2509. */
  2510. ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
  2511. alloc_end - alloc_start);
  2512. if (ret < 0)
  2513. return ret;
  2514. inode_lock(inode);
  2515. if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) {
  2516. ret = inode_newsize_ok(inode, offset + len);
  2517. if (ret)
  2518. goto out;
  2519. }
  2520. /*
  2521. * TODO: Move these two operations after we have checked
  2522. * accurate reserved space, or fallocate can still fail but
  2523. * with page truncated or size expanded.
  2524. *
  2525. * But that's a minor problem and won't do much harm BTW.
  2526. */
  2527. if (alloc_start > inode->i_size) {
  2528. ret = btrfs_cont_expand(inode, i_size_read(inode),
  2529. alloc_start);
  2530. if (ret)
  2531. goto out;
  2532. } else if (offset + len > inode->i_size) {
  2533. /*
  2534. * If we are fallocating from the end of the file onward we
  2535. * need to zero out the end of the block if i_size lands in the
  2536. * middle of a block.
  2537. */
  2538. ret = btrfs_truncate_block(inode, inode->i_size, 0, 0);
  2539. if (ret)
  2540. goto out;
  2541. }
  2542. /*
  2543. * wait for ordered IO before we have any locks. We'll loop again
  2544. * below with the locks held.
  2545. */
  2546. ret = btrfs_wait_ordered_range(inode, alloc_start,
  2547. alloc_end - alloc_start);
  2548. if (ret)
  2549. goto out;
  2550. locked_end = alloc_end - 1;
  2551. while (1) {
  2552. struct btrfs_ordered_extent *ordered;
  2553. /* the extent lock is ordered inside the running
  2554. * transaction
  2555. */
  2556. lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
  2557. locked_end, &cached_state);
  2558. ordered = btrfs_lookup_first_ordered_extent(inode,
  2559. alloc_end - 1);
  2560. if (ordered &&
  2561. ordered->file_offset + ordered->len > alloc_start &&
  2562. ordered->file_offset < alloc_end) {
  2563. btrfs_put_ordered_extent(ordered);
  2564. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  2565. alloc_start, locked_end,
  2566. &cached_state, GFP_KERNEL);
  2567. /*
  2568. * we can't wait on the range with the transaction
  2569. * running or with the extent lock held
  2570. */
  2571. ret = btrfs_wait_ordered_range(inode, alloc_start,
  2572. alloc_end - alloc_start);
  2573. if (ret)
  2574. goto out;
  2575. } else {
  2576. if (ordered)
  2577. btrfs_put_ordered_extent(ordered);
  2578. break;
  2579. }
  2580. }
  2581. /* First, check if we exceed the qgroup limit */
  2582. INIT_LIST_HEAD(&reserve_list);
  2583. while (1) {
  2584. em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
  2585. alloc_end - cur_offset, 0);
  2586. if (IS_ERR(em)) {
  2587. ret = PTR_ERR(em);
  2588. break;
  2589. }
  2590. last_byte = min(extent_map_end(em), alloc_end);
  2591. actual_end = min_t(u64, extent_map_end(em), offset + len);
  2592. last_byte = ALIGN(last_byte, blocksize);
  2593. if (em->block_start == EXTENT_MAP_HOLE ||
  2594. (cur_offset >= inode->i_size &&
  2595. !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
  2596. ret = add_falloc_range(&reserve_list, cur_offset,
  2597. last_byte - cur_offset);
  2598. if (ret < 0) {
  2599. free_extent_map(em);
  2600. break;
  2601. }
  2602. ret = btrfs_qgroup_reserve_data(inode, cur_offset,
  2603. last_byte - cur_offset);
  2604. if (ret < 0) {
  2605. free_extent_map(em);
  2606. break;
  2607. }
  2608. } else {
  2609. /*
  2610. * Do not need to reserve unwritten extent for this
  2611. * range, free reserved data space first, otherwise
  2612. * it'll result in false ENOSPC error.
  2613. */
  2614. btrfs_free_reserved_data_space(inode, cur_offset,
  2615. last_byte - cur_offset);
  2616. }
  2617. free_extent_map(em);
  2618. cur_offset = last_byte;
  2619. if (cur_offset >= alloc_end)
  2620. break;
  2621. }
  2622. /*
  2623. * If ret is still 0, means we're OK to fallocate.
  2624. * Or just cleanup the list and exit.
  2625. */
  2626. list_for_each_entry_safe(range, tmp, &reserve_list, list) {
  2627. if (!ret)
  2628. ret = btrfs_prealloc_file_range(inode, mode,
  2629. range->start,
  2630. range->len, i_blocksize(inode),
  2631. offset + len, &alloc_hint);
  2632. else
  2633. btrfs_free_reserved_data_space(inode, range->start,
  2634. range->len);
  2635. list_del(&range->list);
  2636. kfree(range);
  2637. }
  2638. if (ret < 0)
  2639. goto out_unlock;
  2640. if (actual_end > inode->i_size &&
  2641. !(mode & FALLOC_FL_KEEP_SIZE)) {
  2642. struct btrfs_trans_handle *trans;
  2643. struct btrfs_root *root = BTRFS_I(inode)->root;
  2644. /*
  2645. * We didn't need to allocate any more space, but we
  2646. * still extended the size of the file so we need to
  2647. * update i_size and the inode item.
  2648. */
  2649. trans = btrfs_start_transaction(root, 1);
  2650. if (IS_ERR(trans)) {
  2651. ret = PTR_ERR(trans);
  2652. } else {
  2653. inode->i_ctime = current_time(inode);
  2654. i_size_write(inode, actual_end);
  2655. btrfs_ordered_update_i_size(inode, actual_end, NULL);
  2656. ret = btrfs_update_inode(trans, root, inode);
  2657. if (ret)
  2658. btrfs_end_transaction(trans);
  2659. else
  2660. ret = btrfs_end_transaction(trans);
  2661. }
  2662. }
  2663. out_unlock:
  2664. unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
  2665. &cached_state, GFP_KERNEL);
  2666. out:
  2667. inode_unlock(inode);
  2668. /* Let go of our reservation. */
  2669. if (ret != 0)
  2670. btrfs_free_reserved_data_space(inode, alloc_start,
  2671. alloc_end - cur_offset);
  2672. return ret;
  2673. }
  2674. static int find_desired_extent(struct inode *inode, loff_t *offset, int whence)
  2675. {
  2676. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2677. struct extent_map *em = NULL;
  2678. struct extent_state *cached_state = NULL;
  2679. u64 lockstart;
  2680. u64 lockend;
  2681. u64 start;
  2682. u64 len;
  2683. int ret = 0;
  2684. if (inode->i_size == 0)
  2685. return -ENXIO;
  2686. /*
  2687. * *offset can be negative, in this case we start finding DATA/HOLE from
  2688. * the very start of the file.
  2689. */
  2690. start = max_t(loff_t, 0, *offset);
  2691. lockstart = round_down(start, fs_info->sectorsize);
  2692. lockend = round_up(i_size_read(inode),
  2693. fs_info->sectorsize);
  2694. if (lockend <= lockstart)
  2695. lockend = lockstart + fs_info->sectorsize;
  2696. lockend--;
  2697. len = lockend - lockstart + 1;
  2698. lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  2699. &cached_state);
  2700. while (start < inode->i_size) {
  2701. em = btrfs_get_extent_fiemap(BTRFS_I(inode), NULL, 0,
  2702. start, len, 0);
  2703. if (IS_ERR(em)) {
  2704. ret = PTR_ERR(em);
  2705. em = NULL;
  2706. break;
  2707. }
  2708. if (whence == SEEK_HOLE &&
  2709. (em->block_start == EXTENT_MAP_HOLE ||
  2710. test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
  2711. break;
  2712. else if (whence == SEEK_DATA &&
  2713. (em->block_start != EXTENT_MAP_HOLE &&
  2714. !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
  2715. break;
  2716. start = em->start + em->len;
  2717. free_extent_map(em);
  2718. em = NULL;
  2719. cond_resched();
  2720. }
  2721. free_extent_map(em);
  2722. if (!ret) {
  2723. if (whence == SEEK_DATA && start >= inode->i_size)
  2724. ret = -ENXIO;
  2725. else
  2726. *offset = min_t(loff_t, start, inode->i_size);
  2727. }
  2728. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  2729. &cached_state, GFP_NOFS);
  2730. return ret;
  2731. }
  2732. static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence)
  2733. {
  2734. struct inode *inode = file->f_mapping->host;
  2735. int ret;
  2736. inode_lock(inode);
  2737. switch (whence) {
  2738. case SEEK_END:
  2739. case SEEK_CUR:
  2740. offset = generic_file_llseek(file, offset, whence);
  2741. goto out;
  2742. case SEEK_DATA:
  2743. case SEEK_HOLE:
  2744. if (offset >= i_size_read(inode)) {
  2745. inode_unlock(inode);
  2746. return -ENXIO;
  2747. }
  2748. ret = find_desired_extent(inode, &offset, whence);
  2749. if (ret) {
  2750. inode_unlock(inode);
  2751. return ret;
  2752. }
  2753. }
  2754. offset = vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
  2755. out:
  2756. inode_unlock(inode);
  2757. return offset;
  2758. }
  2759. const struct file_operations btrfs_file_operations = {
  2760. .llseek = btrfs_file_llseek,
  2761. .read_iter = generic_file_read_iter,
  2762. .splice_read = generic_file_splice_read,
  2763. .write_iter = btrfs_file_write_iter,
  2764. .mmap = btrfs_file_mmap,
  2765. .open = generic_file_open,
  2766. .release = btrfs_release_file,
  2767. .fsync = btrfs_sync_file,
  2768. .fallocate = btrfs_fallocate,
  2769. .unlocked_ioctl = btrfs_ioctl,
  2770. #ifdef CONFIG_COMPAT
  2771. .compat_ioctl = btrfs_compat_ioctl,
  2772. #endif
  2773. .clone_file_range = btrfs_clone_file_range,
  2774. .dedupe_file_range = btrfs_dedupe_file_range,
  2775. };
  2776. void btrfs_auto_defrag_exit(void)
  2777. {
  2778. kmem_cache_destroy(btrfs_inode_defrag_cachep);
  2779. }
  2780. int btrfs_auto_defrag_init(void)
  2781. {
  2782. btrfs_inode_defrag_cachep = kmem_cache_create("btrfs_inode_defrag",
  2783. sizeof(struct inode_defrag), 0,
  2784. SLAB_MEM_SPREAD,
  2785. NULL);
  2786. if (!btrfs_inode_defrag_cachep)
  2787. return -ENOMEM;
  2788. return 0;
  2789. }
  2790. int btrfs_fdatawrite_range(struct inode *inode, loff_t start, loff_t end)
  2791. {
  2792. int ret;
  2793. /*
  2794. * So with compression we will find and lock a dirty page and clear the
  2795. * first one as dirty, setup an async extent, and immediately return
  2796. * with the entire range locked but with nobody actually marked with
  2797. * writeback. So we can't just filemap_write_and_wait_range() and
  2798. * expect it to work since it will just kick off a thread to do the
  2799. * actual work. So we need to call filemap_fdatawrite_range _again_
  2800. * since it will wait on the page lock, which won't be unlocked until
  2801. * after the pages have been marked as writeback and so we're good to go
  2802. * from there. We have to do this otherwise we'll miss the ordered
  2803. * extents and that results in badness. Please Josef, do not think you
  2804. * know better and pull this out at some point in the future, it is
  2805. * right and you are wrong.
  2806. */
  2807. ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
  2808. if (!ret && test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
  2809. &BTRFS_I(inode)->runtime_flags))
  2810. ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
  2811. return ret;
  2812. }