inode.c 289 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932993399349935993699379938993999409941994299439944994599469947994899499950995199529953995499559956995799589959996099619962996399649965996699679968996999709971997299739974997599769977997899799980998199829983998499859986998799889989999099919992999399949995999699979998999910000100011000210003100041000510006100071000810009100101001110012100131001410015100161001710018100191002010021100221002310024100251002610027100281002910030100311003210033100341003510036100371003810039100401004110042100431004410045100461004710048100491005010051100521005310054100551005610057100581005910060100611006210063100641006510066100671006810069100701007110072100731007410075100761007710078100791008010081100821008310084100851008610087100881008910090100911009210093100941009510096100971009810099101001010110102101031010410105101061010710108101091011010111101121011310114101151011610117101181011910120101211012210123101241012510126101271012810129101301013110132101331013410135101361013710138101391014010141101421014310144101451014610147101481014910150101511015210153101541015510156101571015810159101601016110162101631016410165101661016710168101691017010171101721017310174101751017610177101781017910180101811018210183101841018510186101871018810189101901019110192101931019410195101961019710198101991020010201102021020310204102051020610207102081020910210102111021210213102141021510216102171021810219102201022110222102231022410225102261022710228102291023010231102321023310234102351023610237102381023910240102411024210243102441024510246102471024810249102501025110252102531025410255102561025710258102591026010261102621026310264102651026610267102681026910270102711027210273102741027510276102771027810279102801028110282102831028410285102861028710288102891029010291102921029310294102951029610297102981029910300103011030210303103041030510306103071030810309103101031110312103131031410315103161031710318103191032010321103221032310324103251032610327103281032910330103311033210333103341033510336103371033810339103401034110342103431034410345103461034710348103491035010351103521035310354103551035610357103581035910360103611036210363103641036510366103671036810369103701037110372103731037410375103761037710378103791038010381103821038310384103851038610387103881038910390103911039210393103941039510396103971039810399104001040110402104031040410405104061040710408104091041010411104121041310414104151041610417104181041910420104211042210423104241042510426104271042810429104301043110432104331043410435104361043710438104391044010441104421044310444104451044610447104481044910450104511045210453104541045510456104571045810459104601046110462104631046410465104661046710468104691047010471104721047310474104751047610477104781047910480104811048210483104841048510486104871048810489104901049110492104931049410495104961049710498104991050010501105021050310504105051050610507105081050910510105111051210513105141051510516105171051810519105201052110522105231052410525105261052710528105291053010531105321053310534105351053610537105381053910540105411054210543105441054510546105471054810549105501055110552105531055410555105561055710558105591056010561105621056310564105651056610567105681056910570105711057210573105741057510576105771057810579105801058110582105831058410585105861058710588105891059010591105921059310594105951059610597105981059910600106011060210603106041060510606106071060810609106101061110612106131061410615106161061710618106191062010621106221062310624106251062610627106281062910630106311063210633106341063510636106371063810639106401064110642106431064410645106461064710648106491065010651106521065310654106551065610657106581065910660106611066210663106641066510666106671066810669106701067110672106731067410675106761067710678106791068010681106821068310684106851068610687106881068910690106911069210693106941069510696106971069810699107001070110702107031070410705107061070710708107091071010711107121071310714107151071610717107181071910720107211072210723107241072510726107271072810729107301073110732107331073410735107361073710738107391074010741107421074310744107451074610747107481074910750107511075210753107541075510756107571075810759107601076110762107631076410765107661076710768107691077010771107721077310774107751077610777107781077910780107811078210783107841078510786107871078810789107901079110792107931079410795107961079710798107991080010801108021080310804108051080610807108081080910810108111081210813108141081510816108171081810819108201082110822108231082410825
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/kernel.h>
  19. #include <linux/bio.h>
  20. #include <linux/buffer_head.h>
  21. #include <linux/file.h>
  22. #include <linux/fs.h>
  23. #include <linux/pagemap.h>
  24. #include <linux/highmem.h>
  25. #include <linux/time.h>
  26. #include <linux/init.h>
  27. #include <linux/string.h>
  28. #include <linux/backing-dev.h>
  29. #include <linux/mpage.h>
  30. #include <linux/swap.h>
  31. #include <linux/writeback.h>
  32. #include <linux/compat.h>
  33. #include <linux/bit_spinlock.h>
  34. #include <linux/xattr.h>
  35. #include <linux/posix_acl.h>
  36. #include <linux/falloc.h>
  37. #include <linux/slab.h>
  38. #include <linux/ratelimit.h>
  39. #include <linux/mount.h>
  40. #include <linux/btrfs.h>
  41. #include <linux/blkdev.h>
  42. #include <linux/posix_acl_xattr.h>
  43. #include <linux/uio.h>
  44. #include "ctree.h"
  45. #include "disk-io.h"
  46. #include "transaction.h"
  47. #include "btrfs_inode.h"
  48. #include "print-tree.h"
  49. #include "ordered-data.h"
  50. #include "xattr.h"
  51. #include "tree-log.h"
  52. #include "volumes.h"
  53. #include "compression.h"
  54. #include "locking.h"
  55. #include "free-space-cache.h"
  56. #include "inode-map.h"
  57. #include "backref.h"
  58. #include "hash.h"
  59. #include "props.h"
  60. #include "qgroup.h"
  61. #include "dedupe.h"
  62. struct btrfs_iget_args {
  63. struct btrfs_key *location;
  64. struct btrfs_root *root;
  65. };
  66. struct btrfs_dio_data {
  67. u64 outstanding_extents;
  68. u64 reserve;
  69. u64 unsubmitted_oe_range_start;
  70. u64 unsubmitted_oe_range_end;
  71. int overwrite;
  72. };
  73. static const struct inode_operations btrfs_dir_inode_operations;
  74. static const struct inode_operations btrfs_symlink_inode_operations;
  75. static const struct inode_operations btrfs_dir_ro_inode_operations;
  76. static const struct inode_operations btrfs_special_inode_operations;
  77. static const struct inode_operations btrfs_file_inode_operations;
  78. static const struct address_space_operations btrfs_aops;
  79. static const struct address_space_operations btrfs_symlink_aops;
  80. static const struct file_operations btrfs_dir_file_operations;
  81. static const struct extent_io_ops btrfs_extent_io_ops;
  82. static struct kmem_cache *btrfs_inode_cachep;
  83. struct kmem_cache *btrfs_trans_handle_cachep;
  84. struct kmem_cache *btrfs_path_cachep;
  85. struct kmem_cache *btrfs_free_space_cachep;
  86. #define S_SHIFT 12
  87. static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = {
  88. [S_IFREG >> S_SHIFT] = BTRFS_FT_REG_FILE,
  89. [S_IFDIR >> S_SHIFT] = BTRFS_FT_DIR,
  90. [S_IFCHR >> S_SHIFT] = BTRFS_FT_CHRDEV,
  91. [S_IFBLK >> S_SHIFT] = BTRFS_FT_BLKDEV,
  92. [S_IFIFO >> S_SHIFT] = BTRFS_FT_FIFO,
  93. [S_IFSOCK >> S_SHIFT] = BTRFS_FT_SOCK,
  94. [S_IFLNK >> S_SHIFT] = BTRFS_FT_SYMLINK,
  95. };
  96. static int btrfs_setsize(struct inode *inode, struct iattr *attr);
  97. static int btrfs_truncate(struct inode *inode);
  98. static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
  99. static noinline int cow_file_range(struct inode *inode,
  100. struct page *locked_page,
  101. u64 start, u64 end, u64 delalloc_end,
  102. int *page_started, unsigned long *nr_written,
  103. int unlock, struct btrfs_dedupe_hash *hash);
  104. static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
  105. u64 orig_start, u64 block_start,
  106. u64 block_len, u64 orig_block_len,
  107. u64 ram_bytes, int compress_type,
  108. int type);
  109. static void __endio_write_update_ordered(struct inode *inode,
  110. const u64 offset, const u64 bytes,
  111. const bool uptodate);
  112. /*
  113. * Cleanup all submitted ordered extents in specified range to handle errors
  114. * from the fill_dellaloc() callback.
  115. *
  116. * NOTE: caller must ensure that when an error happens, it can not call
  117. * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
  118. * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
  119. * to be released, which we want to happen only when finishing the ordered
  120. * extent (btrfs_finish_ordered_io()). Also note that the caller of the
  121. * fill_delalloc() callback already does proper cleanup for the first page of
  122. * the range, that is, it invokes the callback writepage_end_io_hook() for the
  123. * range of the first page.
  124. */
  125. static inline void btrfs_cleanup_ordered_extents(struct inode *inode,
  126. const u64 offset,
  127. const u64 bytes)
  128. {
  129. return __endio_write_update_ordered(inode, offset + PAGE_SIZE,
  130. bytes - PAGE_SIZE, false);
  131. }
  132. static int btrfs_dirty_inode(struct inode *inode);
  133. #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
  134. void btrfs_test_inode_set_ops(struct inode *inode)
  135. {
  136. BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
  137. }
  138. #endif
  139. static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
  140. struct inode *inode, struct inode *dir,
  141. const struct qstr *qstr)
  142. {
  143. int err;
  144. err = btrfs_init_acl(trans, inode, dir);
  145. if (!err)
  146. err = btrfs_xattr_security_init(trans, inode, dir, qstr);
  147. return err;
  148. }
  149. /*
  150. * this does all the hard work for inserting an inline extent into
  151. * the btree. The caller should have done a btrfs_drop_extents so that
  152. * no overlapping inline items exist in the btree
  153. */
  154. static int insert_inline_extent(struct btrfs_trans_handle *trans,
  155. struct btrfs_path *path, int extent_inserted,
  156. struct btrfs_root *root, struct inode *inode,
  157. u64 start, size_t size, size_t compressed_size,
  158. int compress_type,
  159. struct page **compressed_pages)
  160. {
  161. struct extent_buffer *leaf;
  162. struct page *page = NULL;
  163. char *kaddr;
  164. unsigned long ptr;
  165. struct btrfs_file_extent_item *ei;
  166. int err = 0;
  167. int ret;
  168. size_t cur_size = size;
  169. unsigned long offset;
  170. if (compressed_size && compressed_pages)
  171. cur_size = compressed_size;
  172. inode_add_bytes(inode, size);
  173. if (!extent_inserted) {
  174. struct btrfs_key key;
  175. size_t datasize;
  176. key.objectid = btrfs_ino(BTRFS_I(inode));
  177. key.offset = start;
  178. key.type = BTRFS_EXTENT_DATA_KEY;
  179. datasize = btrfs_file_extent_calc_inline_size(cur_size);
  180. path->leave_spinning = 1;
  181. ret = btrfs_insert_empty_item(trans, root, path, &key,
  182. datasize);
  183. if (ret) {
  184. err = ret;
  185. goto fail;
  186. }
  187. }
  188. leaf = path->nodes[0];
  189. ei = btrfs_item_ptr(leaf, path->slots[0],
  190. struct btrfs_file_extent_item);
  191. btrfs_set_file_extent_generation(leaf, ei, trans->transid);
  192. btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
  193. btrfs_set_file_extent_encryption(leaf, ei, 0);
  194. btrfs_set_file_extent_other_encoding(leaf, ei, 0);
  195. btrfs_set_file_extent_ram_bytes(leaf, ei, size);
  196. ptr = btrfs_file_extent_inline_start(ei);
  197. if (compress_type != BTRFS_COMPRESS_NONE) {
  198. struct page *cpage;
  199. int i = 0;
  200. while (compressed_size > 0) {
  201. cpage = compressed_pages[i];
  202. cur_size = min_t(unsigned long, compressed_size,
  203. PAGE_SIZE);
  204. kaddr = kmap_atomic(cpage);
  205. write_extent_buffer(leaf, kaddr, ptr, cur_size);
  206. kunmap_atomic(kaddr);
  207. i++;
  208. ptr += cur_size;
  209. compressed_size -= cur_size;
  210. }
  211. btrfs_set_file_extent_compression(leaf, ei,
  212. compress_type);
  213. } else {
  214. page = find_get_page(inode->i_mapping,
  215. start >> PAGE_SHIFT);
  216. btrfs_set_file_extent_compression(leaf, ei, 0);
  217. kaddr = kmap_atomic(page);
  218. offset = start & (PAGE_SIZE - 1);
  219. write_extent_buffer(leaf, kaddr + offset, ptr, size);
  220. kunmap_atomic(kaddr);
  221. put_page(page);
  222. }
  223. btrfs_mark_buffer_dirty(leaf);
  224. btrfs_release_path(path);
  225. /*
  226. * we're an inline extent, so nobody can
  227. * extend the file past i_size without locking
  228. * a page we already have locked.
  229. *
  230. * We must do any isize and inode updates
  231. * before we unlock the pages. Otherwise we
  232. * could end up racing with unlink.
  233. */
  234. BTRFS_I(inode)->disk_i_size = inode->i_size;
  235. ret = btrfs_update_inode(trans, root, inode);
  236. return ret;
  237. fail:
  238. return err;
  239. }
  240. /*
  241. * conditionally insert an inline extent into the file. This
  242. * does the checks required to make sure the data is small enough
  243. * to fit as an inline extent.
  244. */
  245. static noinline int cow_file_range_inline(struct btrfs_root *root,
  246. struct inode *inode, u64 start,
  247. u64 end, size_t compressed_size,
  248. int compress_type,
  249. struct page **compressed_pages)
  250. {
  251. struct btrfs_fs_info *fs_info = root->fs_info;
  252. struct btrfs_trans_handle *trans;
  253. u64 isize = i_size_read(inode);
  254. u64 actual_end = min(end + 1, isize);
  255. u64 inline_len = actual_end - start;
  256. u64 aligned_end = ALIGN(end, fs_info->sectorsize);
  257. u64 data_len = inline_len;
  258. int ret;
  259. struct btrfs_path *path;
  260. int extent_inserted = 0;
  261. u32 extent_item_size;
  262. if (compressed_size)
  263. data_len = compressed_size;
  264. if (start > 0 ||
  265. actual_end > fs_info->sectorsize ||
  266. data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) ||
  267. (!compressed_size &&
  268. (actual_end & (fs_info->sectorsize - 1)) == 0) ||
  269. end + 1 < isize ||
  270. data_len > fs_info->max_inline) {
  271. return 1;
  272. }
  273. path = btrfs_alloc_path();
  274. if (!path)
  275. return -ENOMEM;
  276. trans = btrfs_join_transaction(root);
  277. if (IS_ERR(trans)) {
  278. btrfs_free_path(path);
  279. return PTR_ERR(trans);
  280. }
  281. trans->block_rsv = &fs_info->delalloc_block_rsv;
  282. if (compressed_size && compressed_pages)
  283. extent_item_size = btrfs_file_extent_calc_inline_size(
  284. compressed_size);
  285. else
  286. extent_item_size = btrfs_file_extent_calc_inline_size(
  287. inline_len);
  288. ret = __btrfs_drop_extents(trans, root, inode, path,
  289. start, aligned_end, NULL,
  290. 1, 1, extent_item_size, &extent_inserted);
  291. if (ret) {
  292. btrfs_abort_transaction(trans, ret);
  293. goto out;
  294. }
  295. if (isize > actual_end)
  296. inline_len = min_t(u64, isize, actual_end);
  297. ret = insert_inline_extent(trans, path, extent_inserted,
  298. root, inode, start,
  299. inline_len, compressed_size,
  300. compress_type, compressed_pages);
  301. if (ret && ret != -ENOSPC) {
  302. btrfs_abort_transaction(trans, ret);
  303. goto out;
  304. } else if (ret == -ENOSPC) {
  305. ret = 1;
  306. goto out;
  307. }
  308. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
  309. btrfs_delalloc_release_metadata(BTRFS_I(inode), end + 1 - start);
  310. btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0);
  311. out:
  312. /*
  313. * Don't forget to free the reserved space, as for inlined extent
  314. * it won't count as data extent, free them directly here.
  315. * And at reserve time, it's always aligned to page size, so
  316. * just free one page here.
  317. */
  318. btrfs_qgroup_free_data(inode, 0, PAGE_SIZE);
  319. btrfs_free_path(path);
  320. btrfs_end_transaction(trans);
  321. return ret;
  322. }
  323. struct async_extent {
  324. u64 start;
  325. u64 ram_size;
  326. u64 compressed_size;
  327. struct page **pages;
  328. unsigned long nr_pages;
  329. int compress_type;
  330. struct list_head list;
  331. };
  332. struct async_cow {
  333. struct inode *inode;
  334. struct btrfs_root *root;
  335. struct page *locked_page;
  336. u64 start;
  337. u64 end;
  338. struct list_head extents;
  339. struct btrfs_work work;
  340. };
  341. static noinline int add_async_extent(struct async_cow *cow,
  342. u64 start, u64 ram_size,
  343. u64 compressed_size,
  344. struct page **pages,
  345. unsigned long nr_pages,
  346. int compress_type)
  347. {
  348. struct async_extent *async_extent;
  349. async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
  350. BUG_ON(!async_extent); /* -ENOMEM */
  351. async_extent->start = start;
  352. async_extent->ram_size = ram_size;
  353. async_extent->compressed_size = compressed_size;
  354. async_extent->pages = pages;
  355. async_extent->nr_pages = nr_pages;
  356. async_extent->compress_type = compress_type;
  357. list_add_tail(&async_extent->list, &cow->extents);
  358. return 0;
  359. }
  360. static inline int inode_need_compress(struct inode *inode)
  361. {
  362. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  363. /* force compress */
  364. if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
  365. return 1;
  366. /* bad compression ratios */
  367. if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
  368. return 0;
  369. if (btrfs_test_opt(fs_info, COMPRESS) ||
  370. BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
  371. BTRFS_I(inode)->force_compress)
  372. return 1;
  373. return 0;
  374. }
  375. static inline void inode_should_defrag(struct btrfs_inode *inode,
  376. u64 start, u64 end, u64 num_bytes, u64 small_write)
  377. {
  378. /* If this is a small write inside eof, kick off a defrag */
  379. if (num_bytes < small_write &&
  380. (start > 0 || end + 1 < inode->disk_i_size))
  381. btrfs_add_inode_defrag(NULL, inode);
  382. }
  383. /*
  384. * we create compressed extents in two phases. The first
  385. * phase compresses a range of pages that have already been
  386. * locked (both pages and state bits are locked).
  387. *
  388. * This is done inside an ordered work queue, and the compression
  389. * is spread across many cpus. The actual IO submission is step
  390. * two, and the ordered work queue takes care of making sure that
  391. * happens in the same order things were put onto the queue by
  392. * writepages and friends.
  393. *
  394. * If this code finds it can't get good compression, it puts an
  395. * entry onto the work queue to write the uncompressed bytes. This
  396. * makes sure that both compressed inodes and uncompressed inodes
  397. * are written in the same order that the flusher thread sent them
  398. * down.
  399. */
  400. static noinline void compress_file_range(struct inode *inode,
  401. struct page *locked_page,
  402. u64 start, u64 end,
  403. struct async_cow *async_cow,
  404. int *num_added)
  405. {
  406. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  407. struct btrfs_root *root = BTRFS_I(inode)->root;
  408. u64 num_bytes;
  409. u64 blocksize = fs_info->sectorsize;
  410. u64 actual_end;
  411. u64 isize = i_size_read(inode);
  412. int ret = 0;
  413. struct page **pages = NULL;
  414. unsigned long nr_pages;
  415. unsigned long total_compressed = 0;
  416. unsigned long total_in = 0;
  417. int i;
  418. int will_compress;
  419. int compress_type = fs_info->compress_type;
  420. int redirty = 0;
  421. inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1,
  422. SZ_16K);
  423. actual_end = min_t(u64, isize, end + 1);
  424. again:
  425. will_compress = 0;
  426. nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
  427. BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0);
  428. nr_pages = min_t(unsigned long, nr_pages,
  429. BTRFS_MAX_COMPRESSED / PAGE_SIZE);
  430. /*
  431. * we don't want to send crud past the end of i_size through
  432. * compression, that's just a waste of CPU time. So, if the
  433. * end of the file is before the start of our current
  434. * requested range of bytes, we bail out to the uncompressed
  435. * cleanup code that can deal with all of this.
  436. *
  437. * It isn't really the fastest way to fix things, but this is a
  438. * very uncommon corner.
  439. */
  440. if (actual_end <= start)
  441. goto cleanup_and_bail_uncompressed;
  442. total_compressed = actual_end - start;
  443. /*
  444. * skip compression for a small file range(<=blocksize) that
  445. * isn't an inline extent, since it doesn't save disk space at all.
  446. */
  447. if (total_compressed <= blocksize &&
  448. (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
  449. goto cleanup_and_bail_uncompressed;
  450. total_compressed = min_t(unsigned long, total_compressed,
  451. BTRFS_MAX_UNCOMPRESSED);
  452. num_bytes = ALIGN(end - start + 1, blocksize);
  453. num_bytes = max(blocksize, num_bytes);
  454. total_in = 0;
  455. ret = 0;
  456. /*
  457. * we do compression for mount -o compress and when the
  458. * inode has not been flagged as nocompress. This flag can
  459. * change at any time if we discover bad compression ratios.
  460. */
  461. if (inode_need_compress(inode)) {
  462. WARN_ON(pages);
  463. pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
  464. if (!pages) {
  465. /* just bail out to the uncompressed code */
  466. goto cont;
  467. }
  468. if (BTRFS_I(inode)->force_compress)
  469. compress_type = BTRFS_I(inode)->force_compress;
  470. /*
  471. * we need to call clear_page_dirty_for_io on each
  472. * page in the range. Otherwise applications with the file
  473. * mmap'd can wander in and change the page contents while
  474. * we are compressing them.
  475. *
  476. * If the compression fails for any reason, we set the pages
  477. * dirty again later on.
  478. */
  479. extent_range_clear_dirty_for_io(inode, start, end);
  480. redirty = 1;
  481. ret = btrfs_compress_pages(compress_type,
  482. inode->i_mapping, start,
  483. pages,
  484. &nr_pages,
  485. &total_in,
  486. &total_compressed);
  487. if (!ret) {
  488. unsigned long offset = total_compressed &
  489. (PAGE_SIZE - 1);
  490. struct page *page = pages[nr_pages - 1];
  491. char *kaddr;
  492. /* zero the tail end of the last page, we might be
  493. * sending it down to disk
  494. */
  495. if (offset) {
  496. kaddr = kmap_atomic(page);
  497. memset(kaddr + offset, 0,
  498. PAGE_SIZE - offset);
  499. kunmap_atomic(kaddr);
  500. }
  501. will_compress = 1;
  502. }
  503. }
  504. cont:
  505. if (start == 0) {
  506. /* lets try to make an inline extent */
  507. if (ret || total_in < (actual_end - start)) {
  508. /* we didn't compress the entire range, try
  509. * to make an uncompressed inline extent.
  510. */
  511. ret = cow_file_range_inline(root, inode, start, end,
  512. 0, BTRFS_COMPRESS_NONE, NULL);
  513. } else {
  514. /* try making a compressed inline extent */
  515. ret = cow_file_range_inline(root, inode, start, end,
  516. total_compressed,
  517. compress_type, pages);
  518. }
  519. if (ret <= 0) {
  520. unsigned long clear_flags = EXTENT_DELALLOC |
  521. EXTENT_DELALLOC_NEW | EXTENT_DEFRAG;
  522. unsigned long page_error_op;
  523. clear_flags |= (ret < 0) ? EXTENT_DO_ACCOUNTING : 0;
  524. page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
  525. /*
  526. * inline extent creation worked or returned error,
  527. * we don't need to create any more async work items.
  528. * Unlock and free up our temp pages.
  529. */
  530. extent_clear_unlock_delalloc(inode, start, end, end,
  531. NULL, clear_flags,
  532. PAGE_UNLOCK |
  533. PAGE_CLEAR_DIRTY |
  534. PAGE_SET_WRITEBACK |
  535. page_error_op |
  536. PAGE_END_WRITEBACK);
  537. if (ret == 0)
  538. btrfs_free_reserved_data_space_noquota(inode,
  539. start,
  540. end - start + 1);
  541. goto free_pages_out;
  542. }
  543. }
  544. if (will_compress) {
  545. /*
  546. * we aren't doing an inline extent round the compressed size
  547. * up to a block size boundary so the allocator does sane
  548. * things
  549. */
  550. total_compressed = ALIGN(total_compressed, blocksize);
  551. /*
  552. * one last check to make sure the compression is really a
  553. * win, compare the page count read with the blocks on disk
  554. */
  555. total_in = ALIGN(total_in, PAGE_SIZE);
  556. if (total_compressed >= total_in) {
  557. will_compress = 0;
  558. } else {
  559. num_bytes = total_in;
  560. *num_added += 1;
  561. /*
  562. * The async work queues will take care of doing actual
  563. * allocation on disk for these compressed pages, and
  564. * will submit them to the elevator.
  565. */
  566. add_async_extent(async_cow, start, num_bytes,
  567. total_compressed, pages, nr_pages,
  568. compress_type);
  569. if (start + num_bytes < end) {
  570. start += num_bytes;
  571. pages = NULL;
  572. cond_resched();
  573. goto again;
  574. }
  575. return;
  576. }
  577. }
  578. if (pages) {
  579. /*
  580. * the compression code ran but failed to make things smaller,
  581. * free any pages it allocated and our page pointer array
  582. */
  583. for (i = 0; i < nr_pages; i++) {
  584. WARN_ON(pages[i]->mapping);
  585. put_page(pages[i]);
  586. }
  587. kfree(pages);
  588. pages = NULL;
  589. total_compressed = 0;
  590. nr_pages = 0;
  591. /* flag the file so we don't compress in the future */
  592. if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) &&
  593. !(BTRFS_I(inode)->force_compress)) {
  594. BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
  595. }
  596. }
  597. cleanup_and_bail_uncompressed:
  598. /*
  599. * No compression, but we still need to write the pages in the file
  600. * we've been given so far. redirty the locked page if it corresponds
  601. * to our extent and set things up for the async work queue to run
  602. * cow_file_range to do the normal delalloc dance.
  603. */
  604. if (page_offset(locked_page) >= start &&
  605. page_offset(locked_page) <= end)
  606. __set_page_dirty_nobuffers(locked_page);
  607. /* unlocked later on in the async handlers */
  608. if (redirty)
  609. extent_range_redirty_for_io(inode, start, end);
  610. add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0,
  611. BTRFS_COMPRESS_NONE);
  612. *num_added += 1;
  613. return;
  614. free_pages_out:
  615. for (i = 0; i < nr_pages; i++) {
  616. WARN_ON(pages[i]->mapping);
  617. put_page(pages[i]);
  618. }
  619. kfree(pages);
  620. }
  621. static void free_async_extent_pages(struct async_extent *async_extent)
  622. {
  623. int i;
  624. if (!async_extent->pages)
  625. return;
  626. for (i = 0; i < async_extent->nr_pages; i++) {
  627. WARN_ON(async_extent->pages[i]->mapping);
  628. put_page(async_extent->pages[i]);
  629. }
  630. kfree(async_extent->pages);
  631. async_extent->nr_pages = 0;
  632. async_extent->pages = NULL;
  633. }
  634. /*
  635. * phase two of compressed writeback. This is the ordered portion
  636. * of the code, which only gets called in the order the work was
  637. * queued. We walk all the async extents created by compress_file_range
  638. * and send them down to the disk.
  639. */
  640. static noinline void submit_compressed_extents(struct inode *inode,
  641. struct async_cow *async_cow)
  642. {
  643. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  644. struct async_extent *async_extent;
  645. u64 alloc_hint = 0;
  646. struct btrfs_key ins;
  647. struct extent_map *em;
  648. struct btrfs_root *root = BTRFS_I(inode)->root;
  649. struct extent_io_tree *io_tree;
  650. int ret = 0;
  651. again:
  652. while (!list_empty(&async_cow->extents)) {
  653. async_extent = list_entry(async_cow->extents.next,
  654. struct async_extent, list);
  655. list_del(&async_extent->list);
  656. io_tree = &BTRFS_I(inode)->io_tree;
  657. retry:
  658. /* did the compression code fall back to uncompressed IO? */
  659. if (!async_extent->pages) {
  660. int page_started = 0;
  661. unsigned long nr_written = 0;
  662. lock_extent(io_tree, async_extent->start,
  663. async_extent->start +
  664. async_extent->ram_size - 1);
  665. /* allocate blocks */
  666. ret = cow_file_range(inode, async_cow->locked_page,
  667. async_extent->start,
  668. async_extent->start +
  669. async_extent->ram_size - 1,
  670. async_extent->start +
  671. async_extent->ram_size - 1,
  672. &page_started, &nr_written, 0,
  673. NULL);
  674. /* JDM XXX */
  675. /*
  676. * if page_started, cow_file_range inserted an
  677. * inline extent and took care of all the unlocking
  678. * and IO for us. Otherwise, we need to submit
  679. * all those pages down to the drive.
  680. */
  681. if (!page_started && !ret)
  682. extent_write_locked_range(io_tree,
  683. inode, async_extent->start,
  684. async_extent->start +
  685. async_extent->ram_size - 1,
  686. btrfs_get_extent,
  687. WB_SYNC_ALL);
  688. else if (ret)
  689. unlock_page(async_cow->locked_page);
  690. kfree(async_extent);
  691. cond_resched();
  692. continue;
  693. }
  694. lock_extent(io_tree, async_extent->start,
  695. async_extent->start + async_extent->ram_size - 1);
  696. ret = btrfs_reserve_extent(root, async_extent->ram_size,
  697. async_extent->compressed_size,
  698. async_extent->compressed_size,
  699. 0, alloc_hint, &ins, 1, 1);
  700. if (ret) {
  701. free_async_extent_pages(async_extent);
  702. if (ret == -ENOSPC) {
  703. unlock_extent(io_tree, async_extent->start,
  704. async_extent->start +
  705. async_extent->ram_size - 1);
  706. /*
  707. * we need to redirty the pages if we decide to
  708. * fallback to uncompressed IO, otherwise we
  709. * will not submit these pages down to lower
  710. * layers.
  711. */
  712. extent_range_redirty_for_io(inode,
  713. async_extent->start,
  714. async_extent->start +
  715. async_extent->ram_size - 1);
  716. goto retry;
  717. }
  718. goto out_free;
  719. }
  720. /*
  721. * here we're doing allocation and writeback of the
  722. * compressed pages
  723. */
  724. em = create_io_em(inode, async_extent->start,
  725. async_extent->ram_size, /* len */
  726. async_extent->start, /* orig_start */
  727. ins.objectid, /* block_start */
  728. ins.offset, /* block_len */
  729. ins.offset, /* orig_block_len */
  730. async_extent->ram_size, /* ram_bytes */
  731. async_extent->compress_type,
  732. BTRFS_ORDERED_COMPRESSED);
  733. if (IS_ERR(em))
  734. /* ret value is not necessary due to void function */
  735. goto out_free_reserve;
  736. free_extent_map(em);
  737. ret = btrfs_add_ordered_extent_compress(inode,
  738. async_extent->start,
  739. ins.objectid,
  740. async_extent->ram_size,
  741. ins.offset,
  742. BTRFS_ORDERED_COMPRESSED,
  743. async_extent->compress_type);
  744. if (ret) {
  745. btrfs_drop_extent_cache(BTRFS_I(inode),
  746. async_extent->start,
  747. async_extent->start +
  748. async_extent->ram_size - 1, 0);
  749. goto out_free_reserve;
  750. }
  751. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  752. /*
  753. * clear dirty, set writeback and unlock the pages.
  754. */
  755. extent_clear_unlock_delalloc(inode, async_extent->start,
  756. async_extent->start +
  757. async_extent->ram_size - 1,
  758. async_extent->start +
  759. async_extent->ram_size - 1,
  760. NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
  761. PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
  762. PAGE_SET_WRITEBACK);
  763. ret = btrfs_submit_compressed_write(inode,
  764. async_extent->start,
  765. async_extent->ram_size,
  766. ins.objectid,
  767. ins.offset, async_extent->pages,
  768. async_extent->nr_pages);
  769. if (ret) {
  770. struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
  771. struct page *p = async_extent->pages[0];
  772. const u64 start = async_extent->start;
  773. const u64 end = start + async_extent->ram_size - 1;
  774. p->mapping = inode->i_mapping;
  775. tree->ops->writepage_end_io_hook(p, start, end,
  776. NULL, 0);
  777. p->mapping = NULL;
  778. extent_clear_unlock_delalloc(inode, start, end, end,
  779. NULL, 0,
  780. PAGE_END_WRITEBACK |
  781. PAGE_SET_ERROR);
  782. free_async_extent_pages(async_extent);
  783. }
  784. alloc_hint = ins.objectid + ins.offset;
  785. kfree(async_extent);
  786. cond_resched();
  787. }
  788. return;
  789. out_free_reserve:
  790. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  791. btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
  792. out_free:
  793. extent_clear_unlock_delalloc(inode, async_extent->start,
  794. async_extent->start +
  795. async_extent->ram_size - 1,
  796. async_extent->start +
  797. async_extent->ram_size - 1,
  798. NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
  799. EXTENT_DELALLOC_NEW |
  800. EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
  801. PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
  802. PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
  803. PAGE_SET_ERROR);
  804. free_async_extent_pages(async_extent);
  805. kfree(async_extent);
  806. goto again;
  807. }
  808. static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
  809. u64 num_bytes)
  810. {
  811. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  812. struct extent_map *em;
  813. u64 alloc_hint = 0;
  814. read_lock(&em_tree->lock);
  815. em = search_extent_mapping(em_tree, start, num_bytes);
  816. if (em) {
  817. /*
  818. * if block start isn't an actual block number then find the
  819. * first block in this inode and use that as a hint. If that
  820. * block is also bogus then just don't worry about it.
  821. */
  822. if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
  823. free_extent_map(em);
  824. em = search_extent_mapping(em_tree, 0, 0);
  825. if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
  826. alloc_hint = em->block_start;
  827. if (em)
  828. free_extent_map(em);
  829. } else {
  830. alloc_hint = em->block_start;
  831. free_extent_map(em);
  832. }
  833. }
  834. read_unlock(&em_tree->lock);
  835. return alloc_hint;
  836. }
  837. /*
  838. * when extent_io.c finds a delayed allocation range in the file,
  839. * the call backs end up in this code. The basic idea is to
  840. * allocate extents on disk for the range, and create ordered data structs
  841. * in ram to track those extents.
  842. *
  843. * locked_page is the page that writepage had locked already. We use
  844. * it to make sure we don't do extra locks or unlocks.
  845. *
  846. * *page_started is set to one if we unlock locked_page and do everything
  847. * required to start IO on it. It may be clean and already done with
  848. * IO when we return.
  849. */
  850. static noinline int cow_file_range(struct inode *inode,
  851. struct page *locked_page,
  852. u64 start, u64 end, u64 delalloc_end,
  853. int *page_started, unsigned long *nr_written,
  854. int unlock, struct btrfs_dedupe_hash *hash)
  855. {
  856. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  857. struct btrfs_root *root = BTRFS_I(inode)->root;
  858. u64 alloc_hint = 0;
  859. u64 num_bytes;
  860. unsigned long ram_size;
  861. u64 disk_num_bytes;
  862. u64 cur_alloc_size = 0;
  863. u64 blocksize = fs_info->sectorsize;
  864. struct btrfs_key ins;
  865. struct extent_map *em;
  866. unsigned clear_bits;
  867. unsigned long page_ops;
  868. bool extent_reserved = false;
  869. int ret = 0;
  870. if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
  871. WARN_ON_ONCE(1);
  872. ret = -EINVAL;
  873. goto out_unlock;
  874. }
  875. num_bytes = ALIGN(end - start + 1, blocksize);
  876. num_bytes = max(blocksize, num_bytes);
  877. disk_num_bytes = num_bytes;
  878. inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K);
  879. if (start == 0) {
  880. /* lets try to make an inline extent */
  881. ret = cow_file_range_inline(root, inode, start, end, 0,
  882. BTRFS_COMPRESS_NONE, NULL);
  883. if (ret == 0) {
  884. extent_clear_unlock_delalloc(inode, start, end,
  885. delalloc_end, NULL,
  886. EXTENT_LOCKED | EXTENT_DELALLOC |
  887. EXTENT_DELALLOC_NEW |
  888. EXTENT_DEFRAG, PAGE_UNLOCK |
  889. PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
  890. PAGE_END_WRITEBACK);
  891. btrfs_free_reserved_data_space_noquota(inode, start,
  892. end - start + 1);
  893. *nr_written = *nr_written +
  894. (end - start + PAGE_SIZE) / PAGE_SIZE;
  895. *page_started = 1;
  896. goto out;
  897. } else if (ret < 0) {
  898. goto out_unlock;
  899. }
  900. }
  901. BUG_ON(disk_num_bytes >
  902. btrfs_super_total_bytes(fs_info->super_copy));
  903. alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
  904. btrfs_drop_extent_cache(BTRFS_I(inode), start,
  905. start + num_bytes - 1, 0);
  906. while (disk_num_bytes > 0) {
  907. cur_alloc_size = disk_num_bytes;
  908. ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
  909. fs_info->sectorsize, 0, alloc_hint,
  910. &ins, 1, 1);
  911. if (ret < 0)
  912. goto out_unlock;
  913. cur_alloc_size = ins.offset;
  914. extent_reserved = true;
  915. ram_size = ins.offset;
  916. em = create_io_em(inode, start, ins.offset, /* len */
  917. start, /* orig_start */
  918. ins.objectid, /* block_start */
  919. ins.offset, /* block_len */
  920. ins.offset, /* orig_block_len */
  921. ram_size, /* ram_bytes */
  922. BTRFS_COMPRESS_NONE, /* compress_type */
  923. BTRFS_ORDERED_REGULAR /* type */);
  924. if (IS_ERR(em))
  925. goto out_reserve;
  926. free_extent_map(em);
  927. ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
  928. ram_size, cur_alloc_size, 0);
  929. if (ret)
  930. goto out_drop_extent_cache;
  931. if (root->root_key.objectid ==
  932. BTRFS_DATA_RELOC_TREE_OBJECTID) {
  933. ret = btrfs_reloc_clone_csums(inode, start,
  934. cur_alloc_size);
  935. /*
  936. * Only drop cache here, and process as normal.
  937. *
  938. * We must not allow extent_clear_unlock_delalloc()
  939. * at out_unlock label to free meta of this ordered
  940. * extent, as its meta should be freed by
  941. * btrfs_finish_ordered_io().
  942. *
  943. * So we must continue until @start is increased to
  944. * skip current ordered extent.
  945. */
  946. if (ret)
  947. btrfs_drop_extent_cache(BTRFS_I(inode), start,
  948. start + ram_size - 1, 0);
  949. }
  950. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  951. /* we're not doing compressed IO, don't unlock the first
  952. * page (which the caller expects to stay locked), don't
  953. * clear any dirty bits and don't set any writeback bits
  954. *
  955. * Do set the Private2 bit so we know this page was properly
  956. * setup for writepage
  957. */
  958. page_ops = unlock ? PAGE_UNLOCK : 0;
  959. page_ops |= PAGE_SET_PRIVATE2;
  960. extent_clear_unlock_delalloc(inode, start,
  961. start + ram_size - 1,
  962. delalloc_end, locked_page,
  963. EXTENT_LOCKED | EXTENT_DELALLOC,
  964. page_ops);
  965. if (disk_num_bytes < cur_alloc_size)
  966. disk_num_bytes = 0;
  967. else
  968. disk_num_bytes -= cur_alloc_size;
  969. num_bytes -= cur_alloc_size;
  970. alloc_hint = ins.objectid + ins.offset;
  971. start += cur_alloc_size;
  972. extent_reserved = false;
  973. /*
  974. * btrfs_reloc_clone_csums() error, since start is increased
  975. * extent_clear_unlock_delalloc() at out_unlock label won't
  976. * free metadata of current ordered extent, we're OK to exit.
  977. */
  978. if (ret)
  979. goto out_unlock;
  980. }
  981. out:
  982. return ret;
  983. out_drop_extent_cache:
  984. btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0);
  985. out_reserve:
  986. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  987. btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
  988. out_unlock:
  989. clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
  990. EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;
  991. page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
  992. PAGE_END_WRITEBACK;
  993. /*
  994. * If we reserved an extent for our delalloc range (or a subrange) and
  995. * failed to create the respective ordered extent, then it means that
  996. * when we reserved the extent we decremented the extent's size from
  997. * the data space_info's bytes_may_use counter and incremented the
  998. * space_info's bytes_reserved counter by the same amount. We must make
  999. * sure extent_clear_unlock_delalloc() does not try to decrement again
  1000. * the data space_info's bytes_may_use counter, therefore we do not pass
  1001. * it the flag EXTENT_CLEAR_DATA_RESV.
  1002. */
  1003. if (extent_reserved) {
  1004. extent_clear_unlock_delalloc(inode, start,
  1005. start + cur_alloc_size,
  1006. start + cur_alloc_size,
  1007. locked_page,
  1008. clear_bits,
  1009. page_ops);
  1010. start += cur_alloc_size;
  1011. if (start >= end)
  1012. goto out;
  1013. }
  1014. extent_clear_unlock_delalloc(inode, start, end, delalloc_end,
  1015. locked_page,
  1016. clear_bits | EXTENT_CLEAR_DATA_RESV,
  1017. page_ops);
  1018. goto out;
  1019. }
  1020. /*
  1021. * work queue call back to started compression on a file and pages
  1022. */
  1023. static noinline void async_cow_start(struct btrfs_work *work)
  1024. {
  1025. struct async_cow *async_cow;
  1026. int num_added = 0;
  1027. async_cow = container_of(work, struct async_cow, work);
  1028. compress_file_range(async_cow->inode, async_cow->locked_page,
  1029. async_cow->start, async_cow->end, async_cow,
  1030. &num_added);
  1031. if (num_added == 0) {
  1032. btrfs_add_delayed_iput(async_cow->inode);
  1033. async_cow->inode = NULL;
  1034. }
  1035. }
  1036. /*
  1037. * work queue call back to submit previously compressed pages
  1038. */
  1039. static noinline void async_cow_submit(struct btrfs_work *work)
  1040. {
  1041. struct btrfs_fs_info *fs_info;
  1042. struct async_cow *async_cow;
  1043. struct btrfs_root *root;
  1044. unsigned long nr_pages;
  1045. async_cow = container_of(work, struct async_cow, work);
  1046. root = async_cow->root;
  1047. fs_info = root->fs_info;
  1048. nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >>
  1049. PAGE_SHIFT;
  1050. /*
  1051. * atomic_sub_return implies a barrier for waitqueue_active
  1052. */
  1053. if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
  1054. 5 * SZ_1M &&
  1055. waitqueue_active(&fs_info->async_submit_wait))
  1056. wake_up(&fs_info->async_submit_wait);
  1057. if (async_cow->inode)
  1058. submit_compressed_extents(async_cow->inode, async_cow);
  1059. }
  1060. static noinline void async_cow_free(struct btrfs_work *work)
  1061. {
  1062. struct async_cow *async_cow;
  1063. async_cow = container_of(work, struct async_cow, work);
  1064. if (async_cow->inode)
  1065. btrfs_add_delayed_iput(async_cow->inode);
  1066. kfree(async_cow);
  1067. }
  1068. static int cow_file_range_async(struct inode *inode, struct page *locked_page,
  1069. u64 start, u64 end, int *page_started,
  1070. unsigned long *nr_written)
  1071. {
  1072. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1073. struct async_cow *async_cow;
  1074. struct btrfs_root *root = BTRFS_I(inode)->root;
  1075. unsigned long nr_pages;
  1076. u64 cur_end;
  1077. clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED,
  1078. 1, 0, NULL, GFP_NOFS);
  1079. while (start < end) {
  1080. async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS);
  1081. BUG_ON(!async_cow); /* -ENOMEM */
  1082. async_cow->inode = igrab(inode);
  1083. async_cow->root = root;
  1084. async_cow->locked_page = locked_page;
  1085. async_cow->start = start;
  1086. if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS &&
  1087. !btrfs_test_opt(fs_info, FORCE_COMPRESS))
  1088. cur_end = end;
  1089. else
  1090. cur_end = min(end, start + SZ_512K - 1);
  1091. async_cow->end = cur_end;
  1092. INIT_LIST_HEAD(&async_cow->extents);
  1093. btrfs_init_work(&async_cow->work,
  1094. btrfs_delalloc_helper,
  1095. async_cow_start, async_cow_submit,
  1096. async_cow_free);
  1097. nr_pages = (cur_end - start + PAGE_SIZE) >>
  1098. PAGE_SHIFT;
  1099. atomic_add(nr_pages, &fs_info->async_delalloc_pages);
  1100. btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work);
  1101. while (atomic_read(&fs_info->async_submit_draining) &&
  1102. atomic_read(&fs_info->async_delalloc_pages)) {
  1103. wait_event(fs_info->async_submit_wait,
  1104. (atomic_read(&fs_info->async_delalloc_pages) ==
  1105. 0));
  1106. }
  1107. *nr_written += nr_pages;
  1108. start = cur_end + 1;
  1109. }
  1110. *page_started = 1;
  1111. return 0;
  1112. }
  1113. static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info,
  1114. u64 bytenr, u64 num_bytes)
  1115. {
  1116. int ret;
  1117. struct btrfs_ordered_sum *sums;
  1118. LIST_HEAD(list);
  1119. ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr,
  1120. bytenr + num_bytes - 1, &list, 0);
  1121. if (ret == 0 && list_empty(&list))
  1122. return 0;
  1123. while (!list_empty(&list)) {
  1124. sums = list_entry(list.next, struct btrfs_ordered_sum, list);
  1125. list_del(&sums->list);
  1126. kfree(sums);
  1127. }
  1128. return 1;
  1129. }
  1130. /*
  1131. * when nowcow writeback call back. This checks for snapshots or COW copies
  1132. * of the extents that exist in the file, and COWs the file as required.
  1133. *
  1134. * If no cow copies or snapshots exist, we write directly to the existing
  1135. * blocks on disk
  1136. */
  1137. static noinline int run_delalloc_nocow(struct inode *inode,
  1138. struct page *locked_page,
  1139. u64 start, u64 end, int *page_started, int force,
  1140. unsigned long *nr_written)
  1141. {
  1142. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1143. struct btrfs_root *root = BTRFS_I(inode)->root;
  1144. struct extent_buffer *leaf;
  1145. struct btrfs_path *path;
  1146. struct btrfs_file_extent_item *fi;
  1147. struct btrfs_key found_key;
  1148. struct extent_map *em;
  1149. u64 cow_start;
  1150. u64 cur_offset;
  1151. u64 extent_end;
  1152. u64 extent_offset;
  1153. u64 disk_bytenr;
  1154. u64 num_bytes;
  1155. u64 disk_num_bytes;
  1156. u64 ram_bytes;
  1157. int extent_type;
  1158. int ret, err;
  1159. int type;
  1160. int nocow;
  1161. int check_prev = 1;
  1162. bool nolock;
  1163. u64 ino = btrfs_ino(BTRFS_I(inode));
  1164. path = btrfs_alloc_path();
  1165. if (!path) {
  1166. extent_clear_unlock_delalloc(inode, start, end, end,
  1167. locked_page,
  1168. EXTENT_LOCKED | EXTENT_DELALLOC |
  1169. EXTENT_DO_ACCOUNTING |
  1170. EXTENT_DEFRAG, PAGE_UNLOCK |
  1171. PAGE_CLEAR_DIRTY |
  1172. PAGE_SET_WRITEBACK |
  1173. PAGE_END_WRITEBACK);
  1174. return -ENOMEM;
  1175. }
  1176. nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
  1177. cow_start = (u64)-1;
  1178. cur_offset = start;
  1179. while (1) {
  1180. ret = btrfs_lookup_file_extent(NULL, root, path, ino,
  1181. cur_offset, 0);
  1182. if (ret < 0)
  1183. goto error;
  1184. if (ret > 0 && path->slots[0] > 0 && check_prev) {
  1185. leaf = path->nodes[0];
  1186. btrfs_item_key_to_cpu(leaf, &found_key,
  1187. path->slots[0] - 1);
  1188. if (found_key.objectid == ino &&
  1189. found_key.type == BTRFS_EXTENT_DATA_KEY)
  1190. path->slots[0]--;
  1191. }
  1192. check_prev = 0;
  1193. next_slot:
  1194. leaf = path->nodes[0];
  1195. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  1196. ret = btrfs_next_leaf(root, path);
  1197. if (ret < 0)
  1198. goto error;
  1199. if (ret > 0)
  1200. break;
  1201. leaf = path->nodes[0];
  1202. }
  1203. nocow = 0;
  1204. disk_bytenr = 0;
  1205. num_bytes = 0;
  1206. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  1207. if (found_key.objectid > ino)
  1208. break;
  1209. if (WARN_ON_ONCE(found_key.objectid < ino) ||
  1210. found_key.type < BTRFS_EXTENT_DATA_KEY) {
  1211. path->slots[0]++;
  1212. goto next_slot;
  1213. }
  1214. if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
  1215. found_key.offset > end)
  1216. break;
  1217. if (found_key.offset > cur_offset) {
  1218. extent_end = found_key.offset;
  1219. extent_type = 0;
  1220. goto out_check;
  1221. }
  1222. fi = btrfs_item_ptr(leaf, path->slots[0],
  1223. struct btrfs_file_extent_item);
  1224. extent_type = btrfs_file_extent_type(leaf, fi);
  1225. ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
  1226. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  1227. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  1228. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  1229. extent_offset = btrfs_file_extent_offset(leaf, fi);
  1230. extent_end = found_key.offset +
  1231. btrfs_file_extent_num_bytes(leaf, fi);
  1232. disk_num_bytes =
  1233. btrfs_file_extent_disk_num_bytes(leaf, fi);
  1234. if (extent_end <= start) {
  1235. path->slots[0]++;
  1236. goto next_slot;
  1237. }
  1238. if (disk_bytenr == 0)
  1239. goto out_check;
  1240. if (btrfs_file_extent_compression(leaf, fi) ||
  1241. btrfs_file_extent_encryption(leaf, fi) ||
  1242. btrfs_file_extent_other_encoding(leaf, fi))
  1243. goto out_check;
  1244. if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
  1245. goto out_check;
  1246. if (btrfs_extent_readonly(fs_info, disk_bytenr))
  1247. goto out_check;
  1248. if (btrfs_cross_ref_exist(root, ino,
  1249. found_key.offset -
  1250. extent_offset, disk_bytenr))
  1251. goto out_check;
  1252. disk_bytenr += extent_offset;
  1253. disk_bytenr += cur_offset - found_key.offset;
  1254. num_bytes = min(end + 1, extent_end) - cur_offset;
  1255. /*
  1256. * if there are pending snapshots for this root,
  1257. * we fall into common COW way.
  1258. */
  1259. if (!nolock) {
  1260. err = btrfs_start_write_no_snapshoting(root);
  1261. if (!err)
  1262. goto out_check;
  1263. }
  1264. /*
  1265. * force cow if csum exists in the range.
  1266. * this ensure that csum for a given extent are
  1267. * either valid or do not exist.
  1268. */
  1269. if (csum_exist_in_range(fs_info, disk_bytenr,
  1270. num_bytes)) {
  1271. if (!nolock)
  1272. btrfs_end_write_no_snapshoting(root);
  1273. goto out_check;
  1274. }
  1275. if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) {
  1276. if (!nolock)
  1277. btrfs_end_write_no_snapshoting(root);
  1278. goto out_check;
  1279. }
  1280. nocow = 1;
  1281. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  1282. extent_end = found_key.offset +
  1283. btrfs_file_extent_inline_len(leaf,
  1284. path->slots[0], fi);
  1285. extent_end = ALIGN(extent_end,
  1286. fs_info->sectorsize);
  1287. } else {
  1288. BUG_ON(1);
  1289. }
  1290. out_check:
  1291. if (extent_end <= start) {
  1292. path->slots[0]++;
  1293. if (!nolock && nocow)
  1294. btrfs_end_write_no_snapshoting(root);
  1295. if (nocow)
  1296. btrfs_dec_nocow_writers(fs_info, disk_bytenr);
  1297. goto next_slot;
  1298. }
  1299. if (!nocow) {
  1300. if (cow_start == (u64)-1)
  1301. cow_start = cur_offset;
  1302. cur_offset = extent_end;
  1303. if (cur_offset > end)
  1304. break;
  1305. path->slots[0]++;
  1306. goto next_slot;
  1307. }
  1308. btrfs_release_path(path);
  1309. if (cow_start != (u64)-1) {
  1310. ret = cow_file_range(inode, locked_page,
  1311. cow_start, found_key.offset - 1,
  1312. end, page_started, nr_written, 1,
  1313. NULL);
  1314. if (ret) {
  1315. if (!nolock && nocow)
  1316. btrfs_end_write_no_snapshoting(root);
  1317. if (nocow)
  1318. btrfs_dec_nocow_writers(fs_info,
  1319. disk_bytenr);
  1320. goto error;
  1321. }
  1322. cow_start = (u64)-1;
  1323. }
  1324. if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  1325. u64 orig_start = found_key.offset - extent_offset;
  1326. em = create_io_em(inode, cur_offset, num_bytes,
  1327. orig_start,
  1328. disk_bytenr, /* block_start */
  1329. num_bytes, /* block_len */
  1330. disk_num_bytes, /* orig_block_len */
  1331. ram_bytes, BTRFS_COMPRESS_NONE,
  1332. BTRFS_ORDERED_PREALLOC);
  1333. if (IS_ERR(em)) {
  1334. if (!nolock && nocow)
  1335. btrfs_end_write_no_snapshoting(root);
  1336. if (nocow)
  1337. btrfs_dec_nocow_writers(fs_info,
  1338. disk_bytenr);
  1339. ret = PTR_ERR(em);
  1340. goto error;
  1341. }
  1342. free_extent_map(em);
  1343. }
  1344. if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  1345. type = BTRFS_ORDERED_PREALLOC;
  1346. } else {
  1347. type = BTRFS_ORDERED_NOCOW;
  1348. }
  1349. ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr,
  1350. num_bytes, num_bytes, type);
  1351. if (nocow)
  1352. btrfs_dec_nocow_writers(fs_info, disk_bytenr);
  1353. BUG_ON(ret); /* -ENOMEM */
  1354. if (root->root_key.objectid ==
  1355. BTRFS_DATA_RELOC_TREE_OBJECTID)
  1356. /*
  1357. * Error handled later, as we must prevent
  1358. * extent_clear_unlock_delalloc() in error handler
  1359. * from freeing metadata of created ordered extent.
  1360. */
  1361. ret = btrfs_reloc_clone_csums(inode, cur_offset,
  1362. num_bytes);
  1363. extent_clear_unlock_delalloc(inode, cur_offset,
  1364. cur_offset + num_bytes - 1, end,
  1365. locked_page, EXTENT_LOCKED |
  1366. EXTENT_DELALLOC |
  1367. EXTENT_CLEAR_DATA_RESV,
  1368. PAGE_UNLOCK | PAGE_SET_PRIVATE2);
  1369. if (!nolock && nocow)
  1370. btrfs_end_write_no_snapshoting(root);
  1371. cur_offset = extent_end;
  1372. /*
  1373. * btrfs_reloc_clone_csums() error, now we're OK to call error
  1374. * handler, as metadata for created ordered extent will only
  1375. * be freed by btrfs_finish_ordered_io().
  1376. */
  1377. if (ret)
  1378. goto error;
  1379. if (cur_offset > end)
  1380. break;
  1381. }
  1382. btrfs_release_path(path);
  1383. if (cur_offset <= end && cow_start == (u64)-1) {
  1384. cow_start = cur_offset;
  1385. cur_offset = end;
  1386. }
  1387. if (cow_start != (u64)-1) {
  1388. ret = cow_file_range(inode, locked_page, cow_start, end, end,
  1389. page_started, nr_written, 1, NULL);
  1390. if (ret)
  1391. goto error;
  1392. }
  1393. error:
  1394. if (ret && cur_offset < end)
  1395. extent_clear_unlock_delalloc(inode, cur_offset, end, end,
  1396. locked_page, EXTENT_LOCKED |
  1397. EXTENT_DELALLOC | EXTENT_DEFRAG |
  1398. EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
  1399. PAGE_CLEAR_DIRTY |
  1400. PAGE_SET_WRITEBACK |
  1401. PAGE_END_WRITEBACK);
  1402. btrfs_free_path(path);
  1403. return ret;
  1404. }
  1405. static inline int need_force_cow(struct inode *inode, u64 start, u64 end)
  1406. {
  1407. if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
  1408. !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC))
  1409. return 0;
  1410. /*
  1411. * @defrag_bytes is a hint value, no spinlock held here,
  1412. * if is not zero, it means the file is defragging.
  1413. * Force cow if given extent needs to be defragged.
  1414. */
  1415. if (BTRFS_I(inode)->defrag_bytes &&
  1416. test_range_bit(&BTRFS_I(inode)->io_tree, start, end,
  1417. EXTENT_DEFRAG, 0, NULL))
  1418. return 1;
  1419. return 0;
  1420. }
  1421. /*
  1422. * extent_io.c call back to do delayed allocation processing
  1423. */
  1424. static int run_delalloc_range(void *private_data, struct page *locked_page,
  1425. u64 start, u64 end, int *page_started,
  1426. unsigned long *nr_written)
  1427. {
  1428. struct inode *inode = private_data;
  1429. int ret;
  1430. int force_cow = need_force_cow(inode, start, end);
  1431. if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) {
  1432. ret = run_delalloc_nocow(inode, locked_page, start, end,
  1433. page_started, 1, nr_written);
  1434. } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) {
  1435. ret = run_delalloc_nocow(inode, locked_page, start, end,
  1436. page_started, 0, nr_written);
  1437. } else if (!inode_need_compress(inode)) {
  1438. ret = cow_file_range(inode, locked_page, start, end, end,
  1439. page_started, nr_written, 1, NULL);
  1440. } else {
  1441. set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
  1442. &BTRFS_I(inode)->runtime_flags);
  1443. ret = cow_file_range_async(inode, locked_page, start, end,
  1444. page_started, nr_written);
  1445. }
  1446. if (ret)
  1447. btrfs_cleanup_ordered_extents(inode, start, end - start + 1);
  1448. return ret;
  1449. }
  1450. static void btrfs_split_extent_hook(void *private_data,
  1451. struct extent_state *orig, u64 split)
  1452. {
  1453. struct inode *inode = private_data;
  1454. u64 size;
  1455. /* not delalloc, ignore it */
  1456. if (!(orig->state & EXTENT_DELALLOC))
  1457. return;
  1458. size = orig->end - orig->start + 1;
  1459. if (size > BTRFS_MAX_EXTENT_SIZE) {
  1460. u32 num_extents;
  1461. u64 new_size;
  1462. /*
  1463. * See the explanation in btrfs_merge_extent_hook, the same
  1464. * applies here, just in reverse.
  1465. */
  1466. new_size = orig->end - split + 1;
  1467. num_extents = count_max_extents(new_size);
  1468. new_size = split - orig->start;
  1469. num_extents += count_max_extents(new_size);
  1470. if (count_max_extents(size) >= num_extents)
  1471. return;
  1472. }
  1473. spin_lock(&BTRFS_I(inode)->lock);
  1474. BTRFS_I(inode)->outstanding_extents++;
  1475. spin_unlock(&BTRFS_I(inode)->lock);
  1476. }
  1477. /*
  1478. * extent_io.c merge_extent_hook, used to track merged delayed allocation
  1479. * extents so we can keep track of new extents that are just merged onto old
  1480. * extents, such as when we are doing sequential writes, so we can properly
  1481. * account for the metadata space we'll need.
  1482. */
  1483. static void btrfs_merge_extent_hook(void *private_data,
  1484. struct extent_state *new,
  1485. struct extent_state *other)
  1486. {
  1487. struct inode *inode = private_data;
  1488. u64 new_size, old_size;
  1489. u32 num_extents;
  1490. /* not delalloc, ignore it */
  1491. if (!(other->state & EXTENT_DELALLOC))
  1492. return;
  1493. if (new->start > other->start)
  1494. new_size = new->end - other->start + 1;
  1495. else
  1496. new_size = other->end - new->start + 1;
  1497. /* we're not bigger than the max, unreserve the space and go */
  1498. if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
  1499. spin_lock(&BTRFS_I(inode)->lock);
  1500. BTRFS_I(inode)->outstanding_extents--;
  1501. spin_unlock(&BTRFS_I(inode)->lock);
  1502. return;
  1503. }
  1504. /*
  1505. * We have to add up either side to figure out how many extents were
  1506. * accounted for before we merged into one big extent. If the number of
  1507. * extents we accounted for is <= the amount we need for the new range
  1508. * then we can return, otherwise drop. Think of it like this
  1509. *
  1510. * [ 4k][MAX_SIZE]
  1511. *
  1512. * So we've grown the extent by a MAX_SIZE extent, this would mean we
  1513. * need 2 outstanding extents, on one side we have 1 and the other side
  1514. * we have 1 so they are == and we can return. But in this case
  1515. *
  1516. * [MAX_SIZE+4k][MAX_SIZE+4k]
  1517. *
  1518. * Each range on their own accounts for 2 extents, but merged together
  1519. * they are only 3 extents worth of accounting, so we need to drop in
  1520. * this case.
  1521. */
  1522. old_size = other->end - other->start + 1;
  1523. num_extents = count_max_extents(old_size);
  1524. old_size = new->end - new->start + 1;
  1525. num_extents += count_max_extents(old_size);
  1526. if (count_max_extents(new_size) >= num_extents)
  1527. return;
  1528. spin_lock(&BTRFS_I(inode)->lock);
  1529. BTRFS_I(inode)->outstanding_extents--;
  1530. spin_unlock(&BTRFS_I(inode)->lock);
  1531. }
  1532. static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
  1533. struct inode *inode)
  1534. {
  1535. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1536. spin_lock(&root->delalloc_lock);
  1537. if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
  1538. list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
  1539. &root->delalloc_inodes);
  1540. set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
  1541. &BTRFS_I(inode)->runtime_flags);
  1542. root->nr_delalloc_inodes++;
  1543. if (root->nr_delalloc_inodes == 1) {
  1544. spin_lock(&fs_info->delalloc_root_lock);
  1545. BUG_ON(!list_empty(&root->delalloc_root));
  1546. list_add_tail(&root->delalloc_root,
  1547. &fs_info->delalloc_roots);
  1548. spin_unlock(&fs_info->delalloc_root_lock);
  1549. }
  1550. }
  1551. spin_unlock(&root->delalloc_lock);
  1552. }
  1553. static void btrfs_del_delalloc_inode(struct btrfs_root *root,
  1554. struct btrfs_inode *inode)
  1555. {
  1556. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  1557. spin_lock(&root->delalloc_lock);
  1558. if (!list_empty(&inode->delalloc_inodes)) {
  1559. list_del_init(&inode->delalloc_inodes);
  1560. clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
  1561. &inode->runtime_flags);
  1562. root->nr_delalloc_inodes--;
  1563. if (!root->nr_delalloc_inodes) {
  1564. spin_lock(&fs_info->delalloc_root_lock);
  1565. BUG_ON(list_empty(&root->delalloc_root));
  1566. list_del_init(&root->delalloc_root);
  1567. spin_unlock(&fs_info->delalloc_root_lock);
  1568. }
  1569. }
  1570. spin_unlock(&root->delalloc_lock);
  1571. }
  1572. /*
  1573. * extent_io.c set_bit_hook, used to track delayed allocation
  1574. * bytes in this file, and to maintain the list of inodes that
  1575. * have pending delalloc work to be done.
  1576. */
  1577. static void btrfs_set_bit_hook(void *private_data,
  1578. struct extent_state *state, unsigned *bits)
  1579. {
  1580. struct inode *inode = private_data;
  1581. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1582. if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
  1583. WARN_ON(1);
  1584. /*
  1585. * set_bit and clear bit hooks normally require _irqsave/restore
  1586. * but in this case, we are only testing for the DELALLOC
  1587. * bit, which is only set or cleared with irqs on
  1588. */
  1589. if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
  1590. struct btrfs_root *root = BTRFS_I(inode)->root;
  1591. u64 len = state->end + 1 - state->start;
  1592. bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));
  1593. if (*bits & EXTENT_FIRST_DELALLOC) {
  1594. *bits &= ~EXTENT_FIRST_DELALLOC;
  1595. } else {
  1596. spin_lock(&BTRFS_I(inode)->lock);
  1597. BTRFS_I(inode)->outstanding_extents++;
  1598. spin_unlock(&BTRFS_I(inode)->lock);
  1599. }
  1600. /* For sanity tests */
  1601. if (btrfs_is_testing(fs_info))
  1602. return;
  1603. __percpu_counter_add(&fs_info->delalloc_bytes, len,
  1604. fs_info->delalloc_batch);
  1605. spin_lock(&BTRFS_I(inode)->lock);
  1606. BTRFS_I(inode)->delalloc_bytes += len;
  1607. if (*bits & EXTENT_DEFRAG)
  1608. BTRFS_I(inode)->defrag_bytes += len;
  1609. if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
  1610. &BTRFS_I(inode)->runtime_flags))
  1611. btrfs_add_delalloc_inodes(root, inode);
  1612. spin_unlock(&BTRFS_I(inode)->lock);
  1613. }
  1614. if (!(state->state & EXTENT_DELALLOC_NEW) &&
  1615. (*bits & EXTENT_DELALLOC_NEW)) {
  1616. spin_lock(&BTRFS_I(inode)->lock);
  1617. BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 -
  1618. state->start;
  1619. spin_unlock(&BTRFS_I(inode)->lock);
  1620. }
  1621. }
  1622. /*
  1623. * extent_io.c clear_bit_hook, see set_bit_hook for why
  1624. */
  1625. static void btrfs_clear_bit_hook(void *private_data,
  1626. struct extent_state *state,
  1627. unsigned *bits)
  1628. {
  1629. struct btrfs_inode *inode = BTRFS_I((struct inode *)private_data);
  1630. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  1631. u64 len = state->end + 1 - state->start;
  1632. u32 num_extents = count_max_extents(len);
  1633. spin_lock(&inode->lock);
  1634. if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG))
  1635. inode->defrag_bytes -= len;
  1636. spin_unlock(&inode->lock);
  1637. /*
  1638. * set_bit and clear bit hooks normally require _irqsave/restore
  1639. * but in this case, we are only testing for the DELALLOC
  1640. * bit, which is only set or cleared with irqs on
  1641. */
  1642. if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
  1643. struct btrfs_root *root = inode->root;
  1644. bool do_list = !btrfs_is_free_space_inode(inode);
  1645. if (*bits & EXTENT_FIRST_DELALLOC) {
  1646. *bits &= ~EXTENT_FIRST_DELALLOC;
  1647. } else if (!(*bits & EXTENT_CLEAR_META_RESV)) {
  1648. spin_lock(&inode->lock);
  1649. inode->outstanding_extents -= num_extents;
  1650. spin_unlock(&inode->lock);
  1651. }
  1652. /*
  1653. * We don't reserve metadata space for space cache inodes so we
  1654. * don't need to call dellalloc_release_metadata if there is an
  1655. * error.
  1656. */
  1657. if (*bits & EXTENT_CLEAR_META_RESV &&
  1658. root != fs_info->tree_root)
  1659. btrfs_delalloc_release_metadata(inode, len);
  1660. /* For sanity tests. */
  1661. if (btrfs_is_testing(fs_info))
  1662. return;
  1663. if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID &&
  1664. do_list && !(state->state & EXTENT_NORESERVE) &&
  1665. (*bits & EXTENT_CLEAR_DATA_RESV))
  1666. btrfs_free_reserved_data_space_noquota(
  1667. &inode->vfs_inode,
  1668. state->start, len);
  1669. __percpu_counter_add(&fs_info->delalloc_bytes, -len,
  1670. fs_info->delalloc_batch);
  1671. spin_lock(&inode->lock);
  1672. inode->delalloc_bytes -= len;
  1673. if (do_list && inode->delalloc_bytes == 0 &&
  1674. test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
  1675. &inode->runtime_flags))
  1676. btrfs_del_delalloc_inode(root, inode);
  1677. spin_unlock(&inode->lock);
  1678. }
  1679. if ((state->state & EXTENT_DELALLOC_NEW) &&
  1680. (*bits & EXTENT_DELALLOC_NEW)) {
  1681. spin_lock(&inode->lock);
  1682. ASSERT(inode->new_delalloc_bytes >= len);
  1683. inode->new_delalloc_bytes -= len;
  1684. spin_unlock(&inode->lock);
  1685. }
  1686. }
  1687. /*
  1688. * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
  1689. * we don't create bios that span stripes or chunks
  1690. *
  1691. * return 1 if page cannot be merged to bio
  1692. * return 0 if page can be merged to bio
  1693. * return error otherwise
  1694. */
  1695. int btrfs_merge_bio_hook(struct page *page, unsigned long offset,
  1696. size_t size, struct bio *bio,
  1697. unsigned long bio_flags)
  1698. {
  1699. struct inode *inode = page->mapping->host;
  1700. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1701. u64 logical = (u64)bio->bi_iter.bi_sector << 9;
  1702. u64 length = 0;
  1703. u64 map_length;
  1704. int ret;
  1705. if (bio_flags & EXTENT_BIO_COMPRESSED)
  1706. return 0;
  1707. length = bio->bi_iter.bi_size;
  1708. map_length = length;
  1709. ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length,
  1710. NULL, 0);
  1711. if (ret < 0)
  1712. return ret;
  1713. if (map_length < length + size)
  1714. return 1;
  1715. return 0;
  1716. }
  1717. /*
  1718. * in order to insert checksums into the metadata in large chunks,
  1719. * we wait until bio submission time. All the pages in the bio are
  1720. * checksummed and sums are attached onto the ordered extent record.
  1721. *
  1722. * At IO completion time the cums attached on the ordered extent record
  1723. * are inserted into the btree
  1724. */
  1725. static int __btrfs_submit_bio_start(void *private_data, struct bio *bio,
  1726. int mirror_num, unsigned long bio_flags,
  1727. u64 bio_offset)
  1728. {
  1729. struct inode *inode = private_data;
  1730. int ret = 0;
  1731. ret = btrfs_csum_one_bio(inode, bio, 0, 0);
  1732. BUG_ON(ret); /* -ENOMEM */
  1733. return 0;
  1734. }
  1735. /*
  1736. * in order to insert checksums into the metadata in large chunks,
  1737. * we wait until bio submission time. All the pages in the bio are
  1738. * checksummed and sums are attached onto the ordered extent record.
  1739. *
  1740. * At IO completion time the cums attached on the ordered extent record
  1741. * are inserted into the btree
  1742. */
  1743. static int __btrfs_submit_bio_done(void *private_data, struct bio *bio,
  1744. int mirror_num, unsigned long bio_flags,
  1745. u64 bio_offset)
  1746. {
  1747. struct inode *inode = private_data;
  1748. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1749. int ret;
  1750. ret = btrfs_map_bio(fs_info, bio, mirror_num, 1);
  1751. if (ret) {
  1752. bio->bi_error = ret;
  1753. bio_endio(bio);
  1754. }
  1755. return ret;
  1756. }
  1757. /*
  1758. * extent_io.c submission hook. This does the right thing for csum calculation
  1759. * on write, or reading the csums from the tree before a read
  1760. */
  1761. static int btrfs_submit_bio_hook(void *private_data, struct bio *bio,
  1762. int mirror_num, unsigned long bio_flags,
  1763. u64 bio_offset)
  1764. {
  1765. struct inode *inode = private_data;
  1766. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1767. struct btrfs_root *root = BTRFS_I(inode)->root;
  1768. enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;
  1769. int ret = 0;
  1770. int skip_sum;
  1771. int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
  1772. skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
  1773. if (btrfs_is_free_space_inode(BTRFS_I(inode)))
  1774. metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
  1775. if (bio_op(bio) != REQ_OP_WRITE) {
  1776. ret = btrfs_bio_wq_end_io(fs_info, bio, metadata);
  1777. if (ret)
  1778. goto out;
  1779. if (bio_flags & EXTENT_BIO_COMPRESSED) {
  1780. ret = btrfs_submit_compressed_read(inode, bio,
  1781. mirror_num,
  1782. bio_flags);
  1783. goto out;
  1784. } else if (!skip_sum) {
  1785. ret = btrfs_lookup_bio_sums(inode, bio, NULL);
  1786. if (ret)
  1787. goto out;
  1788. }
  1789. goto mapit;
  1790. } else if (async && !skip_sum) {
  1791. /* csum items have already been cloned */
  1792. if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
  1793. goto mapit;
  1794. /* we're doing a write, do the async checksumming */
  1795. ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags,
  1796. bio_offset, inode,
  1797. __btrfs_submit_bio_start,
  1798. __btrfs_submit_bio_done);
  1799. goto out;
  1800. } else if (!skip_sum) {
  1801. ret = btrfs_csum_one_bio(inode, bio, 0, 0);
  1802. if (ret)
  1803. goto out;
  1804. }
  1805. mapit:
  1806. ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
  1807. out:
  1808. if (ret < 0) {
  1809. bio->bi_error = ret;
  1810. bio_endio(bio);
  1811. }
  1812. return ret;
  1813. }
  1814. /*
  1815. * given a list of ordered sums record them in the inode. This happens
  1816. * at IO completion time based on sums calculated at bio submission time.
  1817. */
  1818. static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
  1819. struct inode *inode, struct list_head *list)
  1820. {
  1821. struct btrfs_ordered_sum *sum;
  1822. list_for_each_entry(sum, list, list) {
  1823. trans->adding_csums = 1;
  1824. btrfs_csum_file_blocks(trans,
  1825. BTRFS_I(inode)->root->fs_info->csum_root, sum);
  1826. trans->adding_csums = 0;
  1827. }
  1828. return 0;
  1829. }
  1830. int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
  1831. struct extent_state **cached_state, int dedupe)
  1832. {
  1833. WARN_ON((end & (PAGE_SIZE - 1)) == 0);
  1834. return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
  1835. cached_state);
  1836. }
  1837. /* see btrfs_writepage_start_hook for details on why this is required */
  1838. struct btrfs_writepage_fixup {
  1839. struct page *page;
  1840. struct btrfs_work work;
  1841. };
  1842. static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
  1843. {
  1844. struct btrfs_writepage_fixup *fixup;
  1845. struct btrfs_ordered_extent *ordered;
  1846. struct extent_state *cached_state = NULL;
  1847. struct page *page;
  1848. struct inode *inode;
  1849. u64 page_start;
  1850. u64 page_end;
  1851. int ret;
  1852. fixup = container_of(work, struct btrfs_writepage_fixup, work);
  1853. page = fixup->page;
  1854. again:
  1855. lock_page(page);
  1856. if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
  1857. ClearPageChecked(page);
  1858. goto out_page;
  1859. }
  1860. inode = page->mapping->host;
  1861. page_start = page_offset(page);
  1862. page_end = page_offset(page) + PAGE_SIZE - 1;
  1863. lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end,
  1864. &cached_state);
  1865. /* already ordered? We're done */
  1866. if (PagePrivate2(page))
  1867. goto out;
  1868. ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
  1869. PAGE_SIZE);
  1870. if (ordered) {
  1871. unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
  1872. page_end, &cached_state, GFP_NOFS);
  1873. unlock_page(page);
  1874. btrfs_start_ordered_extent(inode, ordered, 1);
  1875. btrfs_put_ordered_extent(ordered);
  1876. goto again;
  1877. }
  1878. ret = btrfs_delalloc_reserve_space(inode, page_start,
  1879. PAGE_SIZE);
  1880. if (ret) {
  1881. mapping_set_error(page->mapping, ret);
  1882. end_extent_writepage(page, ret, page_start, page_end);
  1883. ClearPageChecked(page);
  1884. goto out;
  1885. }
  1886. btrfs_set_extent_delalloc(inode, page_start, page_end, &cached_state,
  1887. 0);
  1888. ClearPageChecked(page);
  1889. set_page_dirty(page);
  1890. out:
  1891. unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
  1892. &cached_state, GFP_NOFS);
  1893. out_page:
  1894. unlock_page(page);
  1895. put_page(page);
  1896. kfree(fixup);
  1897. }
  1898. /*
  1899. * There are a few paths in the higher layers of the kernel that directly
  1900. * set the page dirty bit without asking the filesystem if it is a
  1901. * good idea. This causes problems because we want to make sure COW
  1902. * properly happens and the data=ordered rules are followed.
  1903. *
  1904. * In our case any range that doesn't have the ORDERED bit set
  1905. * hasn't been properly setup for IO. We kick off an async process
  1906. * to fix it up. The async helper will wait for ordered extents, set
  1907. * the delalloc bit and make it safe to write the page.
  1908. */
  1909. static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end)
  1910. {
  1911. struct inode *inode = page->mapping->host;
  1912. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1913. struct btrfs_writepage_fixup *fixup;
  1914. /* this page is properly in the ordered list */
  1915. if (TestClearPagePrivate2(page))
  1916. return 0;
  1917. if (PageChecked(page))
  1918. return -EAGAIN;
  1919. fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
  1920. if (!fixup)
  1921. return -EAGAIN;
  1922. SetPageChecked(page);
  1923. get_page(page);
  1924. btrfs_init_work(&fixup->work, btrfs_fixup_helper,
  1925. btrfs_writepage_fixup_worker, NULL, NULL);
  1926. fixup->page = page;
  1927. btrfs_queue_work(fs_info->fixup_workers, &fixup->work);
  1928. return -EBUSY;
  1929. }
  1930. static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
  1931. struct inode *inode, u64 file_pos,
  1932. u64 disk_bytenr, u64 disk_num_bytes,
  1933. u64 num_bytes, u64 ram_bytes,
  1934. u8 compression, u8 encryption,
  1935. u16 other_encoding, int extent_type)
  1936. {
  1937. struct btrfs_root *root = BTRFS_I(inode)->root;
  1938. struct btrfs_file_extent_item *fi;
  1939. struct btrfs_path *path;
  1940. struct extent_buffer *leaf;
  1941. struct btrfs_key ins;
  1942. int extent_inserted = 0;
  1943. int ret;
  1944. path = btrfs_alloc_path();
  1945. if (!path)
  1946. return -ENOMEM;
  1947. /*
  1948. * we may be replacing one extent in the tree with another.
  1949. * The new extent is pinned in the extent map, and we don't want
  1950. * to drop it from the cache until it is completely in the btree.
  1951. *
  1952. * So, tell btrfs_drop_extents to leave this extent in the cache.
  1953. * the caller is expected to unpin it and allow it to be merged
  1954. * with the others.
  1955. */
  1956. ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
  1957. file_pos + num_bytes, NULL, 0,
  1958. 1, sizeof(*fi), &extent_inserted);
  1959. if (ret)
  1960. goto out;
  1961. if (!extent_inserted) {
  1962. ins.objectid = btrfs_ino(BTRFS_I(inode));
  1963. ins.offset = file_pos;
  1964. ins.type = BTRFS_EXTENT_DATA_KEY;
  1965. path->leave_spinning = 1;
  1966. ret = btrfs_insert_empty_item(trans, root, path, &ins,
  1967. sizeof(*fi));
  1968. if (ret)
  1969. goto out;
  1970. }
  1971. leaf = path->nodes[0];
  1972. fi = btrfs_item_ptr(leaf, path->slots[0],
  1973. struct btrfs_file_extent_item);
  1974. btrfs_set_file_extent_generation(leaf, fi, trans->transid);
  1975. btrfs_set_file_extent_type(leaf, fi, extent_type);
  1976. btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
  1977. btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
  1978. btrfs_set_file_extent_offset(leaf, fi, 0);
  1979. btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
  1980. btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
  1981. btrfs_set_file_extent_compression(leaf, fi, compression);
  1982. btrfs_set_file_extent_encryption(leaf, fi, encryption);
  1983. btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
  1984. btrfs_mark_buffer_dirty(leaf);
  1985. btrfs_release_path(path);
  1986. inode_add_bytes(inode, num_bytes);
  1987. ins.objectid = disk_bytenr;
  1988. ins.offset = disk_num_bytes;
  1989. ins.type = BTRFS_EXTENT_ITEM_KEY;
  1990. ret = btrfs_alloc_reserved_file_extent(trans, root->root_key.objectid,
  1991. btrfs_ino(BTRFS_I(inode)), file_pos, ram_bytes, &ins);
  1992. /*
  1993. * Release the reserved range from inode dirty range map, as it is
  1994. * already moved into delayed_ref_head
  1995. */
  1996. btrfs_qgroup_release_data(inode, file_pos, ram_bytes);
  1997. out:
  1998. btrfs_free_path(path);
  1999. return ret;
  2000. }
  2001. /* snapshot-aware defrag */
  2002. struct sa_defrag_extent_backref {
  2003. struct rb_node node;
  2004. struct old_sa_defrag_extent *old;
  2005. u64 root_id;
  2006. u64 inum;
  2007. u64 file_pos;
  2008. u64 extent_offset;
  2009. u64 num_bytes;
  2010. u64 generation;
  2011. };
  2012. struct old_sa_defrag_extent {
  2013. struct list_head list;
  2014. struct new_sa_defrag_extent *new;
  2015. u64 extent_offset;
  2016. u64 bytenr;
  2017. u64 offset;
  2018. u64 len;
  2019. int count;
  2020. };
  2021. struct new_sa_defrag_extent {
  2022. struct rb_root root;
  2023. struct list_head head;
  2024. struct btrfs_path *path;
  2025. struct inode *inode;
  2026. u64 file_pos;
  2027. u64 len;
  2028. u64 bytenr;
  2029. u64 disk_len;
  2030. u8 compress_type;
  2031. };
  2032. static int backref_comp(struct sa_defrag_extent_backref *b1,
  2033. struct sa_defrag_extent_backref *b2)
  2034. {
  2035. if (b1->root_id < b2->root_id)
  2036. return -1;
  2037. else if (b1->root_id > b2->root_id)
  2038. return 1;
  2039. if (b1->inum < b2->inum)
  2040. return -1;
  2041. else if (b1->inum > b2->inum)
  2042. return 1;
  2043. if (b1->file_pos < b2->file_pos)
  2044. return -1;
  2045. else if (b1->file_pos > b2->file_pos)
  2046. return 1;
  2047. /*
  2048. * [------------------------------] ===> (a range of space)
  2049. * |<--->| |<---->| =============> (fs/file tree A)
  2050. * |<---------------------------->| ===> (fs/file tree B)
  2051. *
  2052. * A range of space can refer to two file extents in one tree while
  2053. * refer to only one file extent in another tree.
  2054. *
  2055. * So we may process a disk offset more than one time(two extents in A)
  2056. * and locate at the same extent(one extent in B), then insert two same
  2057. * backrefs(both refer to the extent in B).
  2058. */
  2059. return 0;
  2060. }
  2061. static void backref_insert(struct rb_root *root,
  2062. struct sa_defrag_extent_backref *backref)
  2063. {
  2064. struct rb_node **p = &root->rb_node;
  2065. struct rb_node *parent = NULL;
  2066. struct sa_defrag_extent_backref *entry;
  2067. int ret;
  2068. while (*p) {
  2069. parent = *p;
  2070. entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
  2071. ret = backref_comp(backref, entry);
  2072. if (ret < 0)
  2073. p = &(*p)->rb_left;
  2074. else
  2075. p = &(*p)->rb_right;
  2076. }
  2077. rb_link_node(&backref->node, parent, p);
  2078. rb_insert_color(&backref->node, root);
  2079. }
  2080. /*
  2081. * Note the backref might has changed, and in this case we just return 0.
  2082. */
  2083. static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
  2084. void *ctx)
  2085. {
  2086. struct btrfs_file_extent_item *extent;
  2087. struct old_sa_defrag_extent *old = ctx;
  2088. struct new_sa_defrag_extent *new = old->new;
  2089. struct btrfs_path *path = new->path;
  2090. struct btrfs_key key;
  2091. struct btrfs_root *root;
  2092. struct sa_defrag_extent_backref *backref;
  2093. struct extent_buffer *leaf;
  2094. struct inode *inode = new->inode;
  2095. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2096. int slot;
  2097. int ret;
  2098. u64 extent_offset;
  2099. u64 num_bytes;
  2100. if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
  2101. inum == btrfs_ino(BTRFS_I(inode)))
  2102. return 0;
  2103. key.objectid = root_id;
  2104. key.type = BTRFS_ROOT_ITEM_KEY;
  2105. key.offset = (u64)-1;
  2106. root = btrfs_read_fs_root_no_name(fs_info, &key);
  2107. if (IS_ERR(root)) {
  2108. if (PTR_ERR(root) == -ENOENT)
  2109. return 0;
  2110. WARN_ON(1);
  2111. btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu",
  2112. inum, offset, root_id);
  2113. return PTR_ERR(root);
  2114. }
  2115. key.objectid = inum;
  2116. key.type = BTRFS_EXTENT_DATA_KEY;
  2117. if (offset > (u64)-1 << 32)
  2118. key.offset = 0;
  2119. else
  2120. key.offset = offset;
  2121. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  2122. if (WARN_ON(ret < 0))
  2123. return ret;
  2124. ret = 0;
  2125. while (1) {
  2126. cond_resched();
  2127. leaf = path->nodes[0];
  2128. slot = path->slots[0];
  2129. if (slot >= btrfs_header_nritems(leaf)) {
  2130. ret = btrfs_next_leaf(root, path);
  2131. if (ret < 0) {
  2132. goto out;
  2133. } else if (ret > 0) {
  2134. ret = 0;
  2135. goto out;
  2136. }
  2137. continue;
  2138. }
  2139. path->slots[0]++;
  2140. btrfs_item_key_to_cpu(leaf, &key, slot);
  2141. if (key.objectid > inum)
  2142. goto out;
  2143. if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
  2144. continue;
  2145. extent = btrfs_item_ptr(leaf, slot,
  2146. struct btrfs_file_extent_item);
  2147. if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
  2148. continue;
  2149. /*
  2150. * 'offset' refers to the exact key.offset,
  2151. * NOT the 'offset' field in btrfs_extent_data_ref, ie.
  2152. * (key.offset - extent_offset).
  2153. */
  2154. if (key.offset != offset)
  2155. continue;
  2156. extent_offset = btrfs_file_extent_offset(leaf, extent);
  2157. num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
  2158. if (extent_offset >= old->extent_offset + old->offset +
  2159. old->len || extent_offset + num_bytes <=
  2160. old->extent_offset + old->offset)
  2161. continue;
  2162. break;
  2163. }
  2164. backref = kmalloc(sizeof(*backref), GFP_NOFS);
  2165. if (!backref) {
  2166. ret = -ENOENT;
  2167. goto out;
  2168. }
  2169. backref->root_id = root_id;
  2170. backref->inum = inum;
  2171. backref->file_pos = offset;
  2172. backref->num_bytes = num_bytes;
  2173. backref->extent_offset = extent_offset;
  2174. backref->generation = btrfs_file_extent_generation(leaf, extent);
  2175. backref->old = old;
  2176. backref_insert(&new->root, backref);
  2177. old->count++;
  2178. out:
  2179. btrfs_release_path(path);
  2180. WARN_ON(ret);
  2181. return ret;
  2182. }
  2183. static noinline bool record_extent_backrefs(struct btrfs_path *path,
  2184. struct new_sa_defrag_extent *new)
  2185. {
  2186. struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
  2187. struct old_sa_defrag_extent *old, *tmp;
  2188. int ret;
  2189. new->path = path;
  2190. list_for_each_entry_safe(old, tmp, &new->head, list) {
  2191. ret = iterate_inodes_from_logical(old->bytenr +
  2192. old->extent_offset, fs_info,
  2193. path, record_one_backref,
  2194. old);
  2195. if (ret < 0 && ret != -ENOENT)
  2196. return false;
  2197. /* no backref to be processed for this extent */
  2198. if (!old->count) {
  2199. list_del(&old->list);
  2200. kfree(old);
  2201. }
  2202. }
  2203. if (list_empty(&new->head))
  2204. return false;
  2205. return true;
  2206. }
  2207. static int relink_is_mergable(struct extent_buffer *leaf,
  2208. struct btrfs_file_extent_item *fi,
  2209. struct new_sa_defrag_extent *new)
  2210. {
  2211. if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr)
  2212. return 0;
  2213. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
  2214. return 0;
  2215. if (btrfs_file_extent_compression(leaf, fi) != new->compress_type)
  2216. return 0;
  2217. if (btrfs_file_extent_encryption(leaf, fi) ||
  2218. btrfs_file_extent_other_encoding(leaf, fi))
  2219. return 0;
  2220. return 1;
  2221. }
  2222. /*
  2223. * Note the backref might has changed, and in this case we just return 0.
  2224. */
  2225. static noinline int relink_extent_backref(struct btrfs_path *path,
  2226. struct sa_defrag_extent_backref *prev,
  2227. struct sa_defrag_extent_backref *backref)
  2228. {
  2229. struct btrfs_file_extent_item *extent;
  2230. struct btrfs_file_extent_item *item;
  2231. struct btrfs_ordered_extent *ordered;
  2232. struct btrfs_trans_handle *trans;
  2233. struct btrfs_root *root;
  2234. struct btrfs_key key;
  2235. struct extent_buffer *leaf;
  2236. struct old_sa_defrag_extent *old = backref->old;
  2237. struct new_sa_defrag_extent *new = old->new;
  2238. struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
  2239. struct inode *inode;
  2240. struct extent_state *cached = NULL;
  2241. int ret = 0;
  2242. u64 start;
  2243. u64 len;
  2244. u64 lock_start;
  2245. u64 lock_end;
  2246. bool merge = false;
  2247. int index;
  2248. if (prev && prev->root_id == backref->root_id &&
  2249. prev->inum == backref->inum &&
  2250. prev->file_pos + prev->num_bytes == backref->file_pos)
  2251. merge = true;
  2252. /* step 1: get root */
  2253. key.objectid = backref->root_id;
  2254. key.type = BTRFS_ROOT_ITEM_KEY;
  2255. key.offset = (u64)-1;
  2256. index = srcu_read_lock(&fs_info->subvol_srcu);
  2257. root = btrfs_read_fs_root_no_name(fs_info, &key);
  2258. if (IS_ERR(root)) {
  2259. srcu_read_unlock(&fs_info->subvol_srcu, index);
  2260. if (PTR_ERR(root) == -ENOENT)
  2261. return 0;
  2262. return PTR_ERR(root);
  2263. }
  2264. if (btrfs_root_readonly(root)) {
  2265. srcu_read_unlock(&fs_info->subvol_srcu, index);
  2266. return 0;
  2267. }
  2268. /* step 2: get inode */
  2269. key.objectid = backref->inum;
  2270. key.type = BTRFS_INODE_ITEM_KEY;
  2271. key.offset = 0;
  2272. inode = btrfs_iget(fs_info->sb, &key, root, NULL);
  2273. if (IS_ERR(inode)) {
  2274. srcu_read_unlock(&fs_info->subvol_srcu, index);
  2275. return 0;
  2276. }
  2277. srcu_read_unlock(&fs_info->subvol_srcu, index);
  2278. /* step 3: relink backref */
  2279. lock_start = backref->file_pos;
  2280. lock_end = backref->file_pos + backref->num_bytes - 1;
  2281. lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
  2282. &cached);
  2283. ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
  2284. if (ordered) {
  2285. btrfs_put_ordered_extent(ordered);
  2286. goto out_unlock;
  2287. }
  2288. trans = btrfs_join_transaction(root);
  2289. if (IS_ERR(trans)) {
  2290. ret = PTR_ERR(trans);
  2291. goto out_unlock;
  2292. }
  2293. key.objectid = backref->inum;
  2294. key.type = BTRFS_EXTENT_DATA_KEY;
  2295. key.offset = backref->file_pos;
  2296. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  2297. if (ret < 0) {
  2298. goto out_free_path;
  2299. } else if (ret > 0) {
  2300. ret = 0;
  2301. goto out_free_path;
  2302. }
  2303. extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
  2304. struct btrfs_file_extent_item);
  2305. if (btrfs_file_extent_generation(path->nodes[0], extent) !=
  2306. backref->generation)
  2307. goto out_free_path;
  2308. btrfs_release_path(path);
  2309. start = backref->file_pos;
  2310. if (backref->extent_offset < old->extent_offset + old->offset)
  2311. start += old->extent_offset + old->offset -
  2312. backref->extent_offset;
  2313. len = min(backref->extent_offset + backref->num_bytes,
  2314. old->extent_offset + old->offset + old->len);
  2315. len -= max(backref->extent_offset, old->extent_offset + old->offset);
  2316. ret = btrfs_drop_extents(trans, root, inode, start,
  2317. start + len, 1);
  2318. if (ret)
  2319. goto out_free_path;
  2320. again:
  2321. key.objectid = btrfs_ino(BTRFS_I(inode));
  2322. key.type = BTRFS_EXTENT_DATA_KEY;
  2323. key.offset = start;
  2324. path->leave_spinning = 1;
  2325. if (merge) {
  2326. struct btrfs_file_extent_item *fi;
  2327. u64 extent_len;
  2328. struct btrfs_key found_key;
  2329. ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
  2330. if (ret < 0)
  2331. goto out_free_path;
  2332. path->slots[0]--;
  2333. leaf = path->nodes[0];
  2334. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  2335. fi = btrfs_item_ptr(leaf, path->slots[0],
  2336. struct btrfs_file_extent_item);
  2337. extent_len = btrfs_file_extent_num_bytes(leaf, fi);
  2338. if (extent_len + found_key.offset == start &&
  2339. relink_is_mergable(leaf, fi, new)) {
  2340. btrfs_set_file_extent_num_bytes(leaf, fi,
  2341. extent_len + len);
  2342. btrfs_mark_buffer_dirty(leaf);
  2343. inode_add_bytes(inode, len);
  2344. ret = 1;
  2345. goto out_free_path;
  2346. } else {
  2347. merge = false;
  2348. btrfs_release_path(path);
  2349. goto again;
  2350. }
  2351. }
  2352. ret = btrfs_insert_empty_item(trans, root, path, &key,
  2353. sizeof(*extent));
  2354. if (ret) {
  2355. btrfs_abort_transaction(trans, ret);
  2356. goto out_free_path;
  2357. }
  2358. leaf = path->nodes[0];
  2359. item = btrfs_item_ptr(leaf, path->slots[0],
  2360. struct btrfs_file_extent_item);
  2361. btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
  2362. btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
  2363. btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
  2364. btrfs_set_file_extent_num_bytes(leaf, item, len);
  2365. btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
  2366. btrfs_set_file_extent_generation(leaf, item, trans->transid);
  2367. btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
  2368. btrfs_set_file_extent_compression(leaf, item, new->compress_type);
  2369. btrfs_set_file_extent_encryption(leaf, item, 0);
  2370. btrfs_set_file_extent_other_encoding(leaf, item, 0);
  2371. btrfs_mark_buffer_dirty(leaf);
  2372. inode_add_bytes(inode, len);
  2373. btrfs_release_path(path);
  2374. ret = btrfs_inc_extent_ref(trans, fs_info, new->bytenr,
  2375. new->disk_len, 0,
  2376. backref->root_id, backref->inum,
  2377. new->file_pos); /* start - extent_offset */
  2378. if (ret) {
  2379. btrfs_abort_transaction(trans, ret);
  2380. goto out_free_path;
  2381. }
  2382. ret = 1;
  2383. out_free_path:
  2384. btrfs_release_path(path);
  2385. path->leave_spinning = 0;
  2386. btrfs_end_transaction(trans);
  2387. out_unlock:
  2388. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
  2389. &cached, GFP_NOFS);
  2390. iput(inode);
  2391. return ret;
  2392. }
  2393. static void free_sa_defrag_extent(struct new_sa_defrag_extent *new)
  2394. {
  2395. struct old_sa_defrag_extent *old, *tmp;
  2396. if (!new)
  2397. return;
  2398. list_for_each_entry_safe(old, tmp, &new->head, list) {
  2399. kfree(old);
  2400. }
  2401. kfree(new);
  2402. }
  2403. static void relink_file_extents(struct new_sa_defrag_extent *new)
  2404. {
  2405. struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
  2406. struct btrfs_path *path;
  2407. struct sa_defrag_extent_backref *backref;
  2408. struct sa_defrag_extent_backref *prev = NULL;
  2409. struct inode *inode;
  2410. struct btrfs_root *root;
  2411. struct rb_node *node;
  2412. int ret;
  2413. inode = new->inode;
  2414. root = BTRFS_I(inode)->root;
  2415. path = btrfs_alloc_path();
  2416. if (!path)
  2417. return;
  2418. if (!record_extent_backrefs(path, new)) {
  2419. btrfs_free_path(path);
  2420. goto out;
  2421. }
  2422. btrfs_release_path(path);
  2423. while (1) {
  2424. node = rb_first(&new->root);
  2425. if (!node)
  2426. break;
  2427. rb_erase(node, &new->root);
  2428. backref = rb_entry(node, struct sa_defrag_extent_backref, node);
  2429. ret = relink_extent_backref(path, prev, backref);
  2430. WARN_ON(ret < 0);
  2431. kfree(prev);
  2432. if (ret == 1)
  2433. prev = backref;
  2434. else
  2435. prev = NULL;
  2436. cond_resched();
  2437. }
  2438. kfree(prev);
  2439. btrfs_free_path(path);
  2440. out:
  2441. free_sa_defrag_extent(new);
  2442. atomic_dec(&fs_info->defrag_running);
  2443. wake_up(&fs_info->transaction_wait);
  2444. }
  2445. static struct new_sa_defrag_extent *
  2446. record_old_file_extents(struct inode *inode,
  2447. struct btrfs_ordered_extent *ordered)
  2448. {
  2449. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2450. struct btrfs_root *root = BTRFS_I(inode)->root;
  2451. struct btrfs_path *path;
  2452. struct btrfs_key key;
  2453. struct old_sa_defrag_extent *old;
  2454. struct new_sa_defrag_extent *new;
  2455. int ret;
  2456. new = kmalloc(sizeof(*new), GFP_NOFS);
  2457. if (!new)
  2458. return NULL;
  2459. new->inode = inode;
  2460. new->file_pos = ordered->file_offset;
  2461. new->len = ordered->len;
  2462. new->bytenr = ordered->start;
  2463. new->disk_len = ordered->disk_len;
  2464. new->compress_type = ordered->compress_type;
  2465. new->root = RB_ROOT;
  2466. INIT_LIST_HEAD(&new->head);
  2467. path = btrfs_alloc_path();
  2468. if (!path)
  2469. goto out_kfree;
  2470. key.objectid = btrfs_ino(BTRFS_I(inode));
  2471. key.type = BTRFS_EXTENT_DATA_KEY;
  2472. key.offset = new->file_pos;
  2473. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  2474. if (ret < 0)
  2475. goto out_free_path;
  2476. if (ret > 0 && path->slots[0] > 0)
  2477. path->slots[0]--;
  2478. /* find out all the old extents for the file range */
  2479. while (1) {
  2480. struct btrfs_file_extent_item *extent;
  2481. struct extent_buffer *l;
  2482. int slot;
  2483. u64 num_bytes;
  2484. u64 offset;
  2485. u64 end;
  2486. u64 disk_bytenr;
  2487. u64 extent_offset;
  2488. l = path->nodes[0];
  2489. slot = path->slots[0];
  2490. if (slot >= btrfs_header_nritems(l)) {
  2491. ret = btrfs_next_leaf(root, path);
  2492. if (ret < 0)
  2493. goto out_free_path;
  2494. else if (ret > 0)
  2495. break;
  2496. continue;
  2497. }
  2498. btrfs_item_key_to_cpu(l, &key, slot);
  2499. if (key.objectid != btrfs_ino(BTRFS_I(inode)))
  2500. break;
  2501. if (key.type != BTRFS_EXTENT_DATA_KEY)
  2502. break;
  2503. if (key.offset >= new->file_pos + new->len)
  2504. break;
  2505. extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);
  2506. num_bytes = btrfs_file_extent_num_bytes(l, extent);
  2507. if (key.offset + num_bytes < new->file_pos)
  2508. goto next;
  2509. disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
  2510. if (!disk_bytenr)
  2511. goto next;
  2512. extent_offset = btrfs_file_extent_offset(l, extent);
  2513. old = kmalloc(sizeof(*old), GFP_NOFS);
  2514. if (!old)
  2515. goto out_free_path;
  2516. offset = max(new->file_pos, key.offset);
  2517. end = min(new->file_pos + new->len, key.offset + num_bytes);
  2518. old->bytenr = disk_bytenr;
  2519. old->extent_offset = extent_offset;
  2520. old->offset = offset - key.offset;
  2521. old->len = end - offset;
  2522. old->new = new;
  2523. old->count = 0;
  2524. list_add_tail(&old->list, &new->head);
  2525. next:
  2526. path->slots[0]++;
  2527. cond_resched();
  2528. }
  2529. btrfs_free_path(path);
  2530. atomic_inc(&fs_info->defrag_running);
  2531. return new;
  2532. out_free_path:
  2533. btrfs_free_path(path);
  2534. out_kfree:
  2535. free_sa_defrag_extent(new);
  2536. return NULL;
  2537. }
  2538. static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
  2539. u64 start, u64 len)
  2540. {
  2541. struct btrfs_block_group_cache *cache;
  2542. cache = btrfs_lookup_block_group(fs_info, start);
  2543. ASSERT(cache);
  2544. spin_lock(&cache->lock);
  2545. cache->delalloc_bytes -= len;
  2546. spin_unlock(&cache->lock);
  2547. btrfs_put_block_group(cache);
  2548. }
  2549. /* as ordered data IO finishes, this gets called so we can finish
  2550. * an ordered extent if the range of bytes in the file it covers are
  2551. * fully written.
  2552. */
  2553. static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
  2554. {
  2555. struct inode *inode = ordered_extent->inode;
  2556. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2557. struct btrfs_root *root = BTRFS_I(inode)->root;
  2558. struct btrfs_trans_handle *trans = NULL;
  2559. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  2560. struct extent_state *cached_state = NULL;
  2561. struct new_sa_defrag_extent *new = NULL;
  2562. int compress_type = 0;
  2563. int ret = 0;
  2564. u64 logical_len = ordered_extent->len;
  2565. bool nolock;
  2566. bool truncated = false;
  2567. bool range_locked = false;
  2568. bool clear_new_delalloc_bytes = false;
  2569. if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
  2570. !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
  2571. !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags))
  2572. clear_new_delalloc_bytes = true;
  2573. nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
  2574. if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
  2575. ret = -EIO;
  2576. goto out;
  2577. }
  2578. btrfs_free_io_failure_record(BTRFS_I(inode),
  2579. ordered_extent->file_offset,
  2580. ordered_extent->file_offset +
  2581. ordered_extent->len - 1);
  2582. if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
  2583. truncated = true;
  2584. logical_len = ordered_extent->truncated_len;
  2585. /* Truncated the entire extent, don't bother adding */
  2586. if (!logical_len)
  2587. goto out;
  2588. }
  2589. if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
  2590. BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
  2591. /*
  2592. * For mwrite(mmap + memset to write) case, we still reserve
  2593. * space for NOCOW range.
  2594. * As NOCOW won't cause a new delayed ref, just free the space
  2595. */
  2596. btrfs_qgroup_free_data(inode, ordered_extent->file_offset,
  2597. ordered_extent->len);
  2598. btrfs_ordered_update_i_size(inode, 0, ordered_extent);
  2599. if (nolock)
  2600. trans = btrfs_join_transaction_nolock(root);
  2601. else
  2602. trans = btrfs_join_transaction(root);
  2603. if (IS_ERR(trans)) {
  2604. ret = PTR_ERR(trans);
  2605. trans = NULL;
  2606. goto out;
  2607. }
  2608. trans->block_rsv = &fs_info->delalloc_block_rsv;
  2609. ret = btrfs_update_inode_fallback(trans, root, inode);
  2610. if (ret) /* -ENOMEM or corruption */
  2611. btrfs_abort_transaction(trans, ret);
  2612. goto out;
  2613. }
  2614. range_locked = true;
  2615. lock_extent_bits(io_tree, ordered_extent->file_offset,
  2616. ordered_extent->file_offset + ordered_extent->len - 1,
  2617. &cached_state);
  2618. ret = test_range_bit(io_tree, ordered_extent->file_offset,
  2619. ordered_extent->file_offset + ordered_extent->len - 1,
  2620. EXTENT_DEFRAG, 0, cached_state);
  2621. if (ret) {
  2622. u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
  2623. if (0 && last_snapshot >= BTRFS_I(inode)->generation)
  2624. /* the inode is shared */
  2625. new = record_old_file_extents(inode, ordered_extent);
  2626. clear_extent_bit(io_tree, ordered_extent->file_offset,
  2627. ordered_extent->file_offset + ordered_extent->len - 1,
  2628. EXTENT_DEFRAG, 0, 0, &cached_state, GFP_NOFS);
  2629. }
  2630. if (nolock)
  2631. trans = btrfs_join_transaction_nolock(root);
  2632. else
  2633. trans = btrfs_join_transaction(root);
  2634. if (IS_ERR(trans)) {
  2635. ret = PTR_ERR(trans);
  2636. trans = NULL;
  2637. goto out;
  2638. }
  2639. trans->block_rsv = &fs_info->delalloc_block_rsv;
  2640. if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
  2641. compress_type = ordered_extent->compress_type;
  2642. if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
  2643. BUG_ON(compress_type);
  2644. ret = btrfs_mark_extent_written(trans, BTRFS_I(inode),
  2645. ordered_extent->file_offset,
  2646. ordered_extent->file_offset +
  2647. logical_len);
  2648. } else {
  2649. BUG_ON(root == fs_info->tree_root);
  2650. ret = insert_reserved_file_extent(trans, inode,
  2651. ordered_extent->file_offset,
  2652. ordered_extent->start,
  2653. ordered_extent->disk_len,
  2654. logical_len, logical_len,
  2655. compress_type, 0, 0,
  2656. BTRFS_FILE_EXTENT_REG);
  2657. if (!ret)
  2658. btrfs_release_delalloc_bytes(fs_info,
  2659. ordered_extent->start,
  2660. ordered_extent->disk_len);
  2661. }
  2662. unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
  2663. ordered_extent->file_offset, ordered_extent->len,
  2664. trans->transid);
  2665. if (ret < 0) {
  2666. btrfs_abort_transaction(trans, ret);
  2667. goto out;
  2668. }
  2669. add_pending_csums(trans, inode, &ordered_extent->list);
  2670. btrfs_ordered_update_i_size(inode, 0, ordered_extent);
  2671. ret = btrfs_update_inode_fallback(trans, root, inode);
  2672. if (ret) { /* -ENOMEM or corruption */
  2673. btrfs_abort_transaction(trans, ret);
  2674. goto out;
  2675. }
  2676. ret = 0;
  2677. out:
  2678. if (range_locked || clear_new_delalloc_bytes) {
  2679. unsigned int clear_bits = 0;
  2680. if (range_locked)
  2681. clear_bits |= EXTENT_LOCKED;
  2682. if (clear_new_delalloc_bytes)
  2683. clear_bits |= EXTENT_DELALLOC_NEW;
  2684. clear_extent_bit(&BTRFS_I(inode)->io_tree,
  2685. ordered_extent->file_offset,
  2686. ordered_extent->file_offset +
  2687. ordered_extent->len - 1,
  2688. clear_bits,
  2689. (clear_bits & EXTENT_LOCKED) ? 1 : 0,
  2690. 0, &cached_state, GFP_NOFS);
  2691. }
  2692. if (root != fs_info->tree_root)
  2693. btrfs_delalloc_release_metadata(BTRFS_I(inode),
  2694. ordered_extent->len);
  2695. if (trans)
  2696. btrfs_end_transaction(trans);
  2697. if (ret || truncated) {
  2698. u64 start, end;
  2699. if (truncated)
  2700. start = ordered_extent->file_offset + logical_len;
  2701. else
  2702. start = ordered_extent->file_offset;
  2703. end = ordered_extent->file_offset + ordered_extent->len - 1;
  2704. clear_extent_uptodate(io_tree, start, end, NULL, GFP_NOFS);
  2705. /* Drop the cache for the part of the extent we didn't write. */
  2706. btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0);
  2707. /*
  2708. * If the ordered extent had an IOERR or something else went
  2709. * wrong we need to return the space for this ordered extent
  2710. * back to the allocator. We only free the extent in the
  2711. * truncated case if we didn't write out the extent at all.
  2712. */
  2713. if ((ret || !logical_len) &&
  2714. !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
  2715. !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
  2716. btrfs_free_reserved_extent(fs_info,
  2717. ordered_extent->start,
  2718. ordered_extent->disk_len, 1);
  2719. }
  2720. /*
  2721. * This needs to be done to make sure anybody waiting knows we are done
  2722. * updating everything for this ordered extent.
  2723. */
  2724. btrfs_remove_ordered_extent(inode, ordered_extent);
  2725. /* for snapshot-aware defrag */
  2726. if (new) {
  2727. if (ret) {
  2728. free_sa_defrag_extent(new);
  2729. atomic_dec(&fs_info->defrag_running);
  2730. } else {
  2731. relink_file_extents(new);
  2732. }
  2733. }
  2734. /* once for us */
  2735. btrfs_put_ordered_extent(ordered_extent);
  2736. /* once for the tree */
  2737. btrfs_put_ordered_extent(ordered_extent);
  2738. return ret;
  2739. }
  2740. static void finish_ordered_fn(struct btrfs_work *work)
  2741. {
  2742. struct btrfs_ordered_extent *ordered_extent;
  2743. ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
  2744. btrfs_finish_ordered_io(ordered_extent);
  2745. }
  2746. static void btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
  2747. struct extent_state *state, int uptodate)
  2748. {
  2749. struct inode *inode = page->mapping->host;
  2750. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2751. struct btrfs_ordered_extent *ordered_extent = NULL;
  2752. struct btrfs_workqueue *wq;
  2753. btrfs_work_func_t func;
  2754. trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
  2755. ClearPagePrivate2(page);
  2756. if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
  2757. end - start + 1, uptodate))
  2758. return;
  2759. if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
  2760. wq = fs_info->endio_freespace_worker;
  2761. func = btrfs_freespace_write_helper;
  2762. } else {
  2763. wq = fs_info->endio_write_workers;
  2764. func = btrfs_endio_write_helper;
  2765. }
  2766. btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL,
  2767. NULL);
  2768. btrfs_queue_work(wq, &ordered_extent->work);
  2769. }
  2770. static int __readpage_endio_check(struct inode *inode,
  2771. struct btrfs_io_bio *io_bio,
  2772. int icsum, struct page *page,
  2773. int pgoff, u64 start, size_t len)
  2774. {
  2775. char *kaddr;
  2776. u32 csum_expected;
  2777. u32 csum = ~(u32)0;
  2778. csum_expected = *(((u32 *)io_bio->csum) + icsum);
  2779. kaddr = kmap_atomic(page);
  2780. csum = btrfs_csum_data(kaddr + pgoff, csum, len);
  2781. btrfs_csum_final(csum, (u8 *)&csum);
  2782. if (csum != csum_expected)
  2783. goto zeroit;
  2784. kunmap_atomic(kaddr);
  2785. return 0;
  2786. zeroit:
  2787. btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected,
  2788. io_bio->mirror_num);
  2789. memset(kaddr + pgoff, 1, len);
  2790. flush_dcache_page(page);
  2791. kunmap_atomic(kaddr);
  2792. if (csum_expected == 0)
  2793. return 0;
  2794. return -EIO;
  2795. }
  2796. /*
  2797. * when reads are done, we need to check csums to verify the data is correct
  2798. * if there's a match, we allow the bio to finish. If not, the code in
  2799. * extent_io.c will try to find good copies for us.
  2800. */
  2801. static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
  2802. u64 phy_offset, struct page *page,
  2803. u64 start, u64 end, int mirror)
  2804. {
  2805. size_t offset = start - page_offset(page);
  2806. struct inode *inode = page->mapping->host;
  2807. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  2808. struct btrfs_root *root = BTRFS_I(inode)->root;
  2809. if (PageChecked(page)) {
  2810. ClearPageChecked(page);
  2811. return 0;
  2812. }
  2813. if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
  2814. return 0;
  2815. if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
  2816. test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
  2817. clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM);
  2818. return 0;
  2819. }
  2820. phy_offset >>= inode->i_sb->s_blocksize_bits;
  2821. return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
  2822. start, (size_t)(end - start + 1));
  2823. }
  2824. void btrfs_add_delayed_iput(struct inode *inode)
  2825. {
  2826. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2827. struct btrfs_inode *binode = BTRFS_I(inode);
  2828. if (atomic_add_unless(&inode->i_count, -1, 1))
  2829. return;
  2830. spin_lock(&fs_info->delayed_iput_lock);
  2831. if (binode->delayed_iput_count == 0) {
  2832. ASSERT(list_empty(&binode->delayed_iput));
  2833. list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs);
  2834. } else {
  2835. binode->delayed_iput_count++;
  2836. }
  2837. spin_unlock(&fs_info->delayed_iput_lock);
  2838. }
  2839. void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
  2840. {
  2841. spin_lock(&fs_info->delayed_iput_lock);
  2842. while (!list_empty(&fs_info->delayed_iputs)) {
  2843. struct btrfs_inode *inode;
  2844. inode = list_first_entry(&fs_info->delayed_iputs,
  2845. struct btrfs_inode, delayed_iput);
  2846. if (inode->delayed_iput_count) {
  2847. inode->delayed_iput_count--;
  2848. list_move_tail(&inode->delayed_iput,
  2849. &fs_info->delayed_iputs);
  2850. } else {
  2851. list_del_init(&inode->delayed_iput);
  2852. }
  2853. spin_unlock(&fs_info->delayed_iput_lock);
  2854. iput(&inode->vfs_inode);
  2855. spin_lock(&fs_info->delayed_iput_lock);
  2856. }
  2857. spin_unlock(&fs_info->delayed_iput_lock);
  2858. }
  2859. /*
  2860. * This is called in transaction commit time. If there are no orphan
  2861. * files in the subvolume, it removes orphan item and frees block_rsv
  2862. * structure.
  2863. */
  2864. void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
  2865. struct btrfs_root *root)
  2866. {
  2867. struct btrfs_fs_info *fs_info = root->fs_info;
  2868. struct btrfs_block_rsv *block_rsv;
  2869. int ret;
  2870. if (atomic_read(&root->orphan_inodes) ||
  2871. root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE)
  2872. return;
  2873. spin_lock(&root->orphan_lock);
  2874. if (atomic_read(&root->orphan_inodes)) {
  2875. spin_unlock(&root->orphan_lock);
  2876. return;
  2877. }
  2878. if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) {
  2879. spin_unlock(&root->orphan_lock);
  2880. return;
  2881. }
  2882. block_rsv = root->orphan_block_rsv;
  2883. root->orphan_block_rsv = NULL;
  2884. spin_unlock(&root->orphan_lock);
  2885. if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) &&
  2886. btrfs_root_refs(&root->root_item) > 0) {
  2887. ret = btrfs_del_orphan_item(trans, fs_info->tree_root,
  2888. root->root_key.objectid);
  2889. if (ret)
  2890. btrfs_abort_transaction(trans, ret);
  2891. else
  2892. clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED,
  2893. &root->state);
  2894. }
  2895. if (block_rsv) {
  2896. WARN_ON(block_rsv->size > 0);
  2897. btrfs_free_block_rsv(fs_info, block_rsv);
  2898. }
  2899. }
  2900. /*
  2901. * This creates an orphan entry for the given inode in case something goes
  2902. * wrong in the middle of an unlink/truncate.
  2903. *
  2904. * NOTE: caller of this function should reserve 5 units of metadata for
  2905. * this function.
  2906. */
  2907. int btrfs_orphan_add(struct btrfs_trans_handle *trans,
  2908. struct btrfs_inode *inode)
  2909. {
  2910. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  2911. struct btrfs_root *root = inode->root;
  2912. struct btrfs_block_rsv *block_rsv = NULL;
  2913. int reserve = 0;
  2914. int insert = 0;
  2915. int ret;
  2916. if (!root->orphan_block_rsv) {
  2917. block_rsv = btrfs_alloc_block_rsv(fs_info,
  2918. BTRFS_BLOCK_RSV_TEMP);
  2919. if (!block_rsv)
  2920. return -ENOMEM;
  2921. }
  2922. spin_lock(&root->orphan_lock);
  2923. if (!root->orphan_block_rsv) {
  2924. root->orphan_block_rsv = block_rsv;
  2925. } else if (block_rsv) {
  2926. btrfs_free_block_rsv(fs_info, block_rsv);
  2927. block_rsv = NULL;
  2928. }
  2929. if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
  2930. &inode->runtime_flags)) {
  2931. #if 0
  2932. /*
  2933. * For proper ENOSPC handling, we should do orphan
  2934. * cleanup when mounting. But this introduces backward
  2935. * compatibility issue.
  2936. */
  2937. if (!xchg(&root->orphan_item_inserted, 1))
  2938. insert = 2;
  2939. else
  2940. insert = 1;
  2941. #endif
  2942. insert = 1;
  2943. atomic_inc(&root->orphan_inodes);
  2944. }
  2945. if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
  2946. &inode->runtime_flags))
  2947. reserve = 1;
  2948. spin_unlock(&root->orphan_lock);
  2949. /* grab metadata reservation from transaction handle */
  2950. if (reserve) {
  2951. ret = btrfs_orphan_reserve_metadata(trans, inode);
  2952. ASSERT(!ret);
  2953. if (ret) {
  2954. atomic_dec(&root->orphan_inodes);
  2955. clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
  2956. &inode->runtime_flags);
  2957. if (insert)
  2958. clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
  2959. &inode->runtime_flags);
  2960. return ret;
  2961. }
  2962. }
  2963. /* insert an orphan item to track this unlinked/truncated file */
  2964. if (insert >= 1) {
  2965. ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
  2966. if (ret) {
  2967. atomic_dec(&root->orphan_inodes);
  2968. if (reserve) {
  2969. clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
  2970. &inode->runtime_flags);
  2971. btrfs_orphan_release_metadata(inode);
  2972. }
  2973. if (ret != -EEXIST) {
  2974. clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
  2975. &inode->runtime_flags);
  2976. btrfs_abort_transaction(trans, ret);
  2977. return ret;
  2978. }
  2979. }
  2980. ret = 0;
  2981. }
  2982. /* insert an orphan item to track subvolume contains orphan files */
  2983. if (insert >= 2) {
  2984. ret = btrfs_insert_orphan_item(trans, fs_info->tree_root,
  2985. root->root_key.objectid);
  2986. if (ret && ret != -EEXIST) {
  2987. btrfs_abort_transaction(trans, ret);
  2988. return ret;
  2989. }
  2990. }
  2991. return 0;
  2992. }
  2993. /*
  2994. * We have done the truncate/delete so we can go ahead and remove the orphan
  2995. * item for this particular inode.
  2996. */
  2997. static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
  2998. struct btrfs_inode *inode)
  2999. {
  3000. struct btrfs_root *root = inode->root;
  3001. int delete_item = 0;
  3002. int release_rsv = 0;
  3003. int ret = 0;
  3004. spin_lock(&root->orphan_lock);
  3005. if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
  3006. &inode->runtime_flags))
  3007. delete_item = 1;
  3008. if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
  3009. &inode->runtime_flags))
  3010. release_rsv = 1;
  3011. spin_unlock(&root->orphan_lock);
  3012. if (delete_item) {
  3013. atomic_dec(&root->orphan_inodes);
  3014. if (trans)
  3015. ret = btrfs_del_orphan_item(trans, root,
  3016. btrfs_ino(inode));
  3017. }
  3018. if (release_rsv)
  3019. btrfs_orphan_release_metadata(inode);
  3020. return ret;
  3021. }
  3022. /*
  3023. * this cleans up any orphans that may be left on the list from the last use
  3024. * of this root.
  3025. */
  3026. int btrfs_orphan_cleanup(struct btrfs_root *root)
  3027. {
  3028. struct btrfs_fs_info *fs_info = root->fs_info;
  3029. struct btrfs_path *path;
  3030. struct extent_buffer *leaf;
  3031. struct btrfs_key key, found_key;
  3032. struct btrfs_trans_handle *trans;
  3033. struct inode *inode;
  3034. u64 last_objectid = 0;
  3035. int ret = 0, nr_unlink = 0, nr_truncate = 0;
  3036. if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
  3037. return 0;
  3038. path = btrfs_alloc_path();
  3039. if (!path) {
  3040. ret = -ENOMEM;
  3041. goto out;
  3042. }
  3043. path->reada = READA_BACK;
  3044. key.objectid = BTRFS_ORPHAN_OBJECTID;
  3045. key.type = BTRFS_ORPHAN_ITEM_KEY;
  3046. key.offset = (u64)-1;
  3047. while (1) {
  3048. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  3049. if (ret < 0)
  3050. goto out;
  3051. /*
  3052. * if ret == 0 means we found what we were searching for, which
  3053. * is weird, but possible, so only screw with path if we didn't
  3054. * find the key and see if we have stuff that matches
  3055. */
  3056. if (ret > 0) {
  3057. ret = 0;
  3058. if (path->slots[0] == 0)
  3059. break;
  3060. path->slots[0]--;
  3061. }
  3062. /* pull out the item */
  3063. leaf = path->nodes[0];
  3064. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  3065. /* make sure the item matches what we want */
  3066. if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
  3067. break;
  3068. if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
  3069. break;
  3070. /* release the path since we're done with it */
  3071. btrfs_release_path(path);
  3072. /*
  3073. * this is where we are basically btrfs_lookup, without the
  3074. * crossing root thing. we store the inode number in the
  3075. * offset of the orphan item.
  3076. */
  3077. if (found_key.offset == last_objectid) {
  3078. btrfs_err(fs_info,
  3079. "Error removing orphan entry, stopping orphan cleanup");
  3080. ret = -EINVAL;
  3081. goto out;
  3082. }
  3083. last_objectid = found_key.offset;
  3084. found_key.objectid = found_key.offset;
  3085. found_key.type = BTRFS_INODE_ITEM_KEY;
  3086. found_key.offset = 0;
  3087. inode = btrfs_iget(fs_info->sb, &found_key, root, NULL);
  3088. ret = PTR_ERR_OR_ZERO(inode);
  3089. if (ret && ret != -ENOENT)
  3090. goto out;
  3091. if (ret == -ENOENT && root == fs_info->tree_root) {
  3092. struct btrfs_root *dead_root;
  3093. struct btrfs_fs_info *fs_info = root->fs_info;
  3094. int is_dead_root = 0;
  3095. /*
  3096. * this is an orphan in the tree root. Currently these
  3097. * could come from 2 sources:
  3098. * a) a snapshot deletion in progress
  3099. * b) a free space cache inode
  3100. * We need to distinguish those two, as the snapshot
  3101. * orphan must not get deleted.
  3102. * find_dead_roots already ran before us, so if this
  3103. * is a snapshot deletion, we should find the root
  3104. * in the dead_roots list
  3105. */
  3106. spin_lock(&fs_info->trans_lock);
  3107. list_for_each_entry(dead_root, &fs_info->dead_roots,
  3108. root_list) {
  3109. if (dead_root->root_key.objectid ==
  3110. found_key.objectid) {
  3111. is_dead_root = 1;
  3112. break;
  3113. }
  3114. }
  3115. spin_unlock(&fs_info->trans_lock);
  3116. if (is_dead_root) {
  3117. /* prevent this orphan from being found again */
  3118. key.offset = found_key.objectid - 1;
  3119. continue;
  3120. }
  3121. }
  3122. /*
  3123. * Inode is already gone but the orphan item is still there,
  3124. * kill the orphan item.
  3125. */
  3126. if (ret == -ENOENT) {
  3127. trans = btrfs_start_transaction(root, 1);
  3128. if (IS_ERR(trans)) {
  3129. ret = PTR_ERR(trans);
  3130. goto out;
  3131. }
  3132. btrfs_debug(fs_info, "auto deleting %Lu",
  3133. found_key.objectid);
  3134. ret = btrfs_del_orphan_item(trans, root,
  3135. found_key.objectid);
  3136. btrfs_end_transaction(trans);
  3137. if (ret)
  3138. goto out;
  3139. continue;
  3140. }
  3141. /*
  3142. * add this inode to the orphan list so btrfs_orphan_del does
  3143. * the proper thing when we hit it
  3144. */
  3145. set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
  3146. &BTRFS_I(inode)->runtime_flags);
  3147. atomic_inc(&root->orphan_inodes);
  3148. /* if we have links, this was a truncate, lets do that */
  3149. if (inode->i_nlink) {
  3150. if (WARN_ON(!S_ISREG(inode->i_mode))) {
  3151. iput(inode);
  3152. continue;
  3153. }
  3154. nr_truncate++;
  3155. /* 1 for the orphan item deletion. */
  3156. trans = btrfs_start_transaction(root, 1);
  3157. if (IS_ERR(trans)) {
  3158. iput(inode);
  3159. ret = PTR_ERR(trans);
  3160. goto out;
  3161. }
  3162. ret = btrfs_orphan_add(trans, BTRFS_I(inode));
  3163. btrfs_end_transaction(trans);
  3164. if (ret) {
  3165. iput(inode);
  3166. goto out;
  3167. }
  3168. ret = btrfs_truncate(inode);
  3169. if (ret)
  3170. btrfs_orphan_del(NULL, BTRFS_I(inode));
  3171. } else {
  3172. nr_unlink++;
  3173. }
  3174. /* this will do delete_inode and everything for us */
  3175. iput(inode);
  3176. if (ret)
  3177. goto out;
  3178. }
  3179. /* release the path since we're done with it */
  3180. btrfs_release_path(path);
  3181. root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;
  3182. if (root->orphan_block_rsv)
  3183. btrfs_block_rsv_release(fs_info, root->orphan_block_rsv,
  3184. (u64)-1);
  3185. if (root->orphan_block_rsv ||
  3186. test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
  3187. trans = btrfs_join_transaction(root);
  3188. if (!IS_ERR(trans))
  3189. btrfs_end_transaction(trans);
  3190. }
  3191. if (nr_unlink)
  3192. btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
  3193. if (nr_truncate)
  3194. btrfs_debug(fs_info, "truncated %d orphans", nr_truncate);
  3195. out:
  3196. if (ret)
  3197. btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
  3198. btrfs_free_path(path);
  3199. return ret;
  3200. }
  3201. /*
  3202. * very simple check to peek ahead in the leaf looking for xattrs. If we
  3203. * don't find any xattrs, we know there can't be any acls.
  3204. *
  3205. * slot is the slot the inode is in, objectid is the objectid of the inode
  3206. */
  3207. static noinline int acls_after_inode_item(struct extent_buffer *leaf,
  3208. int slot, u64 objectid,
  3209. int *first_xattr_slot)
  3210. {
  3211. u32 nritems = btrfs_header_nritems(leaf);
  3212. struct btrfs_key found_key;
  3213. static u64 xattr_access = 0;
  3214. static u64 xattr_default = 0;
  3215. int scanned = 0;
  3216. if (!xattr_access) {
  3217. xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
  3218. strlen(XATTR_NAME_POSIX_ACL_ACCESS));
  3219. xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
  3220. strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
  3221. }
  3222. slot++;
  3223. *first_xattr_slot = -1;
  3224. while (slot < nritems) {
  3225. btrfs_item_key_to_cpu(leaf, &found_key, slot);
  3226. /* we found a different objectid, there must not be acls */
  3227. if (found_key.objectid != objectid)
  3228. return 0;
  3229. /* we found an xattr, assume we've got an acl */
  3230. if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
  3231. if (*first_xattr_slot == -1)
  3232. *first_xattr_slot = slot;
  3233. if (found_key.offset == xattr_access ||
  3234. found_key.offset == xattr_default)
  3235. return 1;
  3236. }
  3237. /*
  3238. * we found a key greater than an xattr key, there can't
  3239. * be any acls later on
  3240. */
  3241. if (found_key.type > BTRFS_XATTR_ITEM_KEY)
  3242. return 0;
  3243. slot++;
  3244. scanned++;
  3245. /*
  3246. * it goes inode, inode backrefs, xattrs, extents,
  3247. * so if there are a ton of hard links to an inode there can
  3248. * be a lot of backrefs. Don't waste time searching too hard,
  3249. * this is just an optimization
  3250. */
  3251. if (scanned >= 8)
  3252. break;
  3253. }
  3254. /* we hit the end of the leaf before we found an xattr or
  3255. * something larger than an xattr. We have to assume the inode
  3256. * has acls
  3257. */
  3258. if (*first_xattr_slot == -1)
  3259. *first_xattr_slot = slot;
  3260. return 1;
  3261. }
  3262. /*
  3263. * read an inode from the btree into the in-memory inode
  3264. */
  3265. static int btrfs_read_locked_inode(struct inode *inode)
  3266. {
  3267. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  3268. struct btrfs_path *path;
  3269. struct extent_buffer *leaf;
  3270. struct btrfs_inode_item *inode_item;
  3271. struct btrfs_root *root = BTRFS_I(inode)->root;
  3272. struct btrfs_key location;
  3273. unsigned long ptr;
  3274. int maybe_acls;
  3275. u32 rdev;
  3276. int ret;
  3277. bool filled = false;
  3278. int first_xattr_slot;
  3279. ret = btrfs_fill_inode(inode, &rdev);
  3280. if (!ret)
  3281. filled = true;
  3282. path = btrfs_alloc_path();
  3283. if (!path) {
  3284. ret = -ENOMEM;
  3285. goto make_bad;
  3286. }
  3287. memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
  3288. ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
  3289. if (ret) {
  3290. if (ret > 0)
  3291. ret = -ENOENT;
  3292. goto make_bad;
  3293. }
  3294. leaf = path->nodes[0];
  3295. if (filled)
  3296. goto cache_index;
  3297. inode_item = btrfs_item_ptr(leaf, path->slots[0],
  3298. struct btrfs_inode_item);
  3299. inode->i_mode = btrfs_inode_mode(leaf, inode_item);
  3300. set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
  3301. i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
  3302. i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
  3303. btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));
  3304. inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
  3305. inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
  3306. inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
  3307. inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
  3308. inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
  3309. inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
  3310. BTRFS_I(inode)->i_otime.tv_sec =
  3311. btrfs_timespec_sec(leaf, &inode_item->otime);
  3312. BTRFS_I(inode)->i_otime.tv_nsec =
  3313. btrfs_timespec_nsec(leaf, &inode_item->otime);
  3314. inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
  3315. BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
  3316. BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
  3317. inode->i_version = btrfs_inode_sequence(leaf, inode_item);
  3318. inode->i_generation = BTRFS_I(inode)->generation;
  3319. inode->i_rdev = 0;
  3320. rdev = btrfs_inode_rdev(leaf, inode_item);
  3321. BTRFS_I(inode)->index_cnt = (u64)-1;
  3322. BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);
  3323. cache_index:
  3324. /*
  3325. * If we were modified in the current generation and evicted from memory
  3326. * and then re-read we need to do a full sync since we don't have any
  3327. * idea about which extents were modified before we were evicted from
  3328. * cache.
  3329. *
  3330. * This is required for both inode re-read from disk and delayed inode
  3331. * in delayed_nodes_tree.
  3332. */
  3333. if (BTRFS_I(inode)->last_trans == fs_info->generation)
  3334. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  3335. &BTRFS_I(inode)->runtime_flags);
  3336. /*
  3337. * We don't persist the id of the transaction where an unlink operation
  3338. * against the inode was last made. So here we assume the inode might
  3339. * have been evicted, and therefore the exact value of last_unlink_trans
  3340. * lost, and set it to last_trans to avoid metadata inconsistencies
  3341. * between the inode and its parent if the inode is fsync'ed and the log
  3342. * replayed. For example, in the scenario:
  3343. *
  3344. * touch mydir/foo
  3345. * ln mydir/foo mydir/bar
  3346. * sync
  3347. * unlink mydir/bar
  3348. * echo 2 > /proc/sys/vm/drop_caches # evicts inode
  3349. * xfs_io -c fsync mydir/foo
  3350. * <power failure>
  3351. * mount fs, triggers fsync log replay
  3352. *
  3353. * We must make sure that when we fsync our inode foo we also log its
  3354. * parent inode, otherwise after log replay the parent still has the
  3355. * dentry with the "bar" name but our inode foo has a link count of 1
  3356. * and doesn't have an inode ref with the name "bar" anymore.
  3357. *
  3358. * Setting last_unlink_trans to last_trans is a pessimistic approach,
  3359. * but it guarantees correctness at the expense of occasional full
  3360. * transaction commits on fsync if our inode is a directory, or if our
  3361. * inode is not a directory, logging its parent unnecessarily.
  3362. */
  3363. BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;
  3364. path->slots[0]++;
  3365. if (inode->i_nlink != 1 ||
  3366. path->slots[0] >= btrfs_header_nritems(leaf))
  3367. goto cache_acl;
  3368. btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
  3369. if (location.objectid != btrfs_ino(BTRFS_I(inode)))
  3370. goto cache_acl;
  3371. ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
  3372. if (location.type == BTRFS_INODE_REF_KEY) {
  3373. struct btrfs_inode_ref *ref;
  3374. ref = (struct btrfs_inode_ref *)ptr;
  3375. BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
  3376. } else if (location.type == BTRFS_INODE_EXTREF_KEY) {
  3377. struct btrfs_inode_extref *extref;
  3378. extref = (struct btrfs_inode_extref *)ptr;
  3379. BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
  3380. extref);
  3381. }
  3382. cache_acl:
  3383. /*
  3384. * try to precache a NULL acl entry for files that don't have
  3385. * any xattrs or acls
  3386. */
  3387. maybe_acls = acls_after_inode_item(leaf, path->slots[0],
  3388. btrfs_ino(BTRFS_I(inode)), &first_xattr_slot);
  3389. if (first_xattr_slot != -1) {
  3390. path->slots[0] = first_xattr_slot;
  3391. ret = btrfs_load_inode_props(inode, path);
  3392. if (ret)
  3393. btrfs_err(fs_info,
  3394. "error loading props for ino %llu (root %llu): %d",
  3395. btrfs_ino(BTRFS_I(inode)),
  3396. root->root_key.objectid, ret);
  3397. }
  3398. btrfs_free_path(path);
  3399. if (!maybe_acls)
  3400. cache_no_acl(inode);
  3401. switch (inode->i_mode & S_IFMT) {
  3402. case S_IFREG:
  3403. inode->i_mapping->a_ops = &btrfs_aops;
  3404. BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
  3405. inode->i_fop = &btrfs_file_operations;
  3406. inode->i_op = &btrfs_file_inode_operations;
  3407. break;
  3408. case S_IFDIR:
  3409. inode->i_fop = &btrfs_dir_file_operations;
  3410. inode->i_op = &btrfs_dir_inode_operations;
  3411. break;
  3412. case S_IFLNK:
  3413. inode->i_op = &btrfs_symlink_inode_operations;
  3414. inode_nohighmem(inode);
  3415. inode->i_mapping->a_ops = &btrfs_symlink_aops;
  3416. break;
  3417. default:
  3418. inode->i_op = &btrfs_special_inode_operations;
  3419. init_special_inode(inode, inode->i_mode, rdev);
  3420. break;
  3421. }
  3422. btrfs_update_iflags(inode);
  3423. return 0;
  3424. make_bad:
  3425. btrfs_free_path(path);
  3426. make_bad_inode(inode);
  3427. return ret;
  3428. }
  3429. /*
  3430. * given a leaf and an inode, copy the inode fields into the leaf
  3431. */
  3432. static void fill_inode_item(struct btrfs_trans_handle *trans,
  3433. struct extent_buffer *leaf,
  3434. struct btrfs_inode_item *item,
  3435. struct inode *inode)
  3436. {
  3437. struct btrfs_map_token token;
  3438. btrfs_init_map_token(&token);
  3439. btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
  3440. btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
  3441. btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
  3442. &token);
  3443. btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
  3444. btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
  3445. btrfs_set_token_timespec_sec(leaf, &item->atime,
  3446. inode->i_atime.tv_sec, &token);
  3447. btrfs_set_token_timespec_nsec(leaf, &item->atime,
  3448. inode->i_atime.tv_nsec, &token);
  3449. btrfs_set_token_timespec_sec(leaf, &item->mtime,
  3450. inode->i_mtime.tv_sec, &token);
  3451. btrfs_set_token_timespec_nsec(leaf, &item->mtime,
  3452. inode->i_mtime.tv_nsec, &token);
  3453. btrfs_set_token_timespec_sec(leaf, &item->ctime,
  3454. inode->i_ctime.tv_sec, &token);
  3455. btrfs_set_token_timespec_nsec(leaf, &item->ctime,
  3456. inode->i_ctime.tv_nsec, &token);
  3457. btrfs_set_token_timespec_sec(leaf, &item->otime,
  3458. BTRFS_I(inode)->i_otime.tv_sec, &token);
  3459. btrfs_set_token_timespec_nsec(leaf, &item->otime,
  3460. BTRFS_I(inode)->i_otime.tv_nsec, &token);
  3461. btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
  3462. &token);
  3463. btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
  3464. &token);
  3465. btrfs_set_token_inode_sequence(leaf, item, inode->i_version, &token);
  3466. btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
  3467. btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
  3468. btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
  3469. btrfs_set_token_inode_block_group(leaf, item, 0, &token);
  3470. }
  3471. /*
  3472. * copy everything in the in-memory inode into the btree.
  3473. */
  3474. static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
  3475. struct btrfs_root *root, struct inode *inode)
  3476. {
  3477. struct btrfs_inode_item *inode_item;
  3478. struct btrfs_path *path;
  3479. struct extent_buffer *leaf;
  3480. int ret;
  3481. path = btrfs_alloc_path();
  3482. if (!path)
  3483. return -ENOMEM;
  3484. path->leave_spinning = 1;
  3485. ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
  3486. 1);
  3487. if (ret) {
  3488. if (ret > 0)
  3489. ret = -ENOENT;
  3490. goto failed;
  3491. }
  3492. leaf = path->nodes[0];
  3493. inode_item = btrfs_item_ptr(leaf, path->slots[0],
  3494. struct btrfs_inode_item);
  3495. fill_inode_item(trans, leaf, inode_item, inode);
  3496. btrfs_mark_buffer_dirty(leaf);
  3497. btrfs_set_inode_last_trans(trans, inode);
  3498. ret = 0;
  3499. failed:
  3500. btrfs_free_path(path);
  3501. return ret;
  3502. }
  3503. /*
  3504. * copy everything in the in-memory inode into the btree.
  3505. */
  3506. noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
  3507. struct btrfs_root *root, struct inode *inode)
  3508. {
  3509. struct btrfs_fs_info *fs_info = root->fs_info;
  3510. int ret;
  3511. /*
  3512. * If the inode is a free space inode, we can deadlock during commit
  3513. * if we put it into the delayed code.
  3514. *
  3515. * The data relocation inode should also be directly updated
  3516. * without delay
  3517. */
  3518. if (!btrfs_is_free_space_inode(BTRFS_I(inode))
  3519. && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
  3520. && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
  3521. btrfs_update_root_times(trans, root);
  3522. ret = btrfs_delayed_update_inode(trans, root, inode);
  3523. if (!ret)
  3524. btrfs_set_inode_last_trans(trans, inode);
  3525. return ret;
  3526. }
  3527. return btrfs_update_inode_item(trans, root, inode);
  3528. }
  3529. noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
  3530. struct btrfs_root *root,
  3531. struct inode *inode)
  3532. {
  3533. int ret;
  3534. ret = btrfs_update_inode(trans, root, inode);
  3535. if (ret == -ENOSPC)
  3536. return btrfs_update_inode_item(trans, root, inode);
  3537. return ret;
  3538. }
  3539. /*
  3540. * unlink helper that gets used here in inode.c and in the tree logging
  3541. * recovery code. It remove a link in a directory with a given name, and
  3542. * also drops the back refs in the inode to the directory
  3543. */
  3544. static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
  3545. struct btrfs_root *root,
  3546. struct btrfs_inode *dir,
  3547. struct btrfs_inode *inode,
  3548. const char *name, int name_len)
  3549. {
  3550. struct btrfs_fs_info *fs_info = root->fs_info;
  3551. struct btrfs_path *path;
  3552. int ret = 0;
  3553. struct extent_buffer *leaf;
  3554. struct btrfs_dir_item *di;
  3555. struct btrfs_key key;
  3556. u64 index;
  3557. u64 ino = btrfs_ino(inode);
  3558. u64 dir_ino = btrfs_ino(dir);
  3559. path = btrfs_alloc_path();
  3560. if (!path) {
  3561. ret = -ENOMEM;
  3562. goto out;
  3563. }
  3564. path->leave_spinning = 1;
  3565. di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
  3566. name, name_len, -1);
  3567. if (IS_ERR(di)) {
  3568. ret = PTR_ERR(di);
  3569. goto err;
  3570. }
  3571. if (!di) {
  3572. ret = -ENOENT;
  3573. goto err;
  3574. }
  3575. leaf = path->nodes[0];
  3576. btrfs_dir_item_key_to_cpu(leaf, di, &key);
  3577. ret = btrfs_delete_one_dir_name(trans, root, path, di);
  3578. if (ret)
  3579. goto err;
  3580. btrfs_release_path(path);
  3581. /*
  3582. * If we don't have dir index, we have to get it by looking up
  3583. * the inode ref, since we get the inode ref, remove it directly,
  3584. * it is unnecessary to do delayed deletion.
  3585. *
  3586. * But if we have dir index, needn't search inode ref to get it.
  3587. * Since the inode ref is close to the inode item, it is better
  3588. * that we delay to delete it, and just do this deletion when
  3589. * we update the inode item.
  3590. */
  3591. if (inode->dir_index) {
  3592. ret = btrfs_delayed_delete_inode_ref(inode);
  3593. if (!ret) {
  3594. index = inode->dir_index;
  3595. goto skip_backref;
  3596. }
  3597. }
  3598. ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
  3599. dir_ino, &index);
  3600. if (ret) {
  3601. btrfs_info(fs_info,
  3602. "failed to delete reference to %.*s, inode %llu parent %llu",
  3603. name_len, name, ino, dir_ino);
  3604. btrfs_abort_transaction(trans, ret);
  3605. goto err;
  3606. }
  3607. skip_backref:
  3608. ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index);
  3609. if (ret) {
  3610. btrfs_abort_transaction(trans, ret);
  3611. goto err;
  3612. }
  3613. ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode,
  3614. dir_ino);
  3615. if (ret != 0 && ret != -ENOENT) {
  3616. btrfs_abort_transaction(trans, ret);
  3617. goto err;
  3618. }
  3619. ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir,
  3620. index);
  3621. if (ret == -ENOENT)
  3622. ret = 0;
  3623. else if (ret)
  3624. btrfs_abort_transaction(trans, ret);
  3625. err:
  3626. btrfs_free_path(path);
  3627. if (ret)
  3628. goto out;
  3629. btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);
  3630. inode_inc_iversion(&inode->vfs_inode);
  3631. inode_inc_iversion(&dir->vfs_inode);
  3632. inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime =
  3633. dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode);
  3634. ret = btrfs_update_inode(trans, root, &dir->vfs_inode);
  3635. out:
  3636. return ret;
  3637. }
  3638. int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
  3639. struct btrfs_root *root,
  3640. struct btrfs_inode *dir, struct btrfs_inode *inode,
  3641. const char *name, int name_len)
  3642. {
  3643. int ret;
  3644. ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
  3645. if (!ret) {
  3646. drop_nlink(&inode->vfs_inode);
  3647. ret = btrfs_update_inode(trans, root, &inode->vfs_inode);
  3648. }
  3649. return ret;
  3650. }
  3651. /*
  3652. * helper to start transaction for unlink and rmdir.
  3653. *
  3654. * unlink and rmdir are special in btrfs, they do not always free space, so
  3655. * if we cannot make our reservations the normal way try and see if there is
  3656. * plenty of slack room in the global reserve to migrate, otherwise we cannot
  3657. * allow the unlink to occur.
  3658. */
  3659. static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
  3660. {
  3661. struct btrfs_root *root = BTRFS_I(dir)->root;
  3662. /*
  3663. * 1 for the possible orphan item
  3664. * 1 for the dir item
  3665. * 1 for the dir index
  3666. * 1 for the inode ref
  3667. * 1 for the inode
  3668. */
  3669. return btrfs_start_transaction_fallback_global_rsv(root, 5, 5);
  3670. }
  3671. static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
  3672. {
  3673. struct btrfs_root *root = BTRFS_I(dir)->root;
  3674. struct btrfs_trans_handle *trans;
  3675. struct inode *inode = d_inode(dentry);
  3676. int ret;
  3677. trans = __unlink_start_trans(dir);
  3678. if (IS_ERR(trans))
  3679. return PTR_ERR(trans);
  3680. btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
  3681. 0);
  3682. ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
  3683. BTRFS_I(d_inode(dentry)), dentry->d_name.name,
  3684. dentry->d_name.len);
  3685. if (ret)
  3686. goto out;
  3687. if (inode->i_nlink == 0) {
  3688. ret = btrfs_orphan_add(trans, BTRFS_I(inode));
  3689. if (ret)
  3690. goto out;
  3691. }
  3692. out:
  3693. btrfs_end_transaction(trans);
  3694. btrfs_btree_balance_dirty(root->fs_info);
  3695. return ret;
  3696. }
  3697. int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
  3698. struct btrfs_root *root,
  3699. struct inode *dir, u64 objectid,
  3700. const char *name, int name_len)
  3701. {
  3702. struct btrfs_fs_info *fs_info = root->fs_info;
  3703. struct btrfs_path *path;
  3704. struct extent_buffer *leaf;
  3705. struct btrfs_dir_item *di;
  3706. struct btrfs_key key;
  3707. u64 index;
  3708. int ret;
  3709. u64 dir_ino = btrfs_ino(BTRFS_I(dir));
  3710. path = btrfs_alloc_path();
  3711. if (!path)
  3712. return -ENOMEM;
  3713. di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
  3714. name, name_len, -1);
  3715. if (IS_ERR_OR_NULL(di)) {
  3716. if (!di)
  3717. ret = -ENOENT;
  3718. else
  3719. ret = PTR_ERR(di);
  3720. goto out;
  3721. }
  3722. leaf = path->nodes[0];
  3723. btrfs_dir_item_key_to_cpu(leaf, di, &key);
  3724. WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
  3725. ret = btrfs_delete_one_dir_name(trans, root, path, di);
  3726. if (ret) {
  3727. btrfs_abort_transaction(trans, ret);
  3728. goto out;
  3729. }
  3730. btrfs_release_path(path);
  3731. ret = btrfs_del_root_ref(trans, fs_info, objectid,
  3732. root->root_key.objectid, dir_ino,
  3733. &index, name, name_len);
  3734. if (ret < 0) {
  3735. if (ret != -ENOENT) {
  3736. btrfs_abort_transaction(trans, ret);
  3737. goto out;
  3738. }
  3739. di = btrfs_search_dir_index_item(root, path, dir_ino,
  3740. name, name_len);
  3741. if (IS_ERR_OR_NULL(di)) {
  3742. if (!di)
  3743. ret = -ENOENT;
  3744. else
  3745. ret = PTR_ERR(di);
  3746. btrfs_abort_transaction(trans, ret);
  3747. goto out;
  3748. }
  3749. leaf = path->nodes[0];
  3750. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  3751. btrfs_release_path(path);
  3752. index = key.offset;
  3753. }
  3754. btrfs_release_path(path);
  3755. ret = btrfs_delete_delayed_dir_index(trans, fs_info, BTRFS_I(dir), index);
  3756. if (ret) {
  3757. btrfs_abort_transaction(trans, ret);
  3758. goto out;
  3759. }
  3760. btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);
  3761. inode_inc_iversion(dir);
  3762. dir->i_mtime = dir->i_ctime = current_time(dir);
  3763. ret = btrfs_update_inode_fallback(trans, root, dir);
  3764. if (ret)
  3765. btrfs_abort_transaction(trans, ret);
  3766. out:
  3767. btrfs_free_path(path);
  3768. return ret;
  3769. }
  3770. static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
  3771. {
  3772. struct inode *inode = d_inode(dentry);
  3773. int err = 0;
  3774. struct btrfs_root *root = BTRFS_I(dir)->root;
  3775. struct btrfs_trans_handle *trans;
  3776. u64 last_unlink_trans;
  3777. if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
  3778. return -ENOTEMPTY;
  3779. if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID)
  3780. return -EPERM;
  3781. trans = __unlink_start_trans(dir);
  3782. if (IS_ERR(trans))
  3783. return PTR_ERR(trans);
  3784. if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
  3785. err = btrfs_unlink_subvol(trans, root, dir,
  3786. BTRFS_I(inode)->location.objectid,
  3787. dentry->d_name.name,
  3788. dentry->d_name.len);
  3789. goto out;
  3790. }
  3791. err = btrfs_orphan_add(trans, BTRFS_I(inode));
  3792. if (err)
  3793. goto out;
  3794. last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;
  3795. /* now the directory is empty */
  3796. err = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
  3797. BTRFS_I(d_inode(dentry)), dentry->d_name.name,
  3798. dentry->d_name.len);
  3799. if (!err) {
  3800. btrfs_i_size_write(BTRFS_I(inode), 0);
  3801. /*
  3802. * Propagate the last_unlink_trans value of the deleted dir to
  3803. * its parent directory. This is to prevent an unrecoverable
  3804. * log tree in the case we do something like this:
  3805. * 1) create dir foo
  3806. * 2) create snapshot under dir foo
  3807. * 3) delete the snapshot
  3808. * 4) rmdir foo
  3809. * 5) mkdir foo
  3810. * 6) fsync foo or some file inside foo
  3811. */
  3812. if (last_unlink_trans >= trans->transid)
  3813. BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
  3814. }
  3815. out:
  3816. btrfs_end_transaction(trans);
  3817. btrfs_btree_balance_dirty(root->fs_info);
  3818. return err;
  3819. }
  3820. static int truncate_space_check(struct btrfs_trans_handle *trans,
  3821. struct btrfs_root *root,
  3822. u64 bytes_deleted)
  3823. {
  3824. struct btrfs_fs_info *fs_info = root->fs_info;
  3825. int ret;
  3826. /*
  3827. * This is only used to apply pressure to the enospc system, we don't
  3828. * intend to use this reservation at all.
  3829. */
  3830. bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted);
  3831. bytes_deleted *= fs_info->nodesize;
  3832. ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv,
  3833. bytes_deleted, BTRFS_RESERVE_NO_FLUSH);
  3834. if (!ret) {
  3835. trace_btrfs_space_reservation(fs_info, "transaction",
  3836. trans->transid,
  3837. bytes_deleted, 1);
  3838. trans->bytes_reserved += bytes_deleted;
  3839. }
  3840. return ret;
  3841. }
  3842. static int truncate_inline_extent(struct inode *inode,
  3843. struct btrfs_path *path,
  3844. struct btrfs_key *found_key,
  3845. const u64 item_end,
  3846. const u64 new_size)
  3847. {
  3848. struct extent_buffer *leaf = path->nodes[0];
  3849. int slot = path->slots[0];
  3850. struct btrfs_file_extent_item *fi;
  3851. u32 size = (u32)(new_size - found_key->offset);
  3852. struct btrfs_root *root = BTRFS_I(inode)->root;
  3853. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  3854. if (btrfs_file_extent_compression(leaf, fi) != BTRFS_COMPRESS_NONE) {
  3855. loff_t offset = new_size;
  3856. loff_t page_end = ALIGN(offset, PAGE_SIZE);
  3857. /*
  3858. * Zero out the remaining of the last page of our inline extent,
  3859. * instead of directly truncating our inline extent here - that
  3860. * would be much more complex (decompressing all the data, then
  3861. * compressing the truncated data, which might be bigger than
  3862. * the size of the inline extent, resize the extent, etc).
  3863. * We release the path because to get the page we might need to
  3864. * read the extent item from disk (data not in the page cache).
  3865. */
  3866. btrfs_release_path(path);
  3867. return btrfs_truncate_block(inode, offset, page_end - offset,
  3868. 0);
  3869. }
  3870. btrfs_set_file_extent_ram_bytes(leaf, fi, size);
  3871. size = btrfs_file_extent_calc_inline_size(size);
  3872. btrfs_truncate_item(root->fs_info, path, size, 1);
  3873. if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
  3874. inode_sub_bytes(inode, item_end + 1 - new_size);
  3875. return 0;
  3876. }
  3877. /*
  3878. * this can truncate away extent items, csum items and directory items.
  3879. * It starts at a high offset and removes keys until it can't find
  3880. * any higher than new_size
  3881. *
  3882. * csum items that cross the new i_size are truncated to the new size
  3883. * as well.
  3884. *
  3885. * min_type is the minimum key type to truncate down to. If set to 0, this
  3886. * will kill all the items on this inode, including the INODE_ITEM_KEY.
  3887. */
  3888. int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
  3889. struct btrfs_root *root,
  3890. struct inode *inode,
  3891. u64 new_size, u32 min_type)
  3892. {
  3893. struct btrfs_fs_info *fs_info = root->fs_info;
  3894. struct btrfs_path *path;
  3895. struct extent_buffer *leaf;
  3896. struct btrfs_file_extent_item *fi;
  3897. struct btrfs_key key;
  3898. struct btrfs_key found_key;
  3899. u64 extent_start = 0;
  3900. u64 extent_num_bytes = 0;
  3901. u64 extent_offset = 0;
  3902. u64 item_end = 0;
  3903. u64 last_size = new_size;
  3904. u32 found_type = (u8)-1;
  3905. int found_extent;
  3906. int del_item;
  3907. int pending_del_nr = 0;
  3908. int pending_del_slot = 0;
  3909. int extent_type = -1;
  3910. int ret;
  3911. int err = 0;
  3912. u64 ino = btrfs_ino(BTRFS_I(inode));
  3913. u64 bytes_deleted = 0;
  3914. bool be_nice = 0;
  3915. bool should_throttle = 0;
  3916. bool should_end = 0;
  3917. BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
  3918. /*
  3919. * for non-free space inodes and ref cows, we want to back off from
  3920. * time to time
  3921. */
  3922. if (!btrfs_is_free_space_inode(BTRFS_I(inode)) &&
  3923. test_bit(BTRFS_ROOT_REF_COWS, &root->state))
  3924. be_nice = 1;
  3925. path = btrfs_alloc_path();
  3926. if (!path)
  3927. return -ENOMEM;
  3928. path->reada = READA_BACK;
  3929. /*
  3930. * We want to drop from the next block forward in case this new size is
  3931. * not block aligned since we will be keeping the last block of the
  3932. * extent just the way it is.
  3933. */
  3934. if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
  3935. root == fs_info->tree_root)
  3936. btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size,
  3937. fs_info->sectorsize),
  3938. (u64)-1, 0);
  3939. /*
  3940. * This function is also used to drop the items in the log tree before
  3941. * we relog the inode, so if root != BTRFS_I(inode)->root, it means
  3942. * it is used to drop the loged items. So we shouldn't kill the delayed
  3943. * items.
  3944. */
  3945. if (min_type == 0 && root == BTRFS_I(inode)->root)
  3946. btrfs_kill_delayed_inode_items(BTRFS_I(inode));
  3947. key.objectid = ino;
  3948. key.offset = (u64)-1;
  3949. key.type = (u8)-1;
  3950. search_again:
  3951. /*
  3952. * with a 16K leaf size and 128MB extents, you can actually queue
  3953. * up a huge file in a single leaf. Most of the time that
  3954. * bytes_deleted is > 0, it will be huge by the time we get here
  3955. */
  3956. if (be_nice && bytes_deleted > SZ_32M) {
  3957. if (btrfs_should_end_transaction(trans)) {
  3958. err = -EAGAIN;
  3959. goto error;
  3960. }
  3961. }
  3962. path->leave_spinning = 1;
  3963. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  3964. if (ret < 0) {
  3965. err = ret;
  3966. goto out;
  3967. }
  3968. if (ret > 0) {
  3969. /* there are no items in the tree for us to truncate, we're
  3970. * done
  3971. */
  3972. if (path->slots[0] == 0)
  3973. goto out;
  3974. path->slots[0]--;
  3975. }
  3976. while (1) {
  3977. fi = NULL;
  3978. leaf = path->nodes[0];
  3979. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  3980. found_type = found_key.type;
  3981. if (found_key.objectid != ino)
  3982. break;
  3983. if (found_type < min_type)
  3984. break;
  3985. item_end = found_key.offset;
  3986. if (found_type == BTRFS_EXTENT_DATA_KEY) {
  3987. fi = btrfs_item_ptr(leaf, path->slots[0],
  3988. struct btrfs_file_extent_item);
  3989. extent_type = btrfs_file_extent_type(leaf, fi);
  3990. if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
  3991. item_end +=
  3992. btrfs_file_extent_num_bytes(leaf, fi);
  3993. trace_btrfs_truncate_show_fi_regular(
  3994. BTRFS_I(inode), leaf, fi,
  3995. found_key.offset);
  3996. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  3997. item_end += btrfs_file_extent_inline_len(leaf,
  3998. path->slots[0], fi);
  3999. trace_btrfs_truncate_show_fi_inline(
  4000. BTRFS_I(inode), leaf, fi, path->slots[0],
  4001. found_key.offset);
  4002. }
  4003. item_end--;
  4004. }
  4005. if (found_type > min_type) {
  4006. del_item = 1;
  4007. } else {
  4008. if (item_end < new_size)
  4009. break;
  4010. if (found_key.offset >= new_size)
  4011. del_item = 1;
  4012. else
  4013. del_item = 0;
  4014. }
  4015. found_extent = 0;
  4016. /* FIXME, shrink the extent if the ref count is only 1 */
  4017. if (found_type != BTRFS_EXTENT_DATA_KEY)
  4018. goto delete;
  4019. if (del_item)
  4020. last_size = found_key.offset;
  4021. else
  4022. last_size = new_size;
  4023. if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
  4024. u64 num_dec;
  4025. extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
  4026. if (!del_item) {
  4027. u64 orig_num_bytes =
  4028. btrfs_file_extent_num_bytes(leaf, fi);
  4029. extent_num_bytes = ALIGN(new_size -
  4030. found_key.offset,
  4031. fs_info->sectorsize);
  4032. btrfs_set_file_extent_num_bytes(leaf, fi,
  4033. extent_num_bytes);
  4034. num_dec = (orig_num_bytes -
  4035. extent_num_bytes);
  4036. if (test_bit(BTRFS_ROOT_REF_COWS,
  4037. &root->state) &&
  4038. extent_start != 0)
  4039. inode_sub_bytes(inode, num_dec);
  4040. btrfs_mark_buffer_dirty(leaf);
  4041. } else {
  4042. extent_num_bytes =
  4043. btrfs_file_extent_disk_num_bytes(leaf,
  4044. fi);
  4045. extent_offset = found_key.offset -
  4046. btrfs_file_extent_offset(leaf, fi);
  4047. /* FIXME blocksize != 4096 */
  4048. num_dec = btrfs_file_extent_num_bytes(leaf, fi);
  4049. if (extent_start != 0) {
  4050. found_extent = 1;
  4051. if (test_bit(BTRFS_ROOT_REF_COWS,
  4052. &root->state))
  4053. inode_sub_bytes(inode, num_dec);
  4054. }
  4055. }
  4056. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  4057. /*
  4058. * we can't truncate inline items that have had
  4059. * special encodings
  4060. */
  4061. if (!del_item &&
  4062. btrfs_file_extent_encryption(leaf, fi) == 0 &&
  4063. btrfs_file_extent_other_encoding(leaf, fi) == 0) {
  4064. /*
  4065. * Need to release path in order to truncate a
  4066. * compressed extent. So delete any accumulated
  4067. * extent items so far.
  4068. */
  4069. if (btrfs_file_extent_compression(leaf, fi) !=
  4070. BTRFS_COMPRESS_NONE && pending_del_nr) {
  4071. err = btrfs_del_items(trans, root, path,
  4072. pending_del_slot,
  4073. pending_del_nr);
  4074. if (err) {
  4075. btrfs_abort_transaction(trans,
  4076. err);
  4077. goto error;
  4078. }
  4079. pending_del_nr = 0;
  4080. }
  4081. err = truncate_inline_extent(inode, path,
  4082. &found_key,
  4083. item_end,
  4084. new_size);
  4085. if (err) {
  4086. btrfs_abort_transaction(trans, err);
  4087. goto error;
  4088. }
  4089. } else if (test_bit(BTRFS_ROOT_REF_COWS,
  4090. &root->state)) {
  4091. inode_sub_bytes(inode, item_end + 1 - new_size);
  4092. }
  4093. }
  4094. delete:
  4095. if (del_item) {
  4096. if (!pending_del_nr) {
  4097. /* no pending yet, add ourselves */
  4098. pending_del_slot = path->slots[0];
  4099. pending_del_nr = 1;
  4100. } else if (pending_del_nr &&
  4101. path->slots[0] + 1 == pending_del_slot) {
  4102. /* hop on the pending chunk */
  4103. pending_del_nr++;
  4104. pending_del_slot = path->slots[0];
  4105. } else {
  4106. BUG();
  4107. }
  4108. } else {
  4109. break;
  4110. }
  4111. should_throttle = 0;
  4112. if (found_extent &&
  4113. (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
  4114. root == fs_info->tree_root)) {
  4115. btrfs_set_path_blocking(path);
  4116. bytes_deleted += extent_num_bytes;
  4117. ret = btrfs_free_extent(trans, fs_info, extent_start,
  4118. extent_num_bytes, 0,
  4119. btrfs_header_owner(leaf),
  4120. ino, extent_offset);
  4121. BUG_ON(ret);
  4122. if (btrfs_should_throttle_delayed_refs(trans, fs_info))
  4123. btrfs_async_run_delayed_refs(fs_info,
  4124. trans->delayed_ref_updates * 2,
  4125. trans->transid, 0);
  4126. if (be_nice) {
  4127. if (truncate_space_check(trans, root,
  4128. extent_num_bytes)) {
  4129. should_end = 1;
  4130. }
  4131. if (btrfs_should_throttle_delayed_refs(trans,
  4132. fs_info))
  4133. should_throttle = 1;
  4134. }
  4135. }
  4136. if (found_type == BTRFS_INODE_ITEM_KEY)
  4137. break;
  4138. if (path->slots[0] == 0 ||
  4139. path->slots[0] != pending_del_slot ||
  4140. should_throttle || should_end) {
  4141. if (pending_del_nr) {
  4142. ret = btrfs_del_items(trans, root, path,
  4143. pending_del_slot,
  4144. pending_del_nr);
  4145. if (ret) {
  4146. btrfs_abort_transaction(trans, ret);
  4147. goto error;
  4148. }
  4149. pending_del_nr = 0;
  4150. }
  4151. btrfs_release_path(path);
  4152. if (should_throttle) {
  4153. unsigned long updates = trans->delayed_ref_updates;
  4154. if (updates) {
  4155. trans->delayed_ref_updates = 0;
  4156. ret = btrfs_run_delayed_refs(trans,
  4157. fs_info,
  4158. updates * 2);
  4159. if (ret && !err)
  4160. err = ret;
  4161. }
  4162. }
  4163. /*
  4164. * if we failed to refill our space rsv, bail out
  4165. * and let the transaction restart
  4166. */
  4167. if (should_end) {
  4168. err = -EAGAIN;
  4169. goto error;
  4170. }
  4171. goto search_again;
  4172. } else {
  4173. path->slots[0]--;
  4174. }
  4175. }
  4176. out:
  4177. if (pending_del_nr) {
  4178. ret = btrfs_del_items(trans, root, path, pending_del_slot,
  4179. pending_del_nr);
  4180. if (ret)
  4181. btrfs_abort_transaction(trans, ret);
  4182. }
  4183. error:
  4184. if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
  4185. ASSERT(last_size >= new_size);
  4186. if (!err && last_size > new_size)
  4187. last_size = new_size;
  4188. btrfs_ordered_update_i_size(inode, last_size, NULL);
  4189. }
  4190. btrfs_free_path(path);
  4191. if (be_nice && bytes_deleted > SZ_32M) {
  4192. unsigned long updates = trans->delayed_ref_updates;
  4193. if (updates) {
  4194. trans->delayed_ref_updates = 0;
  4195. ret = btrfs_run_delayed_refs(trans, fs_info,
  4196. updates * 2);
  4197. if (ret && !err)
  4198. err = ret;
  4199. }
  4200. }
  4201. return err;
  4202. }
  4203. /*
  4204. * btrfs_truncate_block - read, zero a chunk and write a block
  4205. * @inode - inode that we're zeroing
  4206. * @from - the offset to start zeroing
  4207. * @len - the length to zero, 0 to zero the entire range respective to the
  4208. * offset
  4209. * @front - zero up to the offset instead of from the offset on
  4210. *
  4211. * This will find the block for the "from" offset and cow the block and zero the
  4212. * part we want to zero. This is used with truncate and hole punching.
  4213. */
  4214. int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len,
  4215. int front)
  4216. {
  4217. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  4218. struct address_space *mapping = inode->i_mapping;
  4219. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  4220. struct btrfs_ordered_extent *ordered;
  4221. struct extent_state *cached_state = NULL;
  4222. char *kaddr;
  4223. u32 blocksize = fs_info->sectorsize;
  4224. pgoff_t index = from >> PAGE_SHIFT;
  4225. unsigned offset = from & (blocksize - 1);
  4226. struct page *page;
  4227. gfp_t mask = btrfs_alloc_write_mask(mapping);
  4228. int ret = 0;
  4229. u64 block_start;
  4230. u64 block_end;
  4231. if ((offset & (blocksize - 1)) == 0 &&
  4232. (!len || ((len & (blocksize - 1)) == 0)))
  4233. goto out;
  4234. ret = btrfs_delalloc_reserve_space(inode,
  4235. round_down(from, blocksize), blocksize);
  4236. if (ret)
  4237. goto out;
  4238. again:
  4239. page = find_or_create_page(mapping, index, mask);
  4240. if (!page) {
  4241. btrfs_delalloc_release_space(inode,
  4242. round_down(from, blocksize),
  4243. blocksize);
  4244. ret = -ENOMEM;
  4245. goto out;
  4246. }
  4247. block_start = round_down(from, blocksize);
  4248. block_end = block_start + blocksize - 1;
  4249. if (!PageUptodate(page)) {
  4250. ret = btrfs_readpage(NULL, page);
  4251. lock_page(page);
  4252. if (page->mapping != mapping) {
  4253. unlock_page(page);
  4254. put_page(page);
  4255. goto again;
  4256. }
  4257. if (!PageUptodate(page)) {
  4258. ret = -EIO;
  4259. goto out_unlock;
  4260. }
  4261. }
  4262. wait_on_page_writeback(page);
  4263. lock_extent_bits(io_tree, block_start, block_end, &cached_state);
  4264. set_page_extent_mapped(page);
  4265. ordered = btrfs_lookup_ordered_extent(inode, block_start);
  4266. if (ordered) {
  4267. unlock_extent_cached(io_tree, block_start, block_end,
  4268. &cached_state, GFP_NOFS);
  4269. unlock_page(page);
  4270. put_page(page);
  4271. btrfs_start_ordered_extent(inode, ordered, 1);
  4272. btrfs_put_ordered_extent(ordered);
  4273. goto again;
  4274. }
  4275. clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end,
  4276. EXTENT_DIRTY | EXTENT_DELALLOC |
  4277. EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
  4278. 0, 0, &cached_state, GFP_NOFS);
  4279. ret = btrfs_set_extent_delalloc(inode, block_start, block_end,
  4280. &cached_state, 0);
  4281. if (ret) {
  4282. unlock_extent_cached(io_tree, block_start, block_end,
  4283. &cached_state, GFP_NOFS);
  4284. goto out_unlock;
  4285. }
  4286. if (offset != blocksize) {
  4287. if (!len)
  4288. len = blocksize - offset;
  4289. kaddr = kmap(page);
  4290. if (front)
  4291. memset(kaddr + (block_start - page_offset(page)),
  4292. 0, offset);
  4293. else
  4294. memset(kaddr + (block_start - page_offset(page)) + offset,
  4295. 0, len);
  4296. flush_dcache_page(page);
  4297. kunmap(page);
  4298. }
  4299. ClearPageChecked(page);
  4300. set_page_dirty(page);
  4301. unlock_extent_cached(io_tree, block_start, block_end, &cached_state,
  4302. GFP_NOFS);
  4303. out_unlock:
  4304. if (ret)
  4305. btrfs_delalloc_release_space(inode, block_start,
  4306. blocksize);
  4307. unlock_page(page);
  4308. put_page(page);
  4309. out:
  4310. return ret;
  4311. }
  4312. static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
  4313. u64 offset, u64 len)
  4314. {
  4315. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  4316. struct btrfs_trans_handle *trans;
  4317. int ret;
  4318. /*
  4319. * Still need to make sure the inode looks like it's been updated so
  4320. * that any holes get logged if we fsync.
  4321. */
  4322. if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
  4323. BTRFS_I(inode)->last_trans = fs_info->generation;
  4324. BTRFS_I(inode)->last_sub_trans = root->log_transid;
  4325. BTRFS_I(inode)->last_log_commit = root->last_log_commit;
  4326. return 0;
  4327. }
  4328. /*
  4329. * 1 - for the one we're dropping
  4330. * 1 - for the one we're adding
  4331. * 1 - for updating the inode.
  4332. */
  4333. trans = btrfs_start_transaction(root, 3);
  4334. if (IS_ERR(trans))
  4335. return PTR_ERR(trans);
  4336. ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
  4337. if (ret) {
  4338. btrfs_abort_transaction(trans, ret);
  4339. btrfs_end_transaction(trans);
  4340. return ret;
  4341. }
  4342. ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)),
  4343. offset, 0, 0, len, 0, len, 0, 0, 0);
  4344. if (ret)
  4345. btrfs_abort_transaction(trans, ret);
  4346. else
  4347. btrfs_update_inode(trans, root, inode);
  4348. btrfs_end_transaction(trans);
  4349. return ret;
  4350. }
  4351. /*
  4352. * This function puts in dummy file extents for the area we're creating a hole
  4353. * for. So if we are truncating this file to a larger size we need to insert
  4354. * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
  4355. * the range between oldsize and size
  4356. */
  4357. int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
  4358. {
  4359. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  4360. struct btrfs_root *root = BTRFS_I(inode)->root;
  4361. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  4362. struct extent_map *em = NULL;
  4363. struct extent_state *cached_state = NULL;
  4364. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  4365. u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
  4366. u64 block_end = ALIGN(size, fs_info->sectorsize);
  4367. u64 last_byte;
  4368. u64 cur_offset;
  4369. u64 hole_size;
  4370. int err = 0;
  4371. /*
  4372. * If our size started in the middle of a block we need to zero out the
  4373. * rest of the block before we expand the i_size, otherwise we could
  4374. * expose stale data.
  4375. */
  4376. err = btrfs_truncate_block(inode, oldsize, 0, 0);
  4377. if (err)
  4378. return err;
  4379. if (size <= hole_start)
  4380. return 0;
  4381. while (1) {
  4382. struct btrfs_ordered_extent *ordered;
  4383. lock_extent_bits(io_tree, hole_start, block_end - 1,
  4384. &cached_state);
  4385. ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start,
  4386. block_end - hole_start);
  4387. if (!ordered)
  4388. break;
  4389. unlock_extent_cached(io_tree, hole_start, block_end - 1,
  4390. &cached_state, GFP_NOFS);
  4391. btrfs_start_ordered_extent(inode, ordered, 1);
  4392. btrfs_put_ordered_extent(ordered);
  4393. }
  4394. cur_offset = hole_start;
  4395. while (1) {
  4396. em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
  4397. block_end - cur_offset, 0);
  4398. if (IS_ERR(em)) {
  4399. err = PTR_ERR(em);
  4400. em = NULL;
  4401. break;
  4402. }
  4403. last_byte = min(extent_map_end(em), block_end);
  4404. last_byte = ALIGN(last_byte, fs_info->sectorsize);
  4405. if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
  4406. struct extent_map *hole_em;
  4407. hole_size = last_byte - cur_offset;
  4408. err = maybe_insert_hole(root, inode, cur_offset,
  4409. hole_size);
  4410. if (err)
  4411. break;
  4412. btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
  4413. cur_offset + hole_size - 1, 0);
  4414. hole_em = alloc_extent_map();
  4415. if (!hole_em) {
  4416. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  4417. &BTRFS_I(inode)->runtime_flags);
  4418. goto next;
  4419. }
  4420. hole_em->start = cur_offset;
  4421. hole_em->len = hole_size;
  4422. hole_em->orig_start = cur_offset;
  4423. hole_em->block_start = EXTENT_MAP_HOLE;
  4424. hole_em->block_len = 0;
  4425. hole_em->orig_block_len = 0;
  4426. hole_em->ram_bytes = hole_size;
  4427. hole_em->bdev = fs_info->fs_devices->latest_bdev;
  4428. hole_em->compress_type = BTRFS_COMPRESS_NONE;
  4429. hole_em->generation = fs_info->generation;
  4430. while (1) {
  4431. write_lock(&em_tree->lock);
  4432. err = add_extent_mapping(em_tree, hole_em, 1);
  4433. write_unlock(&em_tree->lock);
  4434. if (err != -EEXIST)
  4435. break;
  4436. btrfs_drop_extent_cache(BTRFS_I(inode),
  4437. cur_offset,
  4438. cur_offset +
  4439. hole_size - 1, 0);
  4440. }
  4441. free_extent_map(hole_em);
  4442. }
  4443. next:
  4444. free_extent_map(em);
  4445. em = NULL;
  4446. cur_offset = last_byte;
  4447. if (cur_offset >= block_end)
  4448. break;
  4449. }
  4450. free_extent_map(em);
  4451. unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state,
  4452. GFP_NOFS);
  4453. return err;
  4454. }
  4455. static int btrfs_setsize(struct inode *inode, struct iattr *attr)
  4456. {
  4457. struct btrfs_root *root = BTRFS_I(inode)->root;
  4458. struct btrfs_trans_handle *trans;
  4459. loff_t oldsize = i_size_read(inode);
  4460. loff_t newsize = attr->ia_size;
  4461. int mask = attr->ia_valid;
  4462. int ret;
  4463. /*
  4464. * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
  4465. * special case where we need to update the times despite not having
  4466. * these flags set. For all other operations the VFS set these flags
  4467. * explicitly if it wants a timestamp update.
  4468. */
  4469. if (newsize != oldsize) {
  4470. inode_inc_iversion(inode);
  4471. if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
  4472. inode->i_ctime = inode->i_mtime =
  4473. current_time(inode);
  4474. }
  4475. if (newsize > oldsize) {
  4476. /*
  4477. * Don't do an expanding truncate while snapshoting is ongoing.
  4478. * This is to ensure the snapshot captures a fully consistent
  4479. * state of this file - if the snapshot captures this expanding
  4480. * truncation, it must capture all writes that happened before
  4481. * this truncation.
  4482. */
  4483. btrfs_wait_for_snapshot_creation(root);
  4484. ret = btrfs_cont_expand(inode, oldsize, newsize);
  4485. if (ret) {
  4486. btrfs_end_write_no_snapshoting(root);
  4487. return ret;
  4488. }
  4489. trans = btrfs_start_transaction(root, 1);
  4490. if (IS_ERR(trans)) {
  4491. btrfs_end_write_no_snapshoting(root);
  4492. return PTR_ERR(trans);
  4493. }
  4494. i_size_write(inode, newsize);
  4495. btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL);
  4496. pagecache_isize_extended(inode, oldsize, newsize);
  4497. ret = btrfs_update_inode(trans, root, inode);
  4498. btrfs_end_write_no_snapshoting(root);
  4499. btrfs_end_transaction(trans);
  4500. } else {
  4501. /*
  4502. * We're truncating a file that used to have good data down to
  4503. * zero. Make sure it gets into the ordered flush list so that
  4504. * any new writes get down to disk quickly.
  4505. */
  4506. if (newsize == 0)
  4507. set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
  4508. &BTRFS_I(inode)->runtime_flags);
  4509. /*
  4510. * 1 for the orphan item we're going to add
  4511. * 1 for the orphan item deletion.
  4512. */
  4513. trans = btrfs_start_transaction(root, 2);
  4514. if (IS_ERR(trans))
  4515. return PTR_ERR(trans);
  4516. /*
  4517. * We need to do this in case we fail at _any_ point during the
  4518. * actual truncate. Once we do the truncate_setsize we could
  4519. * invalidate pages which forces any outstanding ordered io to
  4520. * be instantly completed which will give us extents that need
  4521. * to be truncated. If we fail to get an orphan inode down we
  4522. * could have left over extents that were never meant to live,
  4523. * so we need to guarantee from this point on that everything
  4524. * will be consistent.
  4525. */
  4526. ret = btrfs_orphan_add(trans, BTRFS_I(inode));
  4527. btrfs_end_transaction(trans);
  4528. if (ret)
  4529. return ret;
  4530. /* we don't support swapfiles, so vmtruncate shouldn't fail */
  4531. truncate_setsize(inode, newsize);
  4532. /* Disable nonlocked read DIO to avoid the end less truncate */
  4533. btrfs_inode_block_unlocked_dio(BTRFS_I(inode));
  4534. inode_dio_wait(inode);
  4535. btrfs_inode_resume_unlocked_dio(BTRFS_I(inode));
  4536. ret = btrfs_truncate(inode);
  4537. if (ret && inode->i_nlink) {
  4538. int err;
  4539. /* To get a stable disk_i_size */
  4540. err = btrfs_wait_ordered_range(inode, 0, (u64)-1);
  4541. if (err) {
  4542. btrfs_orphan_del(NULL, BTRFS_I(inode));
  4543. return err;
  4544. }
  4545. /*
  4546. * failed to truncate, disk_i_size is only adjusted down
  4547. * as we remove extents, so it should represent the true
  4548. * size of the inode, so reset the in memory size and
  4549. * delete our orphan entry.
  4550. */
  4551. trans = btrfs_join_transaction(root);
  4552. if (IS_ERR(trans)) {
  4553. btrfs_orphan_del(NULL, BTRFS_I(inode));
  4554. return ret;
  4555. }
  4556. i_size_write(inode, BTRFS_I(inode)->disk_i_size);
  4557. err = btrfs_orphan_del(trans, BTRFS_I(inode));
  4558. if (err)
  4559. btrfs_abort_transaction(trans, err);
  4560. btrfs_end_transaction(trans);
  4561. }
  4562. }
  4563. return ret;
  4564. }
  4565. static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
  4566. {
  4567. struct inode *inode = d_inode(dentry);
  4568. struct btrfs_root *root = BTRFS_I(inode)->root;
  4569. int err;
  4570. if (btrfs_root_readonly(root))
  4571. return -EROFS;
  4572. err = setattr_prepare(dentry, attr);
  4573. if (err)
  4574. return err;
  4575. if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
  4576. err = btrfs_setsize(inode, attr);
  4577. if (err)
  4578. return err;
  4579. }
  4580. if (attr->ia_valid) {
  4581. setattr_copy(inode, attr);
  4582. inode_inc_iversion(inode);
  4583. err = btrfs_dirty_inode(inode);
  4584. if (!err && attr->ia_valid & ATTR_MODE)
  4585. err = posix_acl_chmod(inode, inode->i_mode);
  4586. }
  4587. return err;
  4588. }
  4589. /*
  4590. * While truncating the inode pages during eviction, we get the VFS calling
  4591. * btrfs_invalidatepage() against each page of the inode. This is slow because
  4592. * the calls to btrfs_invalidatepage() result in a huge amount of calls to
  4593. * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
  4594. * extent_state structures over and over, wasting lots of time.
  4595. *
  4596. * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
  4597. * those expensive operations on a per page basis and do only the ordered io
  4598. * finishing, while we release here the extent_map and extent_state structures,
  4599. * without the excessive merging and splitting.
  4600. */
  4601. static void evict_inode_truncate_pages(struct inode *inode)
  4602. {
  4603. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  4604. struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
  4605. struct rb_node *node;
  4606. ASSERT(inode->i_state & I_FREEING);
  4607. truncate_inode_pages_final(&inode->i_data);
  4608. write_lock(&map_tree->lock);
  4609. while (!RB_EMPTY_ROOT(&map_tree->map)) {
  4610. struct extent_map *em;
  4611. node = rb_first(&map_tree->map);
  4612. em = rb_entry(node, struct extent_map, rb_node);
  4613. clear_bit(EXTENT_FLAG_PINNED, &em->flags);
  4614. clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
  4615. remove_extent_mapping(map_tree, em);
  4616. free_extent_map(em);
  4617. if (need_resched()) {
  4618. write_unlock(&map_tree->lock);
  4619. cond_resched();
  4620. write_lock(&map_tree->lock);
  4621. }
  4622. }
  4623. write_unlock(&map_tree->lock);
  4624. /*
  4625. * Keep looping until we have no more ranges in the io tree.
  4626. * We can have ongoing bios started by readpages (called from readahead)
  4627. * that have their endio callback (extent_io.c:end_bio_extent_readpage)
  4628. * still in progress (unlocked the pages in the bio but did not yet
  4629. * unlocked the ranges in the io tree). Therefore this means some
  4630. * ranges can still be locked and eviction started because before
  4631. * submitting those bios, which are executed by a separate task (work
  4632. * queue kthread), inode references (inode->i_count) were not taken
  4633. * (which would be dropped in the end io callback of each bio).
  4634. * Therefore here we effectively end up waiting for those bios and
  4635. * anyone else holding locked ranges without having bumped the inode's
  4636. * reference count - if we don't do it, when they access the inode's
  4637. * io_tree to unlock a range it may be too late, leading to an
  4638. * use-after-free issue.
  4639. */
  4640. spin_lock(&io_tree->lock);
  4641. while (!RB_EMPTY_ROOT(&io_tree->state)) {
  4642. struct extent_state *state;
  4643. struct extent_state *cached_state = NULL;
  4644. u64 start;
  4645. u64 end;
  4646. node = rb_first(&io_tree->state);
  4647. state = rb_entry(node, struct extent_state, rb_node);
  4648. start = state->start;
  4649. end = state->end;
  4650. spin_unlock(&io_tree->lock);
  4651. lock_extent_bits(io_tree, start, end, &cached_state);
  4652. /*
  4653. * If still has DELALLOC flag, the extent didn't reach disk,
  4654. * and its reserved space won't be freed by delayed_ref.
  4655. * So we need to free its reserved space here.
  4656. * (Refer to comment in btrfs_invalidatepage, case 2)
  4657. *
  4658. * Note, end is the bytenr of last byte, so we need + 1 here.
  4659. */
  4660. if (state->state & EXTENT_DELALLOC)
  4661. btrfs_qgroup_free_data(inode, start, end - start + 1);
  4662. clear_extent_bit(io_tree, start, end,
  4663. EXTENT_LOCKED | EXTENT_DIRTY |
  4664. EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
  4665. EXTENT_DEFRAG, 1, 1,
  4666. &cached_state, GFP_NOFS);
  4667. cond_resched();
  4668. spin_lock(&io_tree->lock);
  4669. }
  4670. spin_unlock(&io_tree->lock);
  4671. }
  4672. void btrfs_evict_inode(struct inode *inode)
  4673. {
  4674. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  4675. struct btrfs_trans_handle *trans;
  4676. struct btrfs_root *root = BTRFS_I(inode)->root;
  4677. struct btrfs_block_rsv *rsv, *global_rsv;
  4678. int steal_from_global = 0;
  4679. u64 min_size;
  4680. int ret;
  4681. trace_btrfs_inode_evict(inode);
  4682. if (!root) {
  4683. kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
  4684. return;
  4685. }
  4686. min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
  4687. evict_inode_truncate_pages(inode);
  4688. if (inode->i_nlink &&
  4689. ((btrfs_root_refs(&root->root_item) != 0 &&
  4690. root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
  4691. btrfs_is_free_space_inode(BTRFS_I(inode))))
  4692. goto no_delete;
  4693. if (is_bad_inode(inode)) {
  4694. btrfs_orphan_del(NULL, BTRFS_I(inode));
  4695. goto no_delete;
  4696. }
  4697. /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
  4698. if (!special_file(inode->i_mode))
  4699. btrfs_wait_ordered_range(inode, 0, (u64)-1);
  4700. btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1);
  4701. if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
  4702. BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
  4703. &BTRFS_I(inode)->runtime_flags));
  4704. goto no_delete;
  4705. }
  4706. if (inode->i_nlink > 0) {
  4707. BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
  4708. root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
  4709. goto no_delete;
  4710. }
  4711. ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
  4712. if (ret) {
  4713. btrfs_orphan_del(NULL, BTRFS_I(inode));
  4714. goto no_delete;
  4715. }
  4716. rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
  4717. if (!rsv) {
  4718. btrfs_orphan_del(NULL, BTRFS_I(inode));
  4719. goto no_delete;
  4720. }
  4721. rsv->size = min_size;
  4722. rsv->failfast = 1;
  4723. global_rsv = &fs_info->global_block_rsv;
  4724. btrfs_i_size_write(BTRFS_I(inode), 0);
  4725. /*
  4726. * This is a bit simpler than btrfs_truncate since we've already
  4727. * reserved our space for our orphan item in the unlink, so we just
  4728. * need to reserve some slack space in case we add bytes and update
  4729. * inode item when doing the truncate.
  4730. */
  4731. while (1) {
  4732. ret = btrfs_block_rsv_refill(root, rsv, min_size,
  4733. BTRFS_RESERVE_FLUSH_LIMIT);
  4734. /*
  4735. * Try and steal from the global reserve since we will
  4736. * likely not use this space anyway, we want to try as
  4737. * hard as possible to get this to work.
  4738. */
  4739. if (ret)
  4740. steal_from_global++;
  4741. else
  4742. steal_from_global = 0;
  4743. ret = 0;
  4744. /*
  4745. * steal_from_global == 0: we reserved stuff, hooray!
  4746. * steal_from_global == 1: we didn't reserve stuff, boo!
  4747. * steal_from_global == 2: we've committed, still not a lot of
  4748. * room but maybe we'll have room in the global reserve this
  4749. * time.
  4750. * steal_from_global == 3: abandon all hope!
  4751. */
  4752. if (steal_from_global > 2) {
  4753. btrfs_warn(fs_info,
  4754. "Could not get space for a delete, will truncate on mount %d",
  4755. ret);
  4756. btrfs_orphan_del(NULL, BTRFS_I(inode));
  4757. btrfs_free_block_rsv(fs_info, rsv);
  4758. goto no_delete;
  4759. }
  4760. trans = btrfs_join_transaction(root);
  4761. if (IS_ERR(trans)) {
  4762. btrfs_orphan_del(NULL, BTRFS_I(inode));
  4763. btrfs_free_block_rsv(fs_info, rsv);
  4764. goto no_delete;
  4765. }
  4766. /*
  4767. * We can't just steal from the global reserve, we need to make
  4768. * sure there is room to do it, if not we need to commit and try
  4769. * again.
  4770. */
  4771. if (steal_from_global) {
  4772. if (!btrfs_check_space_for_delayed_refs(trans, fs_info))
  4773. ret = btrfs_block_rsv_migrate(global_rsv, rsv,
  4774. min_size, 0);
  4775. else
  4776. ret = -ENOSPC;
  4777. }
  4778. /*
  4779. * Couldn't steal from the global reserve, we have too much
  4780. * pending stuff built up, commit the transaction and try it
  4781. * again.
  4782. */
  4783. if (ret) {
  4784. ret = btrfs_commit_transaction(trans);
  4785. if (ret) {
  4786. btrfs_orphan_del(NULL, BTRFS_I(inode));
  4787. btrfs_free_block_rsv(fs_info, rsv);
  4788. goto no_delete;
  4789. }
  4790. continue;
  4791. } else {
  4792. steal_from_global = 0;
  4793. }
  4794. trans->block_rsv = rsv;
  4795. ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
  4796. if (ret != -ENOSPC && ret != -EAGAIN)
  4797. break;
  4798. trans->block_rsv = &fs_info->trans_block_rsv;
  4799. btrfs_end_transaction(trans);
  4800. trans = NULL;
  4801. btrfs_btree_balance_dirty(fs_info);
  4802. }
  4803. btrfs_free_block_rsv(fs_info, rsv);
  4804. /*
  4805. * Errors here aren't a big deal, it just means we leave orphan items
  4806. * in the tree. They will be cleaned up on the next mount.
  4807. */
  4808. if (ret == 0) {
  4809. trans->block_rsv = root->orphan_block_rsv;
  4810. btrfs_orphan_del(trans, BTRFS_I(inode));
  4811. } else {
  4812. btrfs_orphan_del(NULL, BTRFS_I(inode));
  4813. }
  4814. trans->block_rsv = &fs_info->trans_block_rsv;
  4815. if (!(root == fs_info->tree_root ||
  4816. root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
  4817. btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode)));
  4818. btrfs_end_transaction(trans);
  4819. btrfs_btree_balance_dirty(fs_info);
  4820. no_delete:
  4821. btrfs_remove_delayed_node(BTRFS_I(inode));
  4822. clear_inode(inode);
  4823. }
  4824. /*
  4825. * this returns the key found in the dir entry in the location pointer.
  4826. * If no dir entries were found, location->objectid is 0.
  4827. */
  4828. static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
  4829. struct btrfs_key *location)
  4830. {
  4831. const char *name = dentry->d_name.name;
  4832. int namelen = dentry->d_name.len;
  4833. struct btrfs_dir_item *di;
  4834. struct btrfs_path *path;
  4835. struct btrfs_root *root = BTRFS_I(dir)->root;
  4836. int ret = 0;
  4837. path = btrfs_alloc_path();
  4838. if (!path)
  4839. return -ENOMEM;
  4840. di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)),
  4841. name, namelen, 0);
  4842. if (IS_ERR(di))
  4843. ret = PTR_ERR(di);
  4844. if (IS_ERR_OR_NULL(di))
  4845. goto out_err;
  4846. btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
  4847. out:
  4848. btrfs_free_path(path);
  4849. return ret;
  4850. out_err:
  4851. location->objectid = 0;
  4852. goto out;
  4853. }
  4854. /*
  4855. * when we hit a tree root in a directory, the btrfs part of the inode
  4856. * needs to be changed to reflect the root directory of the tree root. This
  4857. * is kind of like crossing a mount point.
  4858. */
  4859. static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
  4860. struct inode *dir,
  4861. struct dentry *dentry,
  4862. struct btrfs_key *location,
  4863. struct btrfs_root **sub_root)
  4864. {
  4865. struct btrfs_path *path;
  4866. struct btrfs_root *new_root;
  4867. struct btrfs_root_ref *ref;
  4868. struct extent_buffer *leaf;
  4869. struct btrfs_key key;
  4870. int ret;
  4871. int err = 0;
  4872. path = btrfs_alloc_path();
  4873. if (!path) {
  4874. err = -ENOMEM;
  4875. goto out;
  4876. }
  4877. err = -ENOENT;
  4878. key.objectid = BTRFS_I(dir)->root->root_key.objectid;
  4879. key.type = BTRFS_ROOT_REF_KEY;
  4880. key.offset = location->objectid;
  4881. ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
  4882. if (ret) {
  4883. if (ret < 0)
  4884. err = ret;
  4885. goto out;
  4886. }
  4887. leaf = path->nodes[0];
  4888. ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
  4889. if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) ||
  4890. btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
  4891. goto out;
  4892. ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
  4893. (unsigned long)(ref + 1),
  4894. dentry->d_name.len);
  4895. if (ret)
  4896. goto out;
  4897. btrfs_release_path(path);
  4898. new_root = btrfs_read_fs_root_no_name(fs_info, location);
  4899. if (IS_ERR(new_root)) {
  4900. err = PTR_ERR(new_root);
  4901. goto out;
  4902. }
  4903. *sub_root = new_root;
  4904. location->objectid = btrfs_root_dirid(&new_root->root_item);
  4905. location->type = BTRFS_INODE_ITEM_KEY;
  4906. location->offset = 0;
  4907. err = 0;
  4908. out:
  4909. btrfs_free_path(path);
  4910. return err;
  4911. }
  4912. static void inode_tree_add(struct inode *inode)
  4913. {
  4914. struct btrfs_root *root = BTRFS_I(inode)->root;
  4915. struct btrfs_inode *entry;
  4916. struct rb_node **p;
  4917. struct rb_node *parent;
  4918. struct rb_node *new = &BTRFS_I(inode)->rb_node;
  4919. u64 ino = btrfs_ino(BTRFS_I(inode));
  4920. if (inode_unhashed(inode))
  4921. return;
  4922. parent = NULL;
  4923. spin_lock(&root->inode_lock);
  4924. p = &root->inode_tree.rb_node;
  4925. while (*p) {
  4926. parent = *p;
  4927. entry = rb_entry(parent, struct btrfs_inode, rb_node);
  4928. if (ino < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
  4929. p = &parent->rb_left;
  4930. else if (ino > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
  4931. p = &parent->rb_right;
  4932. else {
  4933. WARN_ON(!(entry->vfs_inode.i_state &
  4934. (I_WILL_FREE | I_FREEING)));
  4935. rb_replace_node(parent, new, &root->inode_tree);
  4936. RB_CLEAR_NODE(parent);
  4937. spin_unlock(&root->inode_lock);
  4938. return;
  4939. }
  4940. }
  4941. rb_link_node(new, parent, p);
  4942. rb_insert_color(new, &root->inode_tree);
  4943. spin_unlock(&root->inode_lock);
  4944. }
  4945. static void inode_tree_del(struct inode *inode)
  4946. {
  4947. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  4948. struct btrfs_root *root = BTRFS_I(inode)->root;
  4949. int empty = 0;
  4950. spin_lock(&root->inode_lock);
  4951. if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
  4952. rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
  4953. RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
  4954. empty = RB_EMPTY_ROOT(&root->inode_tree);
  4955. }
  4956. spin_unlock(&root->inode_lock);
  4957. if (empty && btrfs_root_refs(&root->root_item) == 0) {
  4958. synchronize_srcu(&fs_info->subvol_srcu);
  4959. spin_lock(&root->inode_lock);
  4960. empty = RB_EMPTY_ROOT(&root->inode_tree);
  4961. spin_unlock(&root->inode_lock);
  4962. if (empty)
  4963. btrfs_add_dead_root(root);
  4964. }
  4965. }
  4966. void btrfs_invalidate_inodes(struct btrfs_root *root)
  4967. {
  4968. struct btrfs_fs_info *fs_info = root->fs_info;
  4969. struct rb_node *node;
  4970. struct rb_node *prev;
  4971. struct btrfs_inode *entry;
  4972. struct inode *inode;
  4973. u64 objectid = 0;
  4974. if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
  4975. WARN_ON(btrfs_root_refs(&root->root_item) != 0);
  4976. spin_lock(&root->inode_lock);
  4977. again:
  4978. node = root->inode_tree.rb_node;
  4979. prev = NULL;
  4980. while (node) {
  4981. prev = node;
  4982. entry = rb_entry(node, struct btrfs_inode, rb_node);
  4983. if (objectid < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
  4984. node = node->rb_left;
  4985. else if (objectid > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
  4986. node = node->rb_right;
  4987. else
  4988. break;
  4989. }
  4990. if (!node) {
  4991. while (prev) {
  4992. entry = rb_entry(prev, struct btrfs_inode, rb_node);
  4993. if (objectid <= btrfs_ino(BTRFS_I(&entry->vfs_inode))) {
  4994. node = prev;
  4995. break;
  4996. }
  4997. prev = rb_next(prev);
  4998. }
  4999. }
  5000. while (node) {
  5001. entry = rb_entry(node, struct btrfs_inode, rb_node);
  5002. objectid = btrfs_ino(BTRFS_I(&entry->vfs_inode)) + 1;
  5003. inode = igrab(&entry->vfs_inode);
  5004. if (inode) {
  5005. spin_unlock(&root->inode_lock);
  5006. if (atomic_read(&inode->i_count) > 1)
  5007. d_prune_aliases(inode);
  5008. /*
  5009. * btrfs_drop_inode will have it removed from
  5010. * the inode cache when its usage count
  5011. * hits zero.
  5012. */
  5013. iput(inode);
  5014. cond_resched();
  5015. spin_lock(&root->inode_lock);
  5016. goto again;
  5017. }
  5018. if (cond_resched_lock(&root->inode_lock))
  5019. goto again;
  5020. node = rb_next(node);
  5021. }
  5022. spin_unlock(&root->inode_lock);
  5023. }
  5024. static int btrfs_init_locked_inode(struct inode *inode, void *p)
  5025. {
  5026. struct btrfs_iget_args *args = p;
  5027. inode->i_ino = args->location->objectid;
  5028. memcpy(&BTRFS_I(inode)->location, args->location,
  5029. sizeof(*args->location));
  5030. BTRFS_I(inode)->root = args->root;
  5031. return 0;
  5032. }
  5033. static int btrfs_find_actor(struct inode *inode, void *opaque)
  5034. {
  5035. struct btrfs_iget_args *args = opaque;
  5036. return args->location->objectid == BTRFS_I(inode)->location.objectid &&
  5037. args->root == BTRFS_I(inode)->root;
  5038. }
  5039. static struct inode *btrfs_iget_locked(struct super_block *s,
  5040. struct btrfs_key *location,
  5041. struct btrfs_root *root)
  5042. {
  5043. struct inode *inode;
  5044. struct btrfs_iget_args args;
  5045. unsigned long hashval = btrfs_inode_hash(location->objectid, root);
  5046. args.location = location;
  5047. args.root = root;
  5048. inode = iget5_locked(s, hashval, btrfs_find_actor,
  5049. btrfs_init_locked_inode,
  5050. (void *)&args);
  5051. return inode;
  5052. }
  5053. /* Get an inode object given its location and corresponding root.
  5054. * Returns in *is_new if the inode was read from disk
  5055. */
  5056. struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
  5057. struct btrfs_root *root, int *new)
  5058. {
  5059. struct inode *inode;
  5060. inode = btrfs_iget_locked(s, location, root);
  5061. if (!inode)
  5062. return ERR_PTR(-ENOMEM);
  5063. if (inode->i_state & I_NEW) {
  5064. int ret;
  5065. ret = btrfs_read_locked_inode(inode);
  5066. if (!is_bad_inode(inode)) {
  5067. inode_tree_add(inode);
  5068. unlock_new_inode(inode);
  5069. if (new)
  5070. *new = 1;
  5071. } else {
  5072. unlock_new_inode(inode);
  5073. iput(inode);
  5074. ASSERT(ret < 0);
  5075. inode = ERR_PTR(ret < 0 ? ret : -ESTALE);
  5076. }
  5077. }
  5078. return inode;
  5079. }
  5080. static struct inode *new_simple_dir(struct super_block *s,
  5081. struct btrfs_key *key,
  5082. struct btrfs_root *root)
  5083. {
  5084. struct inode *inode = new_inode(s);
  5085. if (!inode)
  5086. return ERR_PTR(-ENOMEM);
  5087. BTRFS_I(inode)->root = root;
  5088. memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
  5089. set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
  5090. inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
  5091. inode->i_op = &btrfs_dir_ro_inode_operations;
  5092. inode->i_opflags &= ~IOP_XATTR;
  5093. inode->i_fop = &simple_dir_operations;
  5094. inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
  5095. inode->i_mtime = current_time(inode);
  5096. inode->i_atime = inode->i_mtime;
  5097. inode->i_ctime = inode->i_mtime;
  5098. BTRFS_I(inode)->i_otime = inode->i_mtime;
  5099. return inode;
  5100. }
  5101. struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
  5102. {
  5103. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  5104. struct inode *inode;
  5105. struct btrfs_root *root = BTRFS_I(dir)->root;
  5106. struct btrfs_root *sub_root = root;
  5107. struct btrfs_key location;
  5108. int index;
  5109. int ret = 0;
  5110. if (dentry->d_name.len > BTRFS_NAME_LEN)
  5111. return ERR_PTR(-ENAMETOOLONG);
  5112. ret = btrfs_inode_by_name(dir, dentry, &location);
  5113. if (ret < 0)
  5114. return ERR_PTR(ret);
  5115. if (location.objectid == 0)
  5116. return ERR_PTR(-ENOENT);
  5117. if (location.type == BTRFS_INODE_ITEM_KEY) {
  5118. inode = btrfs_iget(dir->i_sb, &location, root, NULL);
  5119. return inode;
  5120. }
  5121. BUG_ON(location.type != BTRFS_ROOT_ITEM_KEY);
  5122. index = srcu_read_lock(&fs_info->subvol_srcu);
  5123. ret = fixup_tree_root_location(fs_info, dir, dentry,
  5124. &location, &sub_root);
  5125. if (ret < 0) {
  5126. if (ret != -ENOENT)
  5127. inode = ERR_PTR(ret);
  5128. else
  5129. inode = new_simple_dir(dir->i_sb, &location, sub_root);
  5130. } else {
  5131. inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL);
  5132. }
  5133. srcu_read_unlock(&fs_info->subvol_srcu, index);
  5134. if (!IS_ERR(inode) && root != sub_root) {
  5135. down_read(&fs_info->cleanup_work_sem);
  5136. if (!(inode->i_sb->s_flags & MS_RDONLY))
  5137. ret = btrfs_orphan_cleanup(sub_root);
  5138. up_read(&fs_info->cleanup_work_sem);
  5139. if (ret) {
  5140. iput(inode);
  5141. inode = ERR_PTR(ret);
  5142. }
  5143. }
  5144. return inode;
  5145. }
  5146. static int btrfs_dentry_delete(const struct dentry *dentry)
  5147. {
  5148. struct btrfs_root *root;
  5149. struct inode *inode = d_inode(dentry);
  5150. if (!inode && !IS_ROOT(dentry))
  5151. inode = d_inode(dentry->d_parent);
  5152. if (inode) {
  5153. root = BTRFS_I(inode)->root;
  5154. if (btrfs_root_refs(&root->root_item) == 0)
  5155. return 1;
  5156. if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
  5157. return 1;
  5158. }
  5159. return 0;
  5160. }
  5161. static void btrfs_dentry_release(struct dentry *dentry)
  5162. {
  5163. kfree(dentry->d_fsdata);
  5164. }
  5165. static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
  5166. unsigned int flags)
  5167. {
  5168. struct inode *inode;
  5169. inode = btrfs_lookup_dentry(dir, dentry);
  5170. if (IS_ERR(inode)) {
  5171. if (PTR_ERR(inode) == -ENOENT)
  5172. inode = NULL;
  5173. else
  5174. return ERR_CAST(inode);
  5175. }
  5176. return d_splice_alias(inode, dentry);
  5177. }
  5178. unsigned char btrfs_filetype_table[] = {
  5179. DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
  5180. };
  5181. static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
  5182. {
  5183. struct inode *inode = file_inode(file);
  5184. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  5185. struct btrfs_root *root = BTRFS_I(inode)->root;
  5186. struct btrfs_dir_item *di;
  5187. struct btrfs_key key;
  5188. struct btrfs_key found_key;
  5189. struct btrfs_path *path;
  5190. struct list_head ins_list;
  5191. struct list_head del_list;
  5192. int ret;
  5193. struct extent_buffer *leaf;
  5194. int slot;
  5195. unsigned char d_type;
  5196. int over = 0;
  5197. char tmp_name[32];
  5198. char *name_ptr;
  5199. int name_len;
  5200. bool put = false;
  5201. struct btrfs_key location;
  5202. if (!dir_emit_dots(file, ctx))
  5203. return 0;
  5204. path = btrfs_alloc_path();
  5205. if (!path)
  5206. return -ENOMEM;
  5207. path->reada = READA_FORWARD;
  5208. INIT_LIST_HEAD(&ins_list);
  5209. INIT_LIST_HEAD(&del_list);
  5210. put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list);
  5211. key.type = BTRFS_DIR_INDEX_KEY;
  5212. key.offset = ctx->pos;
  5213. key.objectid = btrfs_ino(BTRFS_I(inode));
  5214. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  5215. if (ret < 0)
  5216. goto err;
  5217. while (1) {
  5218. leaf = path->nodes[0];
  5219. slot = path->slots[0];
  5220. if (slot >= btrfs_header_nritems(leaf)) {
  5221. ret = btrfs_next_leaf(root, path);
  5222. if (ret < 0)
  5223. goto err;
  5224. else if (ret > 0)
  5225. break;
  5226. continue;
  5227. }
  5228. btrfs_item_key_to_cpu(leaf, &found_key, slot);
  5229. if (found_key.objectid != key.objectid)
  5230. break;
  5231. if (found_key.type != BTRFS_DIR_INDEX_KEY)
  5232. break;
  5233. if (found_key.offset < ctx->pos)
  5234. goto next;
  5235. if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
  5236. goto next;
  5237. ctx->pos = found_key.offset;
  5238. di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
  5239. if (verify_dir_item(fs_info, leaf, di))
  5240. goto next;
  5241. name_len = btrfs_dir_name_len(leaf, di);
  5242. if (name_len <= sizeof(tmp_name)) {
  5243. name_ptr = tmp_name;
  5244. } else {
  5245. name_ptr = kmalloc(name_len, GFP_KERNEL);
  5246. if (!name_ptr) {
  5247. ret = -ENOMEM;
  5248. goto err;
  5249. }
  5250. }
  5251. read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1),
  5252. name_len);
  5253. d_type = btrfs_filetype_table[btrfs_dir_type(leaf, di)];
  5254. btrfs_dir_item_key_to_cpu(leaf, di, &location);
  5255. over = !dir_emit(ctx, name_ptr, name_len, location.objectid,
  5256. d_type);
  5257. if (name_ptr != tmp_name)
  5258. kfree(name_ptr);
  5259. if (over)
  5260. goto nopos;
  5261. ctx->pos++;
  5262. next:
  5263. path->slots[0]++;
  5264. }
  5265. ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
  5266. if (ret)
  5267. goto nopos;
  5268. /*
  5269. * Stop new entries from being returned after we return the last
  5270. * entry.
  5271. *
  5272. * New directory entries are assigned a strictly increasing
  5273. * offset. This means that new entries created during readdir
  5274. * are *guaranteed* to be seen in the future by that readdir.
  5275. * This has broken buggy programs which operate on names as
  5276. * they're returned by readdir. Until we re-use freed offsets
  5277. * we have this hack to stop new entries from being returned
  5278. * under the assumption that they'll never reach this huge
  5279. * offset.
  5280. *
  5281. * This is being careful not to overflow 32bit loff_t unless the
  5282. * last entry requires it because doing so has broken 32bit apps
  5283. * in the past.
  5284. */
  5285. if (ctx->pos >= INT_MAX)
  5286. ctx->pos = LLONG_MAX;
  5287. else
  5288. ctx->pos = INT_MAX;
  5289. nopos:
  5290. ret = 0;
  5291. err:
  5292. if (put)
  5293. btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list);
  5294. btrfs_free_path(path);
  5295. return ret;
  5296. }
  5297. int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc)
  5298. {
  5299. struct btrfs_root *root = BTRFS_I(inode)->root;
  5300. struct btrfs_trans_handle *trans;
  5301. int ret = 0;
  5302. bool nolock = false;
  5303. if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
  5304. return 0;
  5305. if (btrfs_fs_closing(root->fs_info) &&
  5306. btrfs_is_free_space_inode(BTRFS_I(inode)))
  5307. nolock = true;
  5308. if (wbc->sync_mode == WB_SYNC_ALL) {
  5309. if (nolock)
  5310. trans = btrfs_join_transaction_nolock(root);
  5311. else
  5312. trans = btrfs_join_transaction(root);
  5313. if (IS_ERR(trans))
  5314. return PTR_ERR(trans);
  5315. ret = btrfs_commit_transaction(trans);
  5316. }
  5317. return ret;
  5318. }
  5319. /*
  5320. * This is somewhat expensive, updating the tree every time the
  5321. * inode changes. But, it is most likely to find the inode in cache.
  5322. * FIXME, needs more benchmarking...there are no reasons other than performance
  5323. * to keep or drop this code.
  5324. */
  5325. static int btrfs_dirty_inode(struct inode *inode)
  5326. {
  5327. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  5328. struct btrfs_root *root = BTRFS_I(inode)->root;
  5329. struct btrfs_trans_handle *trans;
  5330. int ret;
  5331. if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
  5332. return 0;
  5333. trans = btrfs_join_transaction(root);
  5334. if (IS_ERR(trans))
  5335. return PTR_ERR(trans);
  5336. ret = btrfs_update_inode(trans, root, inode);
  5337. if (ret && ret == -ENOSPC) {
  5338. /* whoops, lets try again with the full transaction */
  5339. btrfs_end_transaction(trans);
  5340. trans = btrfs_start_transaction(root, 1);
  5341. if (IS_ERR(trans))
  5342. return PTR_ERR(trans);
  5343. ret = btrfs_update_inode(trans, root, inode);
  5344. }
  5345. btrfs_end_transaction(trans);
  5346. if (BTRFS_I(inode)->delayed_node)
  5347. btrfs_balance_delayed_items(fs_info);
  5348. return ret;
  5349. }
  5350. /*
  5351. * This is a copy of file_update_time. We need this so we can return error on
  5352. * ENOSPC for updating the inode in the case of file write and mmap writes.
  5353. */
  5354. static int btrfs_update_time(struct inode *inode, struct timespec *now,
  5355. int flags)
  5356. {
  5357. struct btrfs_root *root = BTRFS_I(inode)->root;
  5358. if (btrfs_root_readonly(root))
  5359. return -EROFS;
  5360. if (flags & S_VERSION)
  5361. inode_inc_iversion(inode);
  5362. if (flags & S_CTIME)
  5363. inode->i_ctime = *now;
  5364. if (flags & S_MTIME)
  5365. inode->i_mtime = *now;
  5366. if (flags & S_ATIME)
  5367. inode->i_atime = *now;
  5368. return btrfs_dirty_inode(inode);
  5369. }
  5370. /*
  5371. * find the highest existing sequence number in a directory
  5372. * and then set the in-memory index_cnt variable to reflect
  5373. * free sequence numbers
  5374. */
  5375. static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
  5376. {
  5377. struct btrfs_root *root = inode->root;
  5378. struct btrfs_key key, found_key;
  5379. struct btrfs_path *path;
  5380. struct extent_buffer *leaf;
  5381. int ret;
  5382. key.objectid = btrfs_ino(inode);
  5383. key.type = BTRFS_DIR_INDEX_KEY;
  5384. key.offset = (u64)-1;
  5385. path = btrfs_alloc_path();
  5386. if (!path)
  5387. return -ENOMEM;
  5388. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  5389. if (ret < 0)
  5390. goto out;
  5391. /* FIXME: we should be able to handle this */
  5392. if (ret == 0)
  5393. goto out;
  5394. ret = 0;
  5395. /*
  5396. * MAGIC NUMBER EXPLANATION:
  5397. * since we search a directory based on f_pos we have to start at 2
  5398. * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
  5399. * else has to start at 2
  5400. */
  5401. if (path->slots[0] == 0) {
  5402. inode->index_cnt = 2;
  5403. goto out;
  5404. }
  5405. path->slots[0]--;
  5406. leaf = path->nodes[0];
  5407. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  5408. if (found_key.objectid != btrfs_ino(inode) ||
  5409. found_key.type != BTRFS_DIR_INDEX_KEY) {
  5410. inode->index_cnt = 2;
  5411. goto out;
  5412. }
  5413. inode->index_cnt = found_key.offset + 1;
  5414. out:
  5415. btrfs_free_path(path);
  5416. return ret;
  5417. }
  5418. /*
  5419. * helper to find a free sequence number in a given directory. This current
  5420. * code is very simple, later versions will do smarter things in the btree
  5421. */
  5422. int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
  5423. {
  5424. int ret = 0;
  5425. if (dir->index_cnt == (u64)-1) {
  5426. ret = btrfs_inode_delayed_dir_index_count(dir);
  5427. if (ret) {
  5428. ret = btrfs_set_inode_index_count(dir);
  5429. if (ret)
  5430. return ret;
  5431. }
  5432. }
  5433. *index = dir->index_cnt;
  5434. dir->index_cnt++;
  5435. return ret;
  5436. }
  5437. static int btrfs_insert_inode_locked(struct inode *inode)
  5438. {
  5439. struct btrfs_iget_args args;
  5440. args.location = &BTRFS_I(inode)->location;
  5441. args.root = BTRFS_I(inode)->root;
  5442. return insert_inode_locked4(inode,
  5443. btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
  5444. btrfs_find_actor, &args);
  5445. }
  5446. static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
  5447. struct btrfs_root *root,
  5448. struct inode *dir,
  5449. const char *name, int name_len,
  5450. u64 ref_objectid, u64 objectid,
  5451. umode_t mode, u64 *index)
  5452. {
  5453. struct btrfs_fs_info *fs_info = root->fs_info;
  5454. struct inode *inode;
  5455. struct btrfs_inode_item *inode_item;
  5456. struct btrfs_key *location;
  5457. struct btrfs_path *path;
  5458. struct btrfs_inode_ref *ref;
  5459. struct btrfs_key key[2];
  5460. u32 sizes[2];
  5461. int nitems = name ? 2 : 1;
  5462. unsigned long ptr;
  5463. int ret;
  5464. path = btrfs_alloc_path();
  5465. if (!path)
  5466. return ERR_PTR(-ENOMEM);
  5467. inode = new_inode(fs_info->sb);
  5468. if (!inode) {
  5469. btrfs_free_path(path);
  5470. return ERR_PTR(-ENOMEM);
  5471. }
  5472. /*
  5473. * O_TMPFILE, set link count to 0, so that after this point,
  5474. * we fill in an inode item with the correct link count.
  5475. */
  5476. if (!name)
  5477. set_nlink(inode, 0);
  5478. /*
  5479. * we have to initialize this early, so we can reclaim the inode
  5480. * number if we fail afterwards in this function.
  5481. */
  5482. inode->i_ino = objectid;
  5483. if (dir && name) {
  5484. trace_btrfs_inode_request(dir);
  5485. ret = btrfs_set_inode_index(BTRFS_I(dir), index);
  5486. if (ret) {
  5487. btrfs_free_path(path);
  5488. iput(inode);
  5489. return ERR_PTR(ret);
  5490. }
  5491. } else if (dir) {
  5492. *index = 0;
  5493. }
  5494. /*
  5495. * index_cnt is ignored for everything but a dir,
  5496. * btrfs_get_inode_index_count has an explanation for the magic
  5497. * number
  5498. */
  5499. BTRFS_I(inode)->index_cnt = 2;
  5500. BTRFS_I(inode)->dir_index = *index;
  5501. BTRFS_I(inode)->root = root;
  5502. BTRFS_I(inode)->generation = trans->transid;
  5503. inode->i_generation = BTRFS_I(inode)->generation;
  5504. /*
  5505. * We could have gotten an inode number from somebody who was fsynced
  5506. * and then removed in this same transaction, so let's just set full
  5507. * sync since it will be a full sync anyway and this will blow away the
  5508. * old info in the log.
  5509. */
  5510. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
  5511. key[0].objectid = objectid;
  5512. key[0].type = BTRFS_INODE_ITEM_KEY;
  5513. key[0].offset = 0;
  5514. sizes[0] = sizeof(struct btrfs_inode_item);
  5515. if (name) {
  5516. /*
  5517. * Start new inodes with an inode_ref. This is slightly more
  5518. * efficient for small numbers of hard links since they will
  5519. * be packed into one item. Extended refs will kick in if we
  5520. * add more hard links than can fit in the ref item.
  5521. */
  5522. key[1].objectid = objectid;
  5523. key[1].type = BTRFS_INODE_REF_KEY;
  5524. key[1].offset = ref_objectid;
  5525. sizes[1] = name_len + sizeof(*ref);
  5526. }
  5527. location = &BTRFS_I(inode)->location;
  5528. location->objectid = objectid;
  5529. location->offset = 0;
  5530. location->type = BTRFS_INODE_ITEM_KEY;
  5531. ret = btrfs_insert_inode_locked(inode);
  5532. if (ret < 0)
  5533. goto fail;
  5534. path->leave_spinning = 1;
  5535. ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
  5536. if (ret != 0)
  5537. goto fail_unlock;
  5538. inode_init_owner(inode, dir, mode);
  5539. inode_set_bytes(inode, 0);
  5540. inode->i_mtime = current_time(inode);
  5541. inode->i_atime = inode->i_mtime;
  5542. inode->i_ctime = inode->i_mtime;
  5543. BTRFS_I(inode)->i_otime = inode->i_mtime;
  5544. inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
  5545. struct btrfs_inode_item);
  5546. memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
  5547. sizeof(*inode_item));
  5548. fill_inode_item(trans, path->nodes[0], inode_item, inode);
  5549. if (name) {
  5550. ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
  5551. struct btrfs_inode_ref);
  5552. btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
  5553. btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
  5554. ptr = (unsigned long)(ref + 1);
  5555. write_extent_buffer(path->nodes[0], name, ptr, name_len);
  5556. }
  5557. btrfs_mark_buffer_dirty(path->nodes[0]);
  5558. btrfs_free_path(path);
  5559. btrfs_inherit_iflags(inode, dir);
  5560. if (S_ISREG(mode)) {
  5561. if (btrfs_test_opt(fs_info, NODATASUM))
  5562. BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
  5563. if (btrfs_test_opt(fs_info, NODATACOW))
  5564. BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
  5565. BTRFS_INODE_NODATASUM;
  5566. }
  5567. inode_tree_add(inode);
  5568. trace_btrfs_inode_new(inode);
  5569. btrfs_set_inode_last_trans(trans, inode);
  5570. btrfs_update_root_times(trans, root);
  5571. ret = btrfs_inode_inherit_props(trans, inode, dir);
  5572. if (ret)
  5573. btrfs_err(fs_info,
  5574. "error inheriting props for ino %llu (root %llu): %d",
  5575. btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret);
  5576. return inode;
  5577. fail_unlock:
  5578. unlock_new_inode(inode);
  5579. fail:
  5580. if (dir && name)
  5581. BTRFS_I(dir)->index_cnt--;
  5582. btrfs_free_path(path);
  5583. iput(inode);
  5584. return ERR_PTR(ret);
  5585. }
  5586. static inline u8 btrfs_inode_type(struct inode *inode)
  5587. {
  5588. return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT];
  5589. }
  5590. /*
  5591. * utility function to add 'inode' into 'parent_inode' with
  5592. * a give name and a given sequence number.
  5593. * if 'add_backref' is true, also insert a backref from the
  5594. * inode to the parent directory.
  5595. */
  5596. int btrfs_add_link(struct btrfs_trans_handle *trans,
  5597. struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
  5598. const char *name, int name_len, int add_backref, u64 index)
  5599. {
  5600. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  5601. int ret = 0;
  5602. struct btrfs_key key;
  5603. struct btrfs_root *root = parent_inode->root;
  5604. u64 ino = btrfs_ino(inode);
  5605. u64 parent_ino = btrfs_ino(parent_inode);
  5606. if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
  5607. memcpy(&key, &inode->root->root_key, sizeof(key));
  5608. } else {
  5609. key.objectid = ino;
  5610. key.type = BTRFS_INODE_ITEM_KEY;
  5611. key.offset = 0;
  5612. }
  5613. if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
  5614. ret = btrfs_add_root_ref(trans, fs_info, key.objectid,
  5615. root->root_key.objectid, parent_ino,
  5616. index, name, name_len);
  5617. } else if (add_backref) {
  5618. ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
  5619. parent_ino, index);
  5620. }
  5621. /* Nothing to clean up yet */
  5622. if (ret)
  5623. return ret;
  5624. ret = btrfs_insert_dir_item(trans, root, name, name_len,
  5625. parent_inode, &key,
  5626. btrfs_inode_type(&inode->vfs_inode), index);
  5627. if (ret == -EEXIST || ret == -EOVERFLOW)
  5628. goto fail_dir_item;
  5629. else if (ret) {
  5630. btrfs_abort_transaction(trans, ret);
  5631. return ret;
  5632. }
  5633. btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
  5634. name_len * 2);
  5635. inode_inc_iversion(&parent_inode->vfs_inode);
  5636. parent_inode->vfs_inode.i_mtime = parent_inode->vfs_inode.i_ctime =
  5637. current_time(&parent_inode->vfs_inode);
  5638. ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode);
  5639. if (ret)
  5640. btrfs_abort_transaction(trans, ret);
  5641. return ret;
  5642. fail_dir_item:
  5643. if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
  5644. u64 local_index;
  5645. int err;
  5646. err = btrfs_del_root_ref(trans, fs_info, key.objectid,
  5647. root->root_key.objectid, parent_ino,
  5648. &local_index, name, name_len);
  5649. } else if (add_backref) {
  5650. u64 local_index;
  5651. int err;
  5652. err = btrfs_del_inode_ref(trans, root, name, name_len,
  5653. ino, parent_ino, &local_index);
  5654. }
  5655. return ret;
  5656. }
  5657. static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
  5658. struct btrfs_inode *dir, struct dentry *dentry,
  5659. struct btrfs_inode *inode, int backref, u64 index)
  5660. {
  5661. int err = btrfs_add_link(trans, dir, inode,
  5662. dentry->d_name.name, dentry->d_name.len,
  5663. backref, index);
  5664. if (err > 0)
  5665. err = -EEXIST;
  5666. return err;
  5667. }
  5668. static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
  5669. umode_t mode, dev_t rdev)
  5670. {
  5671. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  5672. struct btrfs_trans_handle *trans;
  5673. struct btrfs_root *root = BTRFS_I(dir)->root;
  5674. struct inode *inode = NULL;
  5675. int err;
  5676. int drop_inode = 0;
  5677. u64 objectid;
  5678. u64 index = 0;
  5679. /*
  5680. * 2 for inode item and ref
  5681. * 2 for dir items
  5682. * 1 for xattr if selinux is on
  5683. */
  5684. trans = btrfs_start_transaction(root, 5);
  5685. if (IS_ERR(trans))
  5686. return PTR_ERR(trans);
  5687. err = btrfs_find_free_ino(root, &objectid);
  5688. if (err)
  5689. goto out_unlock;
  5690. inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
  5691. dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
  5692. mode, &index);
  5693. if (IS_ERR(inode)) {
  5694. err = PTR_ERR(inode);
  5695. goto out_unlock;
  5696. }
  5697. /*
  5698. * If the active LSM wants to access the inode during
  5699. * d_instantiate it needs these. Smack checks to see
  5700. * if the filesystem supports xattrs by looking at the
  5701. * ops vector.
  5702. */
  5703. inode->i_op = &btrfs_special_inode_operations;
  5704. init_special_inode(inode, inode->i_mode, rdev);
  5705. err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
  5706. if (err)
  5707. goto out_unlock_inode;
  5708. err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
  5709. 0, index);
  5710. if (err) {
  5711. goto out_unlock_inode;
  5712. } else {
  5713. btrfs_update_inode(trans, root, inode);
  5714. unlock_new_inode(inode);
  5715. d_instantiate(dentry, inode);
  5716. }
  5717. out_unlock:
  5718. btrfs_end_transaction(trans);
  5719. btrfs_balance_delayed_items(fs_info);
  5720. btrfs_btree_balance_dirty(fs_info);
  5721. if (drop_inode) {
  5722. inode_dec_link_count(inode);
  5723. iput(inode);
  5724. }
  5725. return err;
  5726. out_unlock_inode:
  5727. drop_inode = 1;
  5728. unlock_new_inode(inode);
  5729. goto out_unlock;
  5730. }
  5731. static int btrfs_create(struct inode *dir, struct dentry *dentry,
  5732. umode_t mode, bool excl)
  5733. {
  5734. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  5735. struct btrfs_trans_handle *trans;
  5736. struct btrfs_root *root = BTRFS_I(dir)->root;
  5737. struct inode *inode = NULL;
  5738. int drop_inode_on_err = 0;
  5739. int err;
  5740. u64 objectid;
  5741. u64 index = 0;
  5742. /*
  5743. * 2 for inode item and ref
  5744. * 2 for dir items
  5745. * 1 for xattr if selinux is on
  5746. */
  5747. trans = btrfs_start_transaction(root, 5);
  5748. if (IS_ERR(trans))
  5749. return PTR_ERR(trans);
  5750. err = btrfs_find_free_ino(root, &objectid);
  5751. if (err)
  5752. goto out_unlock;
  5753. inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
  5754. dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
  5755. mode, &index);
  5756. if (IS_ERR(inode)) {
  5757. err = PTR_ERR(inode);
  5758. goto out_unlock;
  5759. }
  5760. drop_inode_on_err = 1;
  5761. /*
  5762. * If the active LSM wants to access the inode during
  5763. * d_instantiate it needs these. Smack checks to see
  5764. * if the filesystem supports xattrs by looking at the
  5765. * ops vector.
  5766. */
  5767. inode->i_fop = &btrfs_file_operations;
  5768. inode->i_op = &btrfs_file_inode_operations;
  5769. inode->i_mapping->a_ops = &btrfs_aops;
  5770. err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
  5771. if (err)
  5772. goto out_unlock_inode;
  5773. err = btrfs_update_inode(trans, root, inode);
  5774. if (err)
  5775. goto out_unlock_inode;
  5776. err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
  5777. 0, index);
  5778. if (err)
  5779. goto out_unlock_inode;
  5780. BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
  5781. unlock_new_inode(inode);
  5782. d_instantiate(dentry, inode);
  5783. out_unlock:
  5784. btrfs_end_transaction(trans);
  5785. if (err && drop_inode_on_err) {
  5786. inode_dec_link_count(inode);
  5787. iput(inode);
  5788. }
  5789. btrfs_balance_delayed_items(fs_info);
  5790. btrfs_btree_balance_dirty(fs_info);
  5791. return err;
  5792. out_unlock_inode:
  5793. unlock_new_inode(inode);
  5794. goto out_unlock;
  5795. }
  5796. static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
  5797. struct dentry *dentry)
  5798. {
  5799. struct btrfs_trans_handle *trans = NULL;
  5800. struct btrfs_root *root = BTRFS_I(dir)->root;
  5801. struct inode *inode = d_inode(old_dentry);
  5802. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  5803. u64 index;
  5804. int err;
  5805. int drop_inode = 0;
  5806. /* do not allow sys_link's with other subvols of the same device */
  5807. if (root->objectid != BTRFS_I(inode)->root->objectid)
  5808. return -EXDEV;
  5809. if (inode->i_nlink >= BTRFS_LINK_MAX)
  5810. return -EMLINK;
  5811. err = btrfs_set_inode_index(BTRFS_I(dir), &index);
  5812. if (err)
  5813. goto fail;
  5814. /*
  5815. * 2 items for inode and inode ref
  5816. * 2 items for dir items
  5817. * 1 item for parent inode
  5818. */
  5819. trans = btrfs_start_transaction(root, 5);
  5820. if (IS_ERR(trans)) {
  5821. err = PTR_ERR(trans);
  5822. trans = NULL;
  5823. goto fail;
  5824. }
  5825. /* There are several dir indexes for this inode, clear the cache. */
  5826. BTRFS_I(inode)->dir_index = 0ULL;
  5827. inc_nlink(inode);
  5828. inode_inc_iversion(inode);
  5829. inode->i_ctime = current_time(inode);
  5830. ihold(inode);
  5831. set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
  5832. err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
  5833. 1, index);
  5834. if (err) {
  5835. drop_inode = 1;
  5836. } else {
  5837. struct dentry *parent = dentry->d_parent;
  5838. err = btrfs_update_inode(trans, root, inode);
  5839. if (err)
  5840. goto fail;
  5841. if (inode->i_nlink == 1) {
  5842. /*
  5843. * If new hard link count is 1, it's a file created
  5844. * with open(2) O_TMPFILE flag.
  5845. */
  5846. err = btrfs_orphan_del(trans, BTRFS_I(inode));
  5847. if (err)
  5848. goto fail;
  5849. }
  5850. d_instantiate(dentry, inode);
  5851. btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent);
  5852. }
  5853. btrfs_balance_delayed_items(fs_info);
  5854. fail:
  5855. if (trans)
  5856. btrfs_end_transaction(trans);
  5857. if (drop_inode) {
  5858. inode_dec_link_count(inode);
  5859. iput(inode);
  5860. }
  5861. btrfs_btree_balance_dirty(fs_info);
  5862. return err;
  5863. }
  5864. static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
  5865. {
  5866. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  5867. struct inode *inode = NULL;
  5868. struct btrfs_trans_handle *trans;
  5869. struct btrfs_root *root = BTRFS_I(dir)->root;
  5870. int err = 0;
  5871. int drop_on_err = 0;
  5872. u64 objectid = 0;
  5873. u64 index = 0;
  5874. /*
  5875. * 2 items for inode and ref
  5876. * 2 items for dir items
  5877. * 1 for xattr if selinux is on
  5878. */
  5879. trans = btrfs_start_transaction(root, 5);
  5880. if (IS_ERR(trans))
  5881. return PTR_ERR(trans);
  5882. err = btrfs_find_free_ino(root, &objectid);
  5883. if (err)
  5884. goto out_fail;
  5885. inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
  5886. dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
  5887. S_IFDIR | mode, &index);
  5888. if (IS_ERR(inode)) {
  5889. err = PTR_ERR(inode);
  5890. goto out_fail;
  5891. }
  5892. drop_on_err = 1;
  5893. /* these must be set before we unlock the inode */
  5894. inode->i_op = &btrfs_dir_inode_operations;
  5895. inode->i_fop = &btrfs_dir_file_operations;
  5896. err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
  5897. if (err)
  5898. goto out_fail_inode;
  5899. btrfs_i_size_write(BTRFS_I(inode), 0);
  5900. err = btrfs_update_inode(trans, root, inode);
  5901. if (err)
  5902. goto out_fail_inode;
  5903. err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
  5904. dentry->d_name.name,
  5905. dentry->d_name.len, 0, index);
  5906. if (err)
  5907. goto out_fail_inode;
  5908. d_instantiate(dentry, inode);
  5909. /*
  5910. * mkdir is special. We're unlocking after we call d_instantiate
  5911. * to avoid a race with nfsd calling d_instantiate.
  5912. */
  5913. unlock_new_inode(inode);
  5914. drop_on_err = 0;
  5915. out_fail:
  5916. btrfs_end_transaction(trans);
  5917. if (drop_on_err) {
  5918. inode_dec_link_count(inode);
  5919. iput(inode);
  5920. }
  5921. btrfs_balance_delayed_items(fs_info);
  5922. btrfs_btree_balance_dirty(fs_info);
  5923. return err;
  5924. out_fail_inode:
  5925. unlock_new_inode(inode);
  5926. goto out_fail;
  5927. }
  5928. /* Find next extent map of a given extent map, caller needs to ensure locks */
  5929. static struct extent_map *next_extent_map(struct extent_map *em)
  5930. {
  5931. struct rb_node *next;
  5932. next = rb_next(&em->rb_node);
  5933. if (!next)
  5934. return NULL;
  5935. return container_of(next, struct extent_map, rb_node);
  5936. }
  5937. static struct extent_map *prev_extent_map(struct extent_map *em)
  5938. {
  5939. struct rb_node *prev;
  5940. prev = rb_prev(&em->rb_node);
  5941. if (!prev)
  5942. return NULL;
  5943. return container_of(prev, struct extent_map, rb_node);
  5944. }
  5945. /* helper for btfs_get_extent. Given an existing extent in the tree,
  5946. * the existing extent is the nearest extent to map_start,
  5947. * and an extent that you want to insert, deal with overlap and insert
  5948. * the best fitted new extent into the tree.
  5949. */
  5950. static int merge_extent_mapping(struct extent_map_tree *em_tree,
  5951. struct extent_map *existing,
  5952. struct extent_map *em,
  5953. u64 map_start)
  5954. {
  5955. struct extent_map *prev;
  5956. struct extent_map *next;
  5957. u64 start;
  5958. u64 end;
  5959. u64 start_diff;
  5960. BUG_ON(map_start < em->start || map_start >= extent_map_end(em));
  5961. if (existing->start > map_start) {
  5962. next = existing;
  5963. prev = prev_extent_map(next);
  5964. } else {
  5965. prev = existing;
  5966. next = next_extent_map(prev);
  5967. }
  5968. start = prev ? extent_map_end(prev) : em->start;
  5969. start = max_t(u64, start, em->start);
  5970. end = next ? next->start : extent_map_end(em);
  5971. end = min_t(u64, end, extent_map_end(em));
  5972. start_diff = start - em->start;
  5973. em->start = start;
  5974. em->len = end - start;
  5975. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  5976. !test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
  5977. em->block_start += start_diff;
  5978. em->block_len -= start_diff;
  5979. }
  5980. return add_extent_mapping(em_tree, em, 0);
  5981. }
  5982. static noinline int uncompress_inline(struct btrfs_path *path,
  5983. struct page *page,
  5984. size_t pg_offset, u64 extent_offset,
  5985. struct btrfs_file_extent_item *item)
  5986. {
  5987. int ret;
  5988. struct extent_buffer *leaf = path->nodes[0];
  5989. char *tmp;
  5990. size_t max_size;
  5991. unsigned long inline_size;
  5992. unsigned long ptr;
  5993. int compress_type;
  5994. WARN_ON(pg_offset != 0);
  5995. compress_type = btrfs_file_extent_compression(leaf, item);
  5996. max_size = btrfs_file_extent_ram_bytes(leaf, item);
  5997. inline_size = btrfs_file_extent_inline_item_len(leaf,
  5998. btrfs_item_nr(path->slots[0]));
  5999. tmp = kmalloc(inline_size, GFP_NOFS);
  6000. if (!tmp)
  6001. return -ENOMEM;
  6002. ptr = btrfs_file_extent_inline_start(item);
  6003. read_extent_buffer(leaf, tmp, ptr, inline_size);
  6004. max_size = min_t(unsigned long, PAGE_SIZE, max_size);
  6005. ret = btrfs_decompress(compress_type, tmp, page,
  6006. extent_offset, inline_size, max_size);
  6007. /*
  6008. * decompression code contains a memset to fill in any space between the end
  6009. * of the uncompressed data and the end of max_size in case the decompressed
  6010. * data ends up shorter than ram_bytes. That doesn't cover the hole between
  6011. * the end of an inline extent and the beginning of the next block, so we
  6012. * cover that region here.
  6013. */
  6014. if (max_size + pg_offset < PAGE_SIZE) {
  6015. char *map = kmap(page);
  6016. memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset);
  6017. kunmap(page);
  6018. }
  6019. kfree(tmp);
  6020. return ret;
  6021. }
  6022. /*
  6023. * a bit scary, this does extent mapping from logical file offset to the disk.
  6024. * the ugly parts come from merging extents from the disk with the in-ram
  6025. * representation. This gets more complex because of the data=ordered code,
  6026. * where the in-ram extents might be locked pending data=ordered completion.
  6027. *
  6028. * This also copies inline extents directly into the page.
  6029. */
  6030. struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
  6031. struct page *page,
  6032. size_t pg_offset, u64 start, u64 len,
  6033. int create)
  6034. {
  6035. struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
  6036. int ret;
  6037. int err = 0;
  6038. u64 extent_start = 0;
  6039. u64 extent_end = 0;
  6040. u64 objectid = btrfs_ino(inode);
  6041. u32 found_type;
  6042. struct btrfs_path *path = NULL;
  6043. struct btrfs_root *root = inode->root;
  6044. struct btrfs_file_extent_item *item;
  6045. struct extent_buffer *leaf;
  6046. struct btrfs_key found_key;
  6047. struct extent_map *em = NULL;
  6048. struct extent_map_tree *em_tree = &inode->extent_tree;
  6049. struct extent_io_tree *io_tree = &inode->io_tree;
  6050. struct btrfs_trans_handle *trans = NULL;
  6051. const bool new_inline = !page || create;
  6052. again:
  6053. read_lock(&em_tree->lock);
  6054. em = lookup_extent_mapping(em_tree, start, len);
  6055. if (em)
  6056. em->bdev = fs_info->fs_devices->latest_bdev;
  6057. read_unlock(&em_tree->lock);
  6058. if (em) {
  6059. if (em->start > start || em->start + em->len <= start)
  6060. free_extent_map(em);
  6061. else if (em->block_start == EXTENT_MAP_INLINE && page)
  6062. free_extent_map(em);
  6063. else
  6064. goto out;
  6065. }
  6066. em = alloc_extent_map();
  6067. if (!em) {
  6068. err = -ENOMEM;
  6069. goto out;
  6070. }
  6071. em->bdev = fs_info->fs_devices->latest_bdev;
  6072. em->start = EXTENT_MAP_HOLE;
  6073. em->orig_start = EXTENT_MAP_HOLE;
  6074. em->len = (u64)-1;
  6075. em->block_len = (u64)-1;
  6076. if (!path) {
  6077. path = btrfs_alloc_path();
  6078. if (!path) {
  6079. err = -ENOMEM;
  6080. goto out;
  6081. }
  6082. /*
  6083. * Chances are we'll be called again, so go ahead and do
  6084. * readahead
  6085. */
  6086. path->reada = READA_FORWARD;
  6087. }
  6088. ret = btrfs_lookup_file_extent(trans, root, path,
  6089. objectid, start, trans != NULL);
  6090. if (ret < 0) {
  6091. err = ret;
  6092. goto out;
  6093. }
  6094. if (ret != 0) {
  6095. if (path->slots[0] == 0)
  6096. goto not_found;
  6097. path->slots[0]--;
  6098. }
  6099. leaf = path->nodes[0];
  6100. item = btrfs_item_ptr(leaf, path->slots[0],
  6101. struct btrfs_file_extent_item);
  6102. /* are we inside the extent that was found? */
  6103. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  6104. found_type = found_key.type;
  6105. if (found_key.objectid != objectid ||
  6106. found_type != BTRFS_EXTENT_DATA_KEY) {
  6107. /*
  6108. * If we backup past the first extent we want to move forward
  6109. * and see if there is an extent in front of us, otherwise we'll
  6110. * say there is a hole for our whole search range which can
  6111. * cause problems.
  6112. */
  6113. extent_end = start;
  6114. goto next;
  6115. }
  6116. found_type = btrfs_file_extent_type(leaf, item);
  6117. extent_start = found_key.offset;
  6118. if (found_type == BTRFS_FILE_EXTENT_REG ||
  6119. found_type == BTRFS_FILE_EXTENT_PREALLOC) {
  6120. extent_end = extent_start +
  6121. btrfs_file_extent_num_bytes(leaf, item);
  6122. trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,
  6123. extent_start);
  6124. } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
  6125. size_t size;
  6126. size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
  6127. extent_end = ALIGN(extent_start + size,
  6128. fs_info->sectorsize);
  6129. trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,
  6130. path->slots[0],
  6131. extent_start);
  6132. }
  6133. next:
  6134. if (start >= extent_end) {
  6135. path->slots[0]++;
  6136. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  6137. ret = btrfs_next_leaf(root, path);
  6138. if (ret < 0) {
  6139. err = ret;
  6140. goto out;
  6141. }
  6142. if (ret > 0)
  6143. goto not_found;
  6144. leaf = path->nodes[0];
  6145. }
  6146. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  6147. if (found_key.objectid != objectid ||
  6148. found_key.type != BTRFS_EXTENT_DATA_KEY)
  6149. goto not_found;
  6150. if (start + len <= found_key.offset)
  6151. goto not_found;
  6152. if (start > found_key.offset)
  6153. goto next;
  6154. em->start = start;
  6155. em->orig_start = start;
  6156. em->len = found_key.offset - start;
  6157. goto not_found_em;
  6158. }
  6159. btrfs_extent_item_to_extent_map(inode, path, item,
  6160. new_inline, em);
  6161. if (found_type == BTRFS_FILE_EXTENT_REG ||
  6162. found_type == BTRFS_FILE_EXTENT_PREALLOC) {
  6163. goto insert;
  6164. } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
  6165. unsigned long ptr;
  6166. char *map;
  6167. size_t size;
  6168. size_t extent_offset;
  6169. size_t copy_size;
  6170. if (new_inline)
  6171. goto out;
  6172. size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
  6173. extent_offset = page_offset(page) + pg_offset - extent_start;
  6174. copy_size = min_t(u64, PAGE_SIZE - pg_offset,
  6175. size - extent_offset);
  6176. em->start = extent_start + extent_offset;
  6177. em->len = ALIGN(copy_size, fs_info->sectorsize);
  6178. em->orig_block_len = em->len;
  6179. em->orig_start = em->start;
  6180. ptr = btrfs_file_extent_inline_start(item) + extent_offset;
  6181. if (create == 0 && !PageUptodate(page)) {
  6182. if (btrfs_file_extent_compression(leaf, item) !=
  6183. BTRFS_COMPRESS_NONE) {
  6184. ret = uncompress_inline(path, page, pg_offset,
  6185. extent_offset, item);
  6186. if (ret) {
  6187. err = ret;
  6188. goto out;
  6189. }
  6190. } else {
  6191. map = kmap(page);
  6192. read_extent_buffer(leaf, map + pg_offset, ptr,
  6193. copy_size);
  6194. if (pg_offset + copy_size < PAGE_SIZE) {
  6195. memset(map + pg_offset + copy_size, 0,
  6196. PAGE_SIZE - pg_offset -
  6197. copy_size);
  6198. }
  6199. kunmap(page);
  6200. }
  6201. flush_dcache_page(page);
  6202. } else if (create && PageUptodate(page)) {
  6203. BUG();
  6204. if (!trans) {
  6205. kunmap(page);
  6206. free_extent_map(em);
  6207. em = NULL;
  6208. btrfs_release_path(path);
  6209. trans = btrfs_join_transaction(root);
  6210. if (IS_ERR(trans))
  6211. return ERR_CAST(trans);
  6212. goto again;
  6213. }
  6214. map = kmap(page);
  6215. write_extent_buffer(leaf, map + pg_offset, ptr,
  6216. copy_size);
  6217. kunmap(page);
  6218. btrfs_mark_buffer_dirty(leaf);
  6219. }
  6220. set_extent_uptodate(io_tree, em->start,
  6221. extent_map_end(em) - 1, NULL, GFP_NOFS);
  6222. goto insert;
  6223. }
  6224. not_found:
  6225. em->start = start;
  6226. em->orig_start = start;
  6227. em->len = len;
  6228. not_found_em:
  6229. em->block_start = EXTENT_MAP_HOLE;
  6230. set_bit(EXTENT_FLAG_VACANCY, &em->flags);
  6231. insert:
  6232. btrfs_release_path(path);
  6233. if (em->start > start || extent_map_end(em) <= start) {
  6234. btrfs_err(fs_info,
  6235. "bad extent! em: [%llu %llu] passed [%llu %llu]",
  6236. em->start, em->len, start, len);
  6237. err = -EIO;
  6238. goto out;
  6239. }
  6240. err = 0;
  6241. write_lock(&em_tree->lock);
  6242. ret = add_extent_mapping(em_tree, em, 0);
  6243. /* it is possible that someone inserted the extent into the tree
  6244. * while we had the lock dropped. It is also possible that
  6245. * an overlapping map exists in the tree
  6246. */
  6247. if (ret == -EEXIST) {
  6248. struct extent_map *existing;
  6249. ret = 0;
  6250. existing = search_extent_mapping(em_tree, start, len);
  6251. /*
  6252. * existing will always be non-NULL, since there must be
  6253. * extent causing the -EEXIST.
  6254. */
  6255. if (existing->start == em->start &&
  6256. extent_map_end(existing) >= extent_map_end(em) &&
  6257. em->block_start == existing->block_start) {
  6258. /*
  6259. * The existing extent map already encompasses the
  6260. * entire extent map we tried to add.
  6261. */
  6262. free_extent_map(em);
  6263. em = existing;
  6264. err = 0;
  6265. } else if (start >= extent_map_end(existing) ||
  6266. start <= existing->start) {
  6267. /*
  6268. * The existing extent map is the one nearest to
  6269. * the [start, start + len) range which overlaps
  6270. */
  6271. err = merge_extent_mapping(em_tree, existing,
  6272. em, start);
  6273. free_extent_map(existing);
  6274. if (err) {
  6275. free_extent_map(em);
  6276. em = NULL;
  6277. }
  6278. } else {
  6279. free_extent_map(em);
  6280. em = existing;
  6281. err = 0;
  6282. }
  6283. }
  6284. write_unlock(&em_tree->lock);
  6285. out:
  6286. trace_btrfs_get_extent(root, inode, em);
  6287. btrfs_free_path(path);
  6288. if (trans) {
  6289. ret = btrfs_end_transaction(trans);
  6290. if (!err)
  6291. err = ret;
  6292. }
  6293. if (err) {
  6294. free_extent_map(em);
  6295. return ERR_PTR(err);
  6296. }
  6297. BUG_ON(!em); /* Error is always set */
  6298. return em;
  6299. }
  6300. struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
  6301. struct page *page,
  6302. size_t pg_offset, u64 start, u64 len,
  6303. int create)
  6304. {
  6305. struct extent_map *em;
  6306. struct extent_map *hole_em = NULL;
  6307. u64 range_start = start;
  6308. u64 end;
  6309. u64 found;
  6310. u64 found_end;
  6311. int err = 0;
  6312. em = btrfs_get_extent(inode, page, pg_offset, start, len, create);
  6313. if (IS_ERR(em))
  6314. return em;
  6315. /*
  6316. * If our em maps to:
  6317. * - a hole or
  6318. * - a pre-alloc extent,
  6319. * there might actually be delalloc bytes behind it.
  6320. */
  6321. if (em->block_start != EXTENT_MAP_HOLE &&
  6322. !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
  6323. return em;
  6324. else
  6325. hole_em = em;
  6326. /* check to see if we've wrapped (len == -1 or similar) */
  6327. end = start + len;
  6328. if (end < start)
  6329. end = (u64)-1;
  6330. else
  6331. end -= 1;
  6332. em = NULL;
  6333. /* ok, we didn't find anything, lets look for delalloc */
  6334. found = count_range_bits(&inode->io_tree, &range_start,
  6335. end, len, EXTENT_DELALLOC, 1);
  6336. found_end = range_start + found;
  6337. if (found_end < range_start)
  6338. found_end = (u64)-1;
  6339. /*
  6340. * we didn't find anything useful, return
  6341. * the original results from get_extent()
  6342. */
  6343. if (range_start > end || found_end <= start) {
  6344. em = hole_em;
  6345. hole_em = NULL;
  6346. goto out;
  6347. }
  6348. /* adjust the range_start to make sure it doesn't
  6349. * go backwards from the start they passed in
  6350. */
  6351. range_start = max(start, range_start);
  6352. found = found_end - range_start;
  6353. if (found > 0) {
  6354. u64 hole_start = start;
  6355. u64 hole_len = len;
  6356. em = alloc_extent_map();
  6357. if (!em) {
  6358. err = -ENOMEM;
  6359. goto out;
  6360. }
  6361. /*
  6362. * when btrfs_get_extent can't find anything it
  6363. * returns one huge hole
  6364. *
  6365. * make sure what it found really fits our range, and
  6366. * adjust to make sure it is based on the start from
  6367. * the caller
  6368. */
  6369. if (hole_em) {
  6370. u64 calc_end = extent_map_end(hole_em);
  6371. if (calc_end <= start || (hole_em->start > end)) {
  6372. free_extent_map(hole_em);
  6373. hole_em = NULL;
  6374. } else {
  6375. hole_start = max(hole_em->start, start);
  6376. hole_len = calc_end - hole_start;
  6377. }
  6378. }
  6379. em->bdev = NULL;
  6380. if (hole_em && range_start > hole_start) {
  6381. /* our hole starts before our delalloc, so we
  6382. * have to return just the parts of the hole
  6383. * that go until the delalloc starts
  6384. */
  6385. em->len = min(hole_len,
  6386. range_start - hole_start);
  6387. em->start = hole_start;
  6388. em->orig_start = hole_start;
  6389. /*
  6390. * don't adjust block start at all,
  6391. * it is fixed at EXTENT_MAP_HOLE
  6392. */
  6393. em->block_start = hole_em->block_start;
  6394. em->block_len = hole_len;
  6395. if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
  6396. set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
  6397. } else {
  6398. em->start = range_start;
  6399. em->len = found;
  6400. em->orig_start = range_start;
  6401. em->block_start = EXTENT_MAP_DELALLOC;
  6402. em->block_len = found;
  6403. }
  6404. } else if (hole_em) {
  6405. return hole_em;
  6406. }
  6407. out:
  6408. free_extent_map(hole_em);
  6409. if (err) {
  6410. free_extent_map(em);
  6411. return ERR_PTR(err);
  6412. }
  6413. return em;
  6414. }
  6415. static struct extent_map *btrfs_create_dio_extent(struct inode *inode,
  6416. const u64 start,
  6417. const u64 len,
  6418. const u64 orig_start,
  6419. const u64 block_start,
  6420. const u64 block_len,
  6421. const u64 orig_block_len,
  6422. const u64 ram_bytes,
  6423. const int type)
  6424. {
  6425. struct extent_map *em = NULL;
  6426. int ret;
  6427. if (type != BTRFS_ORDERED_NOCOW) {
  6428. em = create_io_em(inode, start, len, orig_start,
  6429. block_start, block_len, orig_block_len,
  6430. ram_bytes,
  6431. BTRFS_COMPRESS_NONE, /* compress_type */
  6432. type);
  6433. if (IS_ERR(em))
  6434. goto out;
  6435. }
  6436. ret = btrfs_add_ordered_extent_dio(inode, start, block_start,
  6437. len, block_len, type);
  6438. if (ret) {
  6439. if (em) {
  6440. free_extent_map(em);
  6441. btrfs_drop_extent_cache(BTRFS_I(inode), start,
  6442. start + len - 1, 0);
  6443. }
  6444. em = ERR_PTR(ret);
  6445. }
  6446. out:
  6447. return em;
  6448. }
  6449. static struct extent_map *btrfs_new_extent_direct(struct inode *inode,
  6450. u64 start, u64 len)
  6451. {
  6452. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6453. struct btrfs_root *root = BTRFS_I(inode)->root;
  6454. struct extent_map *em;
  6455. struct btrfs_key ins;
  6456. u64 alloc_hint;
  6457. int ret;
  6458. alloc_hint = get_extent_allocation_hint(inode, start, len);
  6459. ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
  6460. 0, alloc_hint, &ins, 1, 1);
  6461. if (ret)
  6462. return ERR_PTR(ret);
  6463. em = btrfs_create_dio_extent(inode, start, ins.offset, start,
  6464. ins.objectid, ins.offset, ins.offset,
  6465. ins.offset, BTRFS_ORDERED_REGULAR);
  6466. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  6467. if (IS_ERR(em))
  6468. btrfs_free_reserved_extent(fs_info, ins.objectid,
  6469. ins.offset, 1);
  6470. return em;
  6471. }
  6472. /*
  6473. * returns 1 when the nocow is safe, < 1 on error, 0 if the
  6474. * block must be cow'd
  6475. */
  6476. noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
  6477. u64 *orig_start, u64 *orig_block_len,
  6478. u64 *ram_bytes)
  6479. {
  6480. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6481. struct btrfs_path *path;
  6482. int ret;
  6483. struct extent_buffer *leaf;
  6484. struct btrfs_root *root = BTRFS_I(inode)->root;
  6485. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  6486. struct btrfs_file_extent_item *fi;
  6487. struct btrfs_key key;
  6488. u64 disk_bytenr;
  6489. u64 backref_offset;
  6490. u64 extent_end;
  6491. u64 num_bytes;
  6492. int slot;
  6493. int found_type;
  6494. bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
  6495. path = btrfs_alloc_path();
  6496. if (!path)
  6497. return -ENOMEM;
  6498. ret = btrfs_lookup_file_extent(NULL, root, path,
  6499. btrfs_ino(BTRFS_I(inode)), offset, 0);
  6500. if (ret < 0)
  6501. goto out;
  6502. slot = path->slots[0];
  6503. if (ret == 1) {
  6504. if (slot == 0) {
  6505. /* can't find the item, must cow */
  6506. ret = 0;
  6507. goto out;
  6508. }
  6509. slot--;
  6510. }
  6511. ret = 0;
  6512. leaf = path->nodes[0];
  6513. btrfs_item_key_to_cpu(leaf, &key, slot);
  6514. if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||
  6515. key.type != BTRFS_EXTENT_DATA_KEY) {
  6516. /* not our file or wrong item type, must cow */
  6517. goto out;
  6518. }
  6519. if (key.offset > offset) {
  6520. /* Wrong offset, must cow */
  6521. goto out;
  6522. }
  6523. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  6524. found_type = btrfs_file_extent_type(leaf, fi);
  6525. if (found_type != BTRFS_FILE_EXTENT_REG &&
  6526. found_type != BTRFS_FILE_EXTENT_PREALLOC) {
  6527. /* not a regular extent, must cow */
  6528. goto out;
  6529. }
  6530. if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
  6531. goto out;
  6532. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  6533. if (extent_end <= offset)
  6534. goto out;
  6535. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  6536. if (disk_bytenr == 0)
  6537. goto out;
  6538. if (btrfs_file_extent_compression(leaf, fi) ||
  6539. btrfs_file_extent_encryption(leaf, fi) ||
  6540. btrfs_file_extent_other_encoding(leaf, fi))
  6541. goto out;
  6542. backref_offset = btrfs_file_extent_offset(leaf, fi);
  6543. if (orig_start) {
  6544. *orig_start = key.offset - backref_offset;
  6545. *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
  6546. *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
  6547. }
  6548. if (btrfs_extent_readonly(fs_info, disk_bytenr))
  6549. goto out;
  6550. num_bytes = min(offset + *len, extent_end) - offset;
  6551. if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
  6552. u64 range_end;
  6553. range_end = round_up(offset + num_bytes,
  6554. root->fs_info->sectorsize) - 1;
  6555. ret = test_range_bit(io_tree, offset, range_end,
  6556. EXTENT_DELALLOC, 0, NULL);
  6557. if (ret) {
  6558. ret = -EAGAIN;
  6559. goto out;
  6560. }
  6561. }
  6562. btrfs_release_path(path);
  6563. /*
  6564. * look for other files referencing this extent, if we
  6565. * find any we must cow
  6566. */
  6567. ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)),
  6568. key.offset - backref_offset, disk_bytenr);
  6569. if (ret) {
  6570. ret = 0;
  6571. goto out;
  6572. }
  6573. /*
  6574. * adjust disk_bytenr and num_bytes to cover just the bytes
  6575. * in this extent we are about to write. If there
  6576. * are any csums in that range we have to cow in order
  6577. * to keep the csums correct
  6578. */
  6579. disk_bytenr += backref_offset;
  6580. disk_bytenr += offset - key.offset;
  6581. if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes))
  6582. goto out;
  6583. /*
  6584. * all of the above have passed, it is safe to overwrite this extent
  6585. * without cow
  6586. */
  6587. *len = num_bytes;
  6588. ret = 1;
  6589. out:
  6590. btrfs_free_path(path);
  6591. return ret;
  6592. }
  6593. bool btrfs_page_exists_in_range(struct inode *inode, loff_t start, loff_t end)
  6594. {
  6595. struct radix_tree_root *root = &inode->i_mapping->page_tree;
  6596. bool found = false;
  6597. void **pagep = NULL;
  6598. struct page *page = NULL;
  6599. unsigned long start_idx;
  6600. unsigned long end_idx;
  6601. start_idx = start >> PAGE_SHIFT;
  6602. /*
  6603. * end is the last byte in the last page. end == start is legal
  6604. */
  6605. end_idx = end >> PAGE_SHIFT;
  6606. rcu_read_lock();
  6607. /* Most of the code in this while loop is lifted from
  6608. * find_get_page. It's been modified to begin searching from a
  6609. * page and return just the first page found in that range. If the
  6610. * found idx is less than or equal to the end idx then we know that
  6611. * a page exists. If no pages are found or if those pages are
  6612. * outside of the range then we're fine (yay!) */
  6613. while (page == NULL &&
  6614. radix_tree_gang_lookup_slot(root, &pagep, NULL, start_idx, 1)) {
  6615. page = radix_tree_deref_slot(pagep);
  6616. if (unlikely(!page))
  6617. break;
  6618. if (radix_tree_exception(page)) {
  6619. if (radix_tree_deref_retry(page)) {
  6620. page = NULL;
  6621. continue;
  6622. }
  6623. /*
  6624. * Otherwise, shmem/tmpfs must be storing a swap entry
  6625. * here as an exceptional entry: so return it without
  6626. * attempting to raise page count.
  6627. */
  6628. page = NULL;
  6629. break; /* TODO: Is this relevant for this use case? */
  6630. }
  6631. if (!page_cache_get_speculative(page)) {
  6632. page = NULL;
  6633. continue;
  6634. }
  6635. /*
  6636. * Has the page moved?
  6637. * This is part of the lockless pagecache protocol. See
  6638. * include/linux/pagemap.h for details.
  6639. */
  6640. if (unlikely(page != *pagep)) {
  6641. put_page(page);
  6642. page = NULL;
  6643. }
  6644. }
  6645. if (page) {
  6646. if (page->index <= end_idx)
  6647. found = true;
  6648. put_page(page);
  6649. }
  6650. rcu_read_unlock();
  6651. return found;
  6652. }
  6653. static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
  6654. struct extent_state **cached_state, int writing)
  6655. {
  6656. struct btrfs_ordered_extent *ordered;
  6657. int ret = 0;
  6658. while (1) {
  6659. lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  6660. cached_state);
  6661. /*
  6662. * We're concerned with the entire range that we're going to be
  6663. * doing DIO to, so we need to make sure there's no ordered
  6664. * extents in this range.
  6665. */
  6666. ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,
  6667. lockend - lockstart + 1);
  6668. /*
  6669. * We need to make sure there are no buffered pages in this
  6670. * range either, we could have raced between the invalidate in
  6671. * generic_file_direct_write and locking the extent. The
  6672. * invalidate needs to happen so that reads after a write do not
  6673. * get stale data.
  6674. */
  6675. if (!ordered &&
  6676. (!writing ||
  6677. !btrfs_page_exists_in_range(inode, lockstart, lockend)))
  6678. break;
  6679. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  6680. cached_state, GFP_NOFS);
  6681. if (ordered) {
  6682. /*
  6683. * If we are doing a DIO read and the ordered extent we
  6684. * found is for a buffered write, we can not wait for it
  6685. * to complete and retry, because if we do so we can
  6686. * deadlock with concurrent buffered writes on page
  6687. * locks. This happens only if our DIO read covers more
  6688. * than one extent map, if at this point has already
  6689. * created an ordered extent for a previous extent map
  6690. * and locked its range in the inode's io tree, and a
  6691. * concurrent write against that previous extent map's
  6692. * range and this range started (we unlock the ranges
  6693. * in the io tree only when the bios complete and
  6694. * buffered writes always lock pages before attempting
  6695. * to lock range in the io tree).
  6696. */
  6697. if (writing ||
  6698. test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
  6699. btrfs_start_ordered_extent(inode, ordered, 1);
  6700. else
  6701. ret = -ENOTBLK;
  6702. btrfs_put_ordered_extent(ordered);
  6703. } else {
  6704. /*
  6705. * We could trigger writeback for this range (and wait
  6706. * for it to complete) and then invalidate the pages for
  6707. * this range (through invalidate_inode_pages2_range()),
  6708. * but that can lead us to a deadlock with a concurrent
  6709. * call to readpages() (a buffered read or a defrag call
  6710. * triggered a readahead) on a page lock due to an
  6711. * ordered dio extent we created before but did not have
  6712. * yet a corresponding bio submitted (whence it can not
  6713. * complete), which makes readpages() wait for that
  6714. * ordered extent to complete while holding a lock on
  6715. * that page.
  6716. */
  6717. ret = -ENOTBLK;
  6718. }
  6719. if (ret)
  6720. break;
  6721. cond_resched();
  6722. }
  6723. return ret;
  6724. }
  6725. /* The callers of this must take lock_extent() */
  6726. static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
  6727. u64 orig_start, u64 block_start,
  6728. u64 block_len, u64 orig_block_len,
  6729. u64 ram_bytes, int compress_type,
  6730. int type)
  6731. {
  6732. struct extent_map_tree *em_tree;
  6733. struct extent_map *em;
  6734. struct btrfs_root *root = BTRFS_I(inode)->root;
  6735. int ret;
  6736. ASSERT(type == BTRFS_ORDERED_PREALLOC ||
  6737. type == BTRFS_ORDERED_COMPRESSED ||
  6738. type == BTRFS_ORDERED_NOCOW ||
  6739. type == BTRFS_ORDERED_REGULAR);
  6740. em_tree = &BTRFS_I(inode)->extent_tree;
  6741. em = alloc_extent_map();
  6742. if (!em)
  6743. return ERR_PTR(-ENOMEM);
  6744. em->start = start;
  6745. em->orig_start = orig_start;
  6746. em->len = len;
  6747. em->block_len = block_len;
  6748. em->block_start = block_start;
  6749. em->bdev = root->fs_info->fs_devices->latest_bdev;
  6750. em->orig_block_len = orig_block_len;
  6751. em->ram_bytes = ram_bytes;
  6752. em->generation = -1;
  6753. set_bit(EXTENT_FLAG_PINNED, &em->flags);
  6754. if (type == BTRFS_ORDERED_PREALLOC) {
  6755. set_bit(EXTENT_FLAG_FILLING, &em->flags);
  6756. } else if (type == BTRFS_ORDERED_COMPRESSED) {
  6757. set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  6758. em->compress_type = compress_type;
  6759. }
  6760. do {
  6761. btrfs_drop_extent_cache(BTRFS_I(inode), em->start,
  6762. em->start + em->len - 1, 0);
  6763. write_lock(&em_tree->lock);
  6764. ret = add_extent_mapping(em_tree, em, 1);
  6765. write_unlock(&em_tree->lock);
  6766. /*
  6767. * The caller has taken lock_extent(), who could race with us
  6768. * to add em?
  6769. */
  6770. } while (ret == -EEXIST);
  6771. if (ret) {
  6772. free_extent_map(em);
  6773. return ERR_PTR(ret);
  6774. }
  6775. /* em got 2 refs now, callers needs to do free_extent_map once. */
  6776. return em;
  6777. }
  6778. static void adjust_dio_outstanding_extents(struct inode *inode,
  6779. struct btrfs_dio_data *dio_data,
  6780. const u64 len)
  6781. {
  6782. unsigned num_extents = count_max_extents(len);
  6783. /*
  6784. * If we have an outstanding_extents count still set then we're
  6785. * within our reservation, otherwise we need to adjust our inode
  6786. * counter appropriately.
  6787. */
  6788. if (dio_data->outstanding_extents >= num_extents) {
  6789. dio_data->outstanding_extents -= num_extents;
  6790. } else {
  6791. /*
  6792. * If dio write length has been split due to no large enough
  6793. * contiguous space, we need to compensate our inode counter
  6794. * appropriately.
  6795. */
  6796. u64 num_needed = num_extents - dio_data->outstanding_extents;
  6797. spin_lock(&BTRFS_I(inode)->lock);
  6798. BTRFS_I(inode)->outstanding_extents += num_needed;
  6799. spin_unlock(&BTRFS_I(inode)->lock);
  6800. }
  6801. }
  6802. static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
  6803. struct buffer_head *bh_result, int create)
  6804. {
  6805. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6806. struct extent_map *em;
  6807. struct extent_state *cached_state = NULL;
  6808. struct btrfs_dio_data *dio_data = NULL;
  6809. u64 start = iblock << inode->i_blkbits;
  6810. u64 lockstart, lockend;
  6811. u64 len = bh_result->b_size;
  6812. int unlock_bits = EXTENT_LOCKED;
  6813. int ret = 0;
  6814. if (create)
  6815. unlock_bits |= EXTENT_DIRTY;
  6816. else
  6817. len = min_t(u64, len, fs_info->sectorsize);
  6818. lockstart = start;
  6819. lockend = start + len - 1;
  6820. if (current->journal_info) {
  6821. /*
  6822. * Need to pull our outstanding extents and set journal_info to NULL so
  6823. * that anything that needs to check if there's a transaction doesn't get
  6824. * confused.
  6825. */
  6826. dio_data = current->journal_info;
  6827. current->journal_info = NULL;
  6828. }
  6829. /*
  6830. * If this errors out it's because we couldn't invalidate pagecache for
  6831. * this range and we need to fallback to buffered.
  6832. */
  6833. if (lock_extent_direct(inode, lockstart, lockend, &cached_state,
  6834. create)) {
  6835. ret = -ENOTBLK;
  6836. goto err;
  6837. }
  6838. em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0);
  6839. if (IS_ERR(em)) {
  6840. ret = PTR_ERR(em);
  6841. goto unlock_err;
  6842. }
  6843. /*
  6844. * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
  6845. * io. INLINE is special, and we could probably kludge it in here, but
  6846. * it's still buffered so for safety lets just fall back to the generic
  6847. * buffered path.
  6848. *
  6849. * For COMPRESSED we _have_ to read the entire extent in so we can
  6850. * decompress it, so there will be buffering required no matter what we
  6851. * do, so go ahead and fallback to buffered.
  6852. *
  6853. * We return -ENOTBLK because that's what makes DIO go ahead and go back
  6854. * to buffered IO. Don't blame me, this is the price we pay for using
  6855. * the generic code.
  6856. */
  6857. if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) ||
  6858. em->block_start == EXTENT_MAP_INLINE) {
  6859. free_extent_map(em);
  6860. ret = -ENOTBLK;
  6861. goto unlock_err;
  6862. }
  6863. /* Just a good old fashioned hole, return */
  6864. if (!create && (em->block_start == EXTENT_MAP_HOLE ||
  6865. test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
  6866. free_extent_map(em);
  6867. goto unlock_err;
  6868. }
  6869. /*
  6870. * We don't allocate a new extent in the following cases
  6871. *
  6872. * 1) The inode is marked as NODATACOW. In this case we'll just use the
  6873. * existing extent.
  6874. * 2) The extent is marked as PREALLOC. We're good to go here and can
  6875. * just use the extent.
  6876. *
  6877. */
  6878. if (!create) {
  6879. len = min(len, em->len - (start - em->start));
  6880. lockstart = start + len;
  6881. goto unlock;
  6882. }
  6883. if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
  6884. ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
  6885. em->block_start != EXTENT_MAP_HOLE)) {
  6886. int type;
  6887. u64 block_start, orig_start, orig_block_len, ram_bytes;
  6888. if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
  6889. type = BTRFS_ORDERED_PREALLOC;
  6890. else
  6891. type = BTRFS_ORDERED_NOCOW;
  6892. len = min(len, em->len - (start - em->start));
  6893. block_start = em->block_start + (start - em->start);
  6894. if (can_nocow_extent(inode, start, &len, &orig_start,
  6895. &orig_block_len, &ram_bytes) == 1 &&
  6896. btrfs_inc_nocow_writers(fs_info, block_start)) {
  6897. struct extent_map *em2;
  6898. em2 = btrfs_create_dio_extent(inode, start, len,
  6899. orig_start, block_start,
  6900. len, orig_block_len,
  6901. ram_bytes, type);
  6902. btrfs_dec_nocow_writers(fs_info, block_start);
  6903. if (type == BTRFS_ORDERED_PREALLOC) {
  6904. free_extent_map(em);
  6905. em = em2;
  6906. }
  6907. if (em2 && IS_ERR(em2)) {
  6908. ret = PTR_ERR(em2);
  6909. goto unlock_err;
  6910. }
  6911. /*
  6912. * For inode marked NODATACOW or extent marked PREALLOC,
  6913. * use the existing or preallocated extent, so does not
  6914. * need to adjust btrfs_space_info's bytes_may_use.
  6915. */
  6916. btrfs_free_reserved_data_space_noquota(inode,
  6917. start, len);
  6918. goto unlock;
  6919. }
  6920. }
  6921. /*
  6922. * this will cow the extent, reset the len in case we changed
  6923. * it above
  6924. */
  6925. len = bh_result->b_size;
  6926. free_extent_map(em);
  6927. em = btrfs_new_extent_direct(inode, start, len);
  6928. if (IS_ERR(em)) {
  6929. ret = PTR_ERR(em);
  6930. goto unlock_err;
  6931. }
  6932. len = min(len, em->len - (start - em->start));
  6933. unlock:
  6934. bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
  6935. inode->i_blkbits;
  6936. bh_result->b_size = len;
  6937. bh_result->b_bdev = em->bdev;
  6938. set_buffer_mapped(bh_result);
  6939. if (create) {
  6940. if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
  6941. set_buffer_new(bh_result);
  6942. /*
  6943. * Need to update the i_size under the extent lock so buffered
  6944. * readers will get the updated i_size when we unlock.
  6945. */
  6946. if (!dio_data->overwrite && start + len > i_size_read(inode))
  6947. i_size_write(inode, start + len);
  6948. adjust_dio_outstanding_extents(inode, dio_data, len);
  6949. WARN_ON(dio_data->reserve < len);
  6950. dio_data->reserve -= len;
  6951. dio_data->unsubmitted_oe_range_end = start + len;
  6952. current->journal_info = dio_data;
  6953. }
  6954. /*
  6955. * In the case of write we need to clear and unlock the entire range,
  6956. * in the case of read we need to unlock only the end area that we
  6957. * aren't using if there is any left over space.
  6958. */
  6959. if (lockstart < lockend) {
  6960. clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
  6961. lockend, unlock_bits, 1, 0,
  6962. &cached_state, GFP_NOFS);
  6963. } else {
  6964. free_extent_state(cached_state);
  6965. }
  6966. free_extent_map(em);
  6967. return 0;
  6968. unlock_err:
  6969. clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  6970. unlock_bits, 1, 0, &cached_state, GFP_NOFS);
  6971. err:
  6972. if (dio_data)
  6973. current->journal_info = dio_data;
  6974. /*
  6975. * Compensate the delalloc release we do in btrfs_direct_IO() when we
  6976. * write less data then expected, so that we don't underflow our inode's
  6977. * outstanding extents counter.
  6978. */
  6979. if (create && dio_data)
  6980. adjust_dio_outstanding_extents(inode, dio_data, len);
  6981. return ret;
  6982. }
  6983. static inline int submit_dio_repair_bio(struct inode *inode, struct bio *bio,
  6984. int mirror_num)
  6985. {
  6986. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6987. int ret;
  6988. BUG_ON(bio_op(bio) == REQ_OP_WRITE);
  6989. bio_get(bio);
  6990. ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR);
  6991. if (ret)
  6992. goto err;
  6993. ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
  6994. err:
  6995. bio_put(bio);
  6996. return ret;
  6997. }
  6998. static int btrfs_check_dio_repairable(struct inode *inode,
  6999. struct bio *failed_bio,
  7000. struct io_failure_record *failrec,
  7001. int failed_mirror)
  7002. {
  7003. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  7004. int num_copies;
  7005. num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
  7006. if (num_copies == 1) {
  7007. /*
  7008. * we only have a single copy of the data, so don't bother with
  7009. * all the retry and error correction code that follows. no
  7010. * matter what the error is, it is very likely to persist.
  7011. */
  7012. btrfs_debug(fs_info,
  7013. "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
  7014. num_copies, failrec->this_mirror, failed_mirror);
  7015. return 0;
  7016. }
  7017. failrec->failed_mirror = failed_mirror;
  7018. failrec->this_mirror++;
  7019. if (failrec->this_mirror == failed_mirror)
  7020. failrec->this_mirror++;
  7021. if (failrec->this_mirror > num_copies) {
  7022. btrfs_debug(fs_info,
  7023. "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
  7024. num_copies, failrec->this_mirror, failed_mirror);
  7025. return 0;
  7026. }
  7027. return 1;
  7028. }
  7029. static int dio_read_error(struct inode *inode, struct bio *failed_bio,
  7030. struct page *page, unsigned int pgoff,
  7031. u64 start, u64 end, int failed_mirror,
  7032. bio_end_io_t *repair_endio, void *repair_arg)
  7033. {
  7034. struct io_failure_record *failrec;
  7035. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  7036. struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
  7037. struct bio *bio;
  7038. int isector;
  7039. int read_mode = 0;
  7040. int segs;
  7041. int ret;
  7042. BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
  7043. ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
  7044. if (ret)
  7045. return ret;
  7046. ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
  7047. failed_mirror);
  7048. if (!ret) {
  7049. free_io_failure(failure_tree, io_tree, failrec);
  7050. return -EIO;
  7051. }
  7052. segs = bio_segments(failed_bio);
  7053. if (segs > 1 ||
  7054. (failed_bio->bi_io_vec->bv_len > btrfs_inode_sectorsize(inode)))
  7055. read_mode |= REQ_FAILFAST_DEV;
  7056. isector = start - btrfs_io_bio(failed_bio)->logical;
  7057. isector >>= inode->i_sb->s_blocksize_bits;
  7058. bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
  7059. pgoff, isector, repair_endio, repair_arg);
  7060. if (!bio) {
  7061. free_io_failure(failure_tree, io_tree, failrec);
  7062. return -EIO;
  7063. }
  7064. bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
  7065. btrfs_debug(BTRFS_I(inode)->root->fs_info,
  7066. "Repair DIO Read Error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d\n",
  7067. read_mode, failrec->this_mirror, failrec->in_validation);
  7068. ret = submit_dio_repair_bio(inode, bio, failrec->this_mirror);
  7069. if (ret) {
  7070. free_io_failure(failure_tree, io_tree, failrec);
  7071. bio_put(bio);
  7072. }
  7073. return ret;
  7074. }
  7075. struct btrfs_retry_complete {
  7076. struct completion done;
  7077. struct inode *inode;
  7078. u64 start;
  7079. int uptodate;
  7080. };
  7081. static void btrfs_retry_endio_nocsum(struct bio *bio)
  7082. {
  7083. struct btrfs_retry_complete *done = bio->bi_private;
  7084. struct inode *inode = done->inode;
  7085. struct bio_vec *bvec;
  7086. struct extent_io_tree *io_tree, *failure_tree;
  7087. int i;
  7088. if (bio->bi_error)
  7089. goto end;
  7090. ASSERT(bio->bi_vcnt == 1);
  7091. io_tree = &BTRFS_I(inode)->io_tree;
  7092. failure_tree = &BTRFS_I(inode)->io_failure_tree;
  7093. ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode));
  7094. done->uptodate = 1;
  7095. bio_for_each_segment_all(bvec, bio, i)
  7096. clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree,
  7097. io_tree, done->start, bvec->bv_page,
  7098. btrfs_ino(BTRFS_I(inode)), 0);
  7099. end:
  7100. complete(&done->done);
  7101. bio_put(bio);
  7102. }
  7103. static int __btrfs_correct_data_nocsum(struct inode *inode,
  7104. struct btrfs_io_bio *io_bio)
  7105. {
  7106. struct btrfs_fs_info *fs_info;
  7107. struct bio_vec bvec;
  7108. struct bvec_iter iter;
  7109. struct btrfs_retry_complete done;
  7110. u64 start;
  7111. unsigned int pgoff;
  7112. u32 sectorsize;
  7113. int nr_sectors;
  7114. int ret;
  7115. int err = 0;
  7116. fs_info = BTRFS_I(inode)->root->fs_info;
  7117. sectorsize = fs_info->sectorsize;
  7118. start = io_bio->logical;
  7119. done.inode = inode;
  7120. io_bio->bio.bi_iter = io_bio->iter;
  7121. bio_for_each_segment(bvec, &io_bio->bio, iter) {
  7122. nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
  7123. pgoff = bvec.bv_offset;
  7124. next_block_or_try_again:
  7125. done.uptodate = 0;
  7126. done.start = start;
  7127. init_completion(&done.done);
  7128. ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
  7129. pgoff, start, start + sectorsize - 1,
  7130. io_bio->mirror_num,
  7131. btrfs_retry_endio_nocsum, &done);
  7132. if (ret) {
  7133. err = ret;
  7134. goto next;
  7135. }
  7136. wait_for_completion(&done.done);
  7137. if (!done.uptodate) {
  7138. /* We might have another mirror, so try again */
  7139. goto next_block_or_try_again;
  7140. }
  7141. next:
  7142. start += sectorsize;
  7143. nr_sectors--;
  7144. if (nr_sectors) {
  7145. pgoff += sectorsize;
  7146. ASSERT(pgoff < PAGE_SIZE);
  7147. goto next_block_or_try_again;
  7148. }
  7149. }
  7150. return err;
  7151. }
  7152. static void btrfs_retry_endio(struct bio *bio)
  7153. {
  7154. struct btrfs_retry_complete *done = bio->bi_private;
  7155. struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
  7156. struct extent_io_tree *io_tree, *failure_tree;
  7157. struct inode *inode = done->inode;
  7158. struct bio_vec *bvec;
  7159. int uptodate;
  7160. int ret;
  7161. int i;
  7162. if (bio->bi_error)
  7163. goto end;
  7164. uptodate = 1;
  7165. ASSERT(bio->bi_vcnt == 1);
  7166. ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(done->inode));
  7167. io_tree = &BTRFS_I(inode)->io_tree;
  7168. failure_tree = &BTRFS_I(inode)->io_failure_tree;
  7169. bio_for_each_segment_all(bvec, bio, i) {
  7170. ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
  7171. bvec->bv_offset, done->start,
  7172. bvec->bv_len);
  7173. if (!ret)
  7174. clean_io_failure(BTRFS_I(inode)->root->fs_info,
  7175. failure_tree, io_tree, done->start,
  7176. bvec->bv_page,
  7177. btrfs_ino(BTRFS_I(inode)),
  7178. bvec->bv_offset);
  7179. else
  7180. uptodate = 0;
  7181. }
  7182. done->uptodate = uptodate;
  7183. end:
  7184. complete(&done->done);
  7185. bio_put(bio);
  7186. }
  7187. static int __btrfs_subio_endio_read(struct inode *inode,
  7188. struct btrfs_io_bio *io_bio, int err)
  7189. {
  7190. struct btrfs_fs_info *fs_info;
  7191. struct bio_vec bvec;
  7192. struct bvec_iter iter;
  7193. struct btrfs_retry_complete done;
  7194. u64 start;
  7195. u64 offset = 0;
  7196. u32 sectorsize;
  7197. int nr_sectors;
  7198. unsigned int pgoff;
  7199. int csum_pos;
  7200. int ret;
  7201. fs_info = BTRFS_I(inode)->root->fs_info;
  7202. sectorsize = fs_info->sectorsize;
  7203. err = 0;
  7204. start = io_bio->logical;
  7205. done.inode = inode;
  7206. io_bio->bio.bi_iter = io_bio->iter;
  7207. bio_for_each_segment(bvec, &io_bio->bio, iter) {
  7208. nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
  7209. pgoff = bvec.bv_offset;
  7210. next_block:
  7211. csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset);
  7212. ret = __readpage_endio_check(inode, io_bio, csum_pos,
  7213. bvec.bv_page, pgoff, start,
  7214. sectorsize);
  7215. if (likely(!ret))
  7216. goto next;
  7217. try_again:
  7218. done.uptodate = 0;
  7219. done.start = start;
  7220. init_completion(&done.done);
  7221. ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
  7222. pgoff, start, start + sectorsize - 1,
  7223. io_bio->mirror_num,
  7224. btrfs_retry_endio, &done);
  7225. if (ret) {
  7226. err = ret;
  7227. goto next;
  7228. }
  7229. wait_for_completion(&done.done);
  7230. if (!done.uptodate) {
  7231. /* We might have another mirror, so try again */
  7232. goto try_again;
  7233. }
  7234. next:
  7235. offset += sectorsize;
  7236. start += sectorsize;
  7237. ASSERT(nr_sectors);
  7238. nr_sectors--;
  7239. if (nr_sectors) {
  7240. pgoff += sectorsize;
  7241. ASSERT(pgoff < PAGE_SIZE);
  7242. goto next_block;
  7243. }
  7244. }
  7245. return err;
  7246. }
  7247. static int btrfs_subio_endio_read(struct inode *inode,
  7248. struct btrfs_io_bio *io_bio, int err)
  7249. {
  7250. bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
  7251. if (skip_csum) {
  7252. if (unlikely(err))
  7253. return __btrfs_correct_data_nocsum(inode, io_bio);
  7254. else
  7255. return 0;
  7256. } else {
  7257. return __btrfs_subio_endio_read(inode, io_bio, err);
  7258. }
  7259. }
  7260. static void btrfs_endio_direct_read(struct bio *bio)
  7261. {
  7262. struct btrfs_dio_private *dip = bio->bi_private;
  7263. struct inode *inode = dip->inode;
  7264. struct bio *dio_bio;
  7265. struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
  7266. int err = bio->bi_error;
  7267. if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED)
  7268. err = btrfs_subio_endio_read(inode, io_bio, err);
  7269. unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
  7270. dip->logical_offset + dip->bytes - 1);
  7271. dio_bio = dip->dio_bio;
  7272. kfree(dip);
  7273. dio_bio->bi_error = bio->bi_error;
  7274. dio_end_io(dio_bio, bio->bi_error);
  7275. if (io_bio->end_io)
  7276. io_bio->end_io(io_bio, err);
  7277. bio_put(bio);
  7278. }
  7279. static void __endio_write_update_ordered(struct inode *inode,
  7280. const u64 offset, const u64 bytes,
  7281. const bool uptodate)
  7282. {
  7283. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  7284. struct btrfs_ordered_extent *ordered = NULL;
  7285. struct btrfs_workqueue *wq;
  7286. btrfs_work_func_t func;
  7287. u64 ordered_offset = offset;
  7288. u64 ordered_bytes = bytes;
  7289. int ret;
  7290. if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
  7291. wq = fs_info->endio_freespace_worker;
  7292. func = btrfs_freespace_write_helper;
  7293. } else {
  7294. wq = fs_info->endio_write_workers;
  7295. func = btrfs_endio_write_helper;
  7296. }
  7297. again:
  7298. ret = btrfs_dec_test_first_ordered_pending(inode, &ordered,
  7299. &ordered_offset,
  7300. ordered_bytes,
  7301. uptodate);
  7302. if (!ret)
  7303. goto out_test;
  7304. btrfs_init_work(&ordered->work, func, finish_ordered_fn, NULL, NULL);
  7305. btrfs_queue_work(wq, &ordered->work);
  7306. out_test:
  7307. /*
  7308. * our bio might span multiple ordered extents. If we haven't
  7309. * completed the accounting for the whole dio, go back and try again
  7310. */
  7311. if (ordered_offset < offset + bytes) {
  7312. ordered_bytes = offset + bytes - ordered_offset;
  7313. ordered = NULL;
  7314. goto again;
  7315. }
  7316. }
  7317. static void btrfs_endio_direct_write(struct bio *bio)
  7318. {
  7319. struct btrfs_dio_private *dip = bio->bi_private;
  7320. struct bio *dio_bio = dip->dio_bio;
  7321. __endio_write_update_ordered(dip->inode, dip->logical_offset,
  7322. dip->bytes, !bio->bi_error);
  7323. kfree(dip);
  7324. dio_bio->bi_error = bio->bi_error;
  7325. dio_end_io(dio_bio, bio->bi_error);
  7326. bio_put(bio);
  7327. }
  7328. static int __btrfs_submit_bio_start_direct_io(void *private_data,
  7329. struct bio *bio, int mirror_num,
  7330. unsigned long bio_flags, u64 offset)
  7331. {
  7332. struct inode *inode = private_data;
  7333. int ret;
  7334. ret = btrfs_csum_one_bio(inode, bio, offset, 1);
  7335. BUG_ON(ret); /* -ENOMEM */
  7336. return 0;
  7337. }
  7338. static void btrfs_end_dio_bio(struct bio *bio)
  7339. {
  7340. struct btrfs_dio_private *dip = bio->bi_private;
  7341. int err = bio->bi_error;
  7342. if (err)
  7343. btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
  7344. "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d",
  7345. btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio),
  7346. bio->bi_opf,
  7347. (unsigned long long)bio->bi_iter.bi_sector,
  7348. bio->bi_iter.bi_size, err);
  7349. if (dip->subio_endio)
  7350. err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);
  7351. if (err) {
  7352. dip->errors = 1;
  7353. /*
  7354. * before atomic variable goto zero, we must make sure
  7355. * dip->errors is perceived to be set.
  7356. */
  7357. smp_mb__before_atomic();
  7358. }
  7359. /* if there are more bios still pending for this dio, just exit */
  7360. if (!atomic_dec_and_test(&dip->pending_bios))
  7361. goto out;
  7362. if (dip->errors) {
  7363. bio_io_error(dip->orig_bio);
  7364. } else {
  7365. dip->dio_bio->bi_error = 0;
  7366. bio_endio(dip->orig_bio);
  7367. }
  7368. out:
  7369. bio_put(bio);
  7370. }
  7371. static inline int btrfs_lookup_and_bind_dio_csum(struct inode *inode,
  7372. struct btrfs_dio_private *dip,
  7373. struct bio *bio,
  7374. u64 file_offset)
  7375. {
  7376. struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
  7377. struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio);
  7378. int ret;
  7379. /*
  7380. * We load all the csum data we need when we submit
  7381. * the first bio to reduce the csum tree search and
  7382. * contention.
  7383. */
  7384. if (dip->logical_offset == file_offset) {
  7385. ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio,
  7386. file_offset);
  7387. if (ret)
  7388. return ret;
  7389. }
  7390. if (bio == dip->orig_bio)
  7391. return 0;
  7392. file_offset -= dip->logical_offset;
  7393. file_offset >>= inode->i_sb->s_blocksize_bits;
  7394. io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset);
  7395. return 0;
  7396. }
  7397. static inline int __btrfs_submit_dio_bio(struct bio *bio, struct inode *inode,
  7398. u64 file_offset, int skip_sum,
  7399. int async_submit)
  7400. {
  7401. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  7402. struct btrfs_dio_private *dip = bio->bi_private;
  7403. bool write = bio_op(bio) == REQ_OP_WRITE;
  7404. int ret;
  7405. if (async_submit)
  7406. async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);
  7407. bio_get(bio);
  7408. if (!write) {
  7409. ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
  7410. if (ret)
  7411. goto err;
  7412. }
  7413. if (skip_sum)
  7414. goto map;
  7415. if (write && async_submit) {
  7416. ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0,
  7417. file_offset, inode,
  7418. __btrfs_submit_bio_start_direct_io,
  7419. __btrfs_submit_bio_done);
  7420. goto err;
  7421. } else if (write) {
  7422. /*
  7423. * If we aren't doing async submit, calculate the csum of the
  7424. * bio now.
  7425. */
  7426. ret = btrfs_csum_one_bio(inode, bio, file_offset, 1);
  7427. if (ret)
  7428. goto err;
  7429. } else {
  7430. ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio,
  7431. file_offset);
  7432. if (ret)
  7433. goto err;
  7434. }
  7435. map:
  7436. ret = btrfs_map_bio(fs_info, bio, 0, async_submit);
  7437. err:
  7438. bio_put(bio);
  7439. return ret;
  7440. }
  7441. static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip,
  7442. int skip_sum)
  7443. {
  7444. struct inode *inode = dip->inode;
  7445. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  7446. struct bio *bio;
  7447. struct bio *orig_bio = dip->orig_bio;
  7448. u64 start_sector = orig_bio->bi_iter.bi_sector;
  7449. u64 file_offset = dip->logical_offset;
  7450. u64 map_length;
  7451. int async_submit = 0;
  7452. u64 submit_len;
  7453. int clone_offset = 0;
  7454. int clone_len;
  7455. int ret;
  7456. map_length = orig_bio->bi_iter.bi_size;
  7457. submit_len = map_length;
  7458. ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9,
  7459. &map_length, NULL, 0);
  7460. if (ret)
  7461. return -EIO;
  7462. if (map_length >= submit_len) {
  7463. bio = orig_bio;
  7464. dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED;
  7465. goto submit;
  7466. }
  7467. /* async crcs make it difficult to collect full stripe writes. */
  7468. if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK)
  7469. async_submit = 0;
  7470. else
  7471. async_submit = 1;
  7472. /* bio split */
  7473. ASSERT(map_length <= INT_MAX);
  7474. atomic_inc(&dip->pending_bios);
  7475. do {
  7476. clone_len = min_t(int, submit_len, map_length);
  7477. /*
  7478. * This will never fail as it's passing GPF_NOFS and
  7479. * the allocation is backed by btrfs_bioset.
  7480. */
  7481. bio = btrfs_bio_clone_partial(orig_bio, clone_offset,
  7482. clone_len);
  7483. bio->bi_private = dip;
  7484. bio->bi_end_io = btrfs_end_dio_bio;
  7485. btrfs_io_bio(bio)->logical = file_offset;
  7486. ASSERT(submit_len >= clone_len);
  7487. submit_len -= clone_len;
  7488. if (submit_len == 0)
  7489. break;
  7490. /*
  7491. * Increase the count before we submit the bio so we know
  7492. * the end IO handler won't happen before we increase the
  7493. * count. Otherwise, the dip might get freed before we're
  7494. * done setting it up.
  7495. */
  7496. atomic_inc(&dip->pending_bios);
  7497. ret = __btrfs_submit_dio_bio(bio, inode, file_offset, skip_sum,
  7498. async_submit);
  7499. if (ret) {
  7500. bio_put(bio);
  7501. atomic_dec(&dip->pending_bios);
  7502. goto out_err;
  7503. }
  7504. clone_offset += clone_len;
  7505. start_sector += clone_len >> 9;
  7506. file_offset += clone_len;
  7507. map_length = submit_len;
  7508. ret = btrfs_map_block(fs_info, btrfs_op(orig_bio),
  7509. start_sector << 9, &map_length, NULL, 0);
  7510. if (ret)
  7511. goto out_err;
  7512. } while (submit_len > 0);
  7513. submit:
  7514. ret = __btrfs_submit_dio_bio(bio, inode, file_offset, skip_sum,
  7515. async_submit);
  7516. if (!ret)
  7517. return 0;
  7518. bio_put(bio);
  7519. out_err:
  7520. dip->errors = 1;
  7521. /*
  7522. * before atomic variable goto zero, we must
  7523. * make sure dip->errors is perceived to be set.
  7524. */
  7525. smp_mb__before_atomic();
  7526. if (atomic_dec_and_test(&dip->pending_bios))
  7527. bio_io_error(dip->orig_bio);
  7528. /* bio_end_io() will handle error, so we needn't return it */
  7529. return 0;
  7530. }
  7531. static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode,
  7532. loff_t file_offset)
  7533. {
  7534. struct btrfs_dio_private *dip = NULL;
  7535. struct bio *bio = NULL;
  7536. struct btrfs_io_bio *io_bio;
  7537. int skip_sum;
  7538. bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
  7539. int ret = 0;
  7540. skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
  7541. bio = btrfs_bio_clone(dio_bio, GFP_NOFS);
  7542. if (!bio) {
  7543. ret = -ENOMEM;
  7544. goto free_ordered;
  7545. }
  7546. dip = kzalloc(sizeof(*dip), GFP_NOFS);
  7547. if (!dip) {
  7548. ret = -ENOMEM;
  7549. goto free_ordered;
  7550. }
  7551. dip->private = dio_bio->bi_private;
  7552. dip->inode = inode;
  7553. dip->logical_offset = file_offset;
  7554. dip->bytes = dio_bio->bi_iter.bi_size;
  7555. dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
  7556. bio->bi_private = dip;
  7557. dip->orig_bio = bio;
  7558. dip->dio_bio = dio_bio;
  7559. atomic_set(&dip->pending_bios, 0);
  7560. io_bio = btrfs_io_bio(bio);
  7561. io_bio->logical = file_offset;
  7562. if (write) {
  7563. bio->bi_end_io = btrfs_endio_direct_write;
  7564. } else {
  7565. bio->bi_end_io = btrfs_endio_direct_read;
  7566. dip->subio_endio = btrfs_subio_endio_read;
  7567. }
  7568. /*
  7569. * Reset the range for unsubmitted ordered extents (to a 0 length range)
  7570. * even if we fail to submit a bio, because in such case we do the
  7571. * corresponding error handling below and it must not be done a second
  7572. * time by btrfs_direct_IO().
  7573. */
  7574. if (write) {
  7575. struct btrfs_dio_data *dio_data = current->journal_info;
  7576. dio_data->unsubmitted_oe_range_end = dip->logical_offset +
  7577. dip->bytes;
  7578. dio_data->unsubmitted_oe_range_start =
  7579. dio_data->unsubmitted_oe_range_end;
  7580. }
  7581. ret = btrfs_submit_direct_hook(dip, skip_sum);
  7582. if (!ret)
  7583. return;
  7584. if (io_bio->end_io)
  7585. io_bio->end_io(io_bio, ret);
  7586. free_ordered:
  7587. /*
  7588. * If we arrived here it means either we failed to submit the dip
  7589. * or we either failed to clone the dio_bio or failed to allocate the
  7590. * dip. If we cloned the dio_bio and allocated the dip, we can just
  7591. * call bio_endio against our io_bio so that we get proper resource
  7592. * cleanup if we fail to submit the dip, otherwise, we must do the
  7593. * same as btrfs_endio_direct_[write|read] because we can't call these
  7594. * callbacks - they require an allocated dip and a clone of dio_bio.
  7595. */
  7596. if (bio && dip) {
  7597. bio_io_error(bio);
  7598. /*
  7599. * The end io callbacks free our dip, do the final put on bio
  7600. * and all the cleanup and final put for dio_bio (through
  7601. * dio_end_io()).
  7602. */
  7603. dip = NULL;
  7604. bio = NULL;
  7605. } else {
  7606. if (write)
  7607. __endio_write_update_ordered(inode,
  7608. file_offset,
  7609. dio_bio->bi_iter.bi_size,
  7610. false);
  7611. else
  7612. unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
  7613. file_offset + dio_bio->bi_iter.bi_size - 1);
  7614. dio_bio->bi_error = -EIO;
  7615. /*
  7616. * Releases and cleans up our dio_bio, no need to bio_put()
  7617. * nor bio_endio()/bio_io_error() against dio_bio.
  7618. */
  7619. dio_end_io(dio_bio, ret);
  7620. }
  7621. if (bio)
  7622. bio_put(bio);
  7623. kfree(dip);
  7624. }
  7625. static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
  7626. struct kiocb *iocb,
  7627. const struct iov_iter *iter, loff_t offset)
  7628. {
  7629. int seg;
  7630. int i;
  7631. unsigned int blocksize_mask = fs_info->sectorsize - 1;
  7632. ssize_t retval = -EINVAL;
  7633. if (offset & blocksize_mask)
  7634. goto out;
  7635. if (iov_iter_alignment(iter) & blocksize_mask)
  7636. goto out;
  7637. /* If this is a write we don't need to check anymore */
  7638. if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter))
  7639. return 0;
  7640. /*
  7641. * Check to make sure we don't have duplicate iov_base's in this
  7642. * iovec, if so return EINVAL, otherwise we'll get csum errors
  7643. * when reading back.
  7644. */
  7645. for (seg = 0; seg < iter->nr_segs; seg++) {
  7646. for (i = seg + 1; i < iter->nr_segs; i++) {
  7647. if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
  7648. goto out;
  7649. }
  7650. }
  7651. retval = 0;
  7652. out:
  7653. return retval;
  7654. }
  7655. static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
  7656. {
  7657. struct file *file = iocb->ki_filp;
  7658. struct inode *inode = file->f_mapping->host;
  7659. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  7660. struct btrfs_dio_data dio_data = { 0 };
  7661. loff_t offset = iocb->ki_pos;
  7662. size_t count = 0;
  7663. int flags = 0;
  7664. bool wakeup = true;
  7665. bool relock = false;
  7666. ssize_t ret;
  7667. if (check_direct_IO(fs_info, iocb, iter, offset))
  7668. return 0;
  7669. inode_dio_begin(inode);
  7670. smp_mb__after_atomic();
  7671. /*
  7672. * The generic stuff only does filemap_write_and_wait_range, which
  7673. * isn't enough if we've written compressed pages to this area, so
  7674. * we need to flush the dirty pages again to make absolutely sure
  7675. * that any outstanding dirty pages are on disk.
  7676. */
  7677. count = iov_iter_count(iter);
  7678. if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
  7679. &BTRFS_I(inode)->runtime_flags))
  7680. filemap_fdatawrite_range(inode->i_mapping, offset,
  7681. offset + count - 1);
  7682. if (iov_iter_rw(iter) == WRITE) {
  7683. /*
  7684. * If the write DIO is beyond the EOF, we need update
  7685. * the isize, but it is protected by i_mutex. So we can
  7686. * not unlock the i_mutex at this case.
  7687. */
  7688. if (offset + count <= inode->i_size) {
  7689. dio_data.overwrite = 1;
  7690. inode_unlock(inode);
  7691. relock = true;
  7692. }
  7693. ret = btrfs_delalloc_reserve_space(inode, offset, count);
  7694. if (ret)
  7695. goto out;
  7696. dio_data.outstanding_extents = count_max_extents(count);
  7697. /*
  7698. * We need to know how many extents we reserved so that we can
  7699. * do the accounting properly if we go over the number we
  7700. * originally calculated. Abuse current->journal_info for this.
  7701. */
  7702. dio_data.reserve = round_up(count,
  7703. fs_info->sectorsize);
  7704. dio_data.unsubmitted_oe_range_start = (u64)offset;
  7705. dio_data.unsubmitted_oe_range_end = (u64)offset;
  7706. current->journal_info = &dio_data;
  7707. down_read(&BTRFS_I(inode)->dio_sem);
  7708. } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
  7709. &BTRFS_I(inode)->runtime_flags)) {
  7710. inode_dio_end(inode);
  7711. flags = DIO_LOCKING | DIO_SKIP_HOLES;
  7712. wakeup = false;
  7713. }
  7714. ret = __blockdev_direct_IO(iocb, inode,
  7715. fs_info->fs_devices->latest_bdev,
  7716. iter, btrfs_get_blocks_direct, NULL,
  7717. btrfs_submit_direct, flags);
  7718. if (iov_iter_rw(iter) == WRITE) {
  7719. up_read(&BTRFS_I(inode)->dio_sem);
  7720. current->journal_info = NULL;
  7721. if (ret < 0 && ret != -EIOCBQUEUED) {
  7722. if (dio_data.reserve)
  7723. btrfs_delalloc_release_space(inode, offset,
  7724. dio_data.reserve);
  7725. /*
  7726. * On error we might have left some ordered extents
  7727. * without submitting corresponding bios for them, so
  7728. * cleanup them up to avoid other tasks getting them
  7729. * and waiting for them to complete forever.
  7730. */
  7731. if (dio_data.unsubmitted_oe_range_start <
  7732. dio_data.unsubmitted_oe_range_end)
  7733. __endio_write_update_ordered(inode,
  7734. dio_data.unsubmitted_oe_range_start,
  7735. dio_data.unsubmitted_oe_range_end -
  7736. dio_data.unsubmitted_oe_range_start,
  7737. false);
  7738. } else if (ret >= 0 && (size_t)ret < count)
  7739. btrfs_delalloc_release_space(inode, offset,
  7740. count - (size_t)ret);
  7741. }
  7742. out:
  7743. if (wakeup)
  7744. inode_dio_end(inode);
  7745. if (relock)
  7746. inode_lock(inode);
  7747. return ret;
  7748. }
  7749. #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC)
  7750. static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
  7751. __u64 start, __u64 len)
  7752. {
  7753. int ret;
  7754. ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS);
  7755. if (ret)
  7756. return ret;
  7757. return extent_fiemap(inode, fieinfo, start, len, btrfs_get_extent_fiemap);
  7758. }
  7759. int btrfs_readpage(struct file *file, struct page *page)
  7760. {
  7761. struct extent_io_tree *tree;
  7762. tree = &BTRFS_I(page->mapping->host)->io_tree;
  7763. return extent_read_full_page(tree, page, btrfs_get_extent, 0);
  7764. }
  7765. static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
  7766. {
  7767. struct extent_io_tree *tree;
  7768. struct inode *inode = page->mapping->host;
  7769. int ret;
  7770. if (current->flags & PF_MEMALLOC) {
  7771. redirty_page_for_writepage(wbc, page);
  7772. unlock_page(page);
  7773. return 0;
  7774. }
  7775. /*
  7776. * If we are under memory pressure we will call this directly from the
  7777. * VM, we need to make sure we have the inode referenced for the ordered
  7778. * extent. If not just return like we didn't do anything.
  7779. */
  7780. if (!igrab(inode)) {
  7781. redirty_page_for_writepage(wbc, page);
  7782. return AOP_WRITEPAGE_ACTIVATE;
  7783. }
  7784. tree = &BTRFS_I(page->mapping->host)->io_tree;
  7785. ret = extent_write_full_page(tree, page, btrfs_get_extent, wbc);
  7786. btrfs_add_delayed_iput(inode);
  7787. return ret;
  7788. }
  7789. static int btrfs_writepages(struct address_space *mapping,
  7790. struct writeback_control *wbc)
  7791. {
  7792. struct extent_io_tree *tree;
  7793. tree = &BTRFS_I(mapping->host)->io_tree;
  7794. return extent_writepages(tree, mapping, btrfs_get_extent, wbc);
  7795. }
  7796. static int
  7797. btrfs_readpages(struct file *file, struct address_space *mapping,
  7798. struct list_head *pages, unsigned nr_pages)
  7799. {
  7800. struct extent_io_tree *tree;
  7801. tree = &BTRFS_I(mapping->host)->io_tree;
  7802. return extent_readpages(tree, mapping, pages, nr_pages,
  7803. btrfs_get_extent);
  7804. }
  7805. static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags)
  7806. {
  7807. struct extent_io_tree *tree;
  7808. struct extent_map_tree *map;
  7809. int ret;
  7810. tree = &BTRFS_I(page->mapping->host)->io_tree;
  7811. map = &BTRFS_I(page->mapping->host)->extent_tree;
  7812. ret = try_release_extent_mapping(map, tree, page, gfp_flags);
  7813. if (ret == 1) {
  7814. ClearPagePrivate(page);
  7815. set_page_private(page, 0);
  7816. put_page(page);
  7817. }
  7818. return ret;
  7819. }
  7820. static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
  7821. {
  7822. if (PageWriteback(page) || PageDirty(page))
  7823. return 0;
  7824. return __btrfs_releasepage(page, gfp_flags);
  7825. }
  7826. static void btrfs_invalidatepage(struct page *page, unsigned int offset,
  7827. unsigned int length)
  7828. {
  7829. struct inode *inode = page->mapping->host;
  7830. struct extent_io_tree *tree;
  7831. struct btrfs_ordered_extent *ordered;
  7832. struct extent_state *cached_state = NULL;
  7833. u64 page_start = page_offset(page);
  7834. u64 page_end = page_start + PAGE_SIZE - 1;
  7835. u64 start;
  7836. u64 end;
  7837. int inode_evicting = inode->i_state & I_FREEING;
  7838. /*
  7839. * we have the page locked, so new writeback can't start,
  7840. * and the dirty bit won't be cleared while we are here.
  7841. *
  7842. * Wait for IO on this page so that we can safely clear
  7843. * the PagePrivate2 bit and do ordered accounting
  7844. */
  7845. wait_on_page_writeback(page);
  7846. tree = &BTRFS_I(inode)->io_tree;
  7847. if (offset) {
  7848. btrfs_releasepage(page, GFP_NOFS);
  7849. return;
  7850. }
  7851. if (!inode_evicting)
  7852. lock_extent_bits(tree, page_start, page_end, &cached_state);
  7853. again:
  7854. start = page_start;
  7855. ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
  7856. page_end - start + 1);
  7857. if (ordered) {
  7858. end = min(page_end, ordered->file_offset + ordered->len - 1);
  7859. /*
  7860. * IO on this page will never be started, so we need
  7861. * to account for any ordered extents now
  7862. */
  7863. if (!inode_evicting)
  7864. clear_extent_bit(tree, start, end,
  7865. EXTENT_DIRTY | EXTENT_DELALLOC |
  7866. EXTENT_DELALLOC_NEW |
  7867. EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
  7868. EXTENT_DEFRAG, 1, 0, &cached_state,
  7869. GFP_NOFS);
  7870. /*
  7871. * whoever cleared the private bit is responsible
  7872. * for the finish_ordered_io
  7873. */
  7874. if (TestClearPagePrivate2(page)) {
  7875. struct btrfs_ordered_inode_tree *tree;
  7876. u64 new_len;
  7877. tree = &BTRFS_I(inode)->ordered_tree;
  7878. spin_lock_irq(&tree->lock);
  7879. set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
  7880. new_len = start - ordered->file_offset;
  7881. if (new_len < ordered->truncated_len)
  7882. ordered->truncated_len = new_len;
  7883. spin_unlock_irq(&tree->lock);
  7884. if (btrfs_dec_test_ordered_pending(inode, &ordered,
  7885. start,
  7886. end - start + 1, 1))
  7887. btrfs_finish_ordered_io(ordered);
  7888. }
  7889. btrfs_put_ordered_extent(ordered);
  7890. if (!inode_evicting) {
  7891. cached_state = NULL;
  7892. lock_extent_bits(tree, start, end,
  7893. &cached_state);
  7894. }
  7895. start = end + 1;
  7896. if (start < page_end)
  7897. goto again;
  7898. }
  7899. /*
  7900. * Qgroup reserved space handler
  7901. * Page here will be either
  7902. * 1) Already written to disk
  7903. * In this case, its reserved space is released from data rsv map
  7904. * and will be freed by delayed_ref handler finally.
  7905. * So even we call qgroup_free_data(), it won't decrease reserved
  7906. * space.
  7907. * 2) Not written to disk
  7908. * This means the reserved space should be freed here. However,
  7909. * if a truncate invalidates the page (by clearing PageDirty)
  7910. * and the page is accounted for while allocating extent
  7911. * in btrfs_check_data_free_space() we let delayed_ref to
  7912. * free the entire extent.
  7913. */
  7914. if (PageDirty(page))
  7915. btrfs_qgroup_free_data(inode, page_start, PAGE_SIZE);
  7916. if (!inode_evicting) {
  7917. clear_extent_bit(tree, page_start, page_end,
  7918. EXTENT_LOCKED | EXTENT_DIRTY |
  7919. EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
  7920. EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1,
  7921. &cached_state, GFP_NOFS);
  7922. __btrfs_releasepage(page, GFP_NOFS);
  7923. }
  7924. ClearPageChecked(page);
  7925. if (PagePrivate(page)) {
  7926. ClearPagePrivate(page);
  7927. set_page_private(page, 0);
  7928. put_page(page);
  7929. }
  7930. }
  7931. /*
  7932. * btrfs_page_mkwrite() is not allowed to change the file size as it gets
  7933. * called from a page fault handler when a page is first dirtied. Hence we must
  7934. * be careful to check for EOF conditions here. We set the page up correctly
  7935. * for a written page which means we get ENOSPC checking when writing into
  7936. * holes and correct delalloc and unwritten extent mapping on filesystems that
  7937. * support these features.
  7938. *
  7939. * We are not allowed to take the i_mutex here so we have to play games to
  7940. * protect against truncate races as the page could now be beyond EOF. Because
  7941. * vmtruncate() writes the inode size before removing pages, once we have the
  7942. * page lock we can determine safely if the page is beyond EOF. If it is not
  7943. * beyond EOF, then the page is guaranteed safe against truncation until we
  7944. * unlock the page.
  7945. */
  7946. int btrfs_page_mkwrite(struct vm_fault *vmf)
  7947. {
  7948. struct page *page = vmf->page;
  7949. struct inode *inode = file_inode(vmf->vma->vm_file);
  7950. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  7951. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  7952. struct btrfs_ordered_extent *ordered;
  7953. struct extent_state *cached_state = NULL;
  7954. char *kaddr;
  7955. unsigned long zero_start;
  7956. loff_t size;
  7957. int ret;
  7958. int reserved = 0;
  7959. u64 reserved_space;
  7960. u64 page_start;
  7961. u64 page_end;
  7962. u64 end;
  7963. reserved_space = PAGE_SIZE;
  7964. sb_start_pagefault(inode->i_sb);
  7965. page_start = page_offset(page);
  7966. page_end = page_start + PAGE_SIZE - 1;
  7967. end = page_end;
  7968. /*
  7969. * Reserving delalloc space after obtaining the page lock can lead to
  7970. * deadlock. For example, if a dirty page is locked by this function
  7971. * and the call to btrfs_delalloc_reserve_space() ends up triggering
  7972. * dirty page write out, then the btrfs_writepage() function could
  7973. * end up waiting indefinitely to get a lock on the page currently
  7974. * being processed by btrfs_page_mkwrite() function.
  7975. */
  7976. ret = btrfs_delalloc_reserve_space(inode, page_start,
  7977. reserved_space);
  7978. if (!ret) {
  7979. ret = file_update_time(vmf->vma->vm_file);
  7980. reserved = 1;
  7981. }
  7982. if (ret) {
  7983. if (ret == -ENOMEM)
  7984. ret = VM_FAULT_OOM;
  7985. else /* -ENOSPC, -EIO, etc */
  7986. ret = VM_FAULT_SIGBUS;
  7987. if (reserved)
  7988. goto out;
  7989. goto out_noreserve;
  7990. }
  7991. ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
  7992. again:
  7993. lock_page(page);
  7994. size = i_size_read(inode);
  7995. if ((page->mapping != inode->i_mapping) ||
  7996. (page_start >= size)) {
  7997. /* page got truncated out from underneath us */
  7998. goto out_unlock;
  7999. }
  8000. wait_on_page_writeback(page);
  8001. lock_extent_bits(io_tree, page_start, page_end, &cached_state);
  8002. set_page_extent_mapped(page);
  8003. /*
  8004. * we can't set the delalloc bits if there are pending ordered
  8005. * extents. Drop our locks and wait for them to finish
  8006. */
  8007. ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
  8008. PAGE_SIZE);
  8009. if (ordered) {
  8010. unlock_extent_cached(io_tree, page_start, page_end,
  8011. &cached_state, GFP_NOFS);
  8012. unlock_page(page);
  8013. btrfs_start_ordered_extent(inode, ordered, 1);
  8014. btrfs_put_ordered_extent(ordered);
  8015. goto again;
  8016. }
  8017. if (page->index == ((size - 1) >> PAGE_SHIFT)) {
  8018. reserved_space = round_up(size - page_start,
  8019. fs_info->sectorsize);
  8020. if (reserved_space < PAGE_SIZE) {
  8021. end = page_start + reserved_space - 1;
  8022. spin_lock(&BTRFS_I(inode)->lock);
  8023. BTRFS_I(inode)->outstanding_extents++;
  8024. spin_unlock(&BTRFS_I(inode)->lock);
  8025. btrfs_delalloc_release_space(inode, page_start,
  8026. PAGE_SIZE - reserved_space);
  8027. }
  8028. }
  8029. /*
  8030. * page_mkwrite gets called when the page is firstly dirtied after it's
  8031. * faulted in, but write(2) could also dirty a page and set delalloc
  8032. * bits, thus in this case for space account reason, we still need to
  8033. * clear any delalloc bits within this page range since we have to
  8034. * reserve data&meta space before lock_page() (see above comments).
  8035. */
  8036. clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end,
  8037. EXTENT_DIRTY | EXTENT_DELALLOC |
  8038. EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
  8039. 0, 0, &cached_state, GFP_NOFS);
  8040. ret = btrfs_set_extent_delalloc(inode, page_start, end,
  8041. &cached_state, 0);
  8042. if (ret) {
  8043. unlock_extent_cached(io_tree, page_start, page_end,
  8044. &cached_state, GFP_NOFS);
  8045. ret = VM_FAULT_SIGBUS;
  8046. goto out_unlock;
  8047. }
  8048. ret = 0;
  8049. /* page is wholly or partially inside EOF */
  8050. if (page_start + PAGE_SIZE > size)
  8051. zero_start = size & ~PAGE_MASK;
  8052. else
  8053. zero_start = PAGE_SIZE;
  8054. if (zero_start != PAGE_SIZE) {
  8055. kaddr = kmap(page);
  8056. memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start);
  8057. flush_dcache_page(page);
  8058. kunmap(page);
  8059. }
  8060. ClearPageChecked(page);
  8061. set_page_dirty(page);
  8062. SetPageUptodate(page);
  8063. BTRFS_I(inode)->last_trans = fs_info->generation;
  8064. BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid;
  8065. BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit;
  8066. unlock_extent_cached(io_tree, page_start, page_end, &cached_state, GFP_NOFS);
  8067. out_unlock:
  8068. if (!ret) {
  8069. sb_end_pagefault(inode->i_sb);
  8070. return VM_FAULT_LOCKED;
  8071. }
  8072. unlock_page(page);
  8073. out:
  8074. btrfs_delalloc_release_space(inode, page_start, reserved_space);
  8075. out_noreserve:
  8076. sb_end_pagefault(inode->i_sb);
  8077. return ret;
  8078. }
  8079. static int btrfs_truncate(struct inode *inode)
  8080. {
  8081. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  8082. struct btrfs_root *root = BTRFS_I(inode)->root;
  8083. struct btrfs_block_rsv *rsv;
  8084. int ret = 0;
  8085. int err = 0;
  8086. struct btrfs_trans_handle *trans;
  8087. u64 mask = fs_info->sectorsize - 1;
  8088. u64 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
  8089. ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask),
  8090. (u64)-1);
  8091. if (ret)
  8092. return ret;
  8093. /*
  8094. * Yes ladies and gentlemen, this is indeed ugly. The fact is we have
  8095. * 3 things going on here
  8096. *
  8097. * 1) We need to reserve space for our orphan item and the space to
  8098. * delete our orphan item. Lord knows we don't want to have a dangling
  8099. * orphan item because we didn't reserve space to remove it.
  8100. *
  8101. * 2) We need to reserve space to update our inode.
  8102. *
  8103. * 3) We need to have something to cache all the space that is going to
  8104. * be free'd up by the truncate operation, but also have some slack
  8105. * space reserved in case it uses space during the truncate (thank you
  8106. * very much snapshotting).
  8107. *
  8108. * And we need these to all be separate. The fact is we can use a lot of
  8109. * space doing the truncate, and we have no earthly idea how much space
  8110. * we will use, so we need the truncate reservation to be separate so it
  8111. * doesn't end up using space reserved for updating the inode or
  8112. * removing the orphan item. We also need to be able to stop the
  8113. * transaction and start a new one, which means we need to be able to
  8114. * update the inode several times, and we have no idea of knowing how
  8115. * many times that will be, so we can't just reserve 1 item for the
  8116. * entirety of the operation, so that has to be done separately as well.
  8117. * Then there is the orphan item, which does indeed need to be held on
  8118. * to for the whole operation, and we need nobody to touch this reserved
  8119. * space except the orphan code.
  8120. *
  8121. * So that leaves us with
  8122. *
  8123. * 1) root->orphan_block_rsv - for the orphan deletion.
  8124. * 2) rsv - for the truncate reservation, which we will steal from the
  8125. * transaction reservation.
  8126. * 3) fs_info->trans_block_rsv - this will have 1 items worth left for
  8127. * updating the inode.
  8128. */
  8129. rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
  8130. if (!rsv)
  8131. return -ENOMEM;
  8132. rsv->size = min_size;
  8133. rsv->failfast = 1;
  8134. /*
  8135. * 1 for the truncate slack space
  8136. * 1 for updating the inode.
  8137. */
  8138. trans = btrfs_start_transaction(root, 2);
  8139. if (IS_ERR(trans)) {
  8140. err = PTR_ERR(trans);
  8141. goto out;
  8142. }
  8143. /* Migrate the slack space for the truncate to our reserve */
  8144. ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
  8145. min_size, 0);
  8146. BUG_ON(ret);
  8147. /*
  8148. * So if we truncate and then write and fsync we normally would just
  8149. * write the extents that changed, which is a problem if we need to
  8150. * first truncate that entire inode. So set this flag so we write out
  8151. * all of the extents in the inode to the sync log so we're completely
  8152. * safe.
  8153. */
  8154. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
  8155. trans->block_rsv = rsv;
  8156. while (1) {
  8157. ret = btrfs_truncate_inode_items(trans, root, inode,
  8158. inode->i_size,
  8159. BTRFS_EXTENT_DATA_KEY);
  8160. if (ret != -ENOSPC && ret != -EAGAIN) {
  8161. err = ret;
  8162. break;
  8163. }
  8164. trans->block_rsv = &fs_info->trans_block_rsv;
  8165. ret = btrfs_update_inode(trans, root, inode);
  8166. if (ret) {
  8167. err = ret;
  8168. break;
  8169. }
  8170. btrfs_end_transaction(trans);
  8171. btrfs_btree_balance_dirty(fs_info);
  8172. trans = btrfs_start_transaction(root, 2);
  8173. if (IS_ERR(trans)) {
  8174. ret = err = PTR_ERR(trans);
  8175. trans = NULL;
  8176. break;
  8177. }
  8178. btrfs_block_rsv_release(fs_info, rsv, -1);
  8179. ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
  8180. rsv, min_size, 0);
  8181. BUG_ON(ret); /* shouldn't happen */
  8182. trans->block_rsv = rsv;
  8183. }
  8184. if (ret == 0 && inode->i_nlink > 0) {
  8185. trans->block_rsv = root->orphan_block_rsv;
  8186. ret = btrfs_orphan_del(trans, BTRFS_I(inode));
  8187. if (ret)
  8188. err = ret;
  8189. }
  8190. if (trans) {
  8191. trans->block_rsv = &fs_info->trans_block_rsv;
  8192. ret = btrfs_update_inode(trans, root, inode);
  8193. if (ret && !err)
  8194. err = ret;
  8195. ret = btrfs_end_transaction(trans);
  8196. btrfs_btree_balance_dirty(fs_info);
  8197. }
  8198. out:
  8199. btrfs_free_block_rsv(fs_info, rsv);
  8200. if (ret && !err)
  8201. err = ret;
  8202. return err;
  8203. }
  8204. /*
  8205. * create a new subvolume directory/inode (helper for the ioctl).
  8206. */
  8207. int btrfs_create_subvol_root(struct btrfs_trans_handle *trans,
  8208. struct btrfs_root *new_root,
  8209. struct btrfs_root *parent_root,
  8210. u64 new_dirid)
  8211. {
  8212. struct inode *inode;
  8213. int err;
  8214. u64 index = 0;
  8215. inode = btrfs_new_inode(trans, new_root, NULL, "..", 2,
  8216. new_dirid, new_dirid,
  8217. S_IFDIR | (~current_umask() & S_IRWXUGO),
  8218. &index);
  8219. if (IS_ERR(inode))
  8220. return PTR_ERR(inode);
  8221. inode->i_op = &btrfs_dir_inode_operations;
  8222. inode->i_fop = &btrfs_dir_file_operations;
  8223. set_nlink(inode, 1);
  8224. btrfs_i_size_write(BTRFS_I(inode), 0);
  8225. unlock_new_inode(inode);
  8226. err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
  8227. if (err)
  8228. btrfs_err(new_root->fs_info,
  8229. "error inheriting subvolume %llu properties: %d",
  8230. new_root->root_key.objectid, err);
  8231. err = btrfs_update_inode(trans, new_root, inode);
  8232. iput(inode);
  8233. return err;
  8234. }
  8235. struct inode *btrfs_alloc_inode(struct super_block *sb)
  8236. {
  8237. struct btrfs_inode *ei;
  8238. struct inode *inode;
  8239. ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_NOFS);
  8240. if (!ei)
  8241. return NULL;
  8242. ei->root = NULL;
  8243. ei->generation = 0;
  8244. ei->last_trans = 0;
  8245. ei->last_sub_trans = 0;
  8246. ei->logged_trans = 0;
  8247. ei->delalloc_bytes = 0;
  8248. ei->new_delalloc_bytes = 0;
  8249. ei->defrag_bytes = 0;
  8250. ei->disk_i_size = 0;
  8251. ei->flags = 0;
  8252. ei->csum_bytes = 0;
  8253. ei->index_cnt = (u64)-1;
  8254. ei->dir_index = 0;
  8255. ei->last_unlink_trans = 0;
  8256. ei->last_log_commit = 0;
  8257. ei->delayed_iput_count = 0;
  8258. spin_lock_init(&ei->lock);
  8259. ei->outstanding_extents = 0;
  8260. ei->reserved_extents = 0;
  8261. ei->runtime_flags = 0;
  8262. ei->force_compress = BTRFS_COMPRESS_NONE;
  8263. ei->delayed_node = NULL;
  8264. ei->i_otime.tv_sec = 0;
  8265. ei->i_otime.tv_nsec = 0;
  8266. inode = &ei->vfs_inode;
  8267. extent_map_tree_init(&ei->extent_tree);
  8268. extent_io_tree_init(&ei->io_tree, inode);
  8269. extent_io_tree_init(&ei->io_failure_tree, inode);
  8270. ei->io_tree.track_uptodate = 1;
  8271. ei->io_failure_tree.track_uptodate = 1;
  8272. atomic_set(&ei->sync_writers, 0);
  8273. mutex_init(&ei->log_mutex);
  8274. mutex_init(&ei->delalloc_mutex);
  8275. btrfs_ordered_inode_tree_init(&ei->ordered_tree);
  8276. INIT_LIST_HEAD(&ei->delalloc_inodes);
  8277. INIT_LIST_HEAD(&ei->delayed_iput);
  8278. RB_CLEAR_NODE(&ei->rb_node);
  8279. init_rwsem(&ei->dio_sem);
  8280. return inode;
  8281. }
  8282. #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
  8283. void btrfs_test_destroy_inode(struct inode *inode)
  8284. {
  8285. btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
  8286. kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
  8287. }
  8288. #endif
  8289. static void btrfs_i_callback(struct rcu_head *head)
  8290. {
  8291. struct inode *inode = container_of(head, struct inode, i_rcu);
  8292. kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
  8293. }
  8294. void btrfs_destroy_inode(struct inode *inode)
  8295. {
  8296. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  8297. struct btrfs_ordered_extent *ordered;
  8298. struct btrfs_root *root = BTRFS_I(inode)->root;
  8299. WARN_ON(!hlist_empty(&inode->i_dentry));
  8300. WARN_ON(inode->i_data.nrpages);
  8301. WARN_ON(BTRFS_I(inode)->outstanding_extents);
  8302. WARN_ON(BTRFS_I(inode)->reserved_extents);
  8303. WARN_ON(BTRFS_I(inode)->delalloc_bytes);
  8304. WARN_ON(BTRFS_I(inode)->new_delalloc_bytes);
  8305. WARN_ON(BTRFS_I(inode)->csum_bytes);
  8306. WARN_ON(BTRFS_I(inode)->defrag_bytes);
  8307. /*
  8308. * This can happen where we create an inode, but somebody else also
  8309. * created the same inode and we need to destroy the one we already
  8310. * created.
  8311. */
  8312. if (!root)
  8313. goto free;
  8314. if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
  8315. &BTRFS_I(inode)->runtime_flags)) {
  8316. btrfs_info(fs_info, "inode %llu still on the orphan list",
  8317. btrfs_ino(BTRFS_I(inode)));
  8318. atomic_dec(&root->orphan_inodes);
  8319. }
  8320. while (1) {
  8321. ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
  8322. if (!ordered)
  8323. break;
  8324. else {
  8325. btrfs_err(fs_info,
  8326. "found ordered extent %llu %llu on inode cleanup",
  8327. ordered->file_offset, ordered->len);
  8328. btrfs_remove_ordered_extent(inode, ordered);
  8329. btrfs_put_ordered_extent(ordered);
  8330. btrfs_put_ordered_extent(ordered);
  8331. }
  8332. }
  8333. btrfs_qgroup_check_reserved_leak(inode);
  8334. inode_tree_del(inode);
  8335. btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
  8336. free:
  8337. call_rcu(&inode->i_rcu, btrfs_i_callback);
  8338. }
  8339. int btrfs_drop_inode(struct inode *inode)
  8340. {
  8341. struct btrfs_root *root = BTRFS_I(inode)->root;
  8342. if (root == NULL)
  8343. return 1;
  8344. /* the snap/subvol tree is on deleting */
  8345. if (btrfs_root_refs(&root->root_item) == 0)
  8346. return 1;
  8347. else
  8348. return generic_drop_inode(inode);
  8349. }
  8350. static void init_once(void *foo)
  8351. {
  8352. struct btrfs_inode *ei = (struct btrfs_inode *) foo;
  8353. inode_init_once(&ei->vfs_inode);
  8354. }
  8355. void btrfs_destroy_cachep(void)
  8356. {
  8357. /*
  8358. * Make sure all delayed rcu free inodes are flushed before we
  8359. * destroy cache.
  8360. */
  8361. rcu_barrier();
  8362. kmem_cache_destroy(btrfs_inode_cachep);
  8363. kmem_cache_destroy(btrfs_trans_handle_cachep);
  8364. kmem_cache_destroy(btrfs_path_cachep);
  8365. kmem_cache_destroy(btrfs_free_space_cachep);
  8366. }
  8367. int btrfs_init_cachep(void)
  8368. {
  8369. btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
  8370. sizeof(struct btrfs_inode), 0,
  8371. SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT,
  8372. init_once);
  8373. if (!btrfs_inode_cachep)
  8374. goto fail;
  8375. btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle",
  8376. sizeof(struct btrfs_trans_handle), 0,
  8377. SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
  8378. if (!btrfs_trans_handle_cachep)
  8379. goto fail;
  8380. btrfs_path_cachep = kmem_cache_create("btrfs_path",
  8381. sizeof(struct btrfs_path), 0,
  8382. SLAB_MEM_SPREAD, NULL);
  8383. if (!btrfs_path_cachep)
  8384. goto fail;
  8385. btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space",
  8386. sizeof(struct btrfs_free_space), 0,
  8387. SLAB_MEM_SPREAD, NULL);
  8388. if (!btrfs_free_space_cachep)
  8389. goto fail;
  8390. return 0;
  8391. fail:
  8392. btrfs_destroy_cachep();
  8393. return -ENOMEM;
  8394. }
  8395. static int btrfs_getattr(const struct path *path, struct kstat *stat,
  8396. u32 request_mask, unsigned int flags)
  8397. {
  8398. u64 delalloc_bytes;
  8399. struct inode *inode = d_inode(path->dentry);
  8400. u32 blocksize = inode->i_sb->s_blocksize;
  8401. u32 bi_flags = BTRFS_I(inode)->flags;
  8402. stat->result_mask |= STATX_BTIME;
  8403. stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec;
  8404. stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec;
  8405. if (bi_flags & BTRFS_INODE_APPEND)
  8406. stat->attributes |= STATX_ATTR_APPEND;
  8407. if (bi_flags & BTRFS_INODE_COMPRESS)
  8408. stat->attributes |= STATX_ATTR_COMPRESSED;
  8409. if (bi_flags & BTRFS_INODE_IMMUTABLE)
  8410. stat->attributes |= STATX_ATTR_IMMUTABLE;
  8411. if (bi_flags & BTRFS_INODE_NODUMP)
  8412. stat->attributes |= STATX_ATTR_NODUMP;
  8413. stat->attributes_mask |= (STATX_ATTR_APPEND |
  8414. STATX_ATTR_COMPRESSED |
  8415. STATX_ATTR_IMMUTABLE |
  8416. STATX_ATTR_NODUMP);
  8417. generic_fillattr(inode, stat);
  8418. stat->dev = BTRFS_I(inode)->root->anon_dev;
  8419. spin_lock(&BTRFS_I(inode)->lock);
  8420. delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;
  8421. spin_unlock(&BTRFS_I(inode)->lock);
  8422. stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) +
  8423. ALIGN(delalloc_bytes, blocksize)) >> 9;
  8424. return 0;
  8425. }
  8426. static int btrfs_rename_exchange(struct inode *old_dir,
  8427. struct dentry *old_dentry,
  8428. struct inode *new_dir,
  8429. struct dentry *new_dentry)
  8430. {
  8431. struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
  8432. struct btrfs_trans_handle *trans;
  8433. struct btrfs_root *root = BTRFS_I(old_dir)->root;
  8434. struct btrfs_root *dest = BTRFS_I(new_dir)->root;
  8435. struct inode *new_inode = new_dentry->d_inode;
  8436. struct inode *old_inode = old_dentry->d_inode;
  8437. struct timespec ctime = current_time(old_inode);
  8438. struct dentry *parent;
  8439. u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
  8440. u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
  8441. u64 old_idx = 0;
  8442. u64 new_idx = 0;
  8443. u64 root_objectid;
  8444. int ret;
  8445. bool root_log_pinned = false;
  8446. bool dest_log_pinned = false;
  8447. /* we only allow rename subvolume link between subvolumes */
  8448. if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
  8449. return -EXDEV;
  8450. /* close the race window with snapshot create/destroy ioctl */
  8451. if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
  8452. down_read(&fs_info->subvol_sem);
  8453. if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
  8454. down_read(&fs_info->subvol_sem);
  8455. /*
  8456. * We want to reserve the absolute worst case amount of items. So if
  8457. * both inodes are subvols and we need to unlink them then that would
  8458. * require 4 item modifications, but if they are both normal inodes it
  8459. * would require 5 item modifications, so we'll assume their normal
  8460. * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items
  8461. * should cover the worst case number of items we'll modify.
  8462. */
  8463. trans = btrfs_start_transaction(root, 12);
  8464. if (IS_ERR(trans)) {
  8465. ret = PTR_ERR(trans);
  8466. goto out_notrans;
  8467. }
  8468. /*
  8469. * We need to find a free sequence number both in the source and
  8470. * in the destination directory for the exchange.
  8471. */
  8472. ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);
  8473. if (ret)
  8474. goto out_fail;
  8475. ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);
  8476. if (ret)
  8477. goto out_fail;
  8478. BTRFS_I(old_inode)->dir_index = 0ULL;
  8479. BTRFS_I(new_inode)->dir_index = 0ULL;
  8480. /* Reference for the source. */
  8481. if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
  8482. /* force full log commit if subvolume involved. */
  8483. btrfs_set_log_full_commit(fs_info, trans);
  8484. } else {
  8485. btrfs_pin_log_trans(root);
  8486. root_log_pinned = true;
  8487. ret = btrfs_insert_inode_ref(trans, dest,
  8488. new_dentry->d_name.name,
  8489. new_dentry->d_name.len,
  8490. old_ino,
  8491. btrfs_ino(BTRFS_I(new_dir)),
  8492. old_idx);
  8493. if (ret)
  8494. goto out_fail;
  8495. }
  8496. /* And now for the dest. */
  8497. if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
  8498. /* force full log commit if subvolume involved. */
  8499. btrfs_set_log_full_commit(fs_info, trans);
  8500. } else {
  8501. btrfs_pin_log_trans(dest);
  8502. dest_log_pinned = true;
  8503. ret = btrfs_insert_inode_ref(trans, root,
  8504. old_dentry->d_name.name,
  8505. old_dentry->d_name.len,
  8506. new_ino,
  8507. btrfs_ino(BTRFS_I(old_dir)),
  8508. new_idx);
  8509. if (ret)
  8510. goto out_fail;
  8511. }
  8512. /* Update inode version and ctime/mtime. */
  8513. inode_inc_iversion(old_dir);
  8514. inode_inc_iversion(new_dir);
  8515. inode_inc_iversion(old_inode);
  8516. inode_inc_iversion(new_inode);
  8517. old_dir->i_ctime = old_dir->i_mtime = ctime;
  8518. new_dir->i_ctime = new_dir->i_mtime = ctime;
  8519. old_inode->i_ctime = ctime;
  8520. new_inode->i_ctime = ctime;
  8521. if (old_dentry->d_parent != new_dentry->d_parent) {
  8522. btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
  8523. BTRFS_I(old_inode), 1);
  8524. btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),
  8525. BTRFS_I(new_inode), 1);
  8526. }
  8527. /* src is a subvolume */
  8528. if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
  8529. root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
  8530. ret = btrfs_unlink_subvol(trans, root, old_dir,
  8531. root_objectid,
  8532. old_dentry->d_name.name,
  8533. old_dentry->d_name.len);
  8534. } else { /* src is an inode */
  8535. ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
  8536. BTRFS_I(old_dentry->d_inode),
  8537. old_dentry->d_name.name,
  8538. old_dentry->d_name.len);
  8539. if (!ret)
  8540. ret = btrfs_update_inode(trans, root, old_inode);
  8541. }
  8542. if (ret) {
  8543. btrfs_abort_transaction(trans, ret);
  8544. goto out_fail;
  8545. }
  8546. /* dest is a subvolume */
  8547. if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
  8548. root_objectid = BTRFS_I(new_inode)->root->root_key.objectid;
  8549. ret = btrfs_unlink_subvol(trans, dest, new_dir,
  8550. root_objectid,
  8551. new_dentry->d_name.name,
  8552. new_dentry->d_name.len);
  8553. } else { /* dest is an inode */
  8554. ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
  8555. BTRFS_I(new_dentry->d_inode),
  8556. new_dentry->d_name.name,
  8557. new_dentry->d_name.len);
  8558. if (!ret)
  8559. ret = btrfs_update_inode(trans, dest, new_inode);
  8560. }
  8561. if (ret) {
  8562. btrfs_abort_transaction(trans, ret);
  8563. goto out_fail;
  8564. }
  8565. ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
  8566. new_dentry->d_name.name,
  8567. new_dentry->d_name.len, 0, old_idx);
  8568. if (ret) {
  8569. btrfs_abort_transaction(trans, ret);
  8570. goto out_fail;
  8571. }
  8572. ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
  8573. old_dentry->d_name.name,
  8574. old_dentry->d_name.len, 0, new_idx);
  8575. if (ret) {
  8576. btrfs_abort_transaction(trans, ret);
  8577. goto out_fail;
  8578. }
  8579. if (old_inode->i_nlink == 1)
  8580. BTRFS_I(old_inode)->dir_index = old_idx;
  8581. if (new_inode->i_nlink == 1)
  8582. BTRFS_I(new_inode)->dir_index = new_idx;
  8583. if (root_log_pinned) {
  8584. parent = new_dentry->d_parent;
  8585. btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
  8586. parent);
  8587. btrfs_end_log_trans(root);
  8588. root_log_pinned = false;
  8589. }
  8590. if (dest_log_pinned) {
  8591. parent = old_dentry->d_parent;
  8592. btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir),
  8593. parent);
  8594. btrfs_end_log_trans(dest);
  8595. dest_log_pinned = false;
  8596. }
  8597. out_fail:
  8598. /*
  8599. * If we have pinned a log and an error happened, we unpin tasks
  8600. * trying to sync the log and force them to fallback to a transaction
  8601. * commit if the log currently contains any of the inodes involved in
  8602. * this rename operation (to ensure we do not persist a log with an
  8603. * inconsistent state for any of these inodes or leading to any
  8604. * inconsistencies when replayed). If the transaction was aborted, the
  8605. * abortion reason is propagated to userspace when attempting to commit
  8606. * the transaction. If the log does not contain any of these inodes, we
  8607. * allow the tasks to sync it.
  8608. */
  8609. if (ret && (root_log_pinned || dest_log_pinned)) {
  8610. if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
  8611. btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
  8612. btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
  8613. (new_inode &&
  8614. btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
  8615. btrfs_set_log_full_commit(fs_info, trans);
  8616. if (root_log_pinned) {
  8617. btrfs_end_log_trans(root);
  8618. root_log_pinned = false;
  8619. }
  8620. if (dest_log_pinned) {
  8621. btrfs_end_log_trans(dest);
  8622. dest_log_pinned = false;
  8623. }
  8624. }
  8625. ret = btrfs_end_transaction(trans);
  8626. out_notrans:
  8627. if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
  8628. up_read(&fs_info->subvol_sem);
  8629. if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
  8630. up_read(&fs_info->subvol_sem);
  8631. return ret;
  8632. }
  8633. static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans,
  8634. struct btrfs_root *root,
  8635. struct inode *dir,
  8636. struct dentry *dentry)
  8637. {
  8638. int ret;
  8639. struct inode *inode;
  8640. u64 objectid;
  8641. u64 index;
  8642. ret = btrfs_find_free_ino(root, &objectid);
  8643. if (ret)
  8644. return ret;
  8645. inode = btrfs_new_inode(trans, root, dir,
  8646. dentry->d_name.name,
  8647. dentry->d_name.len,
  8648. btrfs_ino(BTRFS_I(dir)),
  8649. objectid,
  8650. S_IFCHR | WHITEOUT_MODE,
  8651. &index);
  8652. if (IS_ERR(inode)) {
  8653. ret = PTR_ERR(inode);
  8654. return ret;
  8655. }
  8656. inode->i_op = &btrfs_special_inode_operations;
  8657. init_special_inode(inode, inode->i_mode,
  8658. WHITEOUT_DEV);
  8659. ret = btrfs_init_inode_security(trans, inode, dir,
  8660. &dentry->d_name);
  8661. if (ret)
  8662. goto out;
  8663. ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
  8664. BTRFS_I(inode), 0, index);
  8665. if (ret)
  8666. goto out;
  8667. ret = btrfs_update_inode(trans, root, inode);
  8668. out:
  8669. unlock_new_inode(inode);
  8670. if (ret)
  8671. inode_dec_link_count(inode);
  8672. iput(inode);
  8673. return ret;
  8674. }
  8675. static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry,
  8676. struct inode *new_dir, struct dentry *new_dentry,
  8677. unsigned int flags)
  8678. {
  8679. struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
  8680. struct btrfs_trans_handle *trans;
  8681. unsigned int trans_num_items;
  8682. struct btrfs_root *root = BTRFS_I(old_dir)->root;
  8683. struct btrfs_root *dest = BTRFS_I(new_dir)->root;
  8684. struct inode *new_inode = d_inode(new_dentry);
  8685. struct inode *old_inode = d_inode(old_dentry);
  8686. u64 index = 0;
  8687. u64 root_objectid;
  8688. int ret;
  8689. u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
  8690. bool log_pinned = false;
  8691. if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
  8692. return -EPERM;
  8693. /* we only allow rename subvolume link between subvolumes */
  8694. if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
  8695. return -EXDEV;
  8696. if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
  8697. (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))
  8698. return -ENOTEMPTY;
  8699. if (S_ISDIR(old_inode->i_mode) && new_inode &&
  8700. new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
  8701. return -ENOTEMPTY;
  8702. /* check for collisions, even if the name isn't there */
  8703. ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino,
  8704. new_dentry->d_name.name,
  8705. new_dentry->d_name.len);
  8706. if (ret) {
  8707. if (ret == -EEXIST) {
  8708. /* we shouldn't get
  8709. * eexist without a new_inode */
  8710. if (WARN_ON(!new_inode)) {
  8711. return ret;
  8712. }
  8713. } else {
  8714. /* maybe -EOVERFLOW */
  8715. return ret;
  8716. }
  8717. }
  8718. ret = 0;
  8719. /*
  8720. * we're using rename to replace one file with another. Start IO on it
  8721. * now so we don't add too much work to the end of the transaction
  8722. */
  8723. if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
  8724. filemap_flush(old_inode->i_mapping);
  8725. /* close the racy window with snapshot create/destroy ioctl */
  8726. if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
  8727. down_read(&fs_info->subvol_sem);
  8728. /*
  8729. * We want to reserve the absolute worst case amount of items. So if
  8730. * both inodes are subvols and we need to unlink them then that would
  8731. * require 4 item modifications, but if they are both normal inodes it
  8732. * would require 5 item modifications, so we'll assume they are normal
  8733. * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items
  8734. * should cover the worst case number of items we'll modify.
  8735. * If our rename has the whiteout flag, we need more 5 units for the
  8736. * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
  8737. * when selinux is enabled).
  8738. */
  8739. trans_num_items = 11;
  8740. if (flags & RENAME_WHITEOUT)
  8741. trans_num_items += 5;
  8742. trans = btrfs_start_transaction(root, trans_num_items);
  8743. if (IS_ERR(trans)) {
  8744. ret = PTR_ERR(trans);
  8745. goto out_notrans;
  8746. }
  8747. if (dest != root)
  8748. btrfs_record_root_in_trans(trans, dest);
  8749. ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);
  8750. if (ret)
  8751. goto out_fail;
  8752. BTRFS_I(old_inode)->dir_index = 0ULL;
  8753. if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
  8754. /* force full log commit if subvolume involved. */
  8755. btrfs_set_log_full_commit(fs_info, trans);
  8756. } else {
  8757. btrfs_pin_log_trans(root);
  8758. log_pinned = true;
  8759. ret = btrfs_insert_inode_ref(trans, dest,
  8760. new_dentry->d_name.name,
  8761. new_dentry->d_name.len,
  8762. old_ino,
  8763. btrfs_ino(BTRFS_I(new_dir)), index);
  8764. if (ret)
  8765. goto out_fail;
  8766. }
  8767. inode_inc_iversion(old_dir);
  8768. inode_inc_iversion(new_dir);
  8769. inode_inc_iversion(old_inode);
  8770. old_dir->i_ctime = old_dir->i_mtime =
  8771. new_dir->i_ctime = new_dir->i_mtime =
  8772. old_inode->i_ctime = current_time(old_dir);
  8773. if (old_dentry->d_parent != new_dentry->d_parent)
  8774. btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
  8775. BTRFS_I(old_inode), 1);
  8776. if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
  8777. root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
  8778. ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid,
  8779. old_dentry->d_name.name,
  8780. old_dentry->d_name.len);
  8781. } else {
  8782. ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
  8783. BTRFS_I(d_inode(old_dentry)),
  8784. old_dentry->d_name.name,
  8785. old_dentry->d_name.len);
  8786. if (!ret)
  8787. ret = btrfs_update_inode(trans, root, old_inode);
  8788. }
  8789. if (ret) {
  8790. btrfs_abort_transaction(trans, ret);
  8791. goto out_fail;
  8792. }
  8793. if (new_inode) {
  8794. inode_inc_iversion(new_inode);
  8795. new_inode->i_ctime = current_time(new_inode);
  8796. if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==
  8797. BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
  8798. root_objectid = BTRFS_I(new_inode)->location.objectid;
  8799. ret = btrfs_unlink_subvol(trans, dest, new_dir,
  8800. root_objectid,
  8801. new_dentry->d_name.name,
  8802. new_dentry->d_name.len);
  8803. BUG_ON(new_inode->i_nlink == 0);
  8804. } else {
  8805. ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
  8806. BTRFS_I(d_inode(new_dentry)),
  8807. new_dentry->d_name.name,
  8808. new_dentry->d_name.len);
  8809. }
  8810. if (!ret && new_inode->i_nlink == 0)
  8811. ret = btrfs_orphan_add(trans,
  8812. BTRFS_I(d_inode(new_dentry)));
  8813. if (ret) {
  8814. btrfs_abort_transaction(trans, ret);
  8815. goto out_fail;
  8816. }
  8817. }
  8818. ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
  8819. new_dentry->d_name.name,
  8820. new_dentry->d_name.len, 0, index);
  8821. if (ret) {
  8822. btrfs_abort_transaction(trans, ret);
  8823. goto out_fail;
  8824. }
  8825. if (old_inode->i_nlink == 1)
  8826. BTRFS_I(old_inode)->dir_index = index;
  8827. if (log_pinned) {
  8828. struct dentry *parent = new_dentry->d_parent;
  8829. btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
  8830. parent);
  8831. btrfs_end_log_trans(root);
  8832. log_pinned = false;
  8833. }
  8834. if (flags & RENAME_WHITEOUT) {
  8835. ret = btrfs_whiteout_for_rename(trans, root, old_dir,
  8836. old_dentry);
  8837. if (ret) {
  8838. btrfs_abort_transaction(trans, ret);
  8839. goto out_fail;
  8840. }
  8841. }
  8842. out_fail:
  8843. /*
  8844. * If we have pinned the log and an error happened, we unpin tasks
  8845. * trying to sync the log and force them to fallback to a transaction
  8846. * commit if the log currently contains any of the inodes involved in
  8847. * this rename operation (to ensure we do not persist a log with an
  8848. * inconsistent state for any of these inodes or leading to any
  8849. * inconsistencies when replayed). If the transaction was aborted, the
  8850. * abortion reason is propagated to userspace when attempting to commit
  8851. * the transaction. If the log does not contain any of these inodes, we
  8852. * allow the tasks to sync it.
  8853. */
  8854. if (ret && log_pinned) {
  8855. if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
  8856. btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
  8857. btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
  8858. (new_inode &&
  8859. btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
  8860. btrfs_set_log_full_commit(fs_info, trans);
  8861. btrfs_end_log_trans(root);
  8862. log_pinned = false;
  8863. }
  8864. btrfs_end_transaction(trans);
  8865. out_notrans:
  8866. if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
  8867. up_read(&fs_info->subvol_sem);
  8868. return ret;
  8869. }
  8870. static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry,
  8871. struct inode *new_dir, struct dentry *new_dentry,
  8872. unsigned int flags)
  8873. {
  8874. if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
  8875. return -EINVAL;
  8876. if (flags & RENAME_EXCHANGE)
  8877. return btrfs_rename_exchange(old_dir, old_dentry, new_dir,
  8878. new_dentry);
  8879. return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
  8880. }
  8881. static void btrfs_run_delalloc_work(struct btrfs_work *work)
  8882. {
  8883. struct btrfs_delalloc_work *delalloc_work;
  8884. struct inode *inode;
  8885. delalloc_work = container_of(work, struct btrfs_delalloc_work,
  8886. work);
  8887. inode = delalloc_work->inode;
  8888. filemap_flush(inode->i_mapping);
  8889. if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
  8890. &BTRFS_I(inode)->runtime_flags))
  8891. filemap_flush(inode->i_mapping);
  8892. if (delalloc_work->delay_iput)
  8893. btrfs_add_delayed_iput(inode);
  8894. else
  8895. iput(inode);
  8896. complete(&delalloc_work->completion);
  8897. }
  8898. struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode,
  8899. int delay_iput)
  8900. {
  8901. struct btrfs_delalloc_work *work;
  8902. work = kmalloc(sizeof(*work), GFP_NOFS);
  8903. if (!work)
  8904. return NULL;
  8905. init_completion(&work->completion);
  8906. INIT_LIST_HEAD(&work->list);
  8907. work->inode = inode;
  8908. work->delay_iput = delay_iput;
  8909. WARN_ON_ONCE(!inode);
  8910. btrfs_init_work(&work->work, btrfs_flush_delalloc_helper,
  8911. btrfs_run_delalloc_work, NULL, NULL);
  8912. return work;
  8913. }
  8914. void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work)
  8915. {
  8916. wait_for_completion(&work->completion);
  8917. kfree(work);
  8918. }
  8919. /*
  8920. * some fairly slow code that needs optimization. This walks the list
  8921. * of all the inodes with pending delalloc and forces them to disk.
  8922. */
  8923. static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput,
  8924. int nr)
  8925. {
  8926. struct btrfs_inode *binode;
  8927. struct inode *inode;
  8928. struct btrfs_delalloc_work *work, *next;
  8929. struct list_head works;
  8930. struct list_head splice;
  8931. int ret = 0;
  8932. INIT_LIST_HEAD(&works);
  8933. INIT_LIST_HEAD(&splice);
  8934. mutex_lock(&root->delalloc_mutex);
  8935. spin_lock(&root->delalloc_lock);
  8936. list_splice_init(&root->delalloc_inodes, &splice);
  8937. while (!list_empty(&splice)) {
  8938. binode = list_entry(splice.next, struct btrfs_inode,
  8939. delalloc_inodes);
  8940. list_move_tail(&binode->delalloc_inodes,
  8941. &root->delalloc_inodes);
  8942. inode = igrab(&binode->vfs_inode);
  8943. if (!inode) {
  8944. cond_resched_lock(&root->delalloc_lock);
  8945. continue;
  8946. }
  8947. spin_unlock(&root->delalloc_lock);
  8948. work = btrfs_alloc_delalloc_work(inode, delay_iput);
  8949. if (!work) {
  8950. if (delay_iput)
  8951. btrfs_add_delayed_iput(inode);
  8952. else
  8953. iput(inode);
  8954. ret = -ENOMEM;
  8955. goto out;
  8956. }
  8957. list_add_tail(&work->list, &works);
  8958. btrfs_queue_work(root->fs_info->flush_workers,
  8959. &work->work);
  8960. ret++;
  8961. if (nr != -1 && ret >= nr)
  8962. goto out;
  8963. cond_resched();
  8964. spin_lock(&root->delalloc_lock);
  8965. }
  8966. spin_unlock(&root->delalloc_lock);
  8967. out:
  8968. list_for_each_entry_safe(work, next, &works, list) {
  8969. list_del_init(&work->list);
  8970. btrfs_wait_and_free_delalloc_work(work);
  8971. }
  8972. if (!list_empty_careful(&splice)) {
  8973. spin_lock(&root->delalloc_lock);
  8974. list_splice_tail(&splice, &root->delalloc_inodes);
  8975. spin_unlock(&root->delalloc_lock);
  8976. }
  8977. mutex_unlock(&root->delalloc_mutex);
  8978. return ret;
  8979. }
  8980. int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput)
  8981. {
  8982. struct btrfs_fs_info *fs_info = root->fs_info;
  8983. int ret;
  8984. if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
  8985. return -EROFS;
  8986. ret = __start_delalloc_inodes(root, delay_iput, -1);
  8987. if (ret > 0)
  8988. ret = 0;
  8989. /*
  8990. * the filemap_flush will queue IO into the worker threads, but
  8991. * we have to make sure the IO is actually started and that
  8992. * ordered extents get created before we return
  8993. */
  8994. atomic_inc(&fs_info->async_submit_draining);
  8995. while (atomic_read(&fs_info->nr_async_submits) ||
  8996. atomic_read(&fs_info->async_delalloc_pages)) {
  8997. wait_event(fs_info->async_submit_wait,
  8998. (atomic_read(&fs_info->nr_async_submits) == 0 &&
  8999. atomic_read(&fs_info->async_delalloc_pages) == 0));
  9000. }
  9001. atomic_dec(&fs_info->async_submit_draining);
  9002. return ret;
  9003. }
  9004. int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput,
  9005. int nr)
  9006. {
  9007. struct btrfs_root *root;
  9008. struct list_head splice;
  9009. int ret;
  9010. if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
  9011. return -EROFS;
  9012. INIT_LIST_HEAD(&splice);
  9013. mutex_lock(&fs_info->delalloc_root_mutex);
  9014. spin_lock(&fs_info->delalloc_root_lock);
  9015. list_splice_init(&fs_info->delalloc_roots, &splice);
  9016. while (!list_empty(&splice) && nr) {
  9017. root = list_first_entry(&splice, struct btrfs_root,
  9018. delalloc_root);
  9019. root = btrfs_grab_fs_root(root);
  9020. BUG_ON(!root);
  9021. list_move_tail(&root->delalloc_root,
  9022. &fs_info->delalloc_roots);
  9023. spin_unlock(&fs_info->delalloc_root_lock);
  9024. ret = __start_delalloc_inodes(root, delay_iput, nr);
  9025. btrfs_put_fs_root(root);
  9026. if (ret < 0)
  9027. goto out;
  9028. if (nr != -1) {
  9029. nr -= ret;
  9030. WARN_ON(nr < 0);
  9031. }
  9032. spin_lock(&fs_info->delalloc_root_lock);
  9033. }
  9034. spin_unlock(&fs_info->delalloc_root_lock);
  9035. ret = 0;
  9036. atomic_inc(&fs_info->async_submit_draining);
  9037. while (atomic_read(&fs_info->nr_async_submits) ||
  9038. atomic_read(&fs_info->async_delalloc_pages)) {
  9039. wait_event(fs_info->async_submit_wait,
  9040. (atomic_read(&fs_info->nr_async_submits) == 0 &&
  9041. atomic_read(&fs_info->async_delalloc_pages) == 0));
  9042. }
  9043. atomic_dec(&fs_info->async_submit_draining);
  9044. out:
  9045. if (!list_empty_careful(&splice)) {
  9046. spin_lock(&fs_info->delalloc_root_lock);
  9047. list_splice_tail(&splice, &fs_info->delalloc_roots);
  9048. spin_unlock(&fs_info->delalloc_root_lock);
  9049. }
  9050. mutex_unlock(&fs_info->delalloc_root_mutex);
  9051. return ret;
  9052. }
  9053. static int btrfs_symlink(struct inode *dir, struct dentry *dentry,
  9054. const char *symname)
  9055. {
  9056. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  9057. struct btrfs_trans_handle *trans;
  9058. struct btrfs_root *root = BTRFS_I(dir)->root;
  9059. struct btrfs_path *path;
  9060. struct btrfs_key key;
  9061. struct inode *inode = NULL;
  9062. int err;
  9063. int drop_inode = 0;
  9064. u64 objectid;
  9065. u64 index = 0;
  9066. int name_len;
  9067. int datasize;
  9068. unsigned long ptr;
  9069. struct btrfs_file_extent_item *ei;
  9070. struct extent_buffer *leaf;
  9071. name_len = strlen(symname);
  9072. if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
  9073. return -ENAMETOOLONG;
  9074. /*
  9075. * 2 items for inode item and ref
  9076. * 2 items for dir items
  9077. * 1 item for updating parent inode item
  9078. * 1 item for the inline extent item
  9079. * 1 item for xattr if selinux is on
  9080. */
  9081. trans = btrfs_start_transaction(root, 7);
  9082. if (IS_ERR(trans))
  9083. return PTR_ERR(trans);
  9084. err = btrfs_find_free_ino(root, &objectid);
  9085. if (err)
  9086. goto out_unlock;
  9087. inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
  9088. dentry->d_name.len, btrfs_ino(BTRFS_I(dir)),
  9089. objectid, S_IFLNK|S_IRWXUGO, &index);
  9090. if (IS_ERR(inode)) {
  9091. err = PTR_ERR(inode);
  9092. goto out_unlock;
  9093. }
  9094. /*
  9095. * If the active LSM wants to access the inode during
  9096. * d_instantiate it needs these. Smack checks to see
  9097. * if the filesystem supports xattrs by looking at the
  9098. * ops vector.
  9099. */
  9100. inode->i_fop = &btrfs_file_operations;
  9101. inode->i_op = &btrfs_file_inode_operations;
  9102. inode->i_mapping->a_ops = &btrfs_aops;
  9103. BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
  9104. err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
  9105. if (err)
  9106. goto out_unlock_inode;
  9107. path = btrfs_alloc_path();
  9108. if (!path) {
  9109. err = -ENOMEM;
  9110. goto out_unlock_inode;
  9111. }
  9112. key.objectid = btrfs_ino(BTRFS_I(inode));
  9113. key.offset = 0;
  9114. key.type = BTRFS_EXTENT_DATA_KEY;
  9115. datasize = btrfs_file_extent_calc_inline_size(name_len);
  9116. err = btrfs_insert_empty_item(trans, root, path, &key,
  9117. datasize);
  9118. if (err) {
  9119. btrfs_free_path(path);
  9120. goto out_unlock_inode;
  9121. }
  9122. leaf = path->nodes[0];
  9123. ei = btrfs_item_ptr(leaf, path->slots[0],
  9124. struct btrfs_file_extent_item);
  9125. btrfs_set_file_extent_generation(leaf, ei, trans->transid);
  9126. btrfs_set_file_extent_type(leaf, ei,
  9127. BTRFS_FILE_EXTENT_INLINE);
  9128. btrfs_set_file_extent_encryption(leaf, ei, 0);
  9129. btrfs_set_file_extent_compression(leaf, ei, 0);
  9130. btrfs_set_file_extent_other_encoding(leaf, ei, 0);
  9131. btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
  9132. ptr = btrfs_file_extent_inline_start(ei);
  9133. write_extent_buffer(leaf, symname, ptr, name_len);
  9134. btrfs_mark_buffer_dirty(leaf);
  9135. btrfs_free_path(path);
  9136. inode->i_op = &btrfs_symlink_inode_operations;
  9137. inode_nohighmem(inode);
  9138. inode->i_mapping->a_ops = &btrfs_symlink_aops;
  9139. inode_set_bytes(inode, name_len);
  9140. btrfs_i_size_write(BTRFS_I(inode), name_len);
  9141. err = btrfs_update_inode(trans, root, inode);
  9142. /*
  9143. * Last step, add directory indexes for our symlink inode. This is the
  9144. * last step to avoid extra cleanup of these indexes if an error happens
  9145. * elsewhere above.
  9146. */
  9147. if (!err)
  9148. err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
  9149. BTRFS_I(inode), 0, index);
  9150. if (err) {
  9151. drop_inode = 1;
  9152. goto out_unlock_inode;
  9153. }
  9154. unlock_new_inode(inode);
  9155. d_instantiate(dentry, inode);
  9156. out_unlock:
  9157. btrfs_end_transaction(trans);
  9158. if (drop_inode) {
  9159. inode_dec_link_count(inode);
  9160. iput(inode);
  9161. }
  9162. btrfs_btree_balance_dirty(fs_info);
  9163. return err;
  9164. out_unlock_inode:
  9165. drop_inode = 1;
  9166. unlock_new_inode(inode);
  9167. goto out_unlock;
  9168. }
  9169. static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
  9170. u64 start, u64 num_bytes, u64 min_size,
  9171. loff_t actual_len, u64 *alloc_hint,
  9172. struct btrfs_trans_handle *trans)
  9173. {
  9174. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  9175. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  9176. struct extent_map *em;
  9177. struct btrfs_root *root = BTRFS_I(inode)->root;
  9178. struct btrfs_key ins;
  9179. u64 cur_offset = start;
  9180. u64 i_size;
  9181. u64 cur_bytes;
  9182. u64 last_alloc = (u64)-1;
  9183. int ret = 0;
  9184. bool own_trans = true;
  9185. u64 end = start + num_bytes - 1;
  9186. if (trans)
  9187. own_trans = false;
  9188. while (num_bytes > 0) {
  9189. if (own_trans) {
  9190. trans = btrfs_start_transaction(root, 3);
  9191. if (IS_ERR(trans)) {
  9192. ret = PTR_ERR(trans);
  9193. break;
  9194. }
  9195. }
  9196. cur_bytes = min_t(u64, num_bytes, SZ_256M);
  9197. cur_bytes = max(cur_bytes, min_size);
  9198. /*
  9199. * If we are severely fragmented we could end up with really
  9200. * small allocations, so if the allocator is returning small
  9201. * chunks lets make its job easier by only searching for those
  9202. * sized chunks.
  9203. */
  9204. cur_bytes = min(cur_bytes, last_alloc);
  9205. ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
  9206. min_size, 0, *alloc_hint, &ins, 1, 0);
  9207. if (ret) {
  9208. if (own_trans)
  9209. btrfs_end_transaction(trans);
  9210. break;
  9211. }
  9212. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  9213. last_alloc = ins.offset;
  9214. ret = insert_reserved_file_extent(trans, inode,
  9215. cur_offset, ins.objectid,
  9216. ins.offset, ins.offset,
  9217. ins.offset, 0, 0, 0,
  9218. BTRFS_FILE_EXTENT_PREALLOC);
  9219. if (ret) {
  9220. btrfs_free_reserved_extent(fs_info, ins.objectid,
  9221. ins.offset, 0);
  9222. btrfs_abort_transaction(trans, ret);
  9223. if (own_trans)
  9224. btrfs_end_transaction(trans);
  9225. break;
  9226. }
  9227. btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
  9228. cur_offset + ins.offset -1, 0);
  9229. em = alloc_extent_map();
  9230. if (!em) {
  9231. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  9232. &BTRFS_I(inode)->runtime_flags);
  9233. goto next;
  9234. }
  9235. em->start = cur_offset;
  9236. em->orig_start = cur_offset;
  9237. em->len = ins.offset;
  9238. em->block_start = ins.objectid;
  9239. em->block_len = ins.offset;
  9240. em->orig_block_len = ins.offset;
  9241. em->ram_bytes = ins.offset;
  9242. em->bdev = fs_info->fs_devices->latest_bdev;
  9243. set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
  9244. em->generation = trans->transid;
  9245. while (1) {
  9246. write_lock(&em_tree->lock);
  9247. ret = add_extent_mapping(em_tree, em, 1);
  9248. write_unlock(&em_tree->lock);
  9249. if (ret != -EEXIST)
  9250. break;
  9251. btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
  9252. cur_offset + ins.offset - 1,
  9253. 0);
  9254. }
  9255. free_extent_map(em);
  9256. next:
  9257. num_bytes -= ins.offset;
  9258. cur_offset += ins.offset;
  9259. *alloc_hint = ins.objectid + ins.offset;
  9260. inode_inc_iversion(inode);
  9261. inode->i_ctime = current_time(inode);
  9262. BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
  9263. if (!(mode & FALLOC_FL_KEEP_SIZE) &&
  9264. (actual_len > inode->i_size) &&
  9265. (cur_offset > inode->i_size)) {
  9266. if (cur_offset > actual_len)
  9267. i_size = actual_len;
  9268. else
  9269. i_size = cur_offset;
  9270. i_size_write(inode, i_size);
  9271. btrfs_ordered_update_i_size(inode, i_size, NULL);
  9272. }
  9273. ret = btrfs_update_inode(trans, root, inode);
  9274. if (ret) {
  9275. btrfs_abort_transaction(trans, ret);
  9276. if (own_trans)
  9277. btrfs_end_transaction(trans);
  9278. break;
  9279. }
  9280. if (own_trans)
  9281. btrfs_end_transaction(trans);
  9282. }
  9283. if (cur_offset < end)
  9284. btrfs_free_reserved_data_space(inode, cur_offset,
  9285. end - cur_offset + 1);
  9286. return ret;
  9287. }
  9288. int btrfs_prealloc_file_range(struct inode *inode, int mode,
  9289. u64 start, u64 num_bytes, u64 min_size,
  9290. loff_t actual_len, u64 *alloc_hint)
  9291. {
  9292. return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
  9293. min_size, actual_len, alloc_hint,
  9294. NULL);
  9295. }
  9296. int btrfs_prealloc_file_range_trans(struct inode *inode,
  9297. struct btrfs_trans_handle *trans, int mode,
  9298. u64 start, u64 num_bytes, u64 min_size,
  9299. loff_t actual_len, u64 *alloc_hint)
  9300. {
  9301. return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
  9302. min_size, actual_len, alloc_hint, trans);
  9303. }
  9304. static int btrfs_set_page_dirty(struct page *page)
  9305. {
  9306. return __set_page_dirty_nobuffers(page);
  9307. }
  9308. static int btrfs_permission(struct inode *inode, int mask)
  9309. {
  9310. struct btrfs_root *root = BTRFS_I(inode)->root;
  9311. umode_t mode = inode->i_mode;
  9312. if (mask & MAY_WRITE &&
  9313. (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
  9314. if (btrfs_root_readonly(root))
  9315. return -EROFS;
  9316. if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
  9317. return -EACCES;
  9318. }
  9319. return generic_permission(inode, mask);
  9320. }
  9321. static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
  9322. {
  9323. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  9324. struct btrfs_trans_handle *trans;
  9325. struct btrfs_root *root = BTRFS_I(dir)->root;
  9326. struct inode *inode = NULL;
  9327. u64 objectid;
  9328. u64 index;
  9329. int ret = 0;
  9330. /*
  9331. * 5 units required for adding orphan entry
  9332. */
  9333. trans = btrfs_start_transaction(root, 5);
  9334. if (IS_ERR(trans))
  9335. return PTR_ERR(trans);
  9336. ret = btrfs_find_free_ino(root, &objectid);
  9337. if (ret)
  9338. goto out;
  9339. inode = btrfs_new_inode(trans, root, dir, NULL, 0,
  9340. btrfs_ino(BTRFS_I(dir)), objectid, mode, &index);
  9341. if (IS_ERR(inode)) {
  9342. ret = PTR_ERR(inode);
  9343. inode = NULL;
  9344. goto out;
  9345. }
  9346. inode->i_fop = &btrfs_file_operations;
  9347. inode->i_op = &btrfs_file_inode_operations;
  9348. inode->i_mapping->a_ops = &btrfs_aops;
  9349. BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
  9350. ret = btrfs_init_inode_security(trans, inode, dir, NULL);
  9351. if (ret)
  9352. goto out_inode;
  9353. ret = btrfs_update_inode(trans, root, inode);
  9354. if (ret)
  9355. goto out_inode;
  9356. ret = btrfs_orphan_add(trans, BTRFS_I(inode));
  9357. if (ret)
  9358. goto out_inode;
  9359. /*
  9360. * We set number of links to 0 in btrfs_new_inode(), and here we set
  9361. * it to 1 because d_tmpfile() will issue a warning if the count is 0,
  9362. * through:
  9363. *
  9364. * d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
  9365. */
  9366. set_nlink(inode, 1);
  9367. unlock_new_inode(inode);
  9368. d_tmpfile(dentry, inode);
  9369. mark_inode_dirty(inode);
  9370. out:
  9371. btrfs_end_transaction(trans);
  9372. if (ret)
  9373. iput(inode);
  9374. btrfs_balance_delayed_items(fs_info);
  9375. btrfs_btree_balance_dirty(fs_info);
  9376. return ret;
  9377. out_inode:
  9378. unlock_new_inode(inode);
  9379. goto out;
  9380. }
  9381. __attribute__((const))
  9382. static int btrfs_readpage_io_failed_hook(struct page *page, int failed_mirror)
  9383. {
  9384. return -EAGAIN;
  9385. }
  9386. static struct btrfs_fs_info *iotree_fs_info(void *private_data)
  9387. {
  9388. struct inode *inode = private_data;
  9389. return btrfs_sb(inode->i_sb);
  9390. }
  9391. static void btrfs_check_extent_io_range(void *private_data, const char *caller,
  9392. u64 start, u64 end)
  9393. {
  9394. struct inode *inode = private_data;
  9395. u64 isize;
  9396. isize = i_size_read(inode);
  9397. if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
  9398. btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
  9399. "%s: ino %llu isize %llu odd range [%llu,%llu]",
  9400. caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
  9401. }
  9402. }
  9403. void btrfs_set_range_writeback(void *private_data, u64 start, u64 end)
  9404. {
  9405. struct inode *inode = private_data;
  9406. unsigned long index = start >> PAGE_SHIFT;
  9407. unsigned long end_index = end >> PAGE_SHIFT;
  9408. struct page *page;
  9409. while (index <= end_index) {
  9410. page = find_get_page(inode->i_mapping, index);
  9411. ASSERT(page); /* Pages should be in the extent_io_tree */
  9412. set_page_writeback(page);
  9413. put_page(page);
  9414. index++;
  9415. }
  9416. }
  9417. static const struct inode_operations btrfs_dir_inode_operations = {
  9418. .getattr = btrfs_getattr,
  9419. .lookup = btrfs_lookup,
  9420. .create = btrfs_create,
  9421. .unlink = btrfs_unlink,
  9422. .link = btrfs_link,
  9423. .mkdir = btrfs_mkdir,
  9424. .rmdir = btrfs_rmdir,
  9425. .rename = btrfs_rename2,
  9426. .symlink = btrfs_symlink,
  9427. .setattr = btrfs_setattr,
  9428. .mknod = btrfs_mknod,
  9429. .listxattr = btrfs_listxattr,
  9430. .permission = btrfs_permission,
  9431. .get_acl = btrfs_get_acl,
  9432. .set_acl = btrfs_set_acl,
  9433. .update_time = btrfs_update_time,
  9434. .tmpfile = btrfs_tmpfile,
  9435. };
  9436. static const struct inode_operations btrfs_dir_ro_inode_operations = {
  9437. .lookup = btrfs_lookup,
  9438. .permission = btrfs_permission,
  9439. .update_time = btrfs_update_time,
  9440. };
  9441. static const struct file_operations btrfs_dir_file_operations = {
  9442. .llseek = generic_file_llseek,
  9443. .read = generic_read_dir,
  9444. .iterate_shared = btrfs_real_readdir,
  9445. .unlocked_ioctl = btrfs_ioctl,
  9446. #ifdef CONFIG_COMPAT
  9447. .compat_ioctl = btrfs_compat_ioctl,
  9448. #endif
  9449. .release = btrfs_release_file,
  9450. .fsync = btrfs_sync_file,
  9451. };
  9452. static const struct extent_io_ops btrfs_extent_io_ops = {
  9453. /* mandatory callbacks */
  9454. .submit_bio_hook = btrfs_submit_bio_hook,
  9455. .readpage_end_io_hook = btrfs_readpage_end_io_hook,
  9456. .merge_bio_hook = btrfs_merge_bio_hook,
  9457. .readpage_io_failed_hook = btrfs_readpage_io_failed_hook,
  9458. .tree_fs_info = iotree_fs_info,
  9459. .set_range_writeback = btrfs_set_range_writeback,
  9460. /* optional callbacks */
  9461. .fill_delalloc = run_delalloc_range,
  9462. .writepage_end_io_hook = btrfs_writepage_end_io_hook,
  9463. .writepage_start_hook = btrfs_writepage_start_hook,
  9464. .set_bit_hook = btrfs_set_bit_hook,
  9465. .clear_bit_hook = btrfs_clear_bit_hook,
  9466. .merge_extent_hook = btrfs_merge_extent_hook,
  9467. .split_extent_hook = btrfs_split_extent_hook,
  9468. .check_extent_io_range = btrfs_check_extent_io_range,
  9469. };
  9470. /*
  9471. * btrfs doesn't support the bmap operation because swapfiles
  9472. * use bmap to make a mapping of extents in the file. They assume
  9473. * these extents won't change over the life of the file and they
  9474. * use the bmap result to do IO directly to the drive.
  9475. *
  9476. * the btrfs bmap call would return logical addresses that aren't
  9477. * suitable for IO and they also will change frequently as COW
  9478. * operations happen. So, swapfile + btrfs == corruption.
  9479. *
  9480. * For now we're avoiding this by dropping bmap.
  9481. */
  9482. static const struct address_space_operations btrfs_aops = {
  9483. .readpage = btrfs_readpage,
  9484. .writepage = btrfs_writepage,
  9485. .writepages = btrfs_writepages,
  9486. .readpages = btrfs_readpages,
  9487. .direct_IO = btrfs_direct_IO,
  9488. .invalidatepage = btrfs_invalidatepage,
  9489. .releasepage = btrfs_releasepage,
  9490. .set_page_dirty = btrfs_set_page_dirty,
  9491. .error_remove_page = generic_error_remove_page,
  9492. };
  9493. static const struct address_space_operations btrfs_symlink_aops = {
  9494. .readpage = btrfs_readpage,
  9495. .writepage = btrfs_writepage,
  9496. .invalidatepage = btrfs_invalidatepage,
  9497. .releasepage = btrfs_releasepage,
  9498. };
  9499. static const struct inode_operations btrfs_file_inode_operations = {
  9500. .getattr = btrfs_getattr,
  9501. .setattr = btrfs_setattr,
  9502. .listxattr = btrfs_listxattr,
  9503. .permission = btrfs_permission,
  9504. .fiemap = btrfs_fiemap,
  9505. .get_acl = btrfs_get_acl,
  9506. .set_acl = btrfs_set_acl,
  9507. .update_time = btrfs_update_time,
  9508. };
  9509. static const struct inode_operations btrfs_special_inode_operations = {
  9510. .getattr = btrfs_getattr,
  9511. .setattr = btrfs_setattr,
  9512. .permission = btrfs_permission,
  9513. .listxattr = btrfs_listxattr,
  9514. .get_acl = btrfs_get_acl,
  9515. .set_acl = btrfs_set_acl,
  9516. .update_time = btrfs_update_time,
  9517. };
  9518. static const struct inode_operations btrfs_symlink_inode_operations = {
  9519. .get_link = page_get_link,
  9520. .getattr = btrfs_getattr,
  9521. .setattr = btrfs_setattr,
  9522. .permission = btrfs_permission,
  9523. .listxattr = btrfs_listxattr,
  9524. .update_time = btrfs_update_time,
  9525. };
  9526. const struct dentry_operations btrfs_dentry_operations = {
  9527. .d_delete = btrfs_dentry_delete,
  9528. .d_release = btrfs_dentry_release,
  9529. };