memory.c 114 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280
  1. /*
  2. * linux/mm/memory.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. */
  6. /*
  7. * demand-loading started 01.12.91 - seems it is high on the list of
  8. * things wanted, and it should be easy to implement. - Linus
  9. */
  10. /*
  11. * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
  12. * pages started 02.12.91, seems to work. - Linus.
  13. *
  14. * Tested sharing by executing about 30 /bin/sh: under the old kernel it
  15. * would have taken more than the 6M I have free, but it worked well as
  16. * far as I could see.
  17. *
  18. * Also corrected some "invalidate()"s - I wasn't doing enough of them.
  19. */
  20. /*
  21. * Real VM (paging to/from disk) started 18.12.91. Much more work and
  22. * thought has to go into this. Oh, well..
  23. * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
  24. * Found it. Everything seems to work now.
  25. * 20.12.91 - Ok, making the swap-device changeable like the root.
  26. */
  27. /*
  28. * 05.04.94 - Multi-page memory management added for v1.1.
  29. * Idea by Alex Bligh (alex@cconcepts.co.uk)
  30. *
  31. * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
  32. * (Gerhard.Wichert@pdb.siemens.de)
  33. *
  34. * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
  35. */
  36. #include <linux/kernel_stat.h>
  37. #include <linux/mm.h>
  38. #include <linux/sched/mm.h>
  39. #include <linux/sched/coredump.h>
  40. #include <linux/sched/numa_balancing.h>
  41. #include <linux/sched/task.h>
  42. #include <linux/hugetlb.h>
  43. #include <linux/mman.h>
  44. #include <linux/swap.h>
  45. #include <linux/highmem.h>
  46. #include <linux/pagemap.h>
  47. #include <linux/ksm.h>
  48. #include <linux/rmap.h>
  49. #include <linux/export.h>
  50. #include <linux/delayacct.h>
  51. #include <linux/init.h>
  52. #include <linux/pfn_t.h>
  53. #include <linux/writeback.h>
  54. #include <linux/memcontrol.h>
  55. #include <linux/mmu_notifier.h>
  56. #include <linux/kallsyms.h>
  57. #include <linux/swapops.h>
  58. #include <linux/elf.h>
  59. #include <linux/gfp.h>
  60. #include <linux/migrate.h>
  61. #include <linux/string.h>
  62. #include <linux/dma-debug.h>
  63. #include <linux/debugfs.h>
  64. #include <linux/userfaultfd_k.h>
  65. #include <linux/dax.h>
  66. #include <asm/io.h>
  67. #include <asm/mmu_context.h>
  68. #include <asm/pgalloc.h>
  69. #include <linux/uaccess.h>
  70. #include <asm/tlb.h>
  71. #include <asm/tlbflush.h>
  72. #include <asm/pgtable.h>
  73. #include "internal.h"
  74. #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
  75. #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
  76. #endif
  77. #ifndef CONFIG_NEED_MULTIPLE_NODES
  78. /* use the per-pgdat data instead for discontigmem - mbligh */
  79. unsigned long max_mapnr;
  80. EXPORT_SYMBOL(max_mapnr);
  81. struct page *mem_map;
  82. EXPORT_SYMBOL(mem_map);
  83. #endif
  84. /*
  85. * A number of key systems in x86 including ioremap() rely on the assumption
  86. * that high_memory defines the upper bound on direct map memory, then end
  87. * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
  88. * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
  89. * and ZONE_HIGHMEM.
  90. */
  91. void *high_memory;
  92. EXPORT_SYMBOL(high_memory);
  93. /*
  94. * Randomize the address space (stacks, mmaps, brk, etc.).
  95. *
  96. * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
  97. * as ancient (libc5 based) binaries can segfault. )
  98. */
  99. int randomize_va_space __read_mostly =
  100. #ifdef CONFIG_COMPAT_BRK
  101. 1;
  102. #else
  103. 2;
  104. #endif
  105. static int __init disable_randmaps(char *s)
  106. {
  107. randomize_va_space = 0;
  108. return 1;
  109. }
  110. __setup("norandmaps", disable_randmaps);
  111. unsigned long zero_pfn __read_mostly;
  112. EXPORT_SYMBOL(zero_pfn);
  113. unsigned long highest_memmap_pfn __read_mostly;
  114. /*
  115. * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
  116. */
  117. static int __init init_zero_pfn(void)
  118. {
  119. zero_pfn = page_to_pfn(ZERO_PAGE(0));
  120. return 0;
  121. }
  122. core_initcall(init_zero_pfn);
  123. #if defined(SPLIT_RSS_COUNTING)
  124. void sync_mm_rss(struct mm_struct *mm)
  125. {
  126. int i;
  127. for (i = 0; i < NR_MM_COUNTERS; i++) {
  128. if (current->rss_stat.count[i]) {
  129. add_mm_counter(mm, i, current->rss_stat.count[i]);
  130. current->rss_stat.count[i] = 0;
  131. }
  132. }
  133. current->rss_stat.events = 0;
  134. }
  135. static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
  136. {
  137. struct task_struct *task = current;
  138. if (likely(task->mm == mm))
  139. task->rss_stat.count[member] += val;
  140. else
  141. add_mm_counter(mm, member, val);
  142. }
  143. #define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)
  144. #define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1)
  145. /* sync counter once per 64 page faults */
  146. #define TASK_RSS_EVENTS_THRESH (64)
  147. static void check_sync_rss_stat(struct task_struct *task)
  148. {
  149. if (unlikely(task != current))
  150. return;
  151. if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))
  152. sync_mm_rss(task->mm);
  153. }
  154. #else /* SPLIT_RSS_COUNTING */
  155. #define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)
  156. #define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member)
  157. static void check_sync_rss_stat(struct task_struct *task)
  158. {
  159. }
  160. #endif /* SPLIT_RSS_COUNTING */
  161. #ifdef HAVE_GENERIC_MMU_GATHER
  162. static bool tlb_next_batch(struct mmu_gather *tlb)
  163. {
  164. struct mmu_gather_batch *batch;
  165. batch = tlb->active;
  166. if (batch->next) {
  167. tlb->active = batch->next;
  168. return true;
  169. }
  170. if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
  171. return false;
  172. batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
  173. if (!batch)
  174. return false;
  175. tlb->batch_count++;
  176. batch->next = NULL;
  177. batch->nr = 0;
  178. batch->max = MAX_GATHER_BATCH;
  179. tlb->active->next = batch;
  180. tlb->active = batch;
  181. return true;
  182. }
  183. /* tlb_gather_mmu
  184. * Called to initialize an (on-stack) mmu_gather structure for page-table
  185. * tear-down from @mm. The @fullmm argument is used when @mm is without
  186. * users and we're going to destroy the full address space (exit/execve).
  187. */
  188. void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)
  189. {
  190. tlb->mm = mm;
  191. /* Is it from 0 to ~0? */
  192. tlb->fullmm = !(start | (end+1));
  193. tlb->need_flush_all = 0;
  194. tlb->local.next = NULL;
  195. tlb->local.nr = 0;
  196. tlb->local.max = ARRAY_SIZE(tlb->__pages);
  197. tlb->active = &tlb->local;
  198. tlb->batch_count = 0;
  199. #ifdef CONFIG_HAVE_RCU_TABLE_FREE
  200. tlb->batch = NULL;
  201. #endif
  202. tlb->page_size = 0;
  203. __tlb_reset_range(tlb);
  204. }
  205. static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
  206. {
  207. if (!tlb->end)
  208. return;
  209. tlb_flush(tlb);
  210. mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end);
  211. #ifdef CONFIG_HAVE_RCU_TABLE_FREE
  212. tlb_table_flush(tlb);
  213. #endif
  214. __tlb_reset_range(tlb);
  215. }
  216. static void tlb_flush_mmu_free(struct mmu_gather *tlb)
  217. {
  218. struct mmu_gather_batch *batch;
  219. for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
  220. free_pages_and_swap_cache(batch->pages, batch->nr);
  221. batch->nr = 0;
  222. }
  223. tlb->active = &tlb->local;
  224. }
  225. void tlb_flush_mmu(struct mmu_gather *tlb)
  226. {
  227. tlb_flush_mmu_tlbonly(tlb);
  228. tlb_flush_mmu_free(tlb);
  229. }
  230. /* tlb_finish_mmu
  231. * Called at the end of the shootdown operation to free up any resources
  232. * that were required.
  233. */
  234. void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
  235. {
  236. struct mmu_gather_batch *batch, *next;
  237. tlb_flush_mmu(tlb);
  238. /* keep the page table cache within bounds */
  239. check_pgt_cache();
  240. for (batch = tlb->local.next; batch; batch = next) {
  241. next = batch->next;
  242. free_pages((unsigned long)batch, 0);
  243. }
  244. tlb->local.next = NULL;
  245. }
  246. /* __tlb_remove_page
  247. * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while
  248. * handling the additional races in SMP caused by other CPUs caching valid
  249. * mappings in their TLBs. Returns the number of free page slots left.
  250. * When out of page slots we must call tlb_flush_mmu().
  251. *returns true if the caller should flush.
  252. */
  253. bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
  254. {
  255. struct mmu_gather_batch *batch;
  256. VM_BUG_ON(!tlb->end);
  257. VM_WARN_ON(tlb->page_size != page_size);
  258. batch = tlb->active;
  259. /*
  260. * Add the page and check if we are full. If so
  261. * force a flush.
  262. */
  263. batch->pages[batch->nr++] = page;
  264. if (batch->nr == batch->max) {
  265. if (!tlb_next_batch(tlb))
  266. return true;
  267. batch = tlb->active;
  268. }
  269. VM_BUG_ON_PAGE(batch->nr > batch->max, page);
  270. return false;
  271. }
  272. #endif /* HAVE_GENERIC_MMU_GATHER */
  273. #ifdef CONFIG_HAVE_RCU_TABLE_FREE
  274. /*
  275. * See the comment near struct mmu_table_batch.
  276. */
  277. static void tlb_remove_table_smp_sync(void *arg)
  278. {
  279. /* Simply deliver the interrupt */
  280. }
  281. static void tlb_remove_table_one(void *table)
  282. {
  283. /*
  284. * This isn't an RCU grace period and hence the page-tables cannot be
  285. * assumed to be actually RCU-freed.
  286. *
  287. * It is however sufficient for software page-table walkers that rely on
  288. * IRQ disabling. See the comment near struct mmu_table_batch.
  289. */
  290. smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
  291. __tlb_remove_table(table);
  292. }
  293. static void tlb_remove_table_rcu(struct rcu_head *head)
  294. {
  295. struct mmu_table_batch *batch;
  296. int i;
  297. batch = container_of(head, struct mmu_table_batch, rcu);
  298. for (i = 0; i < batch->nr; i++)
  299. __tlb_remove_table(batch->tables[i]);
  300. free_page((unsigned long)batch);
  301. }
  302. void tlb_table_flush(struct mmu_gather *tlb)
  303. {
  304. struct mmu_table_batch **batch = &tlb->batch;
  305. if (*batch) {
  306. call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
  307. *batch = NULL;
  308. }
  309. }
  310. void tlb_remove_table(struct mmu_gather *tlb, void *table)
  311. {
  312. struct mmu_table_batch **batch = &tlb->batch;
  313. /*
  314. * When there's less then two users of this mm there cannot be a
  315. * concurrent page-table walk.
  316. */
  317. if (atomic_read(&tlb->mm->mm_users) < 2) {
  318. __tlb_remove_table(table);
  319. return;
  320. }
  321. if (*batch == NULL) {
  322. *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
  323. if (*batch == NULL) {
  324. tlb_remove_table_one(table);
  325. return;
  326. }
  327. (*batch)->nr = 0;
  328. }
  329. (*batch)->tables[(*batch)->nr++] = table;
  330. if ((*batch)->nr == MAX_TABLE_BATCH)
  331. tlb_table_flush(tlb);
  332. }
  333. #endif /* CONFIG_HAVE_RCU_TABLE_FREE */
  334. /*
  335. * Note: this doesn't free the actual pages themselves. That
  336. * has been handled earlier when unmapping all the memory regions.
  337. */
  338. static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
  339. unsigned long addr)
  340. {
  341. pgtable_t token = pmd_pgtable(*pmd);
  342. pmd_clear(pmd);
  343. pte_free_tlb(tlb, token, addr);
  344. atomic_long_dec(&tlb->mm->nr_ptes);
  345. }
  346. static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
  347. unsigned long addr, unsigned long end,
  348. unsigned long floor, unsigned long ceiling)
  349. {
  350. pmd_t *pmd;
  351. unsigned long next;
  352. unsigned long start;
  353. start = addr;
  354. pmd = pmd_offset(pud, addr);
  355. do {
  356. next = pmd_addr_end(addr, end);
  357. if (pmd_none_or_clear_bad(pmd))
  358. continue;
  359. free_pte_range(tlb, pmd, addr);
  360. } while (pmd++, addr = next, addr != end);
  361. start &= PUD_MASK;
  362. if (start < floor)
  363. return;
  364. if (ceiling) {
  365. ceiling &= PUD_MASK;
  366. if (!ceiling)
  367. return;
  368. }
  369. if (end - 1 > ceiling - 1)
  370. return;
  371. pmd = pmd_offset(pud, start);
  372. pud_clear(pud);
  373. pmd_free_tlb(tlb, pmd, start);
  374. mm_dec_nr_pmds(tlb->mm);
  375. }
  376. static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
  377. unsigned long addr, unsigned long end,
  378. unsigned long floor, unsigned long ceiling)
  379. {
  380. pud_t *pud;
  381. unsigned long next;
  382. unsigned long start;
  383. start = addr;
  384. pud = pud_offset(pgd, addr);
  385. do {
  386. next = pud_addr_end(addr, end);
  387. if (pud_none_or_clear_bad(pud))
  388. continue;
  389. free_pmd_range(tlb, pud, addr, next, floor, ceiling);
  390. } while (pud++, addr = next, addr != end);
  391. start &= PGDIR_MASK;
  392. if (start < floor)
  393. return;
  394. if (ceiling) {
  395. ceiling &= PGDIR_MASK;
  396. if (!ceiling)
  397. return;
  398. }
  399. if (end - 1 > ceiling - 1)
  400. return;
  401. pud = pud_offset(pgd, start);
  402. pgd_clear(pgd);
  403. pud_free_tlb(tlb, pud, start);
  404. }
  405. /*
  406. * This function frees user-level page tables of a process.
  407. */
  408. void free_pgd_range(struct mmu_gather *tlb,
  409. unsigned long addr, unsigned long end,
  410. unsigned long floor, unsigned long ceiling)
  411. {
  412. pgd_t *pgd;
  413. unsigned long next;
  414. /*
  415. * The next few lines have given us lots of grief...
  416. *
  417. * Why are we testing PMD* at this top level? Because often
  418. * there will be no work to do at all, and we'd prefer not to
  419. * go all the way down to the bottom just to discover that.
  420. *
  421. * Why all these "- 1"s? Because 0 represents both the bottom
  422. * of the address space and the top of it (using -1 for the
  423. * top wouldn't help much: the masks would do the wrong thing).
  424. * The rule is that addr 0 and floor 0 refer to the bottom of
  425. * the address space, but end 0 and ceiling 0 refer to the top
  426. * Comparisons need to use "end - 1" and "ceiling - 1" (though
  427. * that end 0 case should be mythical).
  428. *
  429. * Wherever addr is brought up or ceiling brought down, we must
  430. * be careful to reject "the opposite 0" before it confuses the
  431. * subsequent tests. But what about where end is brought down
  432. * by PMD_SIZE below? no, end can't go down to 0 there.
  433. *
  434. * Whereas we round start (addr) and ceiling down, by different
  435. * masks at different levels, in order to test whether a table
  436. * now has no other vmas using it, so can be freed, we don't
  437. * bother to round floor or end up - the tests don't need that.
  438. */
  439. addr &= PMD_MASK;
  440. if (addr < floor) {
  441. addr += PMD_SIZE;
  442. if (!addr)
  443. return;
  444. }
  445. if (ceiling) {
  446. ceiling &= PMD_MASK;
  447. if (!ceiling)
  448. return;
  449. }
  450. if (end - 1 > ceiling - 1)
  451. end -= PMD_SIZE;
  452. if (addr > end - 1)
  453. return;
  454. /*
  455. * We add page table cache pages with PAGE_SIZE,
  456. * (see pte_free_tlb()), flush the tlb if we need
  457. */
  458. tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
  459. pgd = pgd_offset(tlb->mm, addr);
  460. do {
  461. next = pgd_addr_end(addr, end);
  462. if (pgd_none_or_clear_bad(pgd))
  463. continue;
  464. free_pud_range(tlb, pgd, addr, next, floor, ceiling);
  465. } while (pgd++, addr = next, addr != end);
  466. }
  467. void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
  468. unsigned long floor, unsigned long ceiling)
  469. {
  470. while (vma) {
  471. struct vm_area_struct *next = vma->vm_next;
  472. unsigned long addr = vma->vm_start;
  473. /*
  474. * Hide vma from rmap and truncate_pagecache before freeing
  475. * pgtables
  476. */
  477. unlink_anon_vmas(vma);
  478. unlink_file_vma(vma);
  479. if (is_vm_hugetlb_page(vma)) {
  480. hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
  481. floor, next ? next->vm_start : ceiling);
  482. } else {
  483. /*
  484. * Optimization: gather nearby vmas into one call down
  485. */
  486. while (next && next->vm_start <= vma->vm_end + PMD_SIZE
  487. && !is_vm_hugetlb_page(next)) {
  488. vma = next;
  489. next = vma->vm_next;
  490. unlink_anon_vmas(vma);
  491. unlink_file_vma(vma);
  492. }
  493. free_pgd_range(tlb, addr, vma->vm_end,
  494. floor, next ? next->vm_start : ceiling);
  495. }
  496. vma = next;
  497. }
  498. }
  499. int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
  500. {
  501. spinlock_t *ptl;
  502. pgtable_t new = pte_alloc_one(mm, address);
  503. if (!new)
  504. return -ENOMEM;
  505. /*
  506. * Ensure all pte setup (eg. pte page lock and page clearing) are
  507. * visible before the pte is made visible to other CPUs by being
  508. * put into page tables.
  509. *
  510. * The other side of the story is the pointer chasing in the page
  511. * table walking code (when walking the page table without locking;
  512. * ie. most of the time). Fortunately, these data accesses consist
  513. * of a chain of data-dependent loads, meaning most CPUs (alpha
  514. * being the notable exception) will already guarantee loads are
  515. * seen in-order. See the alpha page table accessors for the
  516. * smp_read_barrier_depends() barriers in page table walking code.
  517. */
  518. smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
  519. ptl = pmd_lock(mm, pmd);
  520. if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
  521. atomic_long_inc(&mm->nr_ptes);
  522. pmd_populate(mm, pmd, new);
  523. new = NULL;
  524. }
  525. spin_unlock(ptl);
  526. if (new)
  527. pte_free(mm, new);
  528. return 0;
  529. }
  530. int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
  531. {
  532. pte_t *new = pte_alloc_one_kernel(&init_mm, address);
  533. if (!new)
  534. return -ENOMEM;
  535. smp_wmb(); /* See comment in __pte_alloc */
  536. spin_lock(&init_mm.page_table_lock);
  537. if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
  538. pmd_populate_kernel(&init_mm, pmd, new);
  539. new = NULL;
  540. }
  541. spin_unlock(&init_mm.page_table_lock);
  542. if (new)
  543. pte_free_kernel(&init_mm, new);
  544. return 0;
  545. }
  546. static inline void init_rss_vec(int *rss)
  547. {
  548. memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
  549. }
  550. static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
  551. {
  552. int i;
  553. if (current->mm == mm)
  554. sync_mm_rss(mm);
  555. for (i = 0; i < NR_MM_COUNTERS; i++)
  556. if (rss[i])
  557. add_mm_counter(mm, i, rss[i]);
  558. }
  559. /*
  560. * This function is called to print an error when a bad pte
  561. * is found. For example, we might have a PFN-mapped pte in
  562. * a region that doesn't allow it.
  563. *
  564. * The calling function must still handle the error.
  565. */
  566. static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
  567. pte_t pte, struct page *page)
  568. {
  569. pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
  570. pud_t *pud = pud_offset(pgd, addr);
  571. pmd_t *pmd = pmd_offset(pud, addr);
  572. struct address_space *mapping;
  573. pgoff_t index;
  574. static unsigned long resume;
  575. static unsigned long nr_shown;
  576. static unsigned long nr_unshown;
  577. /*
  578. * Allow a burst of 60 reports, then keep quiet for that minute;
  579. * or allow a steady drip of one report per second.
  580. */
  581. if (nr_shown == 60) {
  582. if (time_before(jiffies, resume)) {
  583. nr_unshown++;
  584. return;
  585. }
  586. if (nr_unshown) {
  587. pr_alert("BUG: Bad page map: %lu messages suppressed\n",
  588. nr_unshown);
  589. nr_unshown = 0;
  590. }
  591. nr_shown = 0;
  592. }
  593. if (nr_shown++ == 0)
  594. resume = jiffies + 60 * HZ;
  595. mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
  596. index = linear_page_index(vma, addr);
  597. pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
  598. current->comm,
  599. (long long)pte_val(pte), (long long)pmd_val(*pmd));
  600. if (page)
  601. dump_page(page, "bad pte");
  602. pr_alert("addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
  603. (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
  604. /*
  605. * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y
  606. */
  607. pr_alert("file:%pD fault:%pf mmap:%pf readpage:%pf\n",
  608. vma->vm_file,
  609. vma->vm_ops ? vma->vm_ops->fault : NULL,
  610. vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
  611. mapping ? mapping->a_ops->readpage : NULL);
  612. dump_stack();
  613. add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  614. }
  615. /*
  616. * vm_normal_page -- This function gets the "struct page" associated with a pte.
  617. *
  618. * "Special" mappings do not wish to be associated with a "struct page" (either
  619. * it doesn't exist, or it exists but they don't want to touch it). In this
  620. * case, NULL is returned here. "Normal" mappings do have a struct page.
  621. *
  622. * There are 2 broad cases. Firstly, an architecture may define a pte_special()
  623. * pte bit, in which case this function is trivial. Secondly, an architecture
  624. * may not have a spare pte bit, which requires a more complicated scheme,
  625. * described below.
  626. *
  627. * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
  628. * special mapping (even if there are underlying and valid "struct pages").
  629. * COWed pages of a VM_PFNMAP are always normal.
  630. *
  631. * The way we recognize COWed pages within VM_PFNMAP mappings is through the
  632. * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
  633. * set, and the vm_pgoff will point to the first PFN mapped: thus every special
  634. * mapping will always honor the rule
  635. *
  636. * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
  637. *
  638. * And for normal mappings this is false.
  639. *
  640. * This restricts such mappings to be a linear translation from virtual address
  641. * to pfn. To get around this restriction, we allow arbitrary mappings so long
  642. * as the vma is not a COW mapping; in that case, we know that all ptes are
  643. * special (because none can have been COWed).
  644. *
  645. *
  646. * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
  647. *
  648. * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
  649. * page" backing, however the difference is that _all_ pages with a struct
  650. * page (that is, those where pfn_valid is true) are refcounted and considered
  651. * normal pages by the VM. The disadvantage is that pages are refcounted
  652. * (which can be slower and simply not an option for some PFNMAP users). The
  653. * advantage is that we don't have to follow the strict linearity rule of
  654. * PFNMAP mappings in order to support COWable mappings.
  655. *
  656. */
  657. #ifdef __HAVE_ARCH_PTE_SPECIAL
  658. # define HAVE_PTE_SPECIAL 1
  659. #else
  660. # define HAVE_PTE_SPECIAL 0
  661. #endif
  662. struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
  663. pte_t pte)
  664. {
  665. unsigned long pfn = pte_pfn(pte);
  666. if (HAVE_PTE_SPECIAL) {
  667. if (likely(!pte_special(pte)))
  668. goto check_pfn;
  669. if (vma->vm_ops && vma->vm_ops->find_special_page)
  670. return vma->vm_ops->find_special_page(vma, addr);
  671. if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
  672. return NULL;
  673. if (!is_zero_pfn(pfn))
  674. print_bad_pte(vma, addr, pte, NULL);
  675. return NULL;
  676. }
  677. /* !HAVE_PTE_SPECIAL case follows: */
  678. if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
  679. if (vma->vm_flags & VM_MIXEDMAP) {
  680. if (!pfn_valid(pfn))
  681. return NULL;
  682. goto out;
  683. } else {
  684. unsigned long off;
  685. off = (addr - vma->vm_start) >> PAGE_SHIFT;
  686. if (pfn == vma->vm_pgoff + off)
  687. return NULL;
  688. if (!is_cow_mapping(vma->vm_flags))
  689. return NULL;
  690. }
  691. }
  692. if (is_zero_pfn(pfn))
  693. return NULL;
  694. check_pfn:
  695. if (unlikely(pfn > highest_memmap_pfn)) {
  696. print_bad_pte(vma, addr, pte, NULL);
  697. return NULL;
  698. }
  699. /*
  700. * NOTE! We still have PageReserved() pages in the page tables.
  701. * eg. VDSO mappings can cause them to exist.
  702. */
  703. out:
  704. return pfn_to_page(pfn);
  705. }
  706. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  707. struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
  708. pmd_t pmd)
  709. {
  710. unsigned long pfn = pmd_pfn(pmd);
  711. /*
  712. * There is no pmd_special() but there may be special pmds, e.g.
  713. * in a direct-access (dax) mapping, so let's just replicate the
  714. * !HAVE_PTE_SPECIAL case from vm_normal_page() here.
  715. */
  716. if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
  717. if (vma->vm_flags & VM_MIXEDMAP) {
  718. if (!pfn_valid(pfn))
  719. return NULL;
  720. goto out;
  721. } else {
  722. unsigned long off;
  723. off = (addr - vma->vm_start) >> PAGE_SHIFT;
  724. if (pfn == vma->vm_pgoff + off)
  725. return NULL;
  726. if (!is_cow_mapping(vma->vm_flags))
  727. return NULL;
  728. }
  729. }
  730. if (is_zero_pfn(pfn))
  731. return NULL;
  732. if (unlikely(pfn > highest_memmap_pfn))
  733. return NULL;
  734. /*
  735. * NOTE! We still have PageReserved() pages in the page tables.
  736. * eg. VDSO mappings can cause them to exist.
  737. */
  738. out:
  739. return pfn_to_page(pfn);
  740. }
  741. #endif
  742. /*
  743. * copy one vm_area from one task to the other. Assumes the page tables
  744. * already present in the new task to be cleared in the whole range
  745. * covered by this vma.
  746. */
  747. static inline unsigned long
  748. copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  749. pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
  750. unsigned long addr, int *rss)
  751. {
  752. unsigned long vm_flags = vma->vm_flags;
  753. pte_t pte = *src_pte;
  754. struct page *page;
  755. /* pte contains position in swap or file, so copy. */
  756. if (unlikely(!pte_present(pte))) {
  757. swp_entry_t entry = pte_to_swp_entry(pte);
  758. if (likely(!non_swap_entry(entry))) {
  759. if (swap_duplicate(entry) < 0)
  760. return entry.val;
  761. /* make sure dst_mm is on swapoff's mmlist. */
  762. if (unlikely(list_empty(&dst_mm->mmlist))) {
  763. spin_lock(&mmlist_lock);
  764. if (list_empty(&dst_mm->mmlist))
  765. list_add(&dst_mm->mmlist,
  766. &src_mm->mmlist);
  767. spin_unlock(&mmlist_lock);
  768. }
  769. rss[MM_SWAPENTS]++;
  770. } else if (is_migration_entry(entry)) {
  771. page = migration_entry_to_page(entry);
  772. rss[mm_counter(page)]++;
  773. if (is_write_migration_entry(entry) &&
  774. is_cow_mapping(vm_flags)) {
  775. /*
  776. * COW mappings require pages in both
  777. * parent and child to be set to read.
  778. */
  779. make_migration_entry_read(&entry);
  780. pte = swp_entry_to_pte(entry);
  781. if (pte_swp_soft_dirty(*src_pte))
  782. pte = pte_swp_mksoft_dirty(pte);
  783. set_pte_at(src_mm, addr, src_pte, pte);
  784. }
  785. }
  786. goto out_set_pte;
  787. }
  788. /*
  789. * If it's a COW mapping, write protect it both
  790. * in the parent and the child
  791. */
  792. if (is_cow_mapping(vm_flags)) {
  793. ptep_set_wrprotect(src_mm, addr, src_pte);
  794. pte = pte_wrprotect(pte);
  795. }
  796. /*
  797. * If it's a shared mapping, mark it clean in
  798. * the child
  799. */
  800. if (vm_flags & VM_SHARED)
  801. pte = pte_mkclean(pte);
  802. pte = pte_mkold(pte);
  803. page = vm_normal_page(vma, addr, pte);
  804. if (page) {
  805. get_page(page);
  806. page_dup_rmap(page, false);
  807. rss[mm_counter(page)]++;
  808. }
  809. out_set_pte:
  810. set_pte_at(dst_mm, addr, dst_pte, pte);
  811. return 0;
  812. }
  813. static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  814. pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
  815. unsigned long addr, unsigned long end)
  816. {
  817. pte_t *orig_src_pte, *orig_dst_pte;
  818. pte_t *src_pte, *dst_pte;
  819. spinlock_t *src_ptl, *dst_ptl;
  820. int progress = 0;
  821. int rss[NR_MM_COUNTERS];
  822. swp_entry_t entry = (swp_entry_t){0};
  823. again:
  824. init_rss_vec(rss);
  825. dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
  826. if (!dst_pte)
  827. return -ENOMEM;
  828. src_pte = pte_offset_map(src_pmd, addr);
  829. src_ptl = pte_lockptr(src_mm, src_pmd);
  830. spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
  831. orig_src_pte = src_pte;
  832. orig_dst_pte = dst_pte;
  833. arch_enter_lazy_mmu_mode();
  834. do {
  835. /*
  836. * We are holding two locks at this point - either of them
  837. * could generate latencies in another task on another CPU.
  838. */
  839. if (progress >= 32) {
  840. progress = 0;
  841. if (need_resched() ||
  842. spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
  843. break;
  844. }
  845. if (pte_none(*src_pte)) {
  846. progress++;
  847. continue;
  848. }
  849. entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
  850. vma, addr, rss);
  851. if (entry.val)
  852. break;
  853. progress += 8;
  854. } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
  855. arch_leave_lazy_mmu_mode();
  856. spin_unlock(src_ptl);
  857. pte_unmap(orig_src_pte);
  858. add_mm_rss_vec(dst_mm, rss);
  859. pte_unmap_unlock(orig_dst_pte, dst_ptl);
  860. cond_resched();
  861. if (entry.val) {
  862. if (add_swap_count_continuation(entry, GFP_KERNEL) < 0)
  863. return -ENOMEM;
  864. progress = 0;
  865. }
  866. if (addr != end)
  867. goto again;
  868. return 0;
  869. }
  870. static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  871. pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
  872. unsigned long addr, unsigned long end)
  873. {
  874. pmd_t *src_pmd, *dst_pmd;
  875. unsigned long next;
  876. dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
  877. if (!dst_pmd)
  878. return -ENOMEM;
  879. src_pmd = pmd_offset(src_pud, addr);
  880. do {
  881. next = pmd_addr_end(addr, end);
  882. if (pmd_trans_huge(*src_pmd) || pmd_devmap(*src_pmd)) {
  883. int err;
  884. VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, vma);
  885. err = copy_huge_pmd(dst_mm, src_mm,
  886. dst_pmd, src_pmd, addr, vma);
  887. if (err == -ENOMEM)
  888. return -ENOMEM;
  889. if (!err)
  890. continue;
  891. /* fall through */
  892. }
  893. if (pmd_none_or_clear_bad(src_pmd))
  894. continue;
  895. if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
  896. vma, addr, next))
  897. return -ENOMEM;
  898. } while (dst_pmd++, src_pmd++, addr = next, addr != end);
  899. return 0;
  900. }
  901. static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  902. pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
  903. unsigned long addr, unsigned long end)
  904. {
  905. pud_t *src_pud, *dst_pud;
  906. unsigned long next;
  907. dst_pud = pud_alloc(dst_mm, dst_pgd, addr);
  908. if (!dst_pud)
  909. return -ENOMEM;
  910. src_pud = pud_offset(src_pgd, addr);
  911. do {
  912. next = pud_addr_end(addr, end);
  913. if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
  914. int err;
  915. VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, vma);
  916. err = copy_huge_pud(dst_mm, src_mm,
  917. dst_pud, src_pud, addr, vma);
  918. if (err == -ENOMEM)
  919. return -ENOMEM;
  920. if (!err)
  921. continue;
  922. /* fall through */
  923. }
  924. if (pud_none_or_clear_bad(src_pud))
  925. continue;
  926. if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
  927. vma, addr, next))
  928. return -ENOMEM;
  929. } while (dst_pud++, src_pud++, addr = next, addr != end);
  930. return 0;
  931. }
  932. int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  933. struct vm_area_struct *vma)
  934. {
  935. pgd_t *src_pgd, *dst_pgd;
  936. unsigned long next;
  937. unsigned long addr = vma->vm_start;
  938. unsigned long end = vma->vm_end;
  939. unsigned long mmun_start; /* For mmu_notifiers */
  940. unsigned long mmun_end; /* For mmu_notifiers */
  941. bool is_cow;
  942. int ret;
  943. /*
  944. * Don't copy ptes where a page fault will fill them correctly.
  945. * Fork becomes much lighter when there are big shared or private
  946. * readonly mappings. The tradeoff is that copy_page_range is more
  947. * efficient than faulting.
  948. */
  949. if (!(vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP)) &&
  950. !vma->anon_vma)
  951. return 0;
  952. if (is_vm_hugetlb_page(vma))
  953. return copy_hugetlb_page_range(dst_mm, src_mm, vma);
  954. if (unlikely(vma->vm_flags & VM_PFNMAP)) {
  955. /*
  956. * We do not free on error cases below as remove_vma
  957. * gets called on error from higher level routine
  958. */
  959. ret = track_pfn_copy(vma);
  960. if (ret)
  961. return ret;
  962. }
  963. /*
  964. * We need to invalidate the secondary MMU mappings only when
  965. * there could be a permission downgrade on the ptes of the
  966. * parent mm. And a permission downgrade will only happen if
  967. * is_cow_mapping() returns true.
  968. */
  969. is_cow = is_cow_mapping(vma->vm_flags);
  970. mmun_start = addr;
  971. mmun_end = end;
  972. if (is_cow)
  973. mmu_notifier_invalidate_range_start(src_mm, mmun_start,
  974. mmun_end);
  975. ret = 0;
  976. dst_pgd = pgd_offset(dst_mm, addr);
  977. src_pgd = pgd_offset(src_mm, addr);
  978. do {
  979. next = pgd_addr_end(addr, end);
  980. if (pgd_none_or_clear_bad(src_pgd))
  981. continue;
  982. if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
  983. vma, addr, next))) {
  984. ret = -ENOMEM;
  985. break;
  986. }
  987. } while (dst_pgd++, src_pgd++, addr = next, addr != end);
  988. if (is_cow)
  989. mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end);
  990. return ret;
  991. }
  992. static unsigned long zap_pte_range(struct mmu_gather *tlb,
  993. struct vm_area_struct *vma, pmd_t *pmd,
  994. unsigned long addr, unsigned long end,
  995. struct zap_details *details)
  996. {
  997. struct mm_struct *mm = tlb->mm;
  998. int force_flush = 0;
  999. int rss[NR_MM_COUNTERS];
  1000. spinlock_t *ptl;
  1001. pte_t *start_pte;
  1002. pte_t *pte;
  1003. swp_entry_t entry;
  1004. tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
  1005. again:
  1006. init_rss_vec(rss);
  1007. start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  1008. pte = start_pte;
  1009. arch_enter_lazy_mmu_mode();
  1010. do {
  1011. pte_t ptent = *pte;
  1012. if (pte_none(ptent))
  1013. continue;
  1014. if (pte_present(ptent)) {
  1015. struct page *page;
  1016. page = vm_normal_page(vma, addr, ptent);
  1017. if (unlikely(details) && page) {
  1018. /*
  1019. * unmap_shared_mapping_pages() wants to
  1020. * invalidate cache without truncating:
  1021. * unmap shared but keep private pages.
  1022. */
  1023. if (details->check_mapping &&
  1024. details->check_mapping != page_rmapping(page))
  1025. continue;
  1026. }
  1027. ptent = ptep_get_and_clear_full(mm, addr, pte,
  1028. tlb->fullmm);
  1029. tlb_remove_tlb_entry(tlb, pte, addr);
  1030. if (unlikely(!page))
  1031. continue;
  1032. if (!PageAnon(page)) {
  1033. if (pte_dirty(ptent)) {
  1034. force_flush = 1;
  1035. set_page_dirty(page);
  1036. }
  1037. if (pte_young(ptent) &&
  1038. likely(!(vma->vm_flags & VM_SEQ_READ)))
  1039. mark_page_accessed(page);
  1040. }
  1041. rss[mm_counter(page)]--;
  1042. page_remove_rmap(page, false);
  1043. if (unlikely(page_mapcount(page) < 0))
  1044. print_bad_pte(vma, addr, ptent, page);
  1045. if (unlikely(__tlb_remove_page(tlb, page))) {
  1046. force_flush = 1;
  1047. addr += PAGE_SIZE;
  1048. break;
  1049. }
  1050. continue;
  1051. }
  1052. /* If details->check_mapping, we leave swap entries. */
  1053. if (unlikely(details))
  1054. continue;
  1055. entry = pte_to_swp_entry(ptent);
  1056. if (!non_swap_entry(entry))
  1057. rss[MM_SWAPENTS]--;
  1058. else if (is_migration_entry(entry)) {
  1059. struct page *page;
  1060. page = migration_entry_to_page(entry);
  1061. rss[mm_counter(page)]--;
  1062. }
  1063. if (unlikely(!free_swap_and_cache(entry)))
  1064. print_bad_pte(vma, addr, ptent, NULL);
  1065. pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
  1066. } while (pte++, addr += PAGE_SIZE, addr != end);
  1067. add_mm_rss_vec(mm, rss);
  1068. arch_leave_lazy_mmu_mode();
  1069. /* Do the actual TLB flush before dropping ptl */
  1070. if (force_flush)
  1071. tlb_flush_mmu_tlbonly(tlb);
  1072. pte_unmap_unlock(start_pte, ptl);
  1073. /*
  1074. * If we forced a TLB flush (either due to running out of
  1075. * batch buffers or because we needed to flush dirty TLB
  1076. * entries before releasing the ptl), free the batched
  1077. * memory too. Restart if we didn't do everything.
  1078. */
  1079. if (force_flush) {
  1080. force_flush = 0;
  1081. tlb_flush_mmu_free(tlb);
  1082. if (addr != end)
  1083. goto again;
  1084. }
  1085. return addr;
  1086. }
  1087. static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
  1088. struct vm_area_struct *vma, pud_t *pud,
  1089. unsigned long addr, unsigned long end,
  1090. struct zap_details *details)
  1091. {
  1092. pmd_t *pmd;
  1093. unsigned long next;
  1094. pmd = pmd_offset(pud, addr);
  1095. do {
  1096. next = pmd_addr_end(addr, end);
  1097. if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
  1098. if (next - addr != HPAGE_PMD_SIZE) {
  1099. VM_BUG_ON_VMA(vma_is_anonymous(vma) &&
  1100. !rwsem_is_locked(&tlb->mm->mmap_sem), vma);
  1101. __split_huge_pmd(vma, pmd, addr, false, NULL);
  1102. } else if (zap_huge_pmd(tlb, vma, pmd, addr))
  1103. goto next;
  1104. /* fall through */
  1105. }
  1106. /*
  1107. * Here there can be other concurrent MADV_DONTNEED or
  1108. * trans huge page faults running, and if the pmd is
  1109. * none or trans huge it can change under us. This is
  1110. * because MADV_DONTNEED holds the mmap_sem in read
  1111. * mode.
  1112. */
  1113. if (pmd_none_or_trans_huge_or_clear_bad(pmd))
  1114. goto next;
  1115. next = zap_pte_range(tlb, vma, pmd, addr, next, details);
  1116. next:
  1117. cond_resched();
  1118. } while (pmd++, addr = next, addr != end);
  1119. return addr;
  1120. }
  1121. static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
  1122. struct vm_area_struct *vma, pgd_t *pgd,
  1123. unsigned long addr, unsigned long end,
  1124. struct zap_details *details)
  1125. {
  1126. pud_t *pud;
  1127. unsigned long next;
  1128. pud = pud_offset(pgd, addr);
  1129. do {
  1130. next = pud_addr_end(addr, end);
  1131. if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
  1132. if (next - addr != HPAGE_PUD_SIZE) {
  1133. VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_sem), vma);
  1134. split_huge_pud(vma, pud, addr);
  1135. } else if (zap_huge_pud(tlb, vma, pud, addr))
  1136. goto next;
  1137. /* fall through */
  1138. }
  1139. if (pud_none_or_clear_bad(pud))
  1140. continue;
  1141. next = zap_pmd_range(tlb, vma, pud, addr, next, details);
  1142. next:
  1143. cond_resched();
  1144. } while (pud++, addr = next, addr != end);
  1145. return addr;
  1146. }
  1147. void unmap_page_range(struct mmu_gather *tlb,
  1148. struct vm_area_struct *vma,
  1149. unsigned long addr, unsigned long end,
  1150. struct zap_details *details)
  1151. {
  1152. pgd_t *pgd;
  1153. unsigned long next;
  1154. BUG_ON(addr >= end);
  1155. tlb_start_vma(tlb, vma);
  1156. pgd = pgd_offset(vma->vm_mm, addr);
  1157. do {
  1158. next = pgd_addr_end(addr, end);
  1159. if (pgd_none_or_clear_bad(pgd))
  1160. continue;
  1161. next = zap_pud_range(tlb, vma, pgd, addr, next, details);
  1162. } while (pgd++, addr = next, addr != end);
  1163. tlb_end_vma(tlb, vma);
  1164. }
  1165. static void unmap_single_vma(struct mmu_gather *tlb,
  1166. struct vm_area_struct *vma, unsigned long start_addr,
  1167. unsigned long end_addr,
  1168. struct zap_details *details)
  1169. {
  1170. unsigned long start = max(vma->vm_start, start_addr);
  1171. unsigned long end;
  1172. if (start >= vma->vm_end)
  1173. return;
  1174. end = min(vma->vm_end, end_addr);
  1175. if (end <= vma->vm_start)
  1176. return;
  1177. if (vma->vm_file)
  1178. uprobe_munmap(vma, start, end);
  1179. if (unlikely(vma->vm_flags & VM_PFNMAP))
  1180. untrack_pfn(vma, 0, 0);
  1181. if (start != end) {
  1182. if (unlikely(is_vm_hugetlb_page(vma))) {
  1183. /*
  1184. * It is undesirable to test vma->vm_file as it
  1185. * should be non-null for valid hugetlb area.
  1186. * However, vm_file will be NULL in the error
  1187. * cleanup path of mmap_region. When
  1188. * hugetlbfs ->mmap method fails,
  1189. * mmap_region() nullifies vma->vm_file
  1190. * before calling this function to clean up.
  1191. * Since no pte has actually been setup, it is
  1192. * safe to do nothing in this case.
  1193. */
  1194. if (vma->vm_file) {
  1195. i_mmap_lock_write(vma->vm_file->f_mapping);
  1196. __unmap_hugepage_range_final(tlb, vma, start, end, NULL);
  1197. i_mmap_unlock_write(vma->vm_file->f_mapping);
  1198. }
  1199. } else
  1200. unmap_page_range(tlb, vma, start, end, details);
  1201. }
  1202. }
  1203. /**
  1204. * unmap_vmas - unmap a range of memory covered by a list of vma's
  1205. * @tlb: address of the caller's struct mmu_gather
  1206. * @vma: the starting vma
  1207. * @start_addr: virtual address at which to start unmapping
  1208. * @end_addr: virtual address at which to end unmapping
  1209. *
  1210. * Unmap all pages in the vma list.
  1211. *
  1212. * Only addresses between `start' and `end' will be unmapped.
  1213. *
  1214. * The VMA list must be sorted in ascending virtual address order.
  1215. *
  1216. * unmap_vmas() assumes that the caller will flush the whole unmapped address
  1217. * range after unmap_vmas() returns. So the only responsibility here is to
  1218. * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
  1219. * drops the lock and schedules.
  1220. */
  1221. void unmap_vmas(struct mmu_gather *tlb,
  1222. struct vm_area_struct *vma, unsigned long start_addr,
  1223. unsigned long end_addr)
  1224. {
  1225. struct mm_struct *mm = vma->vm_mm;
  1226. mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
  1227. for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
  1228. unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
  1229. mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
  1230. }
  1231. /**
  1232. * zap_page_range - remove user pages in a given range
  1233. * @vma: vm_area_struct holding the applicable pages
  1234. * @start: starting address of pages to zap
  1235. * @size: number of bytes to zap
  1236. *
  1237. * Caller must protect the VMA list
  1238. */
  1239. void zap_page_range(struct vm_area_struct *vma, unsigned long start,
  1240. unsigned long size)
  1241. {
  1242. struct mm_struct *mm = vma->vm_mm;
  1243. struct mmu_gather tlb;
  1244. unsigned long end = start + size;
  1245. lru_add_drain();
  1246. tlb_gather_mmu(&tlb, mm, start, end);
  1247. update_hiwater_rss(mm);
  1248. mmu_notifier_invalidate_range_start(mm, start, end);
  1249. for ( ; vma && vma->vm_start < end; vma = vma->vm_next)
  1250. unmap_single_vma(&tlb, vma, start, end, NULL);
  1251. mmu_notifier_invalidate_range_end(mm, start, end);
  1252. tlb_finish_mmu(&tlb, start, end);
  1253. }
  1254. /**
  1255. * zap_page_range_single - remove user pages in a given range
  1256. * @vma: vm_area_struct holding the applicable pages
  1257. * @address: starting address of pages to zap
  1258. * @size: number of bytes to zap
  1259. * @details: details of shared cache invalidation
  1260. *
  1261. * The range must fit into one VMA.
  1262. */
  1263. static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
  1264. unsigned long size, struct zap_details *details)
  1265. {
  1266. struct mm_struct *mm = vma->vm_mm;
  1267. struct mmu_gather tlb;
  1268. unsigned long end = address + size;
  1269. lru_add_drain();
  1270. tlb_gather_mmu(&tlb, mm, address, end);
  1271. update_hiwater_rss(mm);
  1272. mmu_notifier_invalidate_range_start(mm, address, end);
  1273. unmap_single_vma(&tlb, vma, address, end, details);
  1274. mmu_notifier_invalidate_range_end(mm, address, end);
  1275. tlb_finish_mmu(&tlb, address, end);
  1276. }
  1277. /**
  1278. * zap_vma_ptes - remove ptes mapping the vma
  1279. * @vma: vm_area_struct holding ptes to be zapped
  1280. * @address: starting address of pages to zap
  1281. * @size: number of bytes to zap
  1282. *
  1283. * This function only unmaps ptes assigned to VM_PFNMAP vmas.
  1284. *
  1285. * The entire address range must be fully contained within the vma.
  1286. *
  1287. * Returns 0 if successful.
  1288. */
  1289. int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
  1290. unsigned long size)
  1291. {
  1292. if (address < vma->vm_start || address + size > vma->vm_end ||
  1293. !(vma->vm_flags & VM_PFNMAP))
  1294. return -1;
  1295. zap_page_range_single(vma, address, size, NULL);
  1296. return 0;
  1297. }
  1298. EXPORT_SYMBOL_GPL(zap_vma_ptes);
  1299. pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
  1300. spinlock_t **ptl)
  1301. {
  1302. pgd_t *pgd = pgd_offset(mm, addr);
  1303. pud_t *pud = pud_alloc(mm, pgd, addr);
  1304. if (pud) {
  1305. pmd_t *pmd = pmd_alloc(mm, pud, addr);
  1306. if (pmd) {
  1307. VM_BUG_ON(pmd_trans_huge(*pmd));
  1308. return pte_alloc_map_lock(mm, pmd, addr, ptl);
  1309. }
  1310. }
  1311. return NULL;
  1312. }
  1313. /*
  1314. * This is the old fallback for page remapping.
  1315. *
  1316. * For historical reasons, it only allows reserved pages. Only
  1317. * old drivers should use this, and they needed to mark their
  1318. * pages reserved for the old functions anyway.
  1319. */
  1320. static int insert_page(struct vm_area_struct *vma, unsigned long addr,
  1321. struct page *page, pgprot_t prot)
  1322. {
  1323. struct mm_struct *mm = vma->vm_mm;
  1324. int retval;
  1325. pte_t *pte;
  1326. spinlock_t *ptl;
  1327. retval = -EINVAL;
  1328. if (PageAnon(page))
  1329. goto out;
  1330. retval = -ENOMEM;
  1331. flush_dcache_page(page);
  1332. pte = get_locked_pte(mm, addr, &ptl);
  1333. if (!pte)
  1334. goto out;
  1335. retval = -EBUSY;
  1336. if (!pte_none(*pte))
  1337. goto out_unlock;
  1338. /* Ok, finally just insert the thing.. */
  1339. get_page(page);
  1340. inc_mm_counter_fast(mm, mm_counter_file(page));
  1341. page_add_file_rmap(page, false);
  1342. set_pte_at(mm, addr, pte, mk_pte(page, prot));
  1343. retval = 0;
  1344. pte_unmap_unlock(pte, ptl);
  1345. return retval;
  1346. out_unlock:
  1347. pte_unmap_unlock(pte, ptl);
  1348. out:
  1349. return retval;
  1350. }
  1351. /**
  1352. * vm_insert_page - insert single page into user vma
  1353. * @vma: user vma to map to
  1354. * @addr: target user address of this page
  1355. * @page: source kernel page
  1356. *
  1357. * This allows drivers to insert individual pages they've allocated
  1358. * into a user vma.
  1359. *
  1360. * The page has to be a nice clean _individual_ kernel allocation.
  1361. * If you allocate a compound page, you need to have marked it as
  1362. * such (__GFP_COMP), or manually just split the page up yourself
  1363. * (see split_page()).
  1364. *
  1365. * NOTE! Traditionally this was done with "remap_pfn_range()" which
  1366. * took an arbitrary page protection parameter. This doesn't allow
  1367. * that. Your vma protection will have to be set up correctly, which
  1368. * means that if you want a shared writable mapping, you'd better
  1369. * ask for a shared writable mapping!
  1370. *
  1371. * The page does not need to be reserved.
  1372. *
  1373. * Usually this function is called from f_op->mmap() handler
  1374. * under mm->mmap_sem write-lock, so it can change vma->vm_flags.
  1375. * Caller must set VM_MIXEDMAP on vma if it wants to call this
  1376. * function from other places, for example from page-fault handler.
  1377. */
  1378. int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
  1379. struct page *page)
  1380. {
  1381. if (addr < vma->vm_start || addr >= vma->vm_end)
  1382. return -EFAULT;
  1383. if (!page_count(page))
  1384. return -EINVAL;
  1385. if (!(vma->vm_flags & VM_MIXEDMAP)) {
  1386. BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem));
  1387. BUG_ON(vma->vm_flags & VM_PFNMAP);
  1388. vma->vm_flags |= VM_MIXEDMAP;
  1389. }
  1390. return insert_page(vma, addr, page, vma->vm_page_prot);
  1391. }
  1392. EXPORT_SYMBOL(vm_insert_page);
  1393. static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1394. pfn_t pfn, pgprot_t prot)
  1395. {
  1396. struct mm_struct *mm = vma->vm_mm;
  1397. int retval;
  1398. pte_t *pte, entry;
  1399. spinlock_t *ptl;
  1400. retval = -ENOMEM;
  1401. pte = get_locked_pte(mm, addr, &ptl);
  1402. if (!pte)
  1403. goto out;
  1404. retval = -EBUSY;
  1405. if (!pte_none(*pte))
  1406. goto out_unlock;
  1407. /* Ok, finally just insert the thing.. */
  1408. if (pfn_t_devmap(pfn))
  1409. entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
  1410. else
  1411. entry = pte_mkspecial(pfn_t_pte(pfn, prot));
  1412. set_pte_at(mm, addr, pte, entry);
  1413. update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
  1414. retval = 0;
  1415. out_unlock:
  1416. pte_unmap_unlock(pte, ptl);
  1417. out:
  1418. return retval;
  1419. }
  1420. /**
  1421. * vm_insert_pfn - insert single pfn into user vma
  1422. * @vma: user vma to map to
  1423. * @addr: target user address of this page
  1424. * @pfn: source kernel pfn
  1425. *
  1426. * Similar to vm_insert_page, this allows drivers to insert individual pages
  1427. * they've allocated into a user vma. Same comments apply.
  1428. *
  1429. * This function should only be called from a vm_ops->fault handler, and
  1430. * in that case the handler should return NULL.
  1431. *
  1432. * vma cannot be a COW mapping.
  1433. *
  1434. * As this is called only for pages that do not currently exist, we
  1435. * do not need to flush old virtual caches or the TLB.
  1436. */
  1437. int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1438. unsigned long pfn)
  1439. {
  1440. return vm_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
  1441. }
  1442. EXPORT_SYMBOL(vm_insert_pfn);
  1443. /**
  1444. * vm_insert_pfn_prot - insert single pfn into user vma with specified pgprot
  1445. * @vma: user vma to map to
  1446. * @addr: target user address of this page
  1447. * @pfn: source kernel pfn
  1448. * @pgprot: pgprot flags for the inserted page
  1449. *
  1450. * This is exactly like vm_insert_pfn, except that it allows drivers to
  1451. * to override pgprot on a per-page basis.
  1452. *
  1453. * This only makes sense for IO mappings, and it makes no sense for
  1454. * cow mappings. In general, using multiple vmas is preferable;
  1455. * vm_insert_pfn_prot should only be used if using multiple VMAs is
  1456. * impractical.
  1457. */
  1458. int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
  1459. unsigned long pfn, pgprot_t pgprot)
  1460. {
  1461. int ret;
  1462. /*
  1463. * Technically, architectures with pte_special can avoid all these
  1464. * restrictions (same for remap_pfn_range). However we would like
  1465. * consistency in testing and feature parity among all, so we should
  1466. * try to keep these invariants in place for everybody.
  1467. */
  1468. BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
  1469. BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
  1470. (VM_PFNMAP|VM_MIXEDMAP));
  1471. BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
  1472. BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
  1473. if (addr < vma->vm_start || addr >= vma->vm_end)
  1474. return -EFAULT;
  1475. track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
  1476. ret = insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot);
  1477. return ret;
  1478. }
  1479. EXPORT_SYMBOL(vm_insert_pfn_prot);
  1480. int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
  1481. pfn_t pfn)
  1482. {
  1483. pgprot_t pgprot = vma->vm_page_prot;
  1484. BUG_ON(!(vma->vm_flags & VM_MIXEDMAP));
  1485. if (addr < vma->vm_start || addr >= vma->vm_end)
  1486. return -EFAULT;
  1487. track_pfn_insert(vma, &pgprot, pfn);
  1488. /*
  1489. * If we don't have pte special, then we have to use the pfn_valid()
  1490. * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
  1491. * refcount the page if pfn_valid is true (hence insert_page rather
  1492. * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP
  1493. * without pte special, it would there be refcounted as a normal page.
  1494. */
  1495. if (!HAVE_PTE_SPECIAL && !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
  1496. struct page *page;
  1497. /*
  1498. * At this point we are committed to insert_page()
  1499. * regardless of whether the caller specified flags that
  1500. * result in pfn_t_has_page() == false.
  1501. */
  1502. page = pfn_to_page(pfn_t_to_pfn(pfn));
  1503. return insert_page(vma, addr, page, pgprot);
  1504. }
  1505. return insert_pfn(vma, addr, pfn, pgprot);
  1506. }
  1507. EXPORT_SYMBOL(vm_insert_mixed);
  1508. /*
  1509. * maps a range of physical memory into the requested pages. the old
  1510. * mappings are removed. any references to nonexistent pages results
  1511. * in null mappings (currently treated as "copy-on-access")
  1512. */
  1513. static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
  1514. unsigned long addr, unsigned long end,
  1515. unsigned long pfn, pgprot_t prot)
  1516. {
  1517. pte_t *pte;
  1518. spinlock_t *ptl;
  1519. pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
  1520. if (!pte)
  1521. return -ENOMEM;
  1522. arch_enter_lazy_mmu_mode();
  1523. do {
  1524. BUG_ON(!pte_none(*pte));
  1525. set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
  1526. pfn++;
  1527. } while (pte++, addr += PAGE_SIZE, addr != end);
  1528. arch_leave_lazy_mmu_mode();
  1529. pte_unmap_unlock(pte - 1, ptl);
  1530. return 0;
  1531. }
  1532. static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
  1533. unsigned long addr, unsigned long end,
  1534. unsigned long pfn, pgprot_t prot)
  1535. {
  1536. pmd_t *pmd;
  1537. unsigned long next;
  1538. pfn -= addr >> PAGE_SHIFT;
  1539. pmd = pmd_alloc(mm, pud, addr);
  1540. if (!pmd)
  1541. return -ENOMEM;
  1542. VM_BUG_ON(pmd_trans_huge(*pmd));
  1543. do {
  1544. next = pmd_addr_end(addr, end);
  1545. if (remap_pte_range(mm, pmd, addr, next,
  1546. pfn + (addr >> PAGE_SHIFT), prot))
  1547. return -ENOMEM;
  1548. } while (pmd++, addr = next, addr != end);
  1549. return 0;
  1550. }
  1551. static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
  1552. unsigned long addr, unsigned long end,
  1553. unsigned long pfn, pgprot_t prot)
  1554. {
  1555. pud_t *pud;
  1556. unsigned long next;
  1557. pfn -= addr >> PAGE_SHIFT;
  1558. pud = pud_alloc(mm, pgd, addr);
  1559. if (!pud)
  1560. return -ENOMEM;
  1561. do {
  1562. next = pud_addr_end(addr, end);
  1563. if (remap_pmd_range(mm, pud, addr, next,
  1564. pfn + (addr >> PAGE_SHIFT), prot))
  1565. return -ENOMEM;
  1566. } while (pud++, addr = next, addr != end);
  1567. return 0;
  1568. }
  1569. /**
  1570. * remap_pfn_range - remap kernel memory to userspace
  1571. * @vma: user vma to map to
  1572. * @addr: target user address to start at
  1573. * @pfn: physical address of kernel memory
  1574. * @size: size of map area
  1575. * @prot: page protection flags for this mapping
  1576. *
  1577. * Note: this is only safe if the mm semaphore is held when called.
  1578. */
  1579. int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
  1580. unsigned long pfn, unsigned long size, pgprot_t prot)
  1581. {
  1582. pgd_t *pgd;
  1583. unsigned long next;
  1584. unsigned long end = addr + PAGE_ALIGN(size);
  1585. struct mm_struct *mm = vma->vm_mm;
  1586. unsigned long remap_pfn = pfn;
  1587. int err;
  1588. /*
  1589. * Physically remapped pages are special. Tell the
  1590. * rest of the world about it:
  1591. * VM_IO tells people not to look at these pages
  1592. * (accesses can have side effects).
  1593. * VM_PFNMAP tells the core MM that the base pages are just
  1594. * raw PFN mappings, and do not have a "struct page" associated
  1595. * with them.
  1596. * VM_DONTEXPAND
  1597. * Disable vma merging and expanding with mremap().
  1598. * VM_DONTDUMP
  1599. * Omit vma from core dump, even when VM_IO turned off.
  1600. *
  1601. * There's a horrible special case to handle copy-on-write
  1602. * behaviour that some programs depend on. We mark the "original"
  1603. * un-COW'ed pages by matching them up with "vma->vm_pgoff".
  1604. * See vm_normal_page() for details.
  1605. */
  1606. if (is_cow_mapping(vma->vm_flags)) {
  1607. if (addr != vma->vm_start || end != vma->vm_end)
  1608. return -EINVAL;
  1609. vma->vm_pgoff = pfn;
  1610. }
  1611. err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size));
  1612. if (err)
  1613. return -EINVAL;
  1614. vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
  1615. BUG_ON(addr >= end);
  1616. pfn -= addr >> PAGE_SHIFT;
  1617. pgd = pgd_offset(mm, addr);
  1618. flush_cache_range(vma, addr, end);
  1619. do {
  1620. next = pgd_addr_end(addr, end);
  1621. err = remap_pud_range(mm, pgd, addr, next,
  1622. pfn + (addr >> PAGE_SHIFT), prot);
  1623. if (err)
  1624. break;
  1625. } while (pgd++, addr = next, addr != end);
  1626. if (err)
  1627. untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size));
  1628. return err;
  1629. }
  1630. EXPORT_SYMBOL(remap_pfn_range);
  1631. /**
  1632. * vm_iomap_memory - remap memory to userspace
  1633. * @vma: user vma to map to
  1634. * @start: start of area
  1635. * @len: size of area
  1636. *
  1637. * This is a simplified io_remap_pfn_range() for common driver use. The
  1638. * driver just needs to give us the physical memory range to be mapped,
  1639. * we'll figure out the rest from the vma information.
  1640. *
  1641. * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
  1642. * whatever write-combining details or similar.
  1643. */
  1644. int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
  1645. {
  1646. unsigned long vm_len, pfn, pages;
  1647. /* Check that the physical memory area passed in looks valid */
  1648. if (start + len < start)
  1649. return -EINVAL;
  1650. /*
  1651. * You *really* shouldn't map things that aren't page-aligned,
  1652. * but we've historically allowed it because IO memory might
  1653. * just have smaller alignment.
  1654. */
  1655. len += start & ~PAGE_MASK;
  1656. pfn = start >> PAGE_SHIFT;
  1657. pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
  1658. if (pfn + pages < pfn)
  1659. return -EINVAL;
  1660. /* We start the mapping 'vm_pgoff' pages into the area */
  1661. if (vma->vm_pgoff > pages)
  1662. return -EINVAL;
  1663. pfn += vma->vm_pgoff;
  1664. pages -= vma->vm_pgoff;
  1665. /* Can we fit all of the mapping? */
  1666. vm_len = vma->vm_end - vma->vm_start;
  1667. if (vm_len >> PAGE_SHIFT > pages)
  1668. return -EINVAL;
  1669. /* Ok, let it rip */
  1670. return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
  1671. }
  1672. EXPORT_SYMBOL(vm_iomap_memory);
  1673. static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
  1674. unsigned long addr, unsigned long end,
  1675. pte_fn_t fn, void *data)
  1676. {
  1677. pte_t *pte;
  1678. int err;
  1679. pgtable_t token;
  1680. spinlock_t *uninitialized_var(ptl);
  1681. pte = (mm == &init_mm) ?
  1682. pte_alloc_kernel(pmd, addr) :
  1683. pte_alloc_map_lock(mm, pmd, addr, &ptl);
  1684. if (!pte)
  1685. return -ENOMEM;
  1686. BUG_ON(pmd_huge(*pmd));
  1687. arch_enter_lazy_mmu_mode();
  1688. token = pmd_pgtable(*pmd);
  1689. do {
  1690. err = fn(pte++, token, addr, data);
  1691. if (err)
  1692. break;
  1693. } while (addr += PAGE_SIZE, addr != end);
  1694. arch_leave_lazy_mmu_mode();
  1695. if (mm != &init_mm)
  1696. pte_unmap_unlock(pte-1, ptl);
  1697. return err;
  1698. }
  1699. static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
  1700. unsigned long addr, unsigned long end,
  1701. pte_fn_t fn, void *data)
  1702. {
  1703. pmd_t *pmd;
  1704. unsigned long next;
  1705. int err;
  1706. BUG_ON(pud_huge(*pud));
  1707. pmd = pmd_alloc(mm, pud, addr);
  1708. if (!pmd)
  1709. return -ENOMEM;
  1710. do {
  1711. next = pmd_addr_end(addr, end);
  1712. err = apply_to_pte_range(mm, pmd, addr, next, fn, data);
  1713. if (err)
  1714. break;
  1715. } while (pmd++, addr = next, addr != end);
  1716. return err;
  1717. }
  1718. static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd,
  1719. unsigned long addr, unsigned long end,
  1720. pte_fn_t fn, void *data)
  1721. {
  1722. pud_t *pud;
  1723. unsigned long next;
  1724. int err;
  1725. pud = pud_alloc(mm, pgd, addr);
  1726. if (!pud)
  1727. return -ENOMEM;
  1728. do {
  1729. next = pud_addr_end(addr, end);
  1730. err = apply_to_pmd_range(mm, pud, addr, next, fn, data);
  1731. if (err)
  1732. break;
  1733. } while (pud++, addr = next, addr != end);
  1734. return err;
  1735. }
  1736. /*
  1737. * Scan a region of virtual memory, filling in page tables as necessary
  1738. * and calling a provided function on each leaf page table.
  1739. */
  1740. int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
  1741. unsigned long size, pte_fn_t fn, void *data)
  1742. {
  1743. pgd_t *pgd;
  1744. unsigned long next;
  1745. unsigned long end = addr + size;
  1746. int err;
  1747. if (WARN_ON(addr >= end))
  1748. return -EINVAL;
  1749. pgd = pgd_offset(mm, addr);
  1750. do {
  1751. next = pgd_addr_end(addr, end);
  1752. err = apply_to_pud_range(mm, pgd, addr, next, fn, data);
  1753. if (err)
  1754. break;
  1755. } while (pgd++, addr = next, addr != end);
  1756. return err;
  1757. }
  1758. EXPORT_SYMBOL_GPL(apply_to_page_range);
  1759. /*
  1760. * handle_pte_fault chooses page fault handler according to an entry which was
  1761. * read non-atomically. Before making any commitment, on those architectures
  1762. * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
  1763. * parts, do_swap_page must check under lock before unmapping the pte and
  1764. * proceeding (but do_wp_page is only called after already making such a check;
  1765. * and do_anonymous_page can safely check later on).
  1766. */
  1767. static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
  1768. pte_t *page_table, pte_t orig_pte)
  1769. {
  1770. int same = 1;
  1771. #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT)
  1772. if (sizeof(pte_t) > sizeof(unsigned long)) {
  1773. spinlock_t *ptl = pte_lockptr(mm, pmd);
  1774. spin_lock(ptl);
  1775. same = pte_same(*page_table, orig_pte);
  1776. spin_unlock(ptl);
  1777. }
  1778. #endif
  1779. pte_unmap(page_table);
  1780. return same;
  1781. }
  1782. static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
  1783. {
  1784. debug_dma_assert_idle(src);
  1785. /*
  1786. * If the source page was a PFN mapping, we don't have
  1787. * a "struct page" for it. We do a best-effort copy by
  1788. * just copying from the original user address. If that
  1789. * fails, we just zero-fill it. Live with it.
  1790. */
  1791. if (unlikely(!src)) {
  1792. void *kaddr = kmap_atomic(dst);
  1793. void __user *uaddr = (void __user *)(va & PAGE_MASK);
  1794. /*
  1795. * This really shouldn't fail, because the page is there
  1796. * in the page tables. But it might just be unreadable,
  1797. * in which case we just give up and fill the result with
  1798. * zeroes.
  1799. */
  1800. if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
  1801. clear_page(kaddr);
  1802. kunmap_atomic(kaddr);
  1803. flush_dcache_page(dst);
  1804. } else
  1805. copy_user_highpage(dst, src, va, vma);
  1806. }
  1807. static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
  1808. {
  1809. struct file *vm_file = vma->vm_file;
  1810. if (vm_file)
  1811. return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
  1812. /*
  1813. * Special mappings (e.g. VDSO) do not have any file so fake
  1814. * a default GFP_KERNEL for them.
  1815. */
  1816. return GFP_KERNEL;
  1817. }
  1818. /*
  1819. * Notify the address space that the page is about to become writable so that
  1820. * it can prohibit this or wait for the page to get into an appropriate state.
  1821. *
  1822. * We do this without the lock held, so that it can sleep if it needs to.
  1823. */
  1824. static int do_page_mkwrite(struct vm_fault *vmf)
  1825. {
  1826. int ret;
  1827. struct page *page = vmf->page;
  1828. unsigned int old_flags = vmf->flags;
  1829. vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
  1830. ret = vmf->vma->vm_ops->page_mkwrite(vmf);
  1831. /* Restore original flags so that caller is not surprised */
  1832. vmf->flags = old_flags;
  1833. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
  1834. return ret;
  1835. if (unlikely(!(ret & VM_FAULT_LOCKED))) {
  1836. lock_page(page);
  1837. if (!page->mapping) {
  1838. unlock_page(page);
  1839. return 0; /* retry */
  1840. }
  1841. ret |= VM_FAULT_LOCKED;
  1842. } else
  1843. VM_BUG_ON_PAGE(!PageLocked(page), page);
  1844. return ret;
  1845. }
  1846. /*
  1847. * Handle dirtying of a page in shared file mapping on a write fault.
  1848. *
  1849. * The function expects the page to be locked and unlocks it.
  1850. */
  1851. static void fault_dirty_shared_page(struct vm_area_struct *vma,
  1852. struct page *page)
  1853. {
  1854. struct address_space *mapping;
  1855. bool dirtied;
  1856. bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
  1857. dirtied = set_page_dirty(page);
  1858. VM_BUG_ON_PAGE(PageAnon(page), page);
  1859. /*
  1860. * Take a local copy of the address_space - page.mapping may be zeroed
  1861. * by truncate after unlock_page(). The address_space itself remains
  1862. * pinned by vma->vm_file's reference. We rely on unlock_page()'s
  1863. * release semantics to prevent the compiler from undoing this copying.
  1864. */
  1865. mapping = page_rmapping(page);
  1866. unlock_page(page);
  1867. if ((dirtied || page_mkwrite) && mapping) {
  1868. /*
  1869. * Some device drivers do not set page.mapping
  1870. * but still dirty their pages
  1871. */
  1872. balance_dirty_pages_ratelimited(mapping);
  1873. }
  1874. if (!page_mkwrite)
  1875. file_update_time(vma->vm_file);
  1876. }
  1877. /*
  1878. * Handle write page faults for pages that can be reused in the current vma
  1879. *
  1880. * This can happen either due to the mapping being with the VM_SHARED flag,
  1881. * or due to us being the last reference standing to the page. In either
  1882. * case, all we need to do here is to mark the page as writable and update
  1883. * any related book-keeping.
  1884. */
  1885. static inline void wp_page_reuse(struct vm_fault *vmf)
  1886. __releases(vmf->ptl)
  1887. {
  1888. struct vm_area_struct *vma = vmf->vma;
  1889. struct page *page = vmf->page;
  1890. pte_t entry;
  1891. /*
  1892. * Clear the pages cpupid information as the existing
  1893. * information potentially belongs to a now completely
  1894. * unrelated process.
  1895. */
  1896. if (page)
  1897. page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
  1898. flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
  1899. entry = pte_mkyoung(vmf->orig_pte);
  1900. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1901. if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
  1902. update_mmu_cache(vma, vmf->address, vmf->pte);
  1903. pte_unmap_unlock(vmf->pte, vmf->ptl);
  1904. }
  1905. /*
  1906. * Handle the case of a page which we actually need to copy to a new page.
  1907. *
  1908. * Called with mmap_sem locked and the old page referenced, but
  1909. * without the ptl held.
  1910. *
  1911. * High level logic flow:
  1912. *
  1913. * - Allocate a page, copy the content of the old page to the new one.
  1914. * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
  1915. * - Take the PTL. If the pte changed, bail out and release the allocated page
  1916. * - If the pte is still the way we remember it, update the page table and all
  1917. * relevant references. This includes dropping the reference the page-table
  1918. * held to the old page, as well as updating the rmap.
  1919. * - In any case, unlock the PTL and drop the reference we took to the old page.
  1920. */
  1921. static int wp_page_copy(struct vm_fault *vmf)
  1922. {
  1923. struct vm_area_struct *vma = vmf->vma;
  1924. struct mm_struct *mm = vma->vm_mm;
  1925. struct page *old_page = vmf->page;
  1926. struct page *new_page = NULL;
  1927. pte_t entry;
  1928. int page_copied = 0;
  1929. const unsigned long mmun_start = vmf->address & PAGE_MASK;
  1930. const unsigned long mmun_end = mmun_start + PAGE_SIZE;
  1931. struct mem_cgroup *memcg;
  1932. if (unlikely(anon_vma_prepare(vma)))
  1933. goto oom;
  1934. if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
  1935. new_page = alloc_zeroed_user_highpage_movable(vma,
  1936. vmf->address);
  1937. if (!new_page)
  1938. goto oom;
  1939. } else {
  1940. new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
  1941. vmf->address);
  1942. if (!new_page)
  1943. goto oom;
  1944. cow_user_page(new_page, old_page, vmf->address, vma);
  1945. }
  1946. if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false))
  1947. goto oom_free_new;
  1948. __SetPageUptodate(new_page);
  1949. mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
  1950. /*
  1951. * Re-check the pte - we dropped the lock
  1952. */
  1953. vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
  1954. if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
  1955. if (old_page) {
  1956. if (!PageAnon(old_page)) {
  1957. dec_mm_counter_fast(mm,
  1958. mm_counter_file(old_page));
  1959. inc_mm_counter_fast(mm, MM_ANONPAGES);
  1960. }
  1961. } else {
  1962. inc_mm_counter_fast(mm, MM_ANONPAGES);
  1963. }
  1964. flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
  1965. entry = mk_pte(new_page, vma->vm_page_prot);
  1966. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1967. /*
  1968. * Clear the pte entry and flush it first, before updating the
  1969. * pte with the new entry. This will avoid a race condition
  1970. * seen in the presence of one thread doing SMC and another
  1971. * thread doing COW.
  1972. */
  1973. ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
  1974. page_add_new_anon_rmap(new_page, vma, vmf->address, false);
  1975. mem_cgroup_commit_charge(new_page, memcg, false, false);
  1976. lru_cache_add_active_or_unevictable(new_page, vma);
  1977. /*
  1978. * We call the notify macro here because, when using secondary
  1979. * mmu page tables (such as kvm shadow page tables), we want the
  1980. * new page to be mapped directly into the secondary page table.
  1981. */
  1982. set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
  1983. update_mmu_cache(vma, vmf->address, vmf->pte);
  1984. if (old_page) {
  1985. /*
  1986. * Only after switching the pte to the new page may
  1987. * we remove the mapcount here. Otherwise another
  1988. * process may come and find the rmap count decremented
  1989. * before the pte is switched to the new page, and
  1990. * "reuse" the old page writing into it while our pte
  1991. * here still points into it and can be read by other
  1992. * threads.
  1993. *
  1994. * The critical issue is to order this
  1995. * page_remove_rmap with the ptp_clear_flush above.
  1996. * Those stores are ordered by (if nothing else,)
  1997. * the barrier present in the atomic_add_negative
  1998. * in page_remove_rmap.
  1999. *
  2000. * Then the TLB flush in ptep_clear_flush ensures that
  2001. * no process can access the old page before the
  2002. * decremented mapcount is visible. And the old page
  2003. * cannot be reused until after the decremented
  2004. * mapcount is visible. So transitively, TLBs to
  2005. * old page will be flushed before it can be reused.
  2006. */
  2007. page_remove_rmap(old_page, false);
  2008. }
  2009. /* Free the old page.. */
  2010. new_page = old_page;
  2011. page_copied = 1;
  2012. } else {
  2013. mem_cgroup_cancel_charge(new_page, memcg, false);
  2014. }
  2015. if (new_page)
  2016. put_page(new_page);
  2017. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2018. mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
  2019. if (old_page) {
  2020. /*
  2021. * Don't let another task, with possibly unlocked vma,
  2022. * keep the mlocked page.
  2023. */
  2024. if (page_copied && (vma->vm_flags & VM_LOCKED)) {
  2025. lock_page(old_page); /* LRU manipulation */
  2026. if (PageMlocked(old_page))
  2027. munlock_vma_page(old_page);
  2028. unlock_page(old_page);
  2029. }
  2030. put_page(old_page);
  2031. }
  2032. return page_copied ? VM_FAULT_WRITE : 0;
  2033. oom_free_new:
  2034. put_page(new_page);
  2035. oom:
  2036. if (old_page)
  2037. put_page(old_page);
  2038. return VM_FAULT_OOM;
  2039. }
  2040. /**
  2041. * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
  2042. * writeable once the page is prepared
  2043. *
  2044. * @vmf: structure describing the fault
  2045. *
  2046. * This function handles all that is needed to finish a write page fault in a
  2047. * shared mapping due to PTE being read-only once the mapped page is prepared.
  2048. * It handles locking of PTE and modifying it. The function returns
  2049. * VM_FAULT_WRITE on success, 0 when PTE got changed before we acquired PTE
  2050. * lock.
  2051. *
  2052. * The function expects the page to be locked or other protection against
  2053. * concurrent faults / writeback (such as DAX radix tree locks).
  2054. */
  2055. int finish_mkwrite_fault(struct vm_fault *vmf)
  2056. {
  2057. WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
  2058. vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
  2059. &vmf->ptl);
  2060. /*
  2061. * We might have raced with another page fault while we released the
  2062. * pte_offset_map_lock.
  2063. */
  2064. if (!pte_same(*vmf->pte, vmf->orig_pte)) {
  2065. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2066. return VM_FAULT_NOPAGE;
  2067. }
  2068. wp_page_reuse(vmf);
  2069. return 0;
  2070. }
  2071. /*
  2072. * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
  2073. * mapping
  2074. */
  2075. static int wp_pfn_shared(struct vm_fault *vmf)
  2076. {
  2077. struct vm_area_struct *vma = vmf->vma;
  2078. if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
  2079. int ret;
  2080. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2081. vmf->flags |= FAULT_FLAG_MKWRITE;
  2082. ret = vma->vm_ops->pfn_mkwrite(vmf);
  2083. if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
  2084. return ret;
  2085. return finish_mkwrite_fault(vmf);
  2086. }
  2087. wp_page_reuse(vmf);
  2088. return VM_FAULT_WRITE;
  2089. }
  2090. static int wp_page_shared(struct vm_fault *vmf)
  2091. __releases(vmf->ptl)
  2092. {
  2093. struct vm_area_struct *vma = vmf->vma;
  2094. get_page(vmf->page);
  2095. if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
  2096. int tmp;
  2097. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2098. tmp = do_page_mkwrite(vmf);
  2099. if (unlikely(!tmp || (tmp &
  2100. (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
  2101. put_page(vmf->page);
  2102. return tmp;
  2103. }
  2104. tmp = finish_mkwrite_fault(vmf);
  2105. if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
  2106. unlock_page(vmf->page);
  2107. put_page(vmf->page);
  2108. return tmp;
  2109. }
  2110. } else {
  2111. wp_page_reuse(vmf);
  2112. lock_page(vmf->page);
  2113. }
  2114. fault_dirty_shared_page(vma, vmf->page);
  2115. put_page(vmf->page);
  2116. return VM_FAULT_WRITE;
  2117. }
  2118. /*
  2119. * This routine handles present pages, when users try to write
  2120. * to a shared page. It is done by copying the page to a new address
  2121. * and decrementing the shared-page counter for the old page.
  2122. *
  2123. * Note that this routine assumes that the protection checks have been
  2124. * done by the caller (the low-level page fault routine in most cases).
  2125. * Thus we can safely just mark it writable once we've done any necessary
  2126. * COW.
  2127. *
  2128. * We also mark the page dirty at this point even though the page will
  2129. * change only once the write actually happens. This avoids a few races,
  2130. * and potentially makes it more efficient.
  2131. *
  2132. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2133. * but allow concurrent faults), with pte both mapped and locked.
  2134. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2135. */
  2136. static int do_wp_page(struct vm_fault *vmf)
  2137. __releases(vmf->ptl)
  2138. {
  2139. struct vm_area_struct *vma = vmf->vma;
  2140. vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
  2141. if (!vmf->page) {
  2142. /*
  2143. * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
  2144. * VM_PFNMAP VMA.
  2145. *
  2146. * We should not cow pages in a shared writeable mapping.
  2147. * Just mark the pages writable and/or call ops->pfn_mkwrite.
  2148. */
  2149. if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
  2150. (VM_WRITE|VM_SHARED))
  2151. return wp_pfn_shared(vmf);
  2152. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2153. return wp_page_copy(vmf);
  2154. }
  2155. /*
  2156. * Take out anonymous pages first, anonymous shared vmas are
  2157. * not dirty accountable.
  2158. */
  2159. if (PageAnon(vmf->page) && !PageKsm(vmf->page)) {
  2160. int total_mapcount;
  2161. if (!trylock_page(vmf->page)) {
  2162. get_page(vmf->page);
  2163. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2164. lock_page(vmf->page);
  2165. vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
  2166. vmf->address, &vmf->ptl);
  2167. if (!pte_same(*vmf->pte, vmf->orig_pte)) {
  2168. unlock_page(vmf->page);
  2169. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2170. put_page(vmf->page);
  2171. return 0;
  2172. }
  2173. put_page(vmf->page);
  2174. }
  2175. if (reuse_swap_page(vmf->page, &total_mapcount)) {
  2176. if (total_mapcount == 1) {
  2177. /*
  2178. * The page is all ours. Move it to
  2179. * our anon_vma so the rmap code will
  2180. * not search our parent or siblings.
  2181. * Protected against the rmap code by
  2182. * the page lock.
  2183. */
  2184. page_move_anon_rmap(vmf->page, vma);
  2185. }
  2186. unlock_page(vmf->page);
  2187. wp_page_reuse(vmf);
  2188. return VM_FAULT_WRITE;
  2189. }
  2190. unlock_page(vmf->page);
  2191. } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
  2192. (VM_WRITE|VM_SHARED))) {
  2193. return wp_page_shared(vmf);
  2194. }
  2195. /*
  2196. * Ok, we need to copy. Oh, well..
  2197. */
  2198. get_page(vmf->page);
  2199. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2200. return wp_page_copy(vmf);
  2201. }
  2202. static void unmap_mapping_range_vma(struct vm_area_struct *vma,
  2203. unsigned long start_addr, unsigned long end_addr,
  2204. struct zap_details *details)
  2205. {
  2206. zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
  2207. }
  2208. static inline void unmap_mapping_range_tree(struct rb_root *root,
  2209. struct zap_details *details)
  2210. {
  2211. struct vm_area_struct *vma;
  2212. pgoff_t vba, vea, zba, zea;
  2213. vma_interval_tree_foreach(vma, root,
  2214. details->first_index, details->last_index) {
  2215. vba = vma->vm_pgoff;
  2216. vea = vba + vma_pages(vma) - 1;
  2217. zba = details->first_index;
  2218. if (zba < vba)
  2219. zba = vba;
  2220. zea = details->last_index;
  2221. if (zea > vea)
  2222. zea = vea;
  2223. unmap_mapping_range_vma(vma,
  2224. ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
  2225. ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
  2226. details);
  2227. }
  2228. }
  2229. /**
  2230. * unmap_mapping_range - unmap the portion of all mmaps in the specified
  2231. * address_space corresponding to the specified page range in the underlying
  2232. * file.
  2233. *
  2234. * @mapping: the address space containing mmaps to be unmapped.
  2235. * @holebegin: byte in first page to unmap, relative to the start of
  2236. * the underlying file. This will be rounded down to a PAGE_SIZE
  2237. * boundary. Note that this is different from truncate_pagecache(), which
  2238. * must keep the partial page. In contrast, we must get rid of
  2239. * partial pages.
  2240. * @holelen: size of prospective hole in bytes. This will be rounded
  2241. * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
  2242. * end of the file.
  2243. * @even_cows: 1 when truncating a file, unmap even private COWed pages;
  2244. * but 0 when invalidating pagecache, don't throw away private data.
  2245. */
  2246. void unmap_mapping_range(struct address_space *mapping,
  2247. loff_t const holebegin, loff_t const holelen, int even_cows)
  2248. {
  2249. struct zap_details details = { };
  2250. pgoff_t hba = holebegin >> PAGE_SHIFT;
  2251. pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  2252. /* Check for overflow. */
  2253. if (sizeof(holelen) > sizeof(hlen)) {
  2254. long long holeend =
  2255. (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  2256. if (holeend & ~(long long)ULONG_MAX)
  2257. hlen = ULONG_MAX - hba + 1;
  2258. }
  2259. details.check_mapping = even_cows ? NULL : mapping;
  2260. details.first_index = hba;
  2261. details.last_index = hba + hlen - 1;
  2262. if (details.last_index < details.first_index)
  2263. details.last_index = ULONG_MAX;
  2264. i_mmap_lock_write(mapping);
  2265. if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap)))
  2266. unmap_mapping_range_tree(&mapping->i_mmap, &details);
  2267. i_mmap_unlock_write(mapping);
  2268. }
  2269. EXPORT_SYMBOL(unmap_mapping_range);
  2270. /*
  2271. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2272. * but allow concurrent faults), and pte mapped but not yet locked.
  2273. * We return with pte unmapped and unlocked.
  2274. *
  2275. * We return with the mmap_sem locked or unlocked in the same cases
  2276. * as does filemap_fault().
  2277. */
  2278. int do_swap_page(struct vm_fault *vmf)
  2279. {
  2280. struct vm_area_struct *vma = vmf->vma;
  2281. struct page *page, *swapcache;
  2282. struct mem_cgroup *memcg;
  2283. swp_entry_t entry;
  2284. pte_t pte;
  2285. int locked;
  2286. int exclusive = 0;
  2287. int ret = 0;
  2288. if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte))
  2289. goto out;
  2290. entry = pte_to_swp_entry(vmf->orig_pte);
  2291. if (unlikely(non_swap_entry(entry))) {
  2292. if (is_migration_entry(entry)) {
  2293. migration_entry_wait(vma->vm_mm, vmf->pmd,
  2294. vmf->address);
  2295. } else if (is_hwpoison_entry(entry)) {
  2296. ret = VM_FAULT_HWPOISON;
  2297. } else {
  2298. print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
  2299. ret = VM_FAULT_SIGBUS;
  2300. }
  2301. goto out;
  2302. }
  2303. delayacct_set_flag(DELAYACCT_PF_SWAPIN);
  2304. page = lookup_swap_cache(entry);
  2305. if (!page) {
  2306. page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vma,
  2307. vmf->address);
  2308. if (!page) {
  2309. /*
  2310. * Back out if somebody else faulted in this pte
  2311. * while we released the pte lock.
  2312. */
  2313. vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
  2314. vmf->address, &vmf->ptl);
  2315. if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
  2316. ret = VM_FAULT_OOM;
  2317. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  2318. goto unlock;
  2319. }
  2320. /* Had to read the page from swap area: Major fault */
  2321. ret = VM_FAULT_MAJOR;
  2322. count_vm_event(PGMAJFAULT);
  2323. mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
  2324. } else if (PageHWPoison(page)) {
  2325. /*
  2326. * hwpoisoned dirty swapcache pages are kept for killing
  2327. * owner processes (which may be unknown at hwpoison time)
  2328. */
  2329. ret = VM_FAULT_HWPOISON;
  2330. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  2331. swapcache = page;
  2332. goto out_release;
  2333. }
  2334. swapcache = page;
  2335. locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
  2336. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  2337. if (!locked) {
  2338. ret |= VM_FAULT_RETRY;
  2339. goto out_release;
  2340. }
  2341. /*
  2342. * Make sure try_to_free_swap or reuse_swap_page or swapoff did not
  2343. * release the swapcache from under us. The page pin, and pte_same
  2344. * test below, are not enough to exclude that. Even if it is still
  2345. * swapcache, we need to check that the page's swap has not changed.
  2346. */
  2347. if (unlikely(!PageSwapCache(page) || page_private(page) != entry.val))
  2348. goto out_page;
  2349. page = ksm_might_need_to_copy(page, vma, vmf->address);
  2350. if (unlikely(!page)) {
  2351. ret = VM_FAULT_OOM;
  2352. page = swapcache;
  2353. goto out_page;
  2354. }
  2355. if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL,
  2356. &memcg, false)) {
  2357. ret = VM_FAULT_OOM;
  2358. goto out_page;
  2359. }
  2360. /*
  2361. * Back out if somebody else already faulted in this pte.
  2362. */
  2363. vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
  2364. &vmf->ptl);
  2365. if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
  2366. goto out_nomap;
  2367. if (unlikely(!PageUptodate(page))) {
  2368. ret = VM_FAULT_SIGBUS;
  2369. goto out_nomap;
  2370. }
  2371. /*
  2372. * The page isn't present yet, go ahead with the fault.
  2373. *
  2374. * Be careful about the sequence of operations here.
  2375. * To get its accounting right, reuse_swap_page() must be called
  2376. * while the page is counted on swap but not yet in mapcount i.e.
  2377. * before page_add_anon_rmap() and swap_free(); try_to_free_swap()
  2378. * must be called after the swap_free(), or it will never succeed.
  2379. */
  2380. inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
  2381. dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
  2382. pte = mk_pte(page, vma->vm_page_prot);
  2383. if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
  2384. pte = maybe_mkwrite(pte_mkdirty(pte), vma);
  2385. vmf->flags &= ~FAULT_FLAG_WRITE;
  2386. ret |= VM_FAULT_WRITE;
  2387. exclusive = RMAP_EXCLUSIVE;
  2388. }
  2389. flush_icache_page(vma, page);
  2390. if (pte_swp_soft_dirty(vmf->orig_pte))
  2391. pte = pte_mksoft_dirty(pte);
  2392. set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
  2393. vmf->orig_pte = pte;
  2394. if (page == swapcache) {
  2395. do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
  2396. mem_cgroup_commit_charge(page, memcg, true, false);
  2397. activate_page(page);
  2398. } else { /* ksm created a completely new copy */
  2399. page_add_new_anon_rmap(page, vma, vmf->address, false);
  2400. mem_cgroup_commit_charge(page, memcg, false, false);
  2401. lru_cache_add_active_or_unevictable(page, vma);
  2402. }
  2403. swap_free(entry);
  2404. if (mem_cgroup_swap_full(page) ||
  2405. (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
  2406. try_to_free_swap(page);
  2407. unlock_page(page);
  2408. if (page != swapcache) {
  2409. /*
  2410. * Hold the lock to avoid the swap entry to be reused
  2411. * until we take the PT lock for the pte_same() check
  2412. * (to avoid false positives from pte_same). For
  2413. * further safety release the lock after the swap_free
  2414. * so that the swap count won't change under a
  2415. * parallel locked swapcache.
  2416. */
  2417. unlock_page(swapcache);
  2418. put_page(swapcache);
  2419. }
  2420. if (vmf->flags & FAULT_FLAG_WRITE) {
  2421. ret |= do_wp_page(vmf);
  2422. if (ret & VM_FAULT_ERROR)
  2423. ret &= VM_FAULT_ERROR;
  2424. goto out;
  2425. }
  2426. /* No need to invalidate - it was non-present before */
  2427. update_mmu_cache(vma, vmf->address, vmf->pte);
  2428. unlock:
  2429. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2430. out:
  2431. return ret;
  2432. out_nomap:
  2433. mem_cgroup_cancel_charge(page, memcg, false);
  2434. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2435. out_page:
  2436. unlock_page(page);
  2437. out_release:
  2438. put_page(page);
  2439. if (page != swapcache) {
  2440. unlock_page(swapcache);
  2441. put_page(swapcache);
  2442. }
  2443. return ret;
  2444. }
  2445. /*
  2446. * This is like a special single-page "expand_{down|up}wards()",
  2447. * except we must first make sure that 'address{-|+}PAGE_SIZE'
  2448. * doesn't hit another vma.
  2449. */
  2450. static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned long address)
  2451. {
  2452. address &= PAGE_MASK;
  2453. if ((vma->vm_flags & VM_GROWSDOWN) && address == vma->vm_start) {
  2454. struct vm_area_struct *prev = vma->vm_prev;
  2455. /*
  2456. * Is there a mapping abutting this one below?
  2457. *
  2458. * That's only ok if it's the same stack mapping
  2459. * that has gotten split..
  2460. */
  2461. if (prev && prev->vm_end == address)
  2462. return prev->vm_flags & VM_GROWSDOWN ? 0 : -ENOMEM;
  2463. return expand_downwards(vma, address - PAGE_SIZE);
  2464. }
  2465. if ((vma->vm_flags & VM_GROWSUP) && address + PAGE_SIZE == vma->vm_end) {
  2466. struct vm_area_struct *next = vma->vm_next;
  2467. /* As VM_GROWSDOWN but s/below/above/ */
  2468. if (next && next->vm_start == address + PAGE_SIZE)
  2469. return next->vm_flags & VM_GROWSUP ? 0 : -ENOMEM;
  2470. return expand_upwards(vma, address + PAGE_SIZE);
  2471. }
  2472. return 0;
  2473. }
  2474. /*
  2475. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2476. * but allow concurrent faults), and pte mapped but not yet locked.
  2477. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2478. */
  2479. static int do_anonymous_page(struct vm_fault *vmf)
  2480. {
  2481. struct vm_area_struct *vma = vmf->vma;
  2482. struct mem_cgroup *memcg;
  2483. struct page *page;
  2484. pte_t entry;
  2485. /* File mapping without ->vm_ops ? */
  2486. if (vma->vm_flags & VM_SHARED)
  2487. return VM_FAULT_SIGBUS;
  2488. /* Check if we need to add a guard page to the stack */
  2489. if (check_stack_guard_page(vma, vmf->address) < 0)
  2490. return VM_FAULT_SIGSEGV;
  2491. /*
  2492. * Use pte_alloc() instead of pte_alloc_map(). We can't run
  2493. * pte_offset_map() on pmds where a huge pmd might be created
  2494. * from a different thread.
  2495. *
  2496. * pte_alloc_map() is safe to use under down_write(mmap_sem) or when
  2497. * parallel threads are excluded by other means.
  2498. *
  2499. * Here we only have down_read(mmap_sem).
  2500. */
  2501. if (pte_alloc(vma->vm_mm, vmf->pmd, vmf->address))
  2502. return VM_FAULT_OOM;
  2503. /* See the comment in pte_alloc_one_map() */
  2504. if (unlikely(pmd_trans_unstable(vmf->pmd)))
  2505. return 0;
  2506. /* Use the zero-page for reads */
  2507. if (!(vmf->flags & FAULT_FLAG_WRITE) &&
  2508. !mm_forbids_zeropage(vma->vm_mm)) {
  2509. entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
  2510. vma->vm_page_prot));
  2511. vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
  2512. vmf->address, &vmf->ptl);
  2513. if (!pte_none(*vmf->pte))
  2514. goto unlock;
  2515. /* Deliver the page fault to userland, check inside PT lock */
  2516. if (userfaultfd_missing(vma)) {
  2517. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2518. return handle_userfault(vmf, VM_UFFD_MISSING);
  2519. }
  2520. goto setpte;
  2521. }
  2522. /* Allocate our own private page. */
  2523. if (unlikely(anon_vma_prepare(vma)))
  2524. goto oom;
  2525. page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
  2526. if (!page)
  2527. goto oom;
  2528. if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false))
  2529. goto oom_free_page;
  2530. /*
  2531. * The memory barrier inside __SetPageUptodate makes sure that
  2532. * preceeding stores to the page contents become visible before
  2533. * the set_pte_at() write.
  2534. */
  2535. __SetPageUptodate(page);
  2536. entry = mk_pte(page, vma->vm_page_prot);
  2537. if (vma->vm_flags & VM_WRITE)
  2538. entry = pte_mkwrite(pte_mkdirty(entry));
  2539. vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
  2540. &vmf->ptl);
  2541. if (!pte_none(*vmf->pte))
  2542. goto release;
  2543. /* Deliver the page fault to userland, check inside PT lock */
  2544. if (userfaultfd_missing(vma)) {
  2545. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2546. mem_cgroup_cancel_charge(page, memcg, false);
  2547. put_page(page);
  2548. return handle_userfault(vmf, VM_UFFD_MISSING);
  2549. }
  2550. inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
  2551. page_add_new_anon_rmap(page, vma, vmf->address, false);
  2552. mem_cgroup_commit_charge(page, memcg, false, false);
  2553. lru_cache_add_active_or_unevictable(page, vma);
  2554. setpte:
  2555. set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
  2556. /* No need to invalidate - it was non-present before */
  2557. update_mmu_cache(vma, vmf->address, vmf->pte);
  2558. unlock:
  2559. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2560. return 0;
  2561. release:
  2562. mem_cgroup_cancel_charge(page, memcg, false);
  2563. put_page(page);
  2564. goto unlock;
  2565. oom_free_page:
  2566. put_page(page);
  2567. oom:
  2568. return VM_FAULT_OOM;
  2569. }
  2570. /*
  2571. * The mmap_sem must have been held on entry, and may have been
  2572. * released depending on flags and vma->vm_ops->fault() return value.
  2573. * See filemap_fault() and __lock_page_retry().
  2574. */
  2575. static int __do_fault(struct vm_fault *vmf)
  2576. {
  2577. struct vm_area_struct *vma = vmf->vma;
  2578. int ret;
  2579. ret = vma->vm_ops->fault(vmf);
  2580. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
  2581. VM_FAULT_DONE_COW)))
  2582. return ret;
  2583. if (unlikely(PageHWPoison(vmf->page))) {
  2584. if (ret & VM_FAULT_LOCKED)
  2585. unlock_page(vmf->page);
  2586. put_page(vmf->page);
  2587. vmf->page = NULL;
  2588. return VM_FAULT_HWPOISON;
  2589. }
  2590. if (unlikely(!(ret & VM_FAULT_LOCKED)))
  2591. lock_page(vmf->page);
  2592. else
  2593. VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
  2594. return ret;
  2595. }
  2596. static int pte_alloc_one_map(struct vm_fault *vmf)
  2597. {
  2598. struct vm_area_struct *vma = vmf->vma;
  2599. if (!pmd_none(*vmf->pmd))
  2600. goto map_pte;
  2601. if (vmf->prealloc_pte) {
  2602. vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
  2603. if (unlikely(!pmd_none(*vmf->pmd))) {
  2604. spin_unlock(vmf->ptl);
  2605. goto map_pte;
  2606. }
  2607. atomic_long_inc(&vma->vm_mm->nr_ptes);
  2608. pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
  2609. spin_unlock(vmf->ptl);
  2610. vmf->prealloc_pte = NULL;
  2611. } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd, vmf->address))) {
  2612. return VM_FAULT_OOM;
  2613. }
  2614. map_pte:
  2615. /*
  2616. * If a huge pmd materialized under us just retry later. Use
  2617. * pmd_trans_unstable() instead of pmd_trans_huge() to ensure the pmd
  2618. * didn't become pmd_trans_huge under us and then back to pmd_none, as
  2619. * a result of MADV_DONTNEED running immediately after a huge pmd fault
  2620. * in a different thread of this mm, in turn leading to a misleading
  2621. * pmd_trans_huge() retval. All we have to ensure is that it is a
  2622. * regular pmd that we can walk with pte_offset_map() and we can do that
  2623. * through an atomic read in C, which is what pmd_trans_unstable()
  2624. * provides.
  2625. */
  2626. if (pmd_trans_unstable(vmf->pmd) || pmd_devmap(*vmf->pmd))
  2627. return VM_FAULT_NOPAGE;
  2628. vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
  2629. &vmf->ptl);
  2630. return 0;
  2631. }
  2632. #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
  2633. #define HPAGE_CACHE_INDEX_MASK (HPAGE_PMD_NR - 1)
  2634. static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
  2635. unsigned long haddr)
  2636. {
  2637. if (((vma->vm_start >> PAGE_SHIFT) & HPAGE_CACHE_INDEX_MASK) !=
  2638. (vma->vm_pgoff & HPAGE_CACHE_INDEX_MASK))
  2639. return false;
  2640. if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
  2641. return false;
  2642. return true;
  2643. }
  2644. static void deposit_prealloc_pte(struct vm_fault *vmf)
  2645. {
  2646. struct vm_area_struct *vma = vmf->vma;
  2647. pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
  2648. /*
  2649. * We are going to consume the prealloc table,
  2650. * count that as nr_ptes.
  2651. */
  2652. atomic_long_inc(&vma->vm_mm->nr_ptes);
  2653. vmf->prealloc_pte = NULL;
  2654. }
  2655. static int do_set_pmd(struct vm_fault *vmf, struct page *page)
  2656. {
  2657. struct vm_area_struct *vma = vmf->vma;
  2658. bool write = vmf->flags & FAULT_FLAG_WRITE;
  2659. unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
  2660. pmd_t entry;
  2661. int i, ret;
  2662. if (!transhuge_vma_suitable(vma, haddr))
  2663. return VM_FAULT_FALLBACK;
  2664. ret = VM_FAULT_FALLBACK;
  2665. page = compound_head(page);
  2666. /*
  2667. * Archs like ppc64 need additonal space to store information
  2668. * related to pte entry. Use the preallocated table for that.
  2669. */
  2670. if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
  2671. vmf->prealloc_pte = pte_alloc_one(vma->vm_mm, vmf->address);
  2672. if (!vmf->prealloc_pte)
  2673. return VM_FAULT_OOM;
  2674. smp_wmb(); /* See comment in __pte_alloc() */
  2675. }
  2676. vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
  2677. if (unlikely(!pmd_none(*vmf->pmd)))
  2678. goto out;
  2679. for (i = 0; i < HPAGE_PMD_NR; i++)
  2680. flush_icache_page(vma, page + i);
  2681. entry = mk_huge_pmd(page, vma->vm_page_prot);
  2682. if (write)
  2683. entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
  2684. add_mm_counter(vma->vm_mm, MM_FILEPAGES, HPAGE_PMD_NR);
  2685. page_add_file_rmap(page, true);
  2686. /*
  2687. * deposit and withdraw with pmd lock held
  2688. */
  2689. if (arch_needs_pgtable_deposit())
  2690. deposit_prealloc_pte(vmf);
  2691. set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
  2692. update_mmu_cache_pmd(vma, haddr, vmf->pmd);
  2693. /* fault is handled */
  2694. ret = 0;
  2695. count_vm_event(THP_FILE_MAPPED);
  2696. out:
  2697. spin_unlock(vmf->ptl);
  2698. return ret;
  2699. }
  2700. #else
  2701. static int do_set_pmd(struct vm_fault *vmf, struct page *page)
  2702. {
  2703. BUILD_BUG();
  2704. return 0;
  2705. }
  2706. #endif
  2707. /**
  2708. * alloc_set_pte - setup new PTE entry for given page and add reverse page
  2709. * mapping. If needed, the fucntion allocates page table or use pre-allocated.
  2710. *
  2711. * @vmf: fault environment
  2712. * @memcg: memcg to charge page (only for private mappings)
  2713. * @page: page to map
  2714. *
  2715. * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on
  2716. * return.
  2717. *
  2718. * Target users are page handler itself and implementations of
  2719. * vm_ops->map_pages.
  2720. */
  2721. int alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
  2722. struct page *page)
  2723. {
  2724. struct vm_area_struct *vma = vmf->vma;
  2725. bool write = vmf->flags & FAULT_FLAG_WRITE;
  2726. pte_t entry;
  2727. int ret;
  2728. if (pmd_none(*vmf->pmd) && PageTransCompound(page) &&
  2729. IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
  2730. /* THP on COW? */
  2731. VM_BUG_ON_PAGE(memcg, page);
  2732. ret = do_set_pmd(vmf, page);
  2733. if (ret != VM_FAULT_FALLBACK)
  2734. return ret;
  2735. }
  2736. if (!vmf->pte) {
  2737. ret = pte_alloc_one_map(vmf);
  2738. if (ret)
  2739. return ret;
  2740. }
  2741. /* Re-check under ptl */
  2742. if (unlikely(!pte_none(*vmf->pte)))
  2743. return VM_FAULT_NOPAGE;
  2744. flush_icache_page(vma, page);
  2745. entry = mk_pte(page, vma->vm_page_prot);
  2746. if (write)
  2747. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  2748. /* copy-on-write page */
  2749. if (write && !(vma->vm_flags & VM_SHARED)) {
  2750. inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
  2751. page_add_new_anon_rmap(page, vma, vmf->address, false);
  2752. mem_cgroup_commit_charge(page, memcg, false, false);
  2753. lru_cache_add_active_or_unevictable(page, vma);
  2754. } else {
  2755. inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
  2756. page_add_file_rmap(page, false);
  2757. }
  2758. set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
  2759. /* no need to invalidate: a not-present page won't be cached */
  2760. update_mmu_cache(vma, vmf->address, vmf->pte);
  2761. return 0;
  2762. }
  2763. /**
  2764. * finish_fault - finish page fault once we have prepared the page to fault
  2765. *
  2766. * @vmf: structure describing the fault
  2767. *
  2768. * This function handles all that is needed to finish a page fault once the
  2769. * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
  2770. * given page, adds reverse page mapping, handles memcg charges and LRU
  2771. * addition. The function returns 0 on success, VM_FAULT_ code in case of
  2772. * error.
  2773. *
  2774. * The function expects the page to be locked and on success it consumes a
  2775. * reference of a page being mapped (for the PTE which maps it).
  2776. */
  2777. int finish_fault(struct vm_fault *vmf)
  2778. {
  2779. struct page *page;
  2780. int ret;
  2781. /* Did we COW the page? */
  2782. if ((vmf->flags & FAULT_FLAG_WRITE) &&
  2783. !(vmf->vma->vm_flags & VM_SHARED))
  2784. page = vmf->cow_page;
  2785. else
  2786. page = vmf->page;
  2787. ret = alloc_set_pte(vmf, vmf->memcg, page);
  2788. if (vmf->pte)
  2789. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2790. return ret;
  2791. }
  2792. static unsigned long fault_around_bytes __read_mostly =
  2793. rounddown_pow_of_two(65536);
  2794. #ifdef CONFIG_DEBUG_FS
  2795. static int fault_around_bytes_get(void *data, u64 *val)
  2796. {
  2797. *val = fault_around_bytes;
  2798. return 0;
  2799. }
  2800. /*
  2801. * fault_around_pages() and fault_around_mask() expects fault_around_bytes
  2802. * rounded down to nearest page order. It's what do_fault_around() expects to
  2803. * see.
  2804. */
  2805. static int fault_around_bytes_set(void *data, u64 val)
  2806. {
  2807. if (val / PAGE_SIZE > PTRS_PER_PTE)
  2808. return -EINVAL;
  2809. if (val > PAGE_SIZE)
  2810. fault_around_bytes = rounddown_pow_of_two(val);
  2811. else
  2812. fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */
  2813. return 0;
  2814. }
  2815. DEFINE_SIMPLE_ATTRIBUTE(fault_around_bytes_fops,
  2816. fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
  2817. static int __init fault_around_debugfs(void)
  2818. {
  2819. void *ret;
  2820. ret = debugfs_create_file("fault_around_bytes", 0644, NULL, NULL,
  2821. &fault_around_bytes_fops);
  2822. if (!ret)
  2823. pr_warn("Failed to create fault_around_bytes in debugfs");
  2824. return 0;
  2825. }
  2826. late_initcall(fault_around_debugfs);
  2827. #endif
  2828. /*
  2829. * do_fault_around() tries to map few pages around the fault address. The hope
  2830. * is that the pages will be needed soon and this will lower the number of
  2831. * faults to handle.
  2832. *
  2833. * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
  2834. * not ready to be mapped: not up-to-date, locked, etc.
  2835. *
  2836. * This function is called with the page table lock taken. In the split ptlock
  2837. * case the page table lock only protects only those entries which belong to
  2838. * the page table corresponding to the fault address.
  2839. *
  2840. * This function doesn't cross the VMA boundaries, in order to call map_pages()
  2841. * only once.
  2842. *
  2843. * fault_around_pages() defines how many pages we'll try to map.
  2844. * do_fault_around() expects it to return a power of two less than or equal to
  2845. * PTRS_PER_PTE.
  2846. *
  2847. * The virtual address of the area that we map is naturally aligned to the
  2848. * fault_around_pages() value (and therefore to page order). This way it's
  2849. * easier to guarantee that we don't cross page table boundaries.
  2850. */
  2851. static int do_fault_around(struct vm_fault *vmf)
  2852. {
  2853. unsigned long address = vmf->address, nr_pages, mask;
  2854. pgoff_t start_pgoff = vmf->pgoff;
  2855. pgoff_t end_pgoff;
  2856. int off, ret = 0;
  2857. nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT;
  2858. mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK;
  2859. vmf->address = max(address & mask, vmf->vma->vm_start);
  2860. off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
  2861. start_pgoff -= off;
  2862. /*
  2863. * end_pgoff is either end of page table or end of vma
  2864. * or fault_around_pages() from start_pgoff, depending what is nearest.
  2865. */
  2866. end_pgoff = start_pgoff -
  2867. ((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
  2868. PTRS_PER_PTE - 1;
  2869. end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
  2870. start_pgoff + nr_pages - 1);
  2871. if (pmd_none(*vmf->pmd)) {
  2872. vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm,
  2873. vmf->address);
  2874. if (!vmf->prealloc_pte)
  2875. goto out;
  2876. smp_wmb(); /* See comment in __pte_alloc() */
  2877. }
  2878. vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff);
  2879. /* Huge page is mapped? Page fault is solved */
  2880. if (pmd_trans_huge(*vmf->pmd)) {
  2881. ret = VM_FAULT_NOPAGE;
  2882. goto out;
  2883. }
  2884. /* ->map_pages() haven't done anything useful. Cold page cache? */
  2885. if (!vmf->pte)
  2886. goto out;
  2887. /* check if the page fault is solved */
  2888. vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT);
  2889. if (!pte_none(*vmf->pte))
  2890. ret = VM_FAULT_NOPAGE;
  2891. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2892. out:
  2893. vmf->address = address;
  2894. vmf->pte = NULL;
  2895. return ret;
  2896. }
  2897. static int do_read_fault(struct vm_fault *vmf)
  2898. {
  2899. struct vm_area_struct *vma = vmf->vma;
  2900. int ret = 0;
  2901. /*
  2902. * Let's call ->map_pages() first and use ->fault() as fallback
  2903. * if page by the offset is not ready to be mapped (cold cache or
  2904. * something).
  2905. */
  2906. if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) {
  2907. ret = do_fault_around(vmf);
  2908. if (ret)
  2909. return ret;
  2910. }
  2911. ret = __do_fault(vmf);
  2912. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  2913. return ret;
  2914. ret |= finish_fault(vmf);
  2915. unlock_page(vmf->page);
  2916. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  2917. put_page(vmf->page);
  2918. return ret;
  2919. }
  2920. static int do_cow_fault(struct vm_fault *vmf)
  2921. {
  2922. struct vm_area_struct *vma = vmf->vma;
  2923. int ret;
  2924. if (unlikely(anon_vma_prepare(vma)))
  2925. return VM_FAULT_OOM;
  2926. vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
  2927. if (!vmf->cow_page)
  2928. return VM_FAULT_OOM;
  2929. if (mem_cgroup_try_charge(vmf->cow_page, vma->vm_mm, GFP_KERNEL,
  2930. &vmf->memcg, false)) {
  2931. put_page(vmf->cow_page);
  2932. return VM_FAULT_OOM;
  2933. }
  2934. ret = __do_fault(vmf);
  2935. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  2936. goto uncharge_out;
  2937. if (ret & VM_FAULT_DONE_COW)
  2938. return ret;
  2939. copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
  2940. __SetPageUptodate(vmf->cow_page);
  2941. ret |= finish_fault(vmf);
  2942. unlock_page(vmf->page);
  2943. put_page(vmf->page);
  2944. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  2945. goto uncharge_out;
  2946. return ret;
  2947. uncharge_out:
  2948. mem_cgroup_cancel_charge(vmf->cow_page, vmf->memcg, false);
  2949. put_page(vmf->cow_page);
  2950. return ret;
  2951. }
  2952. static int do_shared_fault(struct vm_fault *vmf)
  2953. {
  2954. struct vm_area_struct *vma = vmf->vma;
  2955. int ret, tmp;
  2956. ret = __do_fault(vmf);
  2957. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  2958. return ret;
  2959. /*
  2960. * Check if the backing address space wants to know that the page is
  2961. * about to become writable
  2962. */
  2963. if (vma->vm_ops->page_mkwrite) {
  2964. unlock_page(vmf->page);
  2965. tmp = do_page_mkwrite(vmf);
  2966. if (unlikely(!tmp ||
  2967. (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
  2968. put_page(vmf->page);
  2969. return tmp;
  2970. }
  2971. }
  2972. ret |= finish_fault(vmf);
  2973. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
  2974. VM_FAULT_RETRY))) {
  2975. unlock_page(vmf->page);
  2976. put_page(vmf->page);
  2977. return ret;
  2978. }
  2979. fault_dirty_shared_page(vma, vmf->page);
  2980. return ret;
  2981. }
  2982. /*
  2983. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2984. * but allow concurrent faults).
  2985. * The mmap_sem may have been released depending on flags and our
  2986. * return value. See filemap_fault() and __lock_page_or_retry().
  2987. */
  2988. static int do_fault(struct vm_fault *vmf)
  2989. {
  2990. struct vm_area_struct *vma = vmf->vma;
  2991. int ret;
  2992. /* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */
  2993. if (!vma->vm_ops->fault)
  2994. ret = VM_FAULT_SIGBUS;
  2995. else if (!(vmf->flags & FAULT_FLAG_WRITE))
  2996. ret = do_read_fault(vmf);
  2997. else if (!(vma->vm_flags & VM_SHARED))
  2998. ret = do_cow_fault(vmf);
  2999. else
  3000. ret = do_shared_fault(vmf);
  3001. /* preallocated pagetable is unused: free it */
  3002. if (vmf->prealloc_pte) {
  3003. pte_free(vma->vm_mm, vmf->prealloc_pte);
  3004. vmf->prealloc_pte = NULL;
  3005. }
  3006. return ret;
  3007. }
  3008. static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
  3009. unsigned long addr, int page_nid,
  3010. int *flags)
  3011. {
  3012. get_page(page);
  3013. count_vm_numa_event(NUMA_HINT_FAULTS);
  3014. if (page_nid == numa_node_id()) {
  3015. count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
  3016. *flags |= TNF_FAULT_LOCAL;
  3017. }
  3018. return mpol_misplaced(page, vma, addr);
  3019. }
  3020. static int do_numa_page(struct vm_fault *vmf)
  3021. {
  3022. struct vm_area_struct *vma = vmf->vma;
  3023. struct page *page = NULL;
  3024. int page_nid = -1;
  3025. int last_cpupid;
  3026. int target_nid;
  3027. bool migrated = false;
  3028. pte_t pte;
  3029. bool was_writable = pte_savedwrite(vmf->orig_pte);
  3030. int flags = 0;
  3031. /*
  3032. * The "pte" at this point cannot be used safely without
  3033. * validation through pte_unmap_same(). It's of NUMA type but
  3034. * the pfn may be screwed if the read is non atomic.
  3035. */
  3036. vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);
  3037. spin_lock(vmf->ptl);
  3038. if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
  3039. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3040. goto out;
  3041. }
  3042. /*
  3043. * Make it present again, Depending on how arch implementes non
  3044. * accessible ptes, some can allow access by kernel mode.
  3045. */
  3046. pte = ptep_modify_prot_start(vma->vm_mm, vmf->address, vmf->pte);
  3047. pte = pte_modify(pte, vma->vm_page_prot);
  3048. pte = pte_mkyoung(pte);
  3049. if (was_writable)
  3050. pte = pte_mkwrite(pte);
  3051. ptep_modify_prot_commit(vma->vm_mm, vmf->address, vmf->pte, pte);
  3052. update_mmu_cache(vma, vmf->address, vmf->pte);
  3053. page = vm_normal_page(vma, vmf->address, pte);
  3054. if (!page) {
  3055. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3056. return 0;
  3057. }
  3058. /* TODO: handle PTE-mapped THP */
  3059. if (PageCompound(page)) {
  3060. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3061. return 0;
  3062. }
  3063. /*
  3064. * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
  3065. * much anyway since they can be in shared cache state. This misses
  3066. * the case where a mapping is writable but the process never writes
  3067. * to it but pte_write gets cleared during protection updates and
  3068. * pte_dirty has unpredictable behaviour between PTE scan updates,
  3069. * background writeback, dirty balancing and application behaviour.
  3070. */
  3071. if (!pte_write(pte))
  3072. flags |= TNF_NO_GROUP;
  3073. /*
  3074. * Flag if the page is shared between multiple address spaces. This
  3075. * is later used when determining whether to group tasks together
  3076. */
  3077. if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
  3078. flags |= TNF_SHARED;
  3079. last_cpupid = page_cpupid_last(page);
  3080. page_nid = page_to_nid(page);
  3081. target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
  3082. &flags);
  3083. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3084. if (target_nid == -1) {
  3085. put_page(page);
  3086. goto out;
  3087. }
  3088. /* Migrate to the requested node */
  3089. migrated = migrate_misplaced_page(page, vma, target_nid);
  3090. if (migrated) {
  3091. page_nid = target_nid;
  3092. flags |= TNF_MIGRATED;
  3093. } else
  3094. flags |= TNF_MIGRATE_FAIL;
  3095. out:
  3096. if (page_nid != -1)
  3097. task_numa_fault(last_cpupid, page_nid, 1, flags);
  3098. return 0;
  3099. }
  3100. static int create_huge_pmd(struct vm_fault *vmf)
  3101. {
  3102. if (vma_is_anonymous(vmf->vma))
  3103. return do_huge_pmd_anonymous_page(vmf);
  3104. if (vmf->vma->vm_ops->huge_fault)
  3105. return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
  3106. return VM_FAULT_FALLBACK;
  3107. }
  3108. static int wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd)
  3109. {
  3110. if (vma_is_anonymous(vmf->vma))
  3111. return do_huge_pmd_wp_page(vmf, orig_pmd);
  3112. if (vmf->vma->vm_ops->huge_fault)
  3113. return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
  3114. /* COW handled on pte level: split pmd */
  3115. VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma);
  3116. __split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
  3117. return VM_FAULT_FALLBACK;
  3118. }
  3119. static inline bool vma_is_accessible(struct vm_area_struct *vma)
  3120. {
  3121. return vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE);
  3122. }
  3123. static int create_huge_pud(struct vm_fault *vmf)
  3124. {
  3125. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  3126. /* No support for anonymous transparent PUD pages yet */
  3127. if (vma_is_anonymous(vmf->vma))
  3128. return VM_FAULT_FALLBACK;
  3129. if (vmf->vma->vm_ops->huge_fault)
  3130. return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
  3131. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  3132. return VM_FAULT_FALLBACK;
  3133. }
  3134. static int wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
  3135. {
  3136. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  3137. /* No support for anonymous transparent PUD pages yet */
  3138. if (vma_is_anonymous(vmf->vma))
  3139. return VM_FAULT_FALLBACK;
  3140. if (vmf->vma->vm_ops->huge_fault)
  3141. return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
  3142. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  3143. return VM_FAULT_FALLBACK;
  3144. }
  3145. /*
  3146. * These routines also need to handle stuff like marking pages dirty
  3147. * and/or accessed for architectures that don't do it in hardware (most
  3148. * RISC architectures). The early dirtying is also good on the i386.
  3149. *
  3150. * There is also a hook called "update_mmu_cache()" that architectures
  3151. * with external mmu caches can use to update those (ie the Sparc or
  3152. * PowerPC hashed page tables that act as extended TLBs).
  3153. *
  3154. * We enter with non-exclusive mmap_sem (to exclude vma changes, but allow
  3155. * concurrent faults).
  3156. *
  3157. * The mmap_sem may have been released depending on flags and our return value.
  3158. * See filemap_fault() and __lock_page_or_retry().
  3159. */
  3160. static int handle_pte_fault(struct vm_fault *vmf)
  3161. {
  3162. pte_t entry;
  3163. if (unlikely(pmd_none(*vmf->pmd))) {
  3164. /*
  3165. * Leave __pte_alloc() until later: because vm_ops->fault may
  3166. * want to allocate huge page, and if we expose page table
  3167. * for an instant, it will be difficult to retract from
  3168. * concurrent faults and from rmap lookups.
  3169. */
  3170. vmf->pte = NULL;
  3171. } else {
  3172. /* See comment in pte_alloc_one_map() */
  3173. if (pmd_trans_unstable(vmf->pmd) || pmd_devmap(*vmf->pmd))
  3174. return 0;
  3175. /*
  3176. * A regular pmd is established and it can't morph into a huge
  3177. * pmd from under us anymore at this point because we hold the
  3178. * mmap_sem read mode and khugepaged takes it in write mode.
  3179. * So now it's safe to run pte_offset_map().
  3180. */
  3181. vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
  3182. vmf->orig_pte = *vmf->pte;
  3183. /*
  3184. * some architectures can have larger ptes than wordsize,
  3185. * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
  3186. * CONFIG_32BIT=y, so READ_ONCE or ACCESS_ONCE cannot guarantee
  3187. * atomic accesses. The code below just needs a consistent
  3188. * view for the ifs and we later double check anyway with the
  3189. * ptl lock held. So here a barrier will do.
  3190. */
  3191. barrier();
  3192. if (pte_none(vmf->orig_pte)) {
  3193. pte_unmap(vmf->pte);
  3194. vmf->pte = NULL;
  3195. }
  3196. }
  3197. if (!vmf->pte) {
  3198. if (vma_is_anonymous(vmf->vma))
  3199. return do_anonymous_page(vmf);
  3200. else
  3201. return do_fault(vmf);
  3202. }
  3203. if (!pte_present(vmf->orig_pte))
  3204. return do_swap_page(vmf);
  3205. if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
  3206. return do_numa_page(vmf);
  3207. vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
  3208. spin_lock(vmf->ptl);
  3209. entry = vmf->orig_pte;
  3210. if (unlikely(!pte_same(*vmf->pte, entry)))
  3211. goto unlock;
  3212. if (vmf->flags & FAULT_FLAG_WRITE) {
  3213. if (!pte_write(entry))
  3214. return do_wp_page(vmf);
  3215. entry = pte_mkdirty(entry);
  3216. }
  3217. entry = pte_mkyoung(entry);
  3218. if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
  3219. vmf->flags & FAULT_FLAG_WRITE)) {
  3220. update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
  3221. } else {
  3222. /*
  3223. * This is needed only for protection faults but the arch code
  3224. * is not yet telling us if this is a protection fault or not.
  3225. * This still avoids useless tlb flushes for .text page faults
  3226. * with threads.
  3227. */
  3228. if (vmf->flags & FAULT_FLAG_WRITE)
  3229. flush_tlb_fix_spurious_fault(vmf->vma, vmf->address);
  3230. }
  3231. unlock:
  3232. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3233. return 0;
  3234. }
  3235. /*
  3236. * By the time we get here, we already hold the mm semaphore
  3237. *
  3238. * The mmap_sem may have been released depending on flags and our
  3239. * return value. See filemap_fault() and __lock_page_or_retry().
  3240. */
  3241. static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
  3242. unsigned int flags)
  3243. {
  3244. struct vm_fault vmf = {
  3245. .vma = vma,
  3246. .address = address & PAGE_MASK,
  3247. .flags = flags,
  3248. .pgoff = linear_page_index(vma, address),
  3249. .gfp_mask = __get_fault_gfp_mask(vma),
  3250. };
  3251. struct mm_struct *mm = vma->vm_mm;
  3252. pgd_t *pgd;
  3253. int ret;
  3254. pgd = pgd_offset(mm, address);
  3255. vmf.pud = pud_alloc(mm, pgd, address);
  3256. if (!vmf.pud)
  3257. return VM_FAULT_OOM;
  3258. if (pud_none(*vmf.pud) && transparent_hugepage_enabled(vma)) {
  3259. ret = create_huge_pud(&vmf);
  3260. if (!(ret & VM_FAULT_FALLBACK))
  3261. return ret;
  3262. } else {
  3263. pud_t orig_pud = *vmf.pud;
  3264. barrier();
  3265. if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
  3266. unsigned int dirty = flags & FAULT_FLAG_WRITE;
  3267. /* NUMA case for anonymous PUDs would go here */
  3268. if (dirty && !pud_write(orig_pud)) {
  3269. ret = wp_huge_pud(&vmf, orig_pud);
  3270. if (!(ret & VM_FAULT_FALLBACK))
  3271. return ret;
  3272. } else {
  3273. huge_pud_set_accessed(&vmf, orig_pud);
  3274. return 0;
  3275. }
  3276. }
  3277. }
  3278. vmf.pmd = pmd_alloc(mm, vmf.pud, address);
  3279. if (!vmf.pmd)
  3280. return VM_FAULT_OOM;
  3281. if (pmd_none(*vmf.pmd) && transparent_hugepage_enabled(vma)) {
  3282. ret = create_huge_pmd(&vmf);
  3283. if (!(ret & VM_FAULT_FALLBACK))
  3284. return ret;
  3285. } else {
  3286. pmd_t orig_pmd = *vmf.pmd;
  3287. barrier();
  3288. if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) {
  3289. if (pmd_protnone(orig_pmd) && vma_is_accessible(vma))
  3290. return do_huge_pmd_numa_page(&vmf, orig_pmd);
  3291. if ((vmf.flags & FAULT_FLAG_WRITE) &&
  3292. !pmd_write(orig_pmd)) {
  3293. ret = wp_huge_pmd(&vmf, orig_pmd);
  3294. if (!(ret & VM_FAULT_FALLBACK))
  3295. return ret;
  3296. } else {
  3297. huge_pmd_set_accessed(&vmf, orig_pmd);
  3298. return 0;
  3299. }
  3300. }
  3301. }
  3302. return handle_pte_fault(&vmf);
  3303. }
  3304. /*
  3305. * By the time we get here, we already hold the mm semaphore
  3306. *
  3307. * The mmap_sem may have been released depending on flags and our
  3308. * return value. See filemap_fault() and __lock_page_or_retry().
  3309. */
  3310. int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
  3311. unsigned int flags)
  3312. {
  3313. int ret;
  3314. __set_current_state(TASK_RUNNING);
  3315. count_vm_event(PGFAULT);
  3316. mem_cgroup_count_vm_event(vma->vm_mm, PGFAULT);
  3317. /* do counter updates before entering really critical section. */
  3318. check_sync_rss_stat(current);
  3319. /*
  3320. * Enable the memcg OOM handling for faults triggered in user
  3321. * space. Kernel faults are handled more gracefully.
  3322. */
  3323. if (flags & FAULT_FLAG_USER)
  3324. mem_cgroup_oom_enable();
  3325. if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
  3326. flags & FAULT_FLAG_INSTRUCTION,
  3327. flags & FAULT_FLAG_REMOTE))
  3328. return VM_FAULT_SIGSEGV;
  3329. if (unlikely(is_vm_hugetlb_page(vma)))
  3330. ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
  3331. else
  3332. ret = __handle_mm_fault(vma, address, flags);
  3333. if (flags & FAULT_FLAG_USER) {
  3334. mem_cgroup_oom_disable();
  3335. /*
  3336. * The task may have entered a memcg OOM situation but
  3337. * if the allocation error was handled gracefully (no
  3338. * VM_FAULT_OOM), there is no need to kill anything.
  3339. * Just clean up the OOM state peacefully.
  3340. */
  3341. if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
  3342. mem_cgroup_oom_synchronize(false);
  3343. }
  3344. /*
  3345. * This mm has been already reaped by the oom reaper and so the
  3346. * refault cannot be trusted in general. Anonymous refaults would
  3347. * lose data and give a zero page instead e.g. This is especially
  3348. * problem for use_mm() because regular tasks will just die and
  3349. * the corrupted data will not be visible anywhere while kthread
  3350. * will outlive the oom victim and potentially propagate the date
  3351. * further.
  3352. */
  3353. if (unlikely((current->flags & PF_KTHREAD) && !(ret & VM_FAULT_ERROR)
  3354. && test_bit(MMF_UNSTABLE, &vma->vm_mm->flags)))
  3355. ret = VM_FAULT_SIGBUS;
  3356. return ret;
  3357. }
  3358. EXPORT_SYMBOL_GPL(handle_mm_fault);
  3359. #ifndef __PAGETABLE_PUD_FOLDED
  3360. /*
  3361. * Allocate page upper directory.
  3362. * We've already handled the fast-path in-line.
  3363. */
  3364. int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
  3365. {
  3366. pud_t *new = pud_alloc_one(mm, address);
  3367. if (!new)
  3368. return -ENOMEM;
  3369. smp_wmb(); /* See comment in __pte_alloc */
  3370. spin_lock(&mm->page_table_lock);
  3371. if (pgd_present(*pgd)) /* Another has populated it */
  3372. pud_free(mm, new);
  3373. else
  3374. pgd_populate(mm, pgd, new);
  3375. spin_unlock(&mm->page_table_lock);
  3376. return 0;
  3377. }
  3378. #endif /* __PAGETABLE_PUD_FOLDED */
  3379. #ifndef __PAGETABLE_PMD_FOLDED
  3380. /*
  3381. * Allocate page middle directory.
  3382. * We've already handled the fast-path in-line.
  3383. */
  3384. int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
  3385. {
  3386. spinlock_t *ptl;
  3387. pmd_t *new = pmd_alloc_one(mm, address);
  3388. if (!new)
  3389. return -ENOMEM;
  3390. smp_wmb(); /* See comment in __pte_alloc */
  3391. ptl = pud_lock(mm, pud);
  3392. #ifndef __ARCH_HAS_4LEVEL_HACK
  3393. if (!pud_present(*pud)) {
  3394. mm_inc_nr_pmds(mm);
  3395. pud_populate(mm, pud, new);
  3396. } else /* Another has populated it */
  3397. pmd_free(mm, new);
  3398. #else
  3399. if (!pgd_present(*pud)) {
  3400. mm_inc_nr_pmds(mm);
  3401. pgd_populate(mm, pud, new);
  3402. } else /* Another has populated it */
  3403. pmd_free(mm, new);
  3404. #endif /* __ARCH_HAS_4LEVEL_HACK */
  3405. spin_unlock(ptl);
  3406. return 0;
  3407. }
  3408. #endif /* __PAGETABLE_PMD_FOLDED */
  3409. static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
  3410. pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
  3411. {
  3412. pgd_t *pgd;
  3413. pud_t *pud;
  3414. pmd_t *pmd;
  3415. pte_t *ptep;
  3416. pgd = pgd_offset(mm, address);
  3417. if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
  3418. goto out;
  3419. pud = pud_offset(pgd, address);
  3420. if (pud_none(*pud) || unlikely(pud_bad(*pud)))
  3421. goto out;
  3422. pmd = pmd_offset(pud, address);
  3423. VM_BUG_ON(pmd_trans_huge(*pmd));
  3424. if (pmd_huge(*pmd)) {
  3425. if (!pmdpp)
  3426. goto out;
  3427. *ptlp = pmd_lock(mm, pmd);
  3428. if (pmd_huge(*pmd)) {
  3429. *pmdpp = pmd;
  3430. return 0;
  3431. }
  3432. spin_unlock(*ptlp);
  3433. }
  3434. if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
  3435. goto out;
  3436. ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
  3437. if (!ptep)
  3438. goto out;
  3439. if (!pte_present(*ptep))
  3440. goto unlock;
  3441. *ptepp = ptep;
  3442. return 0;
  3443. unlock:
  3444. pte_unmap_unlock(ptep, *ptlp);
  3445. out:
  3446. return -EINVAL;
  3447. }
  3448. static inline int follow_pte(struct mm_struct *mm, unsigned long address,
  3449. pte_t **ptepp, spinlock_t **ptlp)
  3450. {
  3451. int res;
  3452. /* (void) is needed to make gcc happy */
  3453. (void) __cond_lock(*ptlp,
  3454. !(res = __follow_pte_pmd(mm, address, ptepp, NULL,
  3455. ptlp)));
  3456. return res;
  3457. }
  3458. int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
  3459. pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
  3460. {
  3461. int res;
  3462. /* (void) is needed to make gcc happy */
  3463. (void) __cond_lock(*ptlp,
  3464. !(res = __follow_pte_pmd(mm, address, ptepp, pmdpp,
  3465. ptlp)));
  3466. return res;
  3467. }
  3468. EXPORT_SYMBOL(follow_pte_pmd);
  3469. /**
  3470. * follow_pfn - look up PFN at a user virtual address
  3471. * @vma: memory mapping
  3472. * @address: user virtual address
  3473. * @pfn: location to store found PFN
  3474. *
  3475. * Only IO mappings and raw PFN mappings are allowed.
  3476. *
  3477. * Returns zero and the pfn at @pfn on success, -ve otherwise.
  3478. */
  3479. int follow_pfn(struct vm_area_struct *vma, unsigned long address,
  3480. unsigned long *pfn)
  3481. {
  3482. int ret = -EINVAL;
  3483. spinlock_t *ptl;
  3484. pte_t *ptep;
  3485. if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
  3486. return ret;
  3487. ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
  3488. if (ret)
  3489. return ret;
  3490. *pfn = pte_pfn(*ptep);
  3491. pte_unmap_unlock(ptep, ptl);
  3492. return 0;
  3493. }
  3494. EXPORT_SYMBOL(follow_pfn);
  3495. #ifdef CONFIG_HAVE_IOREMAP_PROT
  3496. int follow_phys(struct vm_area_struct *vma,
  3497. unsigned long address, unsigned int flags,
  3498. unsigned long *prot, resource_size_t *phys)
  3499. {
  3500. int ret = -EINVAL;
  3501. pte_t *ptep, pte;
  3502. spinlock_t *ptl;
  3503. if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
  3504. goto out;
  3505. if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
  3506. goto out;
  3507. pte = *ptep;
  3508. if ((flags & FOLL_WRITE) && !pte_write(pte))
  3509. goto unlock;
  3510. *prot = pgprot_val(pte_pgprot(pte));
  3511. *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
  3512. ret = 0;
  3513. unlock:
  3514. pte_unmap_unlock(ptep, ptl);
  3515. out:
  3516. return ret;
  3517. }
  3518. int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
  3519. void *buf, int len, int write)
  3520. {
  3521. resource_size_t phys_addr;
  3522. unsigned long prot = 0;
  3523. void __iomem *maddr;
  3524. int offset = addr & (PAGE_SIZE-1);
  3525. if (follow_phys(vma, addr, write, &prot, &phys_addr))
  3526. return -EINVAL;
  3527. maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
  3528. if (write)
  3529. memcpy_toio(maddr + offset, buf, len);
  3530. else
  3531. memcpy_fromio(buf, maddr + offset, len);
  3532. iounmap(maddr);
  3533. return len;
  3534. }
  3535. EXPORT_SYMBOL_GPL(generic_access_phys);
  3536. #endif
  3537. /*
  3538. * Access another process' address space as given in mm. If non-NULL, use the
  3539. * given task for page fault accounting.
  3540. */
  3541. int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
  3542. unsigned long addr, void *buf, int len, unsigned int gup_flags)
  3543. {
  3544. struct vm_area_struct *vma;
  3545. void *old_buf = buf;
  3546. int write = gup_flags & FOLL_WRITE;
  3547. down_read(&mm->mmap_sem);
  3548. /* ignore errors, just check how much was successfully transferred */
  3549. while (len) {
  3550. int bytes, ret, offset;
  3551. void *maddr;
  3552. struct page *page = NULL;
  3553. ret = get_user_pages_remote(tsk, mm, addr, 1,
  3554. gup_flags, &page, &vma, NULL);
  3555. if (ret <= 0) {
  3556. #ifndef CONFIG_HAVE_IOREMAP_PROT
  3557. break;
  3558. #else
  3559. /*
  3560. * Check if this is a VM_IO | VM_PFNMAP VMA, which
  3561. * we can access using slightly different code.
  3562. */
  3563. vma = find_vma(mm, addr);
  3564. if (!vma || vma->vm_start > addr)
  3565. break;
  3566. if (vma->vm_ops && vma->vm_ops->access)
  3567. ret = vma->vm_ops->access(vma, addr, buf,
  3568. len, write);
  3569. if (ret <= 0)
  3570. break;
  3571. bytes = ret;
  3572. #endif
  3573. } else {
  3574. bytes = len;
  3575. offset = addr & (PAGE_SIZE-1);
  3576. if (bytes > PAGE_SIZE-offset)
  3577. bytes = PAGE_SIZE-offset;
  3578. maddr = kmap(page);
  3579. if (write) {
  3580. copy_to_user_page(vma, page, addr,
  3581. maddr + offset, buf, bytes);
  3582. set_page_dirty_lock(page);
  3583. } else {
  3584. copy_from_user_page(vma, page, addr,
  3585. buf, maddr + offset, bytes);
  3586. }
  3587. kunmap(page);
  3588. put_page(page);
  3589. }
  3590. len -= bytes;
  3591. buf += bytes;
  3592. addr += bytes;
  3593. }
  3594. up_read(&mm->mmap_sem);
  3595. return buf - old_buf;
  3596. }
  3597. /**
  3598. * access_remote_vm - access another process' address space
  3599. * @mm: the mm_struct of the target address space
  3600. * @addr: start address to access
  3601. * @buf: source or destination buffer
  3602. * @len: number of bytes to transfer
  3603. * @gup_flags: flags modifying lookup behaviour
  3604. *
  3605. * The caller must hold a reference on @mm.
  3606. */
  3607. int access_remote_vm(struct mm_struct *mm, unsigned long addr,
  3608. void *buf, int len, unsigned int gup_flags)
  3609. {
  3610. return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
  3611. }
  3612. /*
  3613. * Access another process' address space.
  3614. * Source/target buffer must be kernel space,
  3615. * Do not walk the page table directly, use get_user_pages
  3616. */
  3617. int access_process_vm(struct task_struct *tsk, unsigned long addr,
  3618. void *buf, int len, unsigned int gup_flags)
  3619. {
  3620. struct mm_struct *mm;
  3621. int ret;
  3622. mm = get_task_mm(tsk);
  3623. if (!mm)
  3624. return 0;
  3625. ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
  3626. mmput(mm);
  3627. return ret;
  3628. }
  3629. EXPORT_SYMBOL_GPL(access_process_vm);
  3630. /*
  3631. * Print the name of a VMA.
  3632. */
  3633. void print_vma_addr(char *prefix, unsigned long ip)
  3634. {
  3635. struct mm_struct *mm = current->mm;
  3636. struct vm_area_struct *vma;
  3637. /*
  3638. * Do not print if we are in atomic
  3639. * contexts (in exception stacks, etc.):
  3640. */
  3641. if (preempt_count())
  3642. return;
  3643. down_read(&mm->mmap_sem);
  3644. vma = find_vma(mm, ip);
  3645. if (vma && vma->vm_file) {
  3646. struct file *f = vma->vm_file;
  3647. char *buf = (char *)__get_free_page(GFP_KERNEL);
  3648. if (buf) {
  3649. char *p;
  3650. p = file_path(f, buf, PAGE_SIZE);
  3651. if (IS_ERR(p))
  3652. p = "?";
  3653. printk("%s%s[%lx+%lx]", prefix, kbasename(p),
  3654. vma->vm_start,
  3655. vma->vm_end - vma->vm_start);
  3656. free_page((unsigned long)buf);
  3657. }
  3658. }
  3659. up_read(&mm->mmap_sem);
  3660. }
  3661. #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
  3662. void __might_fault(const char *file, int line)
  3663. {
  3664. /*
  3665. * Some code (nfs/sunrpc) uses socket ops on kernel memory while
  3666. * holding the mmap_sem, this is safe because kernel memory doesn't
  3667. * get paged out, therefore we'll never actually fault, and the
  3668. * below annotations will generate false positives.
  3669. */
  3670. if (uaccess_kernel())
  3671. return;
  3672. if (pagefault_disabled())
  3673. return;
  3674. __might_sleep(file, line, 0);
  3675. #if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
  3676. if (current->mm)
  3677. might_lock_read(&current->mm->mmap_sem);
  3678. #endif
  3679. }
  3680. EXPORT_SYMBOL(__might_fault);
  3681. #endif
  3682. #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
  3683. static void clear_gigantic_page(struct page *page,
  3684. unsigned long addr,
  3685. unsigned int pages_per_huge_page)
  3686. {
  3687. int i;
  3688. struct page *p = page;
  3689. might_sleep();
  3690. for (i = 0; i < pages_per_huge_page;
  3691. i++, p = mem_map_next(p, page, i)) {
  3692. cond_resched();
  3693. clear_user_highpage(p, addr + i * PAGE_SIZE);
  3694. }
  3695. }
  3696. void clear_huge_page(struct page *page,
  3697. unsigned long addr, unsigned int pages_per_huge_page)
  3698. {
  3699. int i;
  3700. if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
  3701. clear_gigantic_page(page, addr, pages_per_huge_page);
  3702. return;
  3703. }
  3704. might_sleep();
  3705. for (i = 0; i < pages_per_huge_page; i++) {
  3706. cond_resched();
  3707. clear_user_highpage(page + i, addr + i * PAGE_SIZE);
  3708. }
  3709. }
  3710. static void copy_user_gigantic_page(struct page *dst, struct page *src,
  3711. unsigned long addr,
  3712. struct vm_area_struct *vma,
  3713. unsigned int pages_per_huge_page)
  3714. {
  3715. int i;
  3716. struct page *dst_base = dst;
  3717. struct page *src_base = src;
  3718. for (i = 0; i < pages_per_huge_page; ) {
  3719. cond_resched();
  3720. copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
  3721. i++;
  3722. dst = mem_map_next(dst, dst_base, i);
  3723. src = mem_map_next(src, src_base, i);
  3724. }
  3725. }
  3726. void copy_user_huge_page(struct page *dst, struct page *src,
  3727. unsigned long addr, struct vm_area_struct *vma,
  3728. unsigned int pages_per_huge_page)
  3729. {
  3730. int i;
  3731. if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
  3732. copy_user_gigantic_page(dst, src, addr, vma,
  3733. pages_per_huge_page);
  3734. return;
  3735. }
  3736. might_sleep();
  3737. for (i = 0; i < pages_per_huge_page; i++) {
  3738. cond_resched();
  3739. copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
  3740. }
  3741. }
  3742. long copy_huge_page_from_user(struct page *dst_page,
  3743. const void __user *usr_src,
  3744. unsigned int pages_per_huge_page,
  3745. bool allow_pagefault)
  3746. {
  3747. void *src = (void *)usr_src;
  3748. void *page_kaddr;
  3749. unsigned long i, rc = 0;
  3750. unsigned long ret_val = pages_per_huge_page * PAGE_SIZE;
  3751. for (i = 0; i < pages_per_huge_page; i++) {
  3752. if (allow_pagefault)
  3753. page_kaddr = kmap(dst_page + i);
  3754. else
  3755. page_kaddr = kmap_atomic(dst_page + i);
  3756. rc = copy_from_user(page_kaddr,
  3757. (const void __user *)(src + i * PAGE_SIZE),
  3758. PAGE_SIZE);
  3759. if (allow_pagefault)
  3760. kunmap(dst_page + i);
  3761. else
  3762. kunmap_atomic(page_kaddr);
  3763. ret_val -= (PAGE_SIZE - rc);
  3764. if (rc)
  3765. break;
  3766. cond_resched();
  3767. }
  3768. return ret_val;
  3769. }
  3770. #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
  3771. #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
  3772. static struct kmem_cache *page_ptl_cachep;
  3773. void __init ptlock_cache_init(void)
  3774. {
  3775. page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
  3776. SLAB_PANIC, NULL);
  3777. }
  3778. bool ptlock_alloc(struct page *page)
  3779. {
  3780. spinlock_t *ptl;
  3781. ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
  3782. if (!ptl)
  3783. return false;
  3784. page->ptl = ptl;
  3785. return true;
  3786. }
  3787. void ptlock_free(struct page *page)
  3788. {
  3789. kmem_cache_free(page_ptl_cachep, page->ptl);
  3790. }
  3791. #endif