shmem.c 90 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468
  1. /*
  2. * Resizable virtual memory filesystem for Linux.
  3. *
  4. * Copyright (C) 2000 Linus Torvalds.
  5. * 2000 Transmeta Corp.
  6. * 2000-2001 Christoph Rohland
  7. * 2000-2001 SAP AG
  8. * 2002 Red Hat Inc.
  9. * Copyright (C) 2002-2011 Hugh Dickins.
  10. * Copyright (C) 2011 Google Inc.
  11. * Copyright (C) 2002-2005 VERITAS Software Corporation.
  12. * Copyright (C) 2004 Andi Kleen, SuSE Labs
  13. *
  14. * Extended attribute support for tmpfs:
  15. * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
  16. * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
  17. *
  18. * tiny-shmem:
  19. * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
  20. *
  21. * This file is released under the GPL.
  22. */
  23. #include <linux/fs.h>
  24. #include <linux/init.h>
  25. #include <linux/vfs.h>
  26. #include <linux/mount.h>
  27. #include <linux/ramfs.h>
  28. #include <linux/pagemap.h>
  29. #include <linux/file.h>
  30. #include <linux/mm.h>
  31. #include <linux/export.h>
  32. #include <linux/swap.h>
  33. #include <linux/uio.h>
  34. static struct vfsmount *shm_mnt;
  35. #ifdef CONFIG_SHMEM
  36. /*
  37. * This virtual memory filesystem is heavily based on the ramfs. It
  38. * extends ramfs by the ability to use swap and honor resource limits
  39. * which makes it a completely usable filesystem.
  40. */
  41. #include <linux/xattr.h>
  42. #include <linux/exportfs.h>
  43. #include <linux/posix_acl.h>
  44. #include <linux/posix_acl_xattr.h>
  45. #include <linux/mman.h>
  46. #include <linux/string.h>
  47. #include <linux/slab.h>
  48. #include <linux/backing-dev.h>
  49. #include <linux/shmem_fs.h>
  50. #include <linux/writeback.h>
  51. #include <linux/blkdev.h>
  52. #include <linux/pagevec.h>
  53. #include <linux/percpu_counter.h>
  54. #include <linux/falloc.h>
  55. #include <linux/splice.h>
  56. #include <linux/security.h>
  57. #include <linux/swapops.h>
  58. #include <linux/mempolicy.h>
  59. #include <linux/namei.h>
  60. #include <linux/ctype.h>
  61. #include <linux/migrate.h>
  62. #include <linux/highmem.h>
  63. #include <linux/seq_file.h>
  64. #include <linux/magic.h>
  65. #include <linux/syscalls.h>
  66. #include <linux/fcntl.h>
  67. #include <uapi/linux/memfd.h>
  68. #include <asm/uaccess.h>
  69. #include <asm/pgtable.h>
  70. #define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
  71. #define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
  72. /* Pretend that each entry is of this size in directory's i_size */
  73. #define BOGO_DIRENT_SIZE 20
  74. /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
  75. #define SHORT_SYMLINK_LEN 128
  76. /*
  77. * shmem_fallocate communicates with shmem_fault or shmem_writepage via
  78. * inode->i_private (with i_mutex making sure that it has only one user at
  79. * a time): we would prefer not to enlarge the shmem inode just for that.
  80. */
  81. struct shmem_falloc {
  82. wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
  83. pgoff_t start; /* start of range currently being fallocated */
  84. pgoff_t next; /* the next page offset to be fallocated */
  85. pgoff_t nr_falloced; /* how many new pages have been fallocated */
  86. pgoff_t nr_unswapped; /* how often writepage refused to swap out */
  87. };
  88. /* Flag allocation requirements to shmem_getpage */
  89. enum sgp_type {
  90. SGP_READ, /* don't exceed i_size, don't allocate page */
  91. SGP_CACHE, /* don't exceed i_size, may allocate page */
  92. SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */
  93. SGP_WRITE, /* may exceed i_size, may allocate !Uptodate page */
  94. SGP_FALLOC, /* like SGP_WRITE, but make existing page Uptodate */
  95. };
  96. #ifdef CONFIG_TMPFS
  97. static unsigned long shmem_default_max_blocks(void)
  98. {
  99. return totalram_pages / 2;
  100. }
  101. static unsigned long shmem_default_max_inodes(void)
  102. {
  103. return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
  104. }
  105. #endif
  106. static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
  107. static int shmem_replace_page(struct page **pagep, gfp_t gfp,
  108. struct shmem_inode_info *info, pgoff_t index);
  109. static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
  110. struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);
  111. static inline int shmem_getpage(struct inode *inode, pgoff_t index,
  112. struct page **pagep, enum sgp_type sgp, int *fault_type)
  113. {
  114. return shmem_getpage_gfp(inode, index, pagep, sgp,
  115. mapping_gfp_mask(inode->i_mapping), fault_type);
  116. }
  117. static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
  118. {
  119. return sb->s_fs_info;
  120. }
  121. /*
  122. * shmem_file_setup pre-accounts the whole fixed size of a VM object,
  123. * for shared memory and for shared anonymous (/dev/zero) mappings
  124. * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
  125. * consistent with the pre-accounting of private mappings ...
  126. */
  127. static inline int shmem_acct_size(unsigned long flags, loff_t size)
  128. {
  129. return (flags & VM_NORESERVE) ?
  130. 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
  131. }
  132. static inline void shmem_unacct_size(unsigned long flags, loff_t size)
  133. {
  134. if (!(flags & VM_NORESERVE))
  135. vm_unacct_memory(VM_ACCT(size));
  136. }
  137. static inline int shmem_reacct_size(unsigned long flags,
  138. loff_t oldsize, loff_t newsize)
  139. {
  140. if (!(flags & VM_NORESERVE)) {
  141. if (VM_ACCT(newsize) > VM_ACCT(oldsize))
  142. return security_vm_enough_memory_mm(current->mm,
  143. VM_ACCT(newsize) - VM_ACCT(oldsize));
  144. else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
  145. vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
  146. }
  147. return 0;
  148. }
  149. /*
  150. * ... whereas tmpfs objects are accounted incrementally as
  151. * pages are allocated, in order to allow huge sparse files.
  152. * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
  153. * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
  154. */
  155. static inline int shmem_acct_block(unsigned long flags)
  156. {
  157. return (flags & VM_NORESERVE) ?
  158. security_vm_enough_memory_mm(current->mm, VM_ACCT(PAGE_CACHE_SIZE)) : 0;
  159. }
  160. static inline void shmem_unacct_blocks(unsigned long flags, long pages)
  161. {
  162. if (flags & VM_NORESERVE)
  163. vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
  164. }
  165. static const struct super_operations shmem_ops;
  166. static const struct address_space_operations shmem_aops;
  167. static const struct file_operations shmem_file_operations;
  168. static const struct inode_operations shmem_inode_operations;
  169. static const struct inode_operations shmem_dir_inode_operations;
  170. static const struct inode_operations shmem_special_inode_operations;
  171. static const struct vm_operations_struct shmem_vm_ops;
  172. static LIST_HEAD(shmem_swaplist);
  173. static DEFINE_MUTEX(shmem_swaplist_mutex);
  174. static int shmem_reserve_inode(struct super_block *sb)
  175. {
  176. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  177. if (sbinfo->max_inodes) {
  178. spin_lock(&sbinfo->stat_lock);
  179. if (!sbinfo->free_inodes) {
  180. spin_unlock(&sbinfo->stat_lock);
  181. return -ENOSPC;
  182. }
  183. sbinfo->free_inodes--;
  184. spin_unlock(&sbinfo->stat_lock);
  185. }
  186. return 0;
  187. }
  188. static void shmem_free_inode(struct super_block *sb)
  189. {
  190. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  191. if (sbinfo->max_inodes) {
  192. spin_lock(&sbinfo->stat_lock);
  193. sbinfo->free_inodes++;
  194. spin_unlock(&sbinfo->stat_lock);
  195. }
  196. }
  197. /**
  198. * shmem_recalc_inode - recalculate the block usage of an inode
  199. * @inode: inode to recalc
  200. *
  201. * We have to calculate the free blocks since the mm can drop
  202. * undirtied hole pages behind our back.
  203. *
  204. * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
  205. * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
  206. *
  207. * It has to be called with the spinlock held.
  208. */
  209. static void shmem_recalc_inode(struct inode *inode)
  210. {
  211. struct shmem_inode_info *info = SHMEM_I(inode);
  212. long freed;
  213. freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
  214. if (freed > 0) {
  215. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  216. if (sbinfo->max_blocks)
  217. percpu_counter_add(&sbinfo->used_blocks, -freed);
  218. info->alloced -= freed;
  219. inode->i_blocks -= freed * BLOCKS_PER_PAGE;
  220. shmem_unacct_blocks(info->flags, freed);
  221. }
  222. }
  223. /*
  224. * Replace item expected in radix tree by a new item, while holding tree lock.
  225. */
  226. static int shmem_radix_tree_replace(struct address_space *mapping,
  227. pgoff_t index, void *expected, void *replacement)
  228. {
  229. void **pslot;
  230. void *item;
  231. VM_BUG_ON(!expected);
  232. VM_BUG_ON(!replacement);
  233. pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
  234. if (!pslot)
  235. return -ENOENT;
  236. item = radix_tree_deref_slot_protected(pslot, &mapping->tree_lock);
  237. if (item != expected)
  238. return -ENOENT;
  239. radix_tree_replace_slot(pslot, replacement);
  240. return 0;
  241. }
  242. /*
  243. * Sometimes, before we decide whether to proceed or to fail, we must check
  244. * that an entry was not already brought back from swap by a racing thread.
  245. *
  246. * Checking page is not enough: by the time a SwapCache page is locked, it
  247. * might be reused, and again be SwapCache, using the same swap as before.
  248. */
  249. static bool shmem_confirm_swap(struct address_space *mapping,
  250. pgoff_t index, swp_entry_t swap)
  251. {
  252. void *item;
  253. rcu_read_lock();
  254. item = radix_tree_lookup(&mapping->page_tree, index);
  255. rcu_read_unlock();
  256. return item == swp_to_radix_entry(swap);
  257. }
  258. /*
  259. * Like add_to_page_cache_locked, but error if expected item has gone.
  260. */
  261. static int shmem_add_to_page_cache(struct page *page,
  262. struct address_space *mapping,
  263. pgoff_t index, void *expected)
  264. {
  265. int error;
  266. VM_BUG_ON_PAGE(!PageLocked(page), page);
  267. VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
  268. page_cache_get(page);
  269. page->mapping = mapping;
  270. page->index = index;
  271. spin_lock_irq(&mapping->tree_lock);
  272. if (!expected)
  273. error = radix_tree_insert(&mapping->page_tree, index, page);
  274. else
  275. error = shmem_radix_tree_replace(mapping, index, expected,
  276. page);
  277. if (!error) {
  278. mapping->nrpages++;
  279. __inc_zone_page_state(page, NR_FILE_PAGES);
  280. __inc_zone_page_state(page, NR_SHMEM);
  281. spin_unlock_irq(&mapping->tree_lock);
  282. } else {
  283. page->mapping = NULL;
  284. spin_unlock_irq(&mapping->tree_lock);
  285. page_cache_release(page);
  286. }
  287. return error;
  288. }
  289. /*
  290. * Like delete_from_page_cache, but substitutes swap for page.
  291. */
  292. static void shmem_delete_from_page_cache(struct page *page, void *radswap)
  293. {
  294. struct address_space *mapping = page->mapping;
  295. int error;
  296. spin_lock_irq(&mapping->tree_lock);
  297. error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
  298. page->mapping = NULL;
  299. mapping->nrpages--;
  300. __dec_zone_page_state(page, NR_FILE_PAGES);
  301. __dec_zone_page_state(page, NR_SHMEM);
  302. spin_unlock_irq(&mapping->tree_lock);
  303. page_cache_release(page);
  304. BUG_ON(error);
  305. }
  306. /*
  307. * Remove swap entry from radix tree, free the swap and its page cache.
  308. */
  309. static int shmem_free_swap(struct address_space *mapping,
  310. pgoff_t index, void *radswap)
  311. {
  312. void *old;
  313. spin_lock_irq(&mapping->tree_lock);
  314. old = radix_tree_delete_item(&mapping->page_tree, index, radswap);
  315. spin_unlock_irq(&mapping->tree_lock);
  316. if (old != radswap)
  317. return -ENOENT;
  318. free_swap_and_cache(radix_to_swp_entry(radswap));
  319. return 0;
  320. }
  321. /*
  322. * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
  323. */
  324. void shmem_unlock_mapping(struct address_space *mapping)
  325. {
  326. struct pagevec pvec;
  327. pgoff_t indices[PAGEVEC_SIZE];
  328. pgoff_t index = 0;
  329. pagevec_init(&pvec, 0);
  330. /*
  331. * Minor point, but we might as well stop if someone else SHM_LOCKs it.
  332. */
  333. while (!mapping_unevictable(mapping)) {
  334. /*
  335. * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
  336. * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
  337. */
  338. pvec.nr = find_get_entries(mapping, index,
  339. PAGEVEC_SIZE, pvec.pages, indices);
  340. if (!pvec.nr)
  341. break;
  342. index = indices[pvec.nr - 1] + 1;
  343. pagevec_remove_exceptionals(&pvec);
  344. check_move_unevictable_pages(pvec.pages, pvec.nr);
  345. pagevec_release(&pvec);
  346. cond_resched();
  347. }
  348. }
  349. /*
  350. * Remove range of pages and swap entries from radix tree, and free them.
  351. * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
  352. */
  353. static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
  354. bool unfalloc)
  355. {
  356. struct address_space *mapping = inode->i_mapping;
  357. struct shmem_inode_info *info = SHMEM_I(inode);
  358. pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  359. pgoff_t end = (lend + 1) >> PAGE_CACHE_SHIFT;
  360. unsigned int partial_start = lstart & (PAGE_CACHE_SIZE - 1);
  361. unsigned int partial_end = (lend + 1) & (PAGE_CACHE_SIZE - 1);
  362. struct pagevec pvec;
  363. pgoff_t indices[PAGEVEC_SIZE];
  364. long nr_swaps_freed = 0;
  365. pgoff_t index;
  366. int i;
  367. if (lend == -1)
  368. end = -1; /* unsigned, so actually very big */
  369. pagevec_init(&pvec, 0);
  370. index = start;
  371. while (index < end) {
  372. pvec.nr = find_get_entries(mapping, index,
  373. min(end - index, (pgoff_t)PAGEVEC_SIZE),
  374. pvec.pages, indices);
  375. if (!pvec.nr)
  376. break;
  377. for (i = 0; i < pagevec_count(&pvec); i++) {
  378. struct page *page = pvec.pages[i];
  379. index = indices[i];
  380. if (index >= end)
  381. break;
  382. if (radix_tree_exceptional_entry(page)) {
  383. if (unfalloc)
  384. continue;
  385. nr_swaps_freed += !shmem_free_swap(mapping,
  386. index, page);
  387. continue;
  388. }
  389. if (!trylock_page(page))
  390. continue;
  391. if (!unfalloc || !PageUptodate(page)) {
  392. if (page->mapping == mapping) {
  393. VM_BUG_ON_PAGE(PageWriteback(page), page);
  394. truncate_inode_page(mapping, page);
  395. }
  396. }
  397. unlock_page(page);
  398. }
  399. pagevec_remove_exceptionals(&pvec);
  400. pagevec_release(&pvec);
  401. cond_resched();
  402. index++;
  403. }
  404. if (partial_start) {
  405. struct page *page = NULL;
  406. shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
  407. if (page) {
  408. unsigned int top = PAGE_CACHE_SIZE;
  409. if (start > end) {
  410. top = partial_end;
  411. partial_end = 0;
  412. }
  413. zero_user_segment(page, partial_start, top);
  414. set_page_dirty(page);
  415. unlock_page(page);
  416. page_cache_release(page);
  417. }
  418. }
  419. if (partial_end) {
  420. struct page *page = NULL;
  421. shmem_getpage(inode, end, &page, SGP_READ, NULL);
  422. if (page) {
  423. zero_user_segment(page, 0, partial_end);
  424. set_page_dirty(page);
  425. unlock_page(page);
  426. page_cache_release(page);
  427. }
  428. }
  429. if (start >= end)
  430. return;
  431. index = start;
  432. while (index < end) {
  433. cond_resched();
  434. pvec.nr = find_get_entries(mapping, index,
  435. min(end - index, (pgoff_t)PAGEVEC_SIZE),
  436. pvec.pages, indices);
  437. if (!pvec.nr) {
  438. /* If all gone or hole-punch or unfalloc, we're done */
  439. if (index == start || end != -1)
  440. break;
  441. /* But if truncating, restart to make sure all gone */
  442. index = start;
  443. continue;
  444. }
  445. for (i = 0; i < pagevec_count(&pvec); i++) {
  446. struct page *page = pvec.pages[i];
  447. index = indices[i];
  448. if (index >= end)
  449. break;
  450. if (radix_tree_exceptional_entry(page)) {
  451. if (unfalloc)
  452. continue;
  453. if (shmem_free_swap(mapping, index, page)) {
  454. /* Swap was replaced by page: retry */
  455. index--;
  456. break;
  457. }
  458. nr_swaps_freed++;
  459. continue;
  460. }
  461. lock_page(page);
  462. if (!unfalloc || !PageUptodate(page)) {
  463. if (page->mapping == mapping) {
  464. VM_BUG_ON_PAGE(PageWriteback(page), page);
  465. truncate_inode_page(mapping, page);
  466. } else {
  467. /* Page was replaced by swap: retry */
  468. unlock_page(page);
  469. index--;
  470. break;
  471. }
  472. }
  473. unlock_page(page);
  474. }
  475. pagevec_remove_exceptionals(&pvec);
  476. pagevec_release(&pvec);
  477. index++;
  478. }
  479. spin_lock(&info->lock);
  480. info->swapped -= nr_swaps_freed;
  481. shmem_recalc_inode(inode);
  482. spin_unlock(&info->lock);
  483. }
  484. void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
  485. {
  486. shmem_undo_range(inode, lstart, lend, false);
  487. inode->i_ctime = inode->i_mtime = CURRENT_TIME;
  488. }
  489. EXPORT_SYMBOL_GPL(shmem_truncate_range);
  490. static int shmem_getattr(struct vfsmount *mnt, struct dentry *dentry,
  491. struct kstat *stat)
  492. {
  493. struct inode *inode = dentry->d_inode;
  494. struct shmem_inode_info *info = SHMEM_I(inode);
  495. spin_lock(&info->lock);
  496. shmem_recalc_inode(inode);
  497. spin_unlock(&info->lock);
  498. generic_fillattr(inode, stat);
  499. return 0;
  500. }
  501. static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
  502. {
  503. struct inode *inode = d_inode(dentry);
  504. struct shmem_inode_info *info = SHMEM_I(inode);
  505. int error;
  506. error = inode_change_ok(inode, attr);
  507. if (error)
  508. return error;
  509. if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
  510. loff_t oldsize = inode->i_size;
  511. loff_t newsize = attr->ia_size;
  512. /* protected by i_mutex */
  513. if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
  514. (newsize > oldsize && (info->seals & F_SEAL_GROW)))
  515. return -EPERM;
  516. if (newsize != oldsize) {
  517. error = shmem_reacct_size(SHMEM_I(inode)->flags,
  518. oldsize, newsize);
  519. if (error)
  520. return error;
  521. i_size_write(inode, newsize);
  522. inode->i_ctime = inode->i_mtime = CURRENT_TIME;
  523. }
  524. if (newsize <= oldsize) {
  525. loff_t holebegin = round_up(newsize, PAGE_SIZE);
  526. unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
  527. shmem_truncate_range(inode, newsize, (loff_t)-1);
  528. /* unmap again to remove racily COWed private pages */
  529. unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
  530. }
  531. }
  532. setattr_copy(inode, attr);
  533. if (attr->ia_valid & ATTR_MODE)
  534. error = posix_acl_chmod(inode, inode->i_mode);
  535. return error;
  536. }
  537. static void shmem_evict_inode(struct inode *inode)
  538. {
  539. struct shmem_inode_info *info = SHMEM_I(inode);
  540. if (inode->i_mapping->a_ops == &shmem_aops) {
  541. shmem_unacct_size(info->flags, inode->i_size);
  542. inode->i_size = 0;
  543. shmem_truncate_range(inode, 0, (loff_t)-1);
  544. if (!list_empty(&info->swaplist)) {
  545. mutex_lock(&shmem_swaplist_mutex);
  546. list_del_init(&info->swaplist);
  547. mutex_unlock(&shmem_swaplist_mutex);
  548. }
  549. } else
  550. kfree(info->symlink);
  551. simple_xattrs_free(&info->xattrs);
  552. WARN_ON(inode->i_blocks);
  553. shmem_free_inode(inode->i_sb);
  554. clear_inode(inode);
  555. }
  556. /*
  557. * If swap found in inode, free it and move page from swapcache to filecache.
  558. */
  559. static int shmem_unuse_inode(struct shmem_inode_info *info,
  560. swp_entry_t swap, struct page **pagep)
  561. {
  562. struct address_space *mapping = info->vfs_inode.i_mapping;
  563. void *radswap;
  564. pgoff_t index;
  565. gfp_t gfp;
  566. int error = 0;
  567. radswap = swp_to_radix_entry(swap);
  568. index = radix_tree_locate_item(&mapping->page_tree, radswap);
  569. if (index == -1)
  570. return -EAGAIN; /* tell shmem_unuse we found nothing */
  571. /*
  572. * Move _head_ to start search for next from here.
  573. * But be careful: shmem_evict_inode checks list_empty without taking
  574. * mutex, and there's an instant in list_move_tail when info->swaplist
  575. * would appear empty, if it were the only one on shmem_swaplist.
  576. */
  577. if (shmem_swaplist.next != &info->swaplist)
  578. list_move_tail(&shmem_swaplist, &info->swaplist);
  579. gfp = mapping_gfp_mask(mapping);
  580. if (shmem_should_replace_page(*pagep, gfp)) {
  581. mutex_unlock(&shmem_swaplist_mutex);
  582. error = shmem_replace_page(pagep, gfp, info, index);
  583. mutex_lock(&shmem_swaplist_mutex);
  584. /*
  585. * We needed to drop mutex to make that restrictive page
  586. * allocation, but the inode might have been freed while we
  587. * dropped it: although a racing shmem_evict_inode() cannot
  588. * complete without emptying the radix_tree, our page lock
  589. * on this swapcache page is not enough to prevent that -
  590. * free_swap_and_cache() of our swap entry will only
  591. * trylock_page(), removing swap from radix_tree whatever.
  592. *
  593. * We must not proceed to shmem_add_to_page_cache() if the
  594. * inode has been freed, but of course we cannot rely on
  595. * inode or mapping or info to check that. However, we can
  596. * safely check if our swap entry is still in use (and here
  597. * it can't have got reused for another page): if it's still
  598. * in use, then the inode cannot have been freed yet, and we
  599. * can safely proceed (if it's no longer in use, that tells
  600. * nothing about the inode, but we don't need to unuse swap).
  601. */
  602. if (!page_swapcount(*pagep))
  603. error = -ENOENT;
  604. }
  605. /*
  606. * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
  607. * but also to hold up shmem_evict_inode(): so inode cannot be freed
  608. * beneath us (pagelock doesn't help until the page is in pagecache).
  609. */
  610. if (!error)
  611. error = shmem_add_to_page_cache(*pagep, mapping, index,
  612. radswap);
  613. if (error != -ENOMEM) {
  614. /*
  615. * Truncation and eviction use free_swap_and_cache(), which
  616. * only does trylock page: if we raced, best clean up here.
  617. */
  618. delete_from_swap_cache(*pagep);
  619. set_page_dirty(*pagep);
  620. if (!error) {
  621. spin_lock(&info->lock);
  622. info->swapped--;
  623. spin_unlock(&info->lock);
  624. swap_free(swap);
  625. }
  626. }
  627. return error;
  628. }
  629. /*
  630. * Search through swapped inodes to find and replace swap by page.
  631. */
  632. int shmem_unuse(swp_entry_t swap, struct page *page)
  633. {
  634. struct list_head *this, *next;
  635. struct shmem_inode_info *info;
  636. struct mem_cgroup *memcg;
  637. int error = 0;
  638. /*
  639. * There's a faint possibility that swap page was replaced before
  640. * caller locked it: caller will come back later with the right page.
  641. */
  642. if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))
  643. goto out;
  644. /*
  645. * Charge page using GFP_KERNEL while we can wait, before taking
  646. * the shmem_swaplist_mutex which might hold up shmem_writepage().
  647. * Charged back to the user (not to caller) when swap account is used.
  648. */
  649. error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg);
  650. if (error)
  651. goto out;
  652. /* No radix_tree_preload: swap entry keeps a place for page in tree */
  653. error = -EAGAIN;
  654. mutex_lock(&shmem_swaplist_mutex);
  655. list_for_each_safe(this, next, &shmem_swaplist) {
  656. info = list_entry(this, struct shmem_inode_info, swaplist);
  657. if (info->swapped)
  658. error = shmem_unuse_inode(info, swap, &page);
  659. else
  660. list_del_init(&info->swaplist);
  661. cond_resched();
  662. if (error != -EAGAIN)
  663. break;
  664. /* found nothing in this: move on to search the next */
  665. }
  666. mutex_unlock(&shmem_swaplist_mutex);
  667. if (error) {
  668. if (error != -ENOMEM)
  669. error = 0;
  670. mem_cgroup_cancel_charge(page, memcg);
  671. } else
  672. mem_cgroup_commit_charge(page, memcg, true);
  673. out:
  674. unlock_page(page);
  675. page_cache_release(page);
  676. return error;
  677. }
  678. /*
  679. * Move the page from the page cache to the swap cache.
  680. */
  681. static int shmem_writepage(struct page *page, struct writeback_control *wbc)
  682. {
  683. struct shmem_inode_info *info;
  684. struct address_space *mapping;
  685. struct inode *inode;
  686. swp_entry_t swap;
  687. pgoff_t index;
  688. BUG_ON(!PageLocked(page));
  689. mapping = page->mapping;
  690. index = page->index;
  691. inode = mapping->host;
  692. info = SHMEM_I(inode);
  693. if (info->flags & VM_LOCKED)
  694. goto redirty;
  695. if (!total_swap_pages)
  696. goto redirty;
  697. /*
  698. * Our capabilities prevent regular writeback or sync from ever calling
  699. * shmem_writepage; but a stacking filesystem might use ->writepage of
  700. * its underlying filesystem, in which case tmpfs should write out to
  701. * swap only in response to memory pressure, and not for the writeback
  702. * threads or sync.
  703. */
  704. if (!wbc->for_reclaim) {
  705. WARN_ON_ONCE(1); /* Still happens? Tell us about it! */
  706. goto redirty;
  707. }
  708. /*
  709. * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
  710. * value into swapfile.c, the only way we can correctly account for a
  711. * fallocated page arriving here is now to initialize it and write it.
  712. *
  713. * That's okay for a page already fallocated earlier, but if we have
  714. * not yet completed the fallocation, then (a) we want to keep track
  715. * of this page in case we have to undo it, and (b) it may not be a
  716. * good idea to continue anyway, once we're pushing into swap. So
  717. * reactivate the page, and let shmem_fallocate() quit when too many.
  718. */
  719. if (!PageUptodate(page)) {
  720. if (inode->i_private) {
  721. struct shmem_falloc *shmem_falloc;
  722. spin_lock(&inode->i_lock);
  723. shmem_falloc = inode->i_private;
  724. if (shmem_falloc &&
  725. !shmem_falloc->waitq &&
  726. index >= shmem_falloc->start &&
  727. index < shmem_falloc->next)
  728. shmem_falloc->nr_unswapped++;
  729. else
  730. shmem_falloc = NULL;
  731. spin_unlock(&inode->i_lock);
  732. if (shmem_falloc)
  733. goto redirty;
  734. }
  735. clear_highpage(page);
  736. flush_dcache_page(page);
  737. SetPageUptodate(page);
  738. }
  739. swap = get_swap_page();
  740. if (!swap.val)
  741. goto redirty;
  742. /*
  743. * Add inode to shmem_unuse()'s list of swapped-out inodes,
  744. * if it's not already there. Do it now before the page is
  745. * moved to swap cache, when its pagelock no longer protects
  746. * the inode from eviction. But don't unlock the mutex until
  747. * we've incremented swapped, because shmem_unuse_inode() will
  748. * prune a !swapped inode from the swaplist under this mutex.
  749. */
  750. mutex_lock(&shmem_swaplist_mutex);
  751. if (list_empty(&info->swaplist))
  752. list_add_tail(&info->swaplist, &shmem_swaplist);
  753. if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
  754. swap_shmem_alloc(swap);
  755. shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
  756. spin_lock(&info->lock);
  757. info->swapped++;
  758. shmem_recalc_inode(inode);
  759. spin_unlock(&info->lock);
  760. mutex_unlock(&shmem_swaplist_mutex);
  761. BUG_ON(page_mapped(page));
  762. swap_writepage(page, wbc);
  763. return 0;
  764. }
  765. mutex_unlock(&shmem_swaplist_mutex);
  766. swapcache_free(swap);
  767. redirty:
  768. set_page_dirty(page);
  769. if (wbc->for_reclaim)
  770. return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
  771. unlock_page(page);
  772. return 0;
  773. }
  774. #ifdef CONFIG_NUMA
  775. #ifdef CONFIG_TMPFS
  776. static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
  777. {
  778. char buffer[64];
  779. if (!mpol || mpol->mode == MPOL_DEFAULT)
  780. return; /* show nothing */
  781. mpol_to_str(buffer, sizeof(buffer), mpol);
  782. seq_printf(seq, ",mpol=%s", buffer);
  783. }
  784. static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
  785. {
  786. struct mempolicy *mpol = NULL;
  787. if (sbinfo->mpol) {
  788. spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
  789. mpol = sbinfo->mpol;
  790. mpol_get(mpol);
  791. spin_unlock(&sbinfo->stat_lock);
  792. }
  793. return mpol;
  794. }
  795. #endif /* CONFIG_TMPFS */
  796. static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
  797. struct shmem_inode_info *info, pgoff_t index)
  798. {
  799. struct vm_area_struct pvma;
  800. struct page *page;
  801. /* Create a pseudo vma that just contains the policy */
  802. pvma.vm_start = 0;
  803. /* Bias interleave by inode number to distribute better across nodes */
  804. pvma.vm_pgoff = index + info->vfs_inode.i_ino;
  805. pvma.vm_ops = NULL;
  806. pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
  807. page = swapin_readahead(swap, gfp, &pvma, 0);
  808. /* Drop reference taken by mpol_shared_policy_lookup() */
  809. mpol_cond_put(pvma.vm_policy);
  810. return page;
  811. }
  812. static struct page *shmem_alloc_page(gfp_t gfp,
  813. struct shmem_inode_info *info, pgoff_t index)
  814. {
  815. struct vm_area_struct pvma;
  816. struct page *page;
  817. /* Create a pseudo vma that just contains the policy */
  818. pvma.vm_start = 0;
  819. /* Bias interleave by inode number to distribute better across nodes */
  820. pvma.vm_pgoff = index + info->vfs_inode.i_ino;
  821. pvma.vm_ops = NULL;
  822. pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
  823. page = alloc_page_vma(gfp, &pvma, 0);
  824. /* Drop reference taken by mpol_shared_policy_lookup() */
  825. mpol_cond_put(pvma.vm_policy);
  826. return page;
  827. }
  828. #else /* !CONFIG_NUMA */
  829. #ifdef CONFIG_TMPFS
  830. static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
  831. {
  832. }
  833. #endif /* CONFIG_TMPFS */
  834. static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
  835. struct shmem_inode_info *info, pgoff_t index)
  836. {
  837. return swapin_readahead(swap, gfp, NULL, 0);
  838. }
  839. static inline struct page *shmem_alloc_page(gfp_t gfp,
  840. struct shmem_inode_info *info, pgoff_t index)
  841. {
  842. return alloc_page(gfp);
  843. }
  844. #endif /* CONFIG_NUMA */
  845. #if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
  846. static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
  847. {
  848. return NULL;
  849. }
  850. #endif
  851. /*
  852. * When a page is moved from swapcache to shmem filecache (either by the
  853. * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
  854. * shmem_unuse_inode()), it may have been read in earlier from swap, in
  855. * ignorance of the mapping it belongs to. If that mapping has special
  856. * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
  857. * we may need to copy to a suitable page before moving to filecache.
  858. *
  859. * In a future release, this may well be extended to respect cpuset and
  860. * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
  861. * but for now it is a simple matter of zone.
  862. */
  863. static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
  864. {
  865. return page_zonenum(page) > gfp_zone(gfp);
  866. }
  867. static int shmem_replace_page(struct page **pagep, gfp_t gfp,
  868. struct shmem_inode_info *info, pgoff_t index)
  869. {
  870. struct page *oldpage, *newpage;
  871. struct address_space *swap_mapping;
  872. pgoff_t swap_index;
  873. int error;
  874. oldpage = *pagep;
  875. swap_index = page_private(oldpage);
  876. swap_mapping = page_mapping(oldpage);
  877. /*
  878. * We have arrived here because our zones are constrained, so don't
  879. * limit chance of success by further cpuset and node constraints.
  880. */
  881. gfp &= ~GFP_CONSTRAINT_MASK;
  882. newpage = shmem_alloc_page(gfp, info, index);
  883. if (!newpage)
  884. return -ENOMEM;
  885. page_cache_get(newpage);
  886. copy_highpage(newpage, oldpage);
  887. flush_dcache_page(newpage);
  888. __set_page_locked(newpage);
  889. SetPageUptodate(newpage);
  890. SetPageSwapBacked(newpage);
  891. set_page_private(newpage, swap_index);
  892. SetPageSwapCache(newpage);
  893. /*
  894. * Our caller will very soon move newpage out of swapcache, but it's
  895. * a nice clean interface for us to replace oldpage by newpage there.
  896. */
  897. spin_lock_irq(&swap_mapping->tree_lock);
  898. error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
  899. newpage);
  900. if (!error) {
  901. __inc_zone_page_state(newpage, NR_FILE_PAGES);
  902. __dec_zone_page_state(oldpage, NR_FILE_PAGES);
  903. }
  904. spin_unlock_irq(&swap_mapping->tree_lock);
  905. if (unlikely(error)) {
  906. /*
  907. * Is this possible? I think not, now that our callers check
  908. * both PageSwapCache and page_private after getting page lock;
  909. * but be defensive. Reverse old to newpage for clear and free.
  910. */
  911. oldpage = newpage;
  912. } else {
  913. mem_cgroup_migrate(oldpage, newpage, true);
  914. lru_cache_add_anon(newpage);
  915. *pagep = newpage;
  916. }
  917. ClearPageSwapCache(oldpage);
  918. set_page_private(oldpage, 0);
  919. unlock_page(oldpage);
  920. page_cache_release(oldpage);
  921. page_cache_release(oldpage);
  922. return error;
  923. }
  924. /*
  925. * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
  926. *
  927. * If we allocate a new one we do not mark it dirty. That's up to the
  928. * vm. If we swap it in we mark it dirty since we also free the swap
  929. * entry since a page cannot live in both the swap and page cache
  930. */
  931. static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
  932. struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
  933. {
  934. struct address_space *mapping = inode->i_mapping;
  935. struct shmem_inode_info *info;
  936. struct shmem_sb_info *sbinfo;
  937. struct mem_cgroup *memcg;
  938. struct page *page;
  939. swp_entry_t swap;
  940. int error;
  941. int once = 0;
  942. int alloced = 0;
  943. if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
  944. return -EFBIG;
  945. repeat:
  946. swap.val = 0;
  947. page = find_lock_entry(mapping, index);
  948. if (radix_tree_exceptional_entry(page)) {
  949. swap = radix_to_swp_entry(page);
  950. page = NULL;
  951. }
  952. if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
  953. ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
  954. error = -EINVAL;
  955. goto failed;
  956. }
  957. if (page && sgp == SGP_WRITE)
  958. mark_page_accessed(page);
  959. /* fallocated page? */
  960. if (page && !PageUptodate(page)) {
  961. if (sgp != SGP_READ)
  962. goto clear;
  963. unlock_page(page);
  964. page_cache_release(page);
  965. page = NULL;
  966. }
  967. if (page || (sgp == SGP_READ && !swap.val)) {
  968. *pagep = page;
  969. return 0;
  970. }
  971. /*
  972. * Fast cache lookup did not find it:
  973. * bring it back from swap or allocate.
  974. */
  975. info = SHMEM_I(inode);
  976. sbinfo = SHMEM_SB(inode->i_sb);
  977. if (swap.val) {
  978. /* Look it up and read it in.. */
  979. page = lookup_swap_cache(swap);
  980. if (!page) {
  981. /* here we actually do the io */
  982. if (fault_type)
  983. *fault_type |= VM_FAULT_MAJOR;
  984. page = shmem_swapin(swap, gfp, info, index);
  985. if (!page) {
  986. error = -ENOMEM;
  987. goto failed;
  988. }
  989. }
  990. /* We have to do this with page locked to prevent races */
  991. lock_page(page);
  992. if (!PageSwapCache(page) || page_private(page) != swap.val ||
  993. !shmem_confirm_swap(mapping, index, swap)) {
  994. error = -EEXIST; /* try again */
  995. goto unlock;
  996. }
  997. if (!PageUptodate(page)) {
  998. error = -EIO;
  999. goto failed;
  1000. }
  1001. wait_on_page_writeback(page);
  1002. if (shmem_should_replace_page(page, gfp)) {
  1003. error = shmem_replace_page(&page, gfp, info, index);
  1004. if (error)
  1005. goto failed;
  1006. }
  1007. error = mem_cgroup_try_charge(page, current->mm, gfp, &memcg);
  1008. if (!error) {
  1009. error = shmem_add_to_page_cache(page, mapping, index,
  1010. swp_to_radix_entry(swap));
  1011. /*
  1012. * We already confirmed swap under page lock, and make
  1013. * no memory allocation here, so usually no possibility
  1014. * of error; but free_swap_and_cache() only trylocks a
  1015. * page, so it is just possible that the entry has been
  1016. * truncated or holepunched since swap was confirmed.
  1017. * shmem_undo_range() will have done some of the
  1018. * unaccounting, now delete_from_swap_cache() will do
  1019. * the rest.
  1020. * Reset swap.val? No, leave it so "failed" goes back to
  1021. * "repeat": reading a hole and writing should succeed.
  1022. */
  1023. if (error) {
  1024. mem_cgroup_cancel_charge(page, memcg);
  1025. delete_from_swap_cache(page);
  1026. }
  1027. }
  1028. if (error)
  1029. goto failed;
  1030. mem_cgroup_commit_charge(page, memcg, true);
  1031. spin_lock(&info->lock);
  1032. info->swapped--;
  1033. shmem_recalc_inode(inode);
  1034. spin_unlock(&info->lock);
  1035. if (sgp == SGP_WRITE)
  1036. mark_page_accessed(page);
  1037. delete_from_swap_cache(page);
  1038. set_page_dirty(page);
  1039. swap_free(swap);
  1040. } else {
  1041. if (shmem_acct_block(info->flags)) {
  1042. error = -ENOSPC;
  1043. goto failed;
  1044. }
  1045. if (sbinfo->max_blocks) {
  1046. if (percpu_counter_compare(&sbinfo->used_blocks,
  1047. sbinfo->max_blocks) >= 0) {
  1048. error = -ENOSPC;
  1049. goto unacct;
  1050. }
  1051. percpu_counter_inc(&sbinfo->used_blocks);
  1052. }
  1053. page = shmem_alloc_page(gfp, info, index);
  1054. if (!page) {
  1055. error = -ENOMEM;
  1056. goto decused;
  1057. }
  1058. __SetPageSwapBacked(page);
  1059. __set_page_locked(page);
  1060. if (sgp == SGP_WRITE)
  1061. __SetPageReferenced(page);
  1062. error = mem_cgroup_try_charge(page, current->mm, gfp, &memcg);
  1063. if (error)
  1064. goto decused;
  1065. error = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK);
  1066. if (!error) {
  1067. error = shmem_add_to_page_cache(page, mapping, index,
  1068. NULL);
  1069. radix_tree_preload_end();
  1070. }
  1071. if (error) {
  1072. mem_cgroup_cancel_charge(page, memcg);
  1073. goto decused;
  1074. }
  1075. mem_cgroup_commit_charge(page, memcg, false);
  1076. lru_cache_add_anon(page);
  1077. spin_lock(&info->lock);
  1078. info->alloced++;
  1079. inode->i_blocks += BLOCKS_PER_PAGE;
  1080. shmem_recalc_inode(inode);
  1081. spin_unlock(&info->lock);
  1082. alloced = true;
  1083. /*
  1084. * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
  1085. */
  1086. if (sgp == SGP_FALLOC)
  1087. sgp = SGP_WRITE;
  1088. clear:
  1089. /*
  1090. * Let SGP_WRITE caller clear ends if write does not fill page;
  1091. * but SGP_FALLOC on a page fallocated earlier must initialize
  1092. * it now, lest undo on failure cancel our earlier guarantee.
  1093. */
  1094. if (sgp != SGP_WRITE) {
  1095. clear_highpage(page);
  1096. flush_dcache_page(page);
  1097. SetPageUptodate(page);
  1098. }
  1099. if (sgp == SGP_DIRTY)
  1100. set_page_dirty(page);
  1101. }
  1102. /* Perhaps the file has been truncated since we checked */
  1103. if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
  1104. ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
  1105. error = -EINVAL;
  1106. if (alloced)
  1107. goto trunc;
  1108. else
  1109. goto failed;
  1110. }
  1111. *pagep = page;
  1112. return 0;
  1113. /*
  1114. * Error recovery.
  1115. */
  1116. trunc:
  1117. info = SHMEM_I(inode);
  1118. ClearPageDirty(page);
  1119. delete_from_page_cache(page);
  1120. spin_lock(&info->lock);
  1121. info->alloced--;
  1122. inode->i_blocks -= BLOCKS_PER_PAGE;
  1123. spin_unlock(&info->lock);
  1124. decused:
  1125. sbinfo = SHMEM_SB(inode->i_sb);
  1126. if (sbinfo->max_blocks)
  1127. percpu_counter_add(&sbinfo->used_blocks, -1);
  1128. unacct:
  1129. shmem_unacct_blocks(info->flags, 1);
  1130. failed:
  1131. if (swap.val && error != -EINVAL &&
  1132. !shmem_confirm_swap(mapping, index, swap))
  1133. error = -EEXIST;
  1134. unlock:
  1135. if (page) {
  1136. unlock_page(page);
  1137. page_cache_release(page);
  1138. }
  1139. if (error == -ENOSPC && !once++) {
  1140. info = SHMEM_I(inode);
  1141. spin_lock(&info->lock);
  1142. shmem_recalc_inode(inode);
  1143. spin_unlock(&info->lock);
  1144. goto repeat;
  1145. }
  1146. if (error == -EEXIST) /* from above or from radix_tree_insert */
  1147. goto repeat;
  1148. return error;
  1149. }
  1150. static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1151. {
  1152. struct inode *inode = file_inode(vma->vm_file);
  1153. int error;
  1154. int ret = VM_FAULT_LOCKED;
  1155. /*
  1156. * Trinity finds that probing a hole which tmpfs is punching can
  1157. * prevent the hole-punch from ever completing: which in turn
  1158. * locks writers out with its hold on i_mutex. So refrain from
  1159. * faulting pages into the hole while it's being punched. Although
  1160. * shmem_undo_range() does remove the additions, it may be unable to
  1161. * keep up, as each new page needs its own unmap_mapping_range() call,
  1162. * and the i_mmap tree grows ever slower to scan if new vmas are added.
  1163. *
  1164. * It does not matter if we sometimes reach this check just before the
  1165. * hole-punch begins, so that one fault then races with the punch:
  1166. * we just need to make racing faults a rare case.
  1167. *
  1168. * The implementation below would be much simpler if we just used a
  1169. * standard mutex or completion: but we cannot take i_mutex in fault,
  1170. * and bloating every shmem inode for this unlikely case would be sad.
  1171. */
  1172. if (unlikely(inode->i_private)) {
  1173. struct shmem_falloc *shmem_falloc;
  1174. spin_lock(&inode->i_lock);
  1175. shmem_falloc = inode->i_private;
  1176. if (shmem_falloc &&
  1177. shmem_falloc->waitq &&
  1178. vmf->pgoff >= shmem_falloc->start &&
  1179. vmf->pgoff < shmem_falloc->next) {
  1180. wait_queue_head_t *shmem_falloc_waitq;
  1181. DEFINE_WAIT(shmem_fault_wait);
  1182. ret = VM_FAULT_NOPAGE;
  1183. if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
  1184. !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
  1185. /* It's polite to up mmap_sem if we can */
  1186. up_read(&vma->vm_mm->mmap_sem);
  1187. ret = VM_FAULT_RETRY;
  1188. }
  1189. shmem_falloc_waitq = shmem_falloc->waitq;
  1190. prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
  1191. TASK_UNINTERRUPTIBLE);
  1192. spin_unlock(&inode->i_lock);
  1193. schedule();
  1194. /*
  1195. * shmem_falloc_waitq points into the shmem_fallocate()
  1196. * stack of the hole-punching task: shmem_falloc_waitq
  1197. * is usually invalid by the time we reach here, but
  1198. * finish_wait() does not dereference it in that case;
  1199. * though i_lock needed lest racing with wake_up_all().
  1200. */
  1201. spin_lock(&inode->i_lock);
  1202. finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
  1203. spin_unlock(&inode->i_lock);
  1204. return ret;
  1205. }
  1206. spin_unlock(&inode->i_lock);
  1207. }
  1208. error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
  1209. if (error)
  1210. return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
  1211. if (ret & VM_FAULT_MAJOR) {
  1212. count_vm_event(PGMAJFAULT);
  1213. mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
  1214. }
  1215. return ret;
  1216. }
  1217. #ifdef CONFIG_NUMA
  1218. static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
  1219. {
  1220. struct inode *inode = file_inode(vma->vm_file);
  1221. return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
  1222. }
  1223. static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
  1224. unsigned long addr)
  1225. {
  1226. struct inode *inode = file_inode(vma->vm_file);
  1227. pgoff_t index;
  1228. index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  1229. return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
  1230. }
  1231. #endif
  1232. int shmem_lock(struct file *file, int lock, struct user_struct *user)
  1233. {
  1234. struct inode *inode = file_inode(file);
  1235. struct shmem_inode_info *info = SHMEM_I(inode);
  1236. int retval = -ENOMEM;
  1237. spin_lock(&info->lock);
  1238. if (lock && !(info->flags & VM_LOCKED)) {
  1239. if (!user_shm_lock(inode->i_size, user))
  1240. goto out_nomem;
  1241. info->flags |= VM_LOCKED;
  1242. mapping_set_unevictable(file->f_mapping);
  1243. }
  1244. if (!lock && (info->flags & VM_LOCKED) && user) {
  1245. user_shm_unlock(inode->i_size, user);
  1246. info->flags &= ~VM_LOCKED;
  1247. mapping_clear_unevictable(file->f_mapping);
  1248. }
  1249. retval = 0;
  1250. out_nomem:
  1251. spin_unlock(&info->lock);
  1252. return retval;
  1253. }
  1254. static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
  1255. {
  1256. file_accessed(file);
  1257. vma->vm_ops = &shmem_vm_ops;
  1258. return 0;
  1259. }
  1260. static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
  1261. umode_t mode, dev_t dev, unsigned long flags)
  1262. {
  1263. struct inode *inode;
  1264. struct shmem_inode_info *info;
  1265. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  1266. if (shmem_reserve_inode(sb))
  1267. return NULL;
  1268. inode = new_inode(sb);
  1269. if (inode) {
  1270. inode->i_ino = get_next_ino();
  1271. inode_init_owner(inode, dir, mode);
  1272. inode->i_blocks = 0;
  1273. inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
  1274. inode->i_generation = get_seconds();
  1275. info = SHMEM_I(inode);
  1276. memset(info, 0, (char *)inode - (char *)info);
  1277. spin_lock_init(&info->lock);
  1278. info->seals = F_SEAL_SEAL;
  1279. info->flags = flags & VM_NORESERVE;
  1280. INIT_LIST_HEAD(&info->swaplist);
  1281. simple_xattrs_init(&info->xattrs);
  1282. cache_no_acl(inode);
  1283. switch (mode & S_IFMT) {
  1284. default:
  1285. inode->i_op = &shmem_special_inode_operations;
  1286. init_special_inode(inode, mode, dev);
  1287. break;
  1288. case S_IFREG:
  1289. inode->i_mapping->a_ops = &shmem_aops;
  1290. inode->i_op = &shmem_inode_operations;
  1291. inode->i_fop = &shmem_file_operations;
  1292. mpol_shared_policy_init(&info->policy,
  1293. shmem_get_sbmpol(sbinfo));
  1294. break;
  1295. case S_IFDIR:
  1296. inc_nlink(inode);
  1297. /* Some things misbehave if size == 0 on a directory */
  1298. inode->i_size = 2 * BOGO_DIRENT_SIZE;
  1299. inode->i_op = &shmem_dir_inode_operations;
  1300. inode->i_fop = &simple_dir_operations;
  1301. break;
  1302. case S_IFLNK:
  1303. /*
  1304. * Must not load anything in the rbtree,
  1305. * mpol_free_shared_policy will not be called.
  1306. */
  1307. mpol_shared_policy_init(&info->policy, NULL);
  1308. break;
  1309. }
  1310. } else
  1311. shmem_free_inode(sb);
  1312. return inode;
  1313. }
  1314. bool shmem_mapping(struct address_space *mapping)
  1315. {
  1316. if (!mapping->host)
  1317. return false;
  1318. return mapping->host->i_sb->s_op == &shmem_ops;
  1319. }
  1320. #ifdef CONFIG_TMPFS
  1321. static const struct inode_operations shmem_symlink_inode_operations;
  1322. static const struct inode_operations shmem_short_symlink_operations;
  1323. #ifdef CONFIG_TMPFS_XATTR
  1324. static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
  1325. #else
  1326. #define shmem_initxattrs NULL
  1327. #endif
  1328. static int
  1329. shmem_write_begin(struct file *file, struct address_space *mapping,
  1330. loff_t pos, unsigned len, unsigned flags,
  1331. struct page **pagep, void **fsdata)
  1332. {
  1333. struct inode *inode = mapping->host;
  1334. struct shmem_inode_info *info = SHMEM_I(inode);
  1335. pgoff_t index = pos >> PAGE_CACHE_SHIFT;
  1336. /* i_mutex is held by caller */
  1337. if (unlikely(info->seals)) {
  1338. if (info->seals & F_SEAL_WRITE)
  1339. return -EPERM;
  1340. if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
  1341. return -EPERM;
  1342. }
  1343. return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
  1344. }
  1345. static int
  1346. shmem_write_end(struct file *file, struct address_space *mapping,
  1347. loff_t pos, unsigned len, unsigned copied,
  1348. struct page *page, void *fsdata)
  1349. {
  1350. struct inode *inode = mapping->host;
  1351. if (pos + copied > inode->i_size)
  1352. i_size_write(inode, pos + copied);
  1353. if (!PageUptodate(page)) {
  1354. if (copied < PAGE_CACHE_SIZE) {
  1355. unsigned from = pos & (PAGE_CACHE_SIZE - 1);
  1356. zero_user_segments(page, 0, from,
  1357. from + copied, PAGE_CACHE_SIZE);
  1358. }
  1359. SetPageUptodate(page);
  1360. }
  1361. set_page_dirty(page);
  1362. unlock_page(page);
  1363. page_cache_release(page);
  1364. return copied;
  1365. }
  1366. static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  1367. {
  1368. struct file *file = iocb->ki_filp;
  1369. struct inode *inode = file_inode(file);
  1370. struct address_space *mapping = inode->i_mapping;
  1371. pgoff_t index;
  1372. unsigned long offset;
  1373. enum sgp_type sgp = SGP_READ;
  1374. int error = 0;
  1375. ssize_t retval = 0;
  1376. loff_t *ppos = &iocb->ki_pos;
  1377. /*
  1378. * Might this read be for a stacking filesystem? Then when reading
  1379. * holes of a sparse file, we actually need to allocate those pages,
  1380. * and even mark them dirty, so it cannot exceed the max_blocks limit.
  1381. */
  1382. if (!iter_is_iovec(to))
  1383. sgp = SGP_DIRTY;
  1384. index = *ppos >> PAGE_CACHE_SHIFT;
  1385. offset = *ppos & ~PAGE_CACHE_MASK;
  1386. for (;;) {
  1387. struct page *page = NULL;
  1388. pgoff_t end_index;
  1389. unsigned long nr, ret;
  1390. loff_t i_size = i_size_read(inode);
  1391. end_index = i_size >> PAGE_CACHE_SHIFT;
  1392. if (index > end_index)
  1393. break;
  1394. if (index == end_index) {
  1395. nr = i_size & ~PAGE_CACHE_MASK;
  1396. if (nr <= offset)
  1397. break;
  1398. }
  1399. error = shmem_getpage(inode, index, &page, sgp, NULL);
  1400. if (error) {
  1401. if (error == -EINVAL)
  1402. error = 0;
  1403. break;
  1404. }
  1405. if (page)
  1406. unlock_page(page);
  1407. /*
  1408. * We must evaluate after, since reads (unlike writes)
  1409. * are called without i_mutex protection against truncate
  1410. */
  1411. nr = PAGE_CACHE_SIZE;
  1412. i_size = i_size_read(inode);
  1413. end_index = i_size >> PAGE_CACHE_SHIFT;
  1414. if (index == end_index) {
  1415. nr = i_size & ~PAGE_CACHE_MASK;
  1416. if (nr <= offset) {
  1417. if (page)
  1418. page_cache_release(page);
  1419. break;
  1420. }
  1421. }
  1422. nr -= offset;
  1423. if (page) {
  1424. /*
  1425. * If users can be writing to this page using arbitrary
  1426. * virtual addresses, take care about potential aliasing
  1427. * before reading the page on the kernel side.
  1428. */
  1429. if (mapping_writably_mapped(mapping))
  1430. flush_dcache_page(page);
  1431. /*
  1432. * Mark the page accessed if we read the beginning.
  1433. */
  1434. if (!offset)
  1435. mark_page_accessed(page);
  1436. } else {
  1437. page = ZERO_PAGE(0);
  1438. page_cache_get(page);
  1439. }
  1440. /*
  1441. * Ok, we have the page, and it's up-to-date, so
  1442. * now we can copy it to user space...
  1443. */
  1444. ret = copy_page_to_iter(page, offset, nr, to);
  1445. retval += ret;
  1446. offset += ret;
  1447. index += offset >> PAGE_CACHE_SHIFT;
  1448. offset &= ~PAGE_CACHE_MASK;
  1449. page_cache_release(page);
  1450. if (!iov_iter_count(to))
  1451. break;
  1452. if (ret < nr) {
  1453. error = -EFAULT;
  1454. break;
  1455. }
  1456. cond_resched();
  1457. }
  1458. *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
  1459. file_accessed(file);
  1460. return retval ? retval : error;
  1461. }
  1462. static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
  1463. struct pipe_inode_info *pipe, size_t len,
  1464. unsigned int flags)
  1465. {
  1466. struct address_space *mapping = in->f_mapping;
  1467. struct inode *inode = mapping->host;
  1468. unsigned int loff, nr_pages, req_pages;
  1469. struct page *pages[PIPE_DEF_BUFFERS];
  1470. struct partial_page partial[PIPE_DEF_BUFFERS];
  1471. struct page *page;
  1472. pgoff_t index, end_index;
  1473. loff_t isize, left;
  1474. int error, page_nr;
  1475. struct splice_pipe_desc spd = {
  1476. .pages = pages,
  1477. .partial = partial,
  1478. .nr_pages_max = PIPE_DEF_BUFFERS,
  1479. .flags = flags,
  1480. .ops = &page_cache_pipe_buf_ops,
  1481. .spd_release = spd_release_page,
  1482. };
  1483. isize = i_size_read(inode);
  1484. if (unlikely(*ppos >= isize))
  1485. return 0;
  1486. left = isize - *ppos;
  1487. if (unlikely(left < len))
  1488. len = left;
  1489. if (splice_grow_spd(pipe, &spd))
  1490. return -ENOMEM;
  1491. index = *ppos >> PAGE_CACHE_SHIFT;
  1492. loff = *ppos & ~PAGE_CACHE_MASK;
  1493. req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1494. nr_pages = min(req_pages, spd.nr_pages_max);
  1495. spd.nr_pages = find_get_pages_contig(mapping, index,
  1496. nr_pages, spd.pages);
  1497. index += spd.nr_pages;
  1498. error = 0;
  1499. while (spd.nr_pages < nr_pages) {
  1500. error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
  1501. if (error)
  1502. break;
  1503. unlock_page(page);
  1504. spd.pages[spd.nr_pages++] = page;
  1505. index++;
  1506. }
  1507. index = *ppos >> PAGE_CACHE_SHIFT;
  1508. nr_pages = spd.nr_pages;
  1509. spd.nr_pages = 0;
  1510. for (page_nr = 0; page_nr < nr_pages; page_nr++) {
  1511. unsigned int this_len;
  1512. if (!len)
  1513. break;
  1514. this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
  1515. page = spd.pages[page_nr];
  1516. if (!PageUptodate(page) || page->mapping != mapping) {
  1517. error = shmem_getpage(inode, index, &page,
  1518. SGP_CACHE, NULL);
  1519. if (error)
  1520. break;
  1521. unlock_page(page);
  1522. page_cache_release(spd.pages[page_nr]);
  1523. spd.pages[page_nr] = page;
  1524. }
  1525. isize = i_size_read(inode);
  1526. end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
  1527. if (unlikely(!isize || index > end_index))
  1528. break;
  1529. if (end_index == index) {
  1530. unsigned int plen;
  1531. plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
  1532. if (plen <= loff)
  1533. break;
  1534. this_len = min(this_len, plen - loff);
  1535. len = this_len;
  1536. }
  1537. spd.partial[page_nr].offset = loff;
  1538. spd.partial[page_nr].len = this_len;
  1539. len -= this_len;
  1540. loff = 0;
  1541. spd.nr_pages++;
  1542. index++;
  1543. }
  1544. while (page_nr < nr_pages)
  1545. page_cache_release(spd.pages[page_nr++]);
  1546. if (spd.nr_pages)
  1547. error = splice_to_pipe(pipe, &spd);
  1548. splice_shrink_spd(&spd);
  1549. if (error > 0) {
  1550. *ppos += error;
  1551. file_accessed(in);
  1552. }
  1553. return error;
  1554. }
  1555. /*
  1556. * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
  1557. */
  1558. static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
  1559. pgoff_t index, pgoff_t end, int whence)
  1560. {
  1561. struct page *page;
  1562. struct pagevec pvec;
  1563. pgoff_t indices[PAGEVEC_SIZE];
  1564. bool done = false;
  1565. int i;
  1566. pagevec_init(&pvec, 0);
  1567. pvec.nr = 1; /* start small: we may be there already */
  1568. while (!done) {
  1569. pvec.nr = find_get_entries(mapping, index,
  1570. pvec.nr, pvec.pages, indices);
  1571. if (!pvec.nr) {
  1572. if (whence == SEEK_DATA)
  1573. index = end;
  1574. break;
  1575. }
  1576. for (i = 0; i < pvec.nr; i++, index++) {
  1577. if (index < indices[i]) {
  1578. if (whence == SEEK_HOLE) {
  1579. done = true;
  1580. break;
  1581. }
  1582. index = indices[i];
  1583. }
  1584. page = pvec.pages[i];
  1585. if (page && !radix_tree_exceptional_entry(page)) {
  1586. if (!PageUptodate(page))
  1587. page = NULL;
  1588. }
  1589. if (index >= end ||
  1590. (page && whence == SEEK_DATA) ||
  1591. (!page && whence == SEEK_HOLE)) {
  1592. done = true;
  1593. break;
  1594. }
  1595. }
  1596. pagevec_remove_exceptionals(&pvec);
  1597. pagevec_release(&pvec);
  1598. pvec.nr = PAGEVEC_SIZE;
  1599. cond_resched();
  1600. }
  1601. return index;
  1602. }
  1603. static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
  1604. {
  1605. struct address_space *mapping = file->f_mapping;
  1606. struct inode *inode = mapping->host;
  1607. pgoff_t start, end;
  1608. loff_t new_offset;
  1609. if (whence != SEEK_DATA && whence != SEEK_HOLE)
  1610. return generic_file_llseek_size(file, offset, whence,
  1611. MAX_LFS_FILESIZE, i_size_read(inode));
  1612. mutex_lock(&inode->i_mutex);
  1613. /* We're holding i_mutex so we can access i_size directly */
  1614. if (offset < 0)
  1615. offset = -EINVAL;
  1616. else if (offset >= inode->i_size)
  1617. offset = -ENXIO;
  1618. else {
  1619. start = offset >> PAGE_CACHE_SHIFT;
  1620. end = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1621. new_offset = shmem_seek_hole_data(mapping, start, end, whence);
  1622. new_offset <<= PAGE_CACHE_SHIFT;
  1623. if (new_offset > offset) {
  1624. if (new_offset < inode->i_size)
  1625. offset = new_offset;
  1626. else if (whence == SEEK_DATA)
  1627. offset = -ENXIO;
  1628. else
  1629. offset = inode->i_size;
  1630. }
  1631. }
  1632. if (offset >= 0)
  1633. offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
  1634. mutex_unlock(&inode->i_mutex);
  1635. return offset;
  1636. }
  1637. /*
  1638. * We need a tag: a new tag would expand every radix_tree_node by 8 bytes,
  1639. * so reuse a tag which we firmly believe is never set or cleared on shmem.
  1640. */
  1641. #define SHMEM_TAG_PINNED PAGECACHE_TAG_TOWRITE
  1642. #define LAST_SCAN 4 /* about 150ms max */
  1643. static void shmem_tag_pins(struct address_space *mapping)
  1644. {
  1645. struct radix_tree_iter iter;
  1646. void **slot;
  1647. pgoff_t start;
  1648. struct page *page;
  1649. lru_add_drain();
  1650. start = 0;
  1651. rcu_read_lock();
  1652. restart:
  1653. radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
  1654. page = radix_tree_deref_slot(slot);
  1655. if (!page || radix_tree_exception(page)) {
  1656. if (radix_tree_deref_retry(page))
  1657. goto restart;
  1658. } else if (page_count(page) - page_mapcount(page) > 1) {
  1659. spin_lock_irq(&mapping->tree_lock);
  1660. radix_tree_tag_set(&mapping->page_tree, iter.index,
  1661. SHMEM_TAG_PINNED);
  1662. spin_unlock_irq(&mapping->tree_lock);
  1663. }
  1664. if (need_resched()) {
  1665. cond_resched_rcu();
  1666. start = iter.index + 1;
  1667. goto restart;
  1668. }
  1669. }
  1670. rcu_read_unlock();
  1671. }
  1672. /*
  1673. * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
  1674. * via get_user_pages(), drivers might have some pending I/O without any active
  1675. * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
  1676. * and see whether it has an elevated ref-count. If so, we tag them and wait for
  1677. * them to be dropped.
  1678. * The caller must guarantee that no new user will acquire writable references
  1679. * to those pages to avoid races.
  1680. */
  1681. static int shmem_wait_for_pins(struct address_space *mapping)
  1682. {
  1683. struct radix_tree_iter iter;
  1684. void **slot;
  1685. pgoff_t start;
  1686. struct page *page;
  1687. int error, scan;
  1688. shmem_tag_pins(mapping);
  1689. error = 0;
  1690. for (scan = 0; scan <= LAST_SCAN; scan++) {
  1691. if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED))
  1692. break;
  1693. if (!scan)
  1694. lru_add_drain_all();
  1695. else if (schedule_timeout_killable((HZ << scan) / 200))
  1696. scan = LAST_SCAN;
  1697. start = 0;
  1698. rcu_read_lock();
  1699. restart:
  1700. radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter,
  1701. start, SHMEM_TAG_PINNED) {
  1702. page = radix_tree_deref_slot(slot);
  1703. if (radix_tree_exception(page)) {
  1704. if (radix_tree_deref_retry(page))
  1705. goto restart;
  1706. page = NULL;
  1707. }
  1708. if (page &&
  1709. page_count(page) - page_mapcount(page) != 1) {
  1710. if (scan < LAST_SCAN)
  1711. goto continue_resched;
  1712. /*
  1713. * On the last scan, we clean up all those tags
  1714. * we inserted; but make a note that we still
  1715. * found pages pinned.
  1716. */
  1717. error = -EBUSY;
  1718. }
  1719. spin_lock_irq(&mapping->tree_lock);
  1720. radix_tree_tag_clear(&mapping->page_tree,
  1721. iter.index, SHMEM_TAG_PINNED);
  1722. spin_unlock_irq(&mapping->tree_lock);
  1723. continue_resched:
  1724. if (need_resched()) {
  1725. cond_resched_rcu();
  1726. start = iter.index + 1;
  1727. goto restart;
  1728. }
  1729. }
  1730. rcu_read_unlock();
  1731. }
  1732. return error;
  1733. }
  1734. #define F_ALL_SEALS (F_SEAL_SEAL | \
  1735. F_SEAL_SHRINK | \
  1736. F_SEAL_GROW | \
  1737. F_SEAL_WRITE)
  1738. int shmem_add_seals(struct file *file, unsigned int seals)
  1739. {
  1740. struct inode *inode = file_inode(file);
  1741. struct shmem_inode_info *info = SHMEM_I(inode);
  1742. int error;
  1743. /*
  1744. * SEALING
  1745. * Sealing allows multiple parties to share a shmem-file but restrict
  1746. * access to a specific subset of file operations. Seals can only be
  1747. * added, but never removed. This way, mutually untrusted parties can
  1748. * share common memory regions with a well-defined policy. A malicious
  1749. * peer can thus never perform unwanted operations on a shared object.
  1750. *
  1751. * Seals are only supported on special shmem-files and always affect
  1752. * the whole underlying inode. Once a seal is set, it may prevent some
  1753. * kinds of access to the file. Currently, the following seals are
  1754. * defined:
  1755. * SEAL_SEAL: Prevent further seals from being set on this file
  1756. * SEAL_SHRINK: Prevent the file from shrinking
  1757. * SEAL_GROW: Prevent the file from growing
  1758. * SEAL_WRITE: Prevent write access to the file
  1759. *
  1760. * As we don't require any trust relationship between two parties, we
  1761. * must prevent seals from being removed. Therefore, sealing a file
  1762. * only adds a given set of seals to the file, it never touches
  1763. * existing seals. Furthermore, the "setting seals"-operation can be
  1764. * sealed itself, which basically prevents any further seal from being
  1765. * added.
  1766. *
  1767. * Semantics of sealing are only defined on volatile files. Only
  1768. * anonymous shmem files support sealing. More importantly, seals are
  1769. * never written to disk. Therefore, there's no plan to support it on
  1770. * other file types.
  1771. */
  1772. if (file->f_op != &shmem_file_operations)
  1773. return -EINVAL;
  1774. if (!(file->f_mode & FMODE_WRITE))
  1775. return -EPERM;
  1776. if (seals & ~(unsigned int)F_ALL_SEALS)
  1777. return -EINVAL;
  1778. mutex_lock(&inode->i_mutex);
  1779. if (info->seals & F_SEAL_SEAL) {
  1780. error = -EPERM;
  1781. goto unlock;
  1782. }
  1783. if ((seals & F_SEAL_WRITE) && !(info->seals & F_SEAL_WRITE)) {
  1784. error = mapping_deny_writable(file->f_mapping);
  1785. if (error)
  1786. goto unlock;
  1787. error = shmem_wait_for_pins(file->f_mapping);
  1788. if (error) {
  1789. mapping_allow_writable(file->f_mapping);
  1790. goto unlock;
  1791. }
  1792. }
  1793. info->seals |= seals;
  1794. error = 0;
  1795. unlock:
  1796. mutex_unlock(&inode->i_mutex);
  1797. return error;
  1798. }
  1799. EXPORT_SYMBOL_GPL(shmem_add_seals);
  1800. int shmem_get_seals(struct file *file)
  1801. {
  1802. if (file->f_op != &shmem_file_operations)
  1803. return -EINVAL;
  1804. return SHMEM_I(file_inode(file))->seals;
  1805. }
  1806. EXPORT_SYMBOL_GPL(shmem_get_seals);
  1807. long shmem_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
  1808. {
  1809. long error;
  1810. switch (cmd) {
  1811. case F_ADD_SEALS:
  1812. /* disallow upper 32bit */
  1813. if (arg > UINT_MAX)
  1814. return -EINVAL;
  1815. error = shmem_add_seals(file, arg);
  1816. break;
  1817. case F_GET_SEALS:
  1818. error = shmem_get_seals(file);
  1819. break;
  1820. default:
  1821. error = -EINVAL;
  1822. break;
  1823. }
  1824. return error;
  1825. }
  1826. static long shmem_fallocate(struct file *file, int mode, loff_t offset,
  1827. loff_t len)
  1828. {
  1829. struct inode *inode = file_inode(file);
  1830. struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
  1831. struct shmem_inode_info *info = SHMEM_I(inode);
  1832. struct shmem_falloc shmem_falloc;
  1833. pgoff_t start, index, end;
  1834. int error;
  1835. if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
  1836. return -EOPNOTSUPP;
  1837. mutex_lock(&inode->i_mutex);
  1838. if (mode & FALLOC_FL_PUNCH_HOLE) {
  1839. struct address_space *mapping = file->f_mapping;
  1840. loff_t unmap_start = round_up(offset, PAGE_SIZE);
  1841. loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
  1842. DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
  1843. /* protected by i_mutex */
  1844. if (info->seals & F_SEAL_WRITE) {
  1845. error = -EPERM;
  1846. goto out;
  1847. }
  1848. shmem_falloc.waitq = &shmem_falloc_waitq;
  1849. shmem_falloc.start = unmap_start >> PAGE_SHIFT;
  1850. shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
  1851. spin_lock(&inode->i_lock);
  1852. inode->i_private = &shmem_falloc;
  1853. spin_unlock(&inode->i_lock);
  1854. if ((u64)unmap_end > (u64)unmap_start)
  1855. unmap_mapping_range(mapping, unmap_start,
  1856. 1 + unmap_end - unmap_start, 0);
  1857. shmem_truncate_range(inode, offset, offset + len - 1);
  1858. /* No need to unmap again: hole-punching leaves COWed pages */
  1859. spin_lock(&inode->i_lock);
  1860. inode->i_private = NULL;
  1861. wake_up_all(&shmem_falloc_waitq);
  1862. spin_unlock(&inode->i_lock);
  1863. error = 0;
  1864. goto out;
  1865. }
  1866. /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
  1867. error = inode_newsize_ok(inode, offset + len);
  1868. if (error)
  1869. goto out;
  1870. if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
  1871. error = -EPERM;
  1872. goto out;
  1873. }
  1874. start = offset >> PAGE_CACHE_SHIFT;
  1875. end = (offset + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1876. /* Try to avoid a swapstorm if len is impossible to satisfy */
  1877. if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
  1878. error = -ENOSPC;
  1879. goto out;
  1880. }
  1881. shmem_falloc.waitq = NULL;
  1882. shmem_falloc.start = start;
  1883. shmem_falloc.next = start;
  1884. shmem_falloc.nr_falloced = 0;
  1885. shmem_falloc.nr_unswapped = 0;
  1886. spin_lock(&inode->i_lock);
  1887. inode->i_private = &shmem_falloc;
  1888. spin_unlock(&inode->i_lock);
  1889. for (index = start; index < end; index++) {
  1890. struct page *page;
  1891. /*
  1892. * Good, the fallocate(2) manpage permits EINTR: we may have
  1893. * been interrupted because we are using up too much memory.
  1894. */
  1895. if (signal_pending(current))
  1896. error = -EINTR;
  1897. else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
  1898. error = -ENOMEM;
  1899. else
  1900. error = shmem_getpage(inode, index, &page, SGP_FALLOC,
  1901. NULL);
  1902. if (error) {
  1903. /* Remove the !PageUptodate pages we added */
  1904. shmem_undo_range(inode,
  1905. (loff_t)start << PAGE_CACHE_SHIFT,
  1906. (loff_t)index << PAGE_CACHE_SHIFT, true);
  1907. goto undone;
  1908. }
  1909. /*
  1910. * Inform shmem_writepage() how far we have reached.
  1911. * No need for lock or barrier: we have the page lock.
  1912. */
  1913. shmem_falloc.next++;
  1914. if (!PageUptodate(page))
  1915. shmem_falloc.nr_falloced++;
  1916. /*
  1917. * If !PageUptodate, leave it that way so that freeable pages
  1918. * can be recognized if we need to rollback on error later.
  1919. * But set_page_dirty so that memory pressure will swap rather
  1920. * than free the pages we are allocating (and SGP_CACHE pages
  1921. * might still be clean: we now need to mark those dirty too).
  1922. */
  1923. set_page_dirty(page);
  1924. unlock_page(page);
  1925. page_cache_release(page);
  1926. cond_resched();
  1927. }
  1928. if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
  1929. i_size_write(inode, offset + len);
  1930. inode->i_ctime = CURRENT_TIME;
  1931. undone:
  1932. spin_lock(&inode->i_lock);
  1933. inode->i_private = NULL;
  1934. spin_unlock(&inode->i_lock);
  1935. out:
  1936. mutex_unlock(&inode->i_mutex);
  1937. return error;
  1938. }
  1939. static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
  1940. {
  1941. struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
  1942. buf->f_type = TMPFS_MAGIC;
  1943. buf->f_bsize = PAGE_CACHE_SIZE;
  1944. buf->f_namelen = NAME_MAX;
  1945. if (sbinfo->max_blocks) {
  1946. buf->f_blocks = sbinfo->max_blocks;
  1947. buf->f_bavail =
  1948. buf->f_bfree = sbinfo->max_blocks -
  1949. percpu_counter_sum(&sbinfo->used_blocks);
  1950. }
  1951. if (sbinfo->max_inodes) {
  1952. buf->f_files = sbinfo->max_inodes;
  1953. buf->f_ffree = sbinfo->free_inodes;
  1954. }
  1955. /* else leave those fields 0 like simple_statfs */
  1956. return 0;
  1957. }
  1958. /*
  1959. * File creation. Allocate an inode, and we're done..
  1960. */
  1961. static int
  1962. shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
  1963. {
  1964. struct inode *inode;
  1965. int error = -ENOSPC;
  1966. inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
  1967. if (inode) {
  1968. error = simple_acl_create(dir, inode);
  1969. if (error)
  1970. goto out_iput;
  1971. error = security_inode_init_security(inode, dir,
  1972. &dentry->d_name,
  1973. shmem_initxattrs, NULL);
  1974. if (error && error != -EOPNOTSUPP)
  1975. goto out_iput;
  1976. error = 0;
  1977. dir->i_size += BOGO_DIRENT_SIZE;
  1978. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  1979. d_instantiate(dentry, inode);
  1980. dget(dentry); /* Extra count - pin the dentry in core */
  1981. }
  1982. return error;
  1983. out_iput:
  1984. iput(inode);
  1985. return error;
  1986. }
  1987. static int
  1988. shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
  1989. {
  1990. struct inode *inode;
  1991. int error = -ENOSPC;
  1992. inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
  1993. if (inode) {
  1994. error = security_inode_init_security(inode, dir,
  1995. NULL,
  1996. shmem_initxattrs, NULL);
  1997. if (error && error != -EOPNOTSUPP)
  1998. goto out_iput;
  1999. error = simple_acl_create(dir, inode);
  2000. if (error)
  2001. goto out_iput;
  2002. d_tmpfile(dentry, inode);
  2003. }
  2004. return error;
  2005. out_iput:
  2006. iput(inode);
  2007. return error;
  2008. }
  2009. static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
  2010. {
  2011. int error;
  2012. if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
  2013. return error;
  2014. inc_nlink(dir);
  2015. return 0;
  2016. }
  2017. static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
  2018. bool excl)
  2019. {
  2020. return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
  2021. }
  2022. /*
  2023. * Link a file..
  2024. */
  2025. static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
  2026. {
  2027. struct inode *inode = d_inode(old_dentry);
  2028. int ret;
  2029. /*
  2030. * No ordinary (disk based) filesystem counts links as inodes;
  2031. * but each new link needs a new dentry, pinning lowmem, and
  2032. * tmpfs dentries cannot be pruned until they are unlinked.
  2033. */
  2034. ret = shmem_reserve_inode(inode->i_sb);
  2035. if (ret)
  2036. goto out;
  2037. dir->i_size += BOGO_DIRENT_SIZE;
  2038. inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  2039. inc_nlink(inode);
  2040. ihold(inode); /* New dentry reference */
  2041. dget(dentry); /* Extra pinning count for the created dentry */
  2042. d_instantiate(dentry, inode);
  2043. out:
  2044. return ret;
  2045. }
  2046. static int shmem_unlink(struct inode *dir, struct dentry *dentry)
  2047. {
  2048. struct inode *inode = d_inode(dentry);
  2049. if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
  2050. shmem_free_inode(inode->i_sb);
  2051. dir->i_size -= BOGO_DIRENT_SIZE;
  2052. inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  2053. drop_nlink(inode);
  2054. dput(dentry); /* Undo the count from "create" - this does all the work */
  2055. return 0;
  2056. }
  2057. static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
  2058. {
  2059. if (!simple_empty(dentry))
  2060. return -ENOTEMPTY;
  2061. drop_nlink(d_inode(dentry));
  2062. drop_nlink(dir);
  2063. return shmem_unlink(dir, dentry);
  2064. }
  2065. static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
  2066. {
  2067. bool old_is_dir = d_is_dir(old_dentry);
  2068. bool new_is_dir = d_is_dir(new_dentry);
  2069. if (old_dir != new_dir && old_is_dir != new_is_dir) {
  2070. if (old_is_dir) {
  2071. drop_nlink(old_dir);
  2072. inc_nlink(new_dir);
  2073. } else {
  2074. drop_nlink(new_dir);
  2075. inc_nlink(old_dir);
  2076. }
  2077. }
  2078. old_dir->i_ctime = old_dir->i_mtime =
  2079. new_dir->i_ctime = new_dir->i_mtime =
  2080. d_inode(old_dentry)->i_ctime =
  2081. d_inode(new_dentry)->i_ctime = CURRENT_TIME;
  2082. return 0;
  2083. }
  2084. static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
  2085. {
  2086. struct dentry *whiteout;
  2087. int error;
  2088. whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
  2089. if (!whiteout)
  2090. return -ENOMEM;
  2091. error = shmem_mknod(old_dir, whiteout,
  2092. S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
  2093. dput(whiteout);
  2094. if (error)
  2095. return error;
  2096. /*
  2097. * Cheat and hash the whiteout while the old dentry is still in
  2098. * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
  2099. *
  2100. * d_lookup() will consistently find one of them at this point,
  2101. * not sure which one, but that isn't even important.
  2102. */
  2103. d_rehash(whiteout);
  2104. return 0;
  2105. }
  2106. /*
  2107. * The VFS layer already does all the dentry stuff for rename,
  2108. * we just have to decrement the usage count for the target if
  2109. * it exists so that the VFS layer correctly free's it when it
  2110. * gets overwritten.
  2111. */
  2112. static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
  2113. {
  2114. struct inode *inode = d_inode(old_dentry);
  2115. int they_are_dirs = S_ISDIR(inode->i_mode);
  2116. if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
  2117. return -EINVAL;
  2118. if (flags & RENAME_EXCHANGE)
  2119. return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
  2120. if (!simple_empty(new_dentry))
  2121. return -ENOTEMPTY;
  2122. if (flags & RENAME_WHITEOUT) {
  2123. int error;
  2124. error = shmem_whiteout(old_dir, old_dentry);
  2125. if (error)
  2126. return error;
  2127. }
  2128. if (d_really_is_positive(new_dentry)) {
  2129. (void) shmem_unlink(new_dir, new_dentry);
  2130. if (they_are_dirs) {
  2131. drop_nlink(d_inode(new_dentry));
  2132. drop_nlink(old_dir);
  2133. }
  2134. } else if (they_are_dirs) {
  2135. drop_nlink(old_dir);
  2136. inc_nlink(new_dir);
  2137. }
  2138. old_dir->i_size -= BOGO_DIRENT_SIZE;
  2139. new_dir->i_size += BOGO_DIRENT_SIZE;
  2140. old_dir->i_ctime = old_dir->i_mtime =
  2141. new_dir->i_ctime = new_dir->i_mtime =
  2142. inode->i_ctime = CURRENT_TIME;
  2143. return 0;
  2144. }
  2145. static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
  2146. {
  2147. int error;
  2148. int len;
  2149. struct inode *inode;
  2150. struct page *page;
  2151. char *kaddr;
  2152. struct shmem_inode_info *info;
  2153. len = strlen(symname) + 1;
  2154. if (len > PAGE_CACHE_SIZE)
  2155. return -ENAMETOOLONG;
  2156. inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
  2157. if (!inode)
  2158. return -ENOSPC;
  2159. error = security_inode_init_security(inode, dir, &dentry->d_name,
  2160. shmem_initxattrs, NULL);
  2161. if (error) {
  2162. if (error != -EOPNOTSUPP) {
  2163. iput(inode);
  2164. return error;
  2165. }
  2166. error = 0;
  2167. }
  2168. info = SHMEM_I(inode);
  2169. inode->i_size = len-1;
  2170. if (len <= SHORT_SYMLINK_LEN) {
  2171. info->symlink = kmemdup(symname, len, GFP_KERNEL);
  2172. if (!info->symlink) {
  2173. iput(inode);
  2174. return -ENOMEM;
  2175. }
  2176. inode->i_op = &shmem_short_symlink_operations;
  2177. inode->i_link = info->symlink;
  2178. } else {
  2179. error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
  2180. if (error) {
  2181. iput(inode);
  2182. return error;
  2183. }
  2184. inode->i_mapping->a_ops = &shmem_aops;
  2185. inode->i_op = &shmem_symlink_inode_operations;
  2186. kaddr = kmap_atomic(page);
  2187. memcpy(kaddr, symname, len);
  2188. kunmap_atomic(kaddr);
  2189. SetPageUptodate(page);
  2190. set_page_dirty(page);
  2191. unlock_page(page);
  2192. page_cache_release(page);
  2193. }
  2194. dir->i_size += BOGO_DIRENT_SIZE;
  2195. dir->i_ctime = dir->i_mtime = CURRENT_TIME;
  2196. d_instantiate(dentry, inode);
  2197. dget(dentry);
  2198. return 0;
  2199. }
  2200. static const char *shmem_follow_link(struct dentry *dentry, void **cookie)
  2201. {
  2202. struct page *page = NULL;
  2203. int error = shmem_getpage(d_inode(dentry), 0, &page, SGP_READ, NULL);
  2204. if (error)
  2205. return ERR_PTR(error);
  2206. unlock_page(page);
  2207. *cookie = page;
  2208. return kmap(page);
  2209. }
  2210. static void shmem_put_link(struct inode *unused, void *cookie)
  2211. {
  2212. struct page *page = cookie;
  2213. kunmap(page);
  2214. mark_page_accessed(page);
  2215. page_cache_release(page);
  2216. }
  2217. #ifdef CONFIG_TMPFS_XATTR
  2218. /*
  2219. * Superblocks without xattr inode operations may get some security.* xattr
  2220. * support from the LSM "for free". As soon as we have any other xattrs
  2221. * like ACLs, we also need to implement the security.* handlers at
  2222. * filesystem level, though.
  2223. */
  2224. /*
  2225. * Callback for security_inode_init_security() for acquiring xattrs.
  2226. */
  2227. static int shmem_initxattrs(struct inode *inode,
  2228. const struct xattr *xattr_array,
  2229. void *fs_info)
  2230. {
  2231. struct shmem_inode_info *info = SHMEM_I(inode);
  2232. const struct xattr *xattr;
  2233. struct simple_xattr *new_xattr;
  2234. size_t len;
  2235. for (xattr = xattr_array; xattr->name != NULL; xattr++) {
  2236. new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
  2237. if (!new_xattr)
  2238. return -ENOMEM;
  2239. len = strlen(xattr->name) + 1;
  2240. new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
  2241. GFP_KERNEL);
  2242. if (!new_xattr->name) {
  2243. kfree(new_xattr);
  2244. return -ENOMEM;
  2245. }
  2246. memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
  2247. XATTR_SECURITY_PREFIX_LEN);
  2248. memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
  2249. xattr->name, len);
  2250. simple_xattr_list_add(&info->xattrs, new_xattr);
  2251. }
  2252. return 0;
  2253. }
  2254. static const struct xattr_handler *shmem_xattr_handlers[] = {
  2255. #ifdef CONFIG_TMPFS_POSIX_ACL
  2256. &posix_acl_access_xattr_handler,
  2257. &posix_acl_default_xattr_handler,
  2258. #endif
  2259. NULL
  2260. };
  2261. static int shmem_xattr_validate(const char *name)
  2262. {
  2263. struct { const char *prefix; size_t len; } arr[] = {
  2264. { XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
  2265. { XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
  2266. };
  2267. int i;
  2268. for (i = 0; i < ARRAY_SIZE(arr); i++) {
  2269. size_t preflen = arr[i].len;
  2270. if (strncmp(name, arr[i].prefix, preflen) == 0) {
  2271. if (!name[preflen])
  2272. return -EINVAL;
  2273. return 0;
  2274. }
  2275. }
  2276. return -EOPNOTSUPP;
  2277. }
  2278. static ssize_t shmem_getxattr(struct dentry *dentry, const char *name,
  2279. void *buffer, size_t size)
  2280. {
  2281. struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
  2282. int err;
  2283. /*
  2284. * If this is a request for a synthetic attribute in the system.*
  2285. * namespace use the generic infrastructure to resolve a handler
  2286. * for it via sb->s_xattr.
  2287. */
  2288. if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
  2289. return generic_getxattr(dentry, name, buffer, size);
  2290. err = shmem_xattr_validate(name);
  2291. if (err)
  2292. return err;
  2293. return simple_xattr_get(&info->xattrs, name, buffer, size);
  2294. }
  2295. static int shmem_setxattr(struct dentry *dentry, const char *name,
  2296. const void *value, size_t size, int flags)
  2297. {
  2298. struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
  2299. int err;
  2300. /*
  2301. * If this is a request for a synthetic attribute in the system.*
  2302. * namespace use the generic infrastructure to resolve a handler
  2303. * for it via sb->s_xattr.
  2304. */
  2305. if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
  2306. return generic_setxattr(dentry, name, value, size, flags);
  2307. err = shmem_xattr_validate(name);
  2308. if (err)
  2309. return err;
  2310. return simple_xattr_set(&info->xattrs, name, value, size, flags);
  2311. }
  2312. static int shmem_removexattr(struct dentry *dentry, const char *name)
  2313. {
  2314. struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
  2315. int err;
  2316. /*
  2317. * If this is a request for a synthetic attribute in the system.*
  2318. * namespace use the generic infrastructure to resolve a handler
  2319. * for it via sb->s_xattr.
  2320. */
  2321. if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
  2322. return generic_removexattr(dentry, name);
  2323. err = shmem_xattr_validate(name);
  2324. if (err)
  2325. return err;
  2326. return simple_xattr_remove(&info->xattrs, name);
  2327. }
  2328. static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
  2329. {
  2330. struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
  2331. return simple_xattr_list(&info->xattrs, buffer, size);
  2332. }
  2333. #endif /* CONFIG_TMPFS_XATTR */
  2334. static const struct inode_operations shmem_short_symlink_operations = {
  2335. .readlink = generic_readlink,
  2336. .follow_link = simple_follow_link,
  2337. #ifdef CONFIG_TMPFS_XATTR
  2338. .setxattr = shmem_setxattr,
  2339. .getxattr = shmem_getxattr,
  2340. .listxattr = shmem_listxattr,
  2341. .removexattr = shmem_removexattr,
  2342. #endif
  2343. };
  2344. static const struct inode_operations shmem_symlink_inode_operations = {
  2345. .readlink = generic_readlink,
  2346. .follow_link = shmem_follow_link,
  2347. .put_link = shmem_put_link,
  2348. #ifdef CONFIG_TMPFS_XATTR
  2349. .setxattr = shmem_setxattr,
  2350. .getxattr = shmem_getxattr,
  2351. .listxattr = shmem_listxattr,
  2352. .removexattr = shmem_removexattr,
  2353. #endif
  2354. };
  2355. static struct dentry *shmem_get_parent(struct dentry *child)
  2356. {
  2357. return ERR_PTR(-ESTALE);
  2358. }
  2359. static int shmem_match(struct inode *ino, void *vfh)
  2360. {
  2361. __u32 *fh = vfh;
  2362. __u64 inum = fh[2];
  2363. inum = (inum << 32) | fh[1];
  2364. return ino->i_ino == inum && fh[0] == ino->i_generation;
  2365. }
  2366. static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
  2367. struct fid *fid, int fh_len, int fh_type)
  2368. {
  2369. struct inode *inode;
  2370. struct dentry *dentry = NULL;
  2371. u64 inum;
  2372. if (fh_len < 3)
  2373. return NULL;
  2374. inum = fid->raw[2];
  2375. inum = (inum << 32) | fid->raw[1];
  2376. inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
  2377. shmem_match, fid->raw);
  2378. if (inode) {
  2379. dentry = d_find_alias(inode);
  2380. iput(inode);
  2381. }
  2382. return dentry;
  2383. }
  2384. static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
  2385. struct inode *parent)
  2386. {
  2387. if (*len < 3) {
  2388. *len = 3;
  2389. return FILEID_INVALID;
  2390. }
  2391. if (inode_unhashed(inode)) {
  2392. /* Unfortunately insert_inode_hash is not idempotent,
  2393. * so as we hash inodes here rather than at creation
  2394. * time, we need a lock to ensure we only try
  2395. * to do it once
  2396. */
  2397. static DEFINE_SPINLOCK(lock);
  2398. spin_lock(&lock);
  2399. if (inode_unhashed(inode))
  2400. __insert_inode_hash(inode,
  2401. inode->i_ino + inode->i_generation);
  2402. spin_unlock(&lock);
  2403. }
  2404. fh[0] = inode->i_generation;
  2405. fh[1] = inode->i_ino;
  2406. fh[2] = ((__u64)inode->i_ino) >> 32;
  2407. *len = 3;
  2408. return 1;
  2409. }
  2410. static const struct export_operations shmem_export_ops = {
  2411. .get_parent = shmem_get_parent,
  2412. .encode_fh = shmem_encode_fh,
  2413. .fh_to_dentry = shmem_fh_to_dentry,
  2414. };
  2415. static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
  2416. bool remount)
  2417. {
  2418. char *this_char, *value, *rest;
  2419. struct mempolicy *mpol = NULL;
  2420. uid_t uid;
  2421. gid_t gid;
  2422. while (options != NULL) {
  2423. this_char = options;
  2424. for (;;) {
  2425. /*
  2426. * NUL-terminate this option: unfortunately,
  2427. * mount options form a comma-separated list,
  2428. * but mpol's nodelist may also contain commas.
  2429. */
  2430. options = strchr(options, ',');
  2431. if (options == NULL)
  2432. break;
  2433. options++;
  2434. if (!isdigit(*options)) {
  2435. options[-1] = '\0';
  2436. break;
  2437. }
  2438. }
  2439. if (!*this_char)
  2440. continue;
  2441. if ((value = strchr(this_char,'=')) != NULL) {
  2442. *value++ = 0;
  2443. } else {
  2444. printk(KERN_ERR
  2445. "tmpfs: No value for mount option '%s'\n",
  2446. this_char);
  2447. goto error;
  2448. }
  2449. if (!strcmp(this_char,"size")) {
  2450. unsigned long long size;
  2451. size = memparse(value,&rest);
  2452. if (*rest == '%') {
  2453. size <<= PAGE_SHIFT;
  2454. size *= totalram_pages;
  2455. do_div(size, 100);
  2456. rest++;
  2457. }
  2458. if (*rest)
  2459. goto bad_val;
  2460. sbinfo->max_blocks =
  2461. DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
  2462. } else if (!strcmp(this_char,"nr_blocks")) {
  2463. sbinfo->max_blocks = memparse(value, &rest);
  2464. if (*rest)
  2465. goto bad_val;
  2466. } else if (!strcmp(this_char,"nr_inodes")) {
  2467. sbinfo->max_inodes = memparse(value, &rest);
  2468. if (*rest)
  2469. goto bad_val;
  2470. } else if (!strcmp(this_char,"mode")) {
  2471. if (remount)
  2472. continue;
  2473. sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
  2474. if (*rest)
  2475. goto bad_val;
  2476. } else if (!strcmp(this_char,"uid")) {
  2477. if (remount)
  2478. continue;
  2479. uid = simple_strtoul(value, &rest, 0);
  2480. if (*rest)
  2481. goto bad_val;
  2482. sbinfo->uid = make_kuid(current_user_ns(), uid);
  2483. if (!uid_valid(sbinfo->uid))
  2484. goto bad_val;
  2485. } else if (!strcmp(this_char,"gid")) {
  2486. if (remount)
  2487. continue;
  2488. gid = simple_strtoul(value, &rest, 0);
  2489. if (*rest)
  2490. goto bad_val;
  2491. sbinfo->gid = make_kgid(current_user_ns(), gid);
  2492. if (!gid_valid(sbinfo->gid))
  2493. goto bad_val;
  2494. } else if (!strcmp(this_char,"mpol")) {
  2495. mpol_put(mpol);
  2496. mpol = NULL;
  2497. if (mpol_parse_str(value, &mpol))
  2498. goto bad_val;
  2499. } else {
  2500. printk(KERN_ERR "tmpfs: Bad mount option %s\n",
  2501. this_char);
  2502. goto error;
  2503. }
  2504. }
  2505. sbinfo->mpol = mpol;
  2506. return 0;
  2507. bad_val:
  2508. printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
  2509. value, this_char);
  2510. error:
  2511. mpol_put(mpol);
  2512. return 1;
  2513. }
  2514. static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
  2515. {
  2516. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  2517. struct shmem_sb_info config = *sbinfo;
  2518. unsigned long inodes;
  2519. int error = -EINVAL;
  2520. config.mpol = NULL;
  2521. if (shmem_parse_options(data, &config, true))
  2522. return error;
  2523. spin_lock(&sbinfo->stat_lock);
  2524. inodes = sbinfo->max_inodes - sbinfo->free_inodes;
  2525. if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
  2526. goto out;
  2527. if (config.max_inodes < inodes)
  2528. goto out;
  2529. /*
  2530. * Those tests disallow limited->unlimited while any are in use;
  2531. * but we must separately disallow unlimited->limited, because
  2532. * in that case we have no record of how much is already in use.
  2533. */
  2534. if (config.max_blocks && !sbinfo->max_blocks)
  2535. goto out;
  2536. if (config.max_inodes && !sbinfo->max_inodes)
  2537. goto out;
  2538. error = 0;
  2539. sbinfo->max_blocks = config.max_blocks;
  2540. sbinfo->max_inodes = config.max_inodes;
  2541. sbinfo->free_inodes = config.max_inodes - inodes;
  2542. /*
  2543. * Preserve previous mempolicy unless mpol remount option was specified.
  2544. */
  2545. if (config.mpol) {
  2546. mpol_put(sbinfo->mpol);
  2547. sbinfo->mpol = config.mpol; /* transfers initial ref */
  2548. }
  2549. out:
  2550. spin_unlock(&sbinfo->stat_lock);
  2551. return error;
  2552. }
  2553. static int shmem_show_options(struct seq_file *seq, struct dentry *root)
  2554. {
  2555. struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
  2556. if (sbinfo->max_blocks != shmem_default_max_blocks())
  2557. seq_printf(seq, ",size=%luk",
  2558. sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
  2559. if (sbinfo->max_inodes != shmem_default_max_inodes())
  2560. seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
  2561. if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
  2562. seq_printf(seq, ",mode=%03ho", sbinfo->mode);
  2563. if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
  2564. seq_printf(seq, ",uid=%u",
  2565. from_kuid_munged(&init_user_ns, sbinfo->uid));
  2566. if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
  2567. seq_printf(seq, ",gid=%u",
  2568. from_kgid_munged(&init_user_ns, sbinfo->gid));
  2569. shmem_show_mpol(seq, sbinfo->mpol);
  2570. return 0;
  2571. }
  2572. #define MFD_NAME_PREFIX "memfd:"
  2573. #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
  2574. #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
  2575. #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING)
  2576. SYSCALL_DEFINE2(memfd_create,
  2577. const char __user *, uname,
  2578. unsigned int, flags)
  2579. {
  2580. struct shmem_inode_info *info;
  2581. struct file *file;
  2582. int fd, error;
  2583. char *name;
  2584. long len;
  2585. if (flags & ~(unsigned int)MFD_ALL_FLAGS)
  2586. return -EINVAL;
  2587. /* length includes terminating zero */
  2588. len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
  2589. if (len <= 0)
  2590. return -EFAULT;
  2591. if (len > MFD_NAME_MAX_LEN + 1)
  2592. return -EINVAL;
  2593. name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_TEMPORARY);
  2594. if (!name)
  2595. return -ENOMEM;
  2596. strcpy(name, MFD_NAME_PREFIX);
  2597. if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
  2598. error = -EFAULT;
  2599. goto err_name;
  2600. }
  2601. /* terminating-zero may have changed after strnlen_user() returned */
  2602. if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
  2603. error = -EFAULT;
  2604. goto err_name;
  2605. }
  2606. fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
  2607. if (fd < 0) {
  2608. error = fd;
  2609. goto err_name;
  2610. }
  2611. file = shmem_file_setup(name, 0, VM_NORESERVE);
  2612. if (IS_ERR(file)) {
  2613. error = PTR_ERR(file);
  2614. goto err_fd;
  2615. }
  2616. info = SHMEM_I(file_inode(file));
  2617. file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
  2618. file->f_flags |= O_RDWR | O_LARGEFILE;
  2619. if (flags & MFD_ALLOW_SEALING)
  2620. info->seals &= ~F_SEAL_SEAL;
  2621. fd_install(fd, file);
  2622. kfree(name);
  2623. return fd;
  2624. err_fd:
  2625. put_unused_fd(fd);
  2626. err_name:
  2627. kfree(name);
  2628. return error;
  2629. }
  2630. #endif /* CONFIG_TMPFS */
  2631. static void shmem_put_super(struct super_block *sb)
  2632. {
  2633. struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
  2634. percpu_counter_destroy(&sbinfo->used_blocks);
  2635. mpol_put(sbinfo->mpol);
  2636. kfree(sbinfo);
  2637. sb->s_fs_info = NULL;
  2638. }
  2639. int shmem_fill_super(struct super_block *sb, void *data, int silent)
  2640. {
  2641. struct inode *inode;
  2642. struct shmem_sb_info *sbinfo;
  2643. int err = -ENOMEM;
  2644. /* Round up to L1_CACHE_BYTES to resist false sharing */
  2645. sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
  2646. L1_CACHE_BYTES), GFP_KERNEL);
  2647. if (!sbinfo)
  2648. return -ENOMEM;
  2649. sbinfo->mode = S_IRWXUGO | S_ISVTX;
  2650. sbinfo->uid = current_fsuid();
  2651. sbinfo->gid = current_fsgid();
  2652. sb->s_fs_info = sbinfo;
  2653. #ifdef CONFIG_TMPFS
  2654. /*
  2655. * Per default we only allow half of the physical ram per
  2656. * tmpfs instance, limiting inodes to one per page of lowmem;
  2657. * but the internal instance is left unlimited.
  2658. */
  2659. if (!(sb->s_flags & MS_KERNMOUNT)) {
  2660. sbinfo->max_blocks = shmem_default_max_blocks();
  2661. sbinfo->max_inodes = shmem_default_max_inodes();
  2662. if (shmem_parse_options(data, sbinfo, false)) {
  2663. err = -EINVAL;
  2664. goto failed;
  2665. }
  2666. } else {
  2667. sb->s_flags |= MS_NOUSER;
  2668. }
  2669. sb->s_export_op = &shmem_export_ops;
  2670. sb->s_flags |= MS_NOSEC;
  2671. #else
  2672. sb->s_flags |= MS_NOUSER;
  2673. #endif
  2674. spin_lock_init(&sbinfo->stat_lock);
  2675. if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
  2676. goto failed;
  2677. sbinfo->free_inodes = sbinfo->max_inodes;
  2678. sb->s_maxbytes = MAX_LFS_FILESIZE;
  2679. sb->s_blocksize = PAGE_CACHE_SIZE;
  2680. sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
  2681. sb->s_magic = TMPFS_MAGIC;
  2682. sb->s_op = &shmem_ops;
  2683. sb->s_time_gran = 1;
  2684. #ifdef CONFIG_TMPFS_XATTR
  2685. sb->s_xattr = shmem_xattr_handlers;
  2686. #endif
  2687. #ifdef CONFIG_TMPFS_POSIX_ACL
  2688. sb->s_flags |= MS_POSIXACL;
  2689. #endif
  2690. inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
  2691. if (!inode)
  2692. goto failed;
  2693. inode->i_uid = sbinfo->uid;
  2694. inode->i_gid = sbinfo->gid;
  2695. sb->s_root = d_make_root(inode);
  2696. if (!sb->s_root)
  2697. goto failed;
  2698. return 0;
  2699. failed:
  2700. shmem_put_super(sb);
  2701. return err;
  2702. }
  2703. static struct kmem_cache *shmem_inode_cachep;
  2704. static struct inode *shmem_alloc_inode(struct super_block *sb)
  2705. {
  2706. struct shmem_inode_info *info;
  2707. info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
  2708. if (!info)
  2709. return NULL;
  2710. return &info->vfs_inode;
  2711. }
  2712. static void shmem_destroy_callback(struct rcu_head *head)
  2713. {
  2714. struct inode *inode = container_of(head, struct inode, i_rcu);
  2715. kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
  2716. }
  2717. static void shmem_destroy_inode(struct inode *inode)
  2718. {
  2719. if (S_ISREG(inode->i_mode))
  2720. mpol_free_shared_policy(&SHMEM_I(inode)->policy);
  2721. call_rcu(&inode->i_rcu, shmem_destroy_callback);
  2722. }
  2723. static void shmem_init_inode(void *foo)
  2724. {
  2725. struct shmem_inode_info *info = foo;
  2726. inode_init_once(&info->vfs_inode);
  2727. }
  2728. static int shmem_init_inodecache(void)
  2729. {
  2730. shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
  2731. sizeof(struct shmem_inode_info),
  2732. 0, SLAB_PANIC, shmem_init_inode);
  2733. return 0;
  2734. }
  2735. static void shmem_destroy_inodecache(void)
  2736. {
  2737. kmem_cache_destroy(shmem_inode_cachep);
  2738. }
  2739. static const struct address_space_operations shmem_aops = {
  2740. .writepage = shmem_writepage,
  2741. .set_page_dirty = __set_page_dirty_no_writeback,
  2742. #ifdef CONFIG_TMPFS
  2743. .write_begin = shmem_write_begin,
  2744. .write_end = shmem_write_end,
  2745. #endif
  2746. #ifdef CONFIG_MIGRATION
  2747. .migratepage = migrate_page,
  2748. #endif
  2749. .error_remove_page = generic_error_remove_page,
  2750. };
  2751. static const struct file_operations shmem_file_operations = {
  2752. .mmap = shmem_mmap,
  2753. #ifdef CONFIG_TMPFS
  2754. .llseek = shmem_file_llseek,
  2755. .read_iter = shmem_file_read_iter,
  2756. .write_iter = generic_file_write_iter,
  2757. .fsync = noop_fsync,
  2758. .splice_read = shmem_file_splice_read,
  2759. .splice_write = iter_file_splice_write,
  2760. .fallocate = shmem_fallocate,
  2761. #endif
  2762. };
  2763. static const struct inode_operations shmem_inode_operations = {
  2764. .getattr = shmem_getattr,
  2765. .setattr = shmem_setattr,
  2766. #ifdef CONFIG_TMPFS_XATTR
  2767. .setxattr = shmem_setxattr,
  2768. .getxattr = shmem_getxattr,
  2769. .listxattr = shmem_listxattr,
  2770. .removexattr = shmem_removexattr,
  2771. .set_acl = simple_set_acl,
  2772. #endif
  2773. };
  2774. static const struct inode_operations shmem_dir_inode_operations = {
  2775. #ifdef CONFIG_TMPFS
  2776. .create = shmem_create,
  2777. .lookup = simple_lookup,
  2778. .link = shmem_link,
  2779. .unlink = shmem_unlink,
  2780. .symlink = shmem_symlink,
  2781. .mkdir = shmem_mkdir,
  2782. .rmdir = shmem_rmdir,
  2783. .mknod = shmem_mknod,
  2784. .rename2 = shmem_rename2,
  2785. .tmpfile = shmem_tmpfile,
  2786. #endif
  2787. #ifdef CONFIG_TMPFS_XATTR
  2788. .setxattr = shmem_setxattr,
  2789. .getxattr = shmem_getxattr,
  2790. .listxattr = shmem_listxattr,
  2791. .removexattr = shmem_removexattr,
  2792. #endif
  2793. #ifdef CONFIG_TMPFS_POSIX_ACL
  2794. .setattr = shmem_setattr,
  2795. .set_acl = simple_set_acl,
  2796. #endif
  2797. };
  2798. static const struct inode_operations shmem_special_inode_operations = {
  2799. #ifdef CONFIG_TMPFS_XATTR
  2800. .setxattr = shmem_setxattr,
  2801. .getxattr = shmem_getxattr,
  2802. .listxattr = shmem_listxattr,
  2803. .removexattr = shmem_removexattr,
  2804. #endif
  2805. #ifdef CONFIG_TMPFS_POSIX_ACL
  2806. .setattr = shmem_setattr,
  2807. .set_acl = simple_set_acl,
  2808. #endif
  2809. };
  2810. static const struct super_operations shmem_ops = {
  2811. .alloc_inode = shmem_alloc_inode,
  2812. .destroy_inode = shmem_destroy_inode,
  2813. #ifdef CONFIG_TMPFS
  2814. .statfs = shmem_statfs,
  2815. .remount_fs = shmem_remount_fs,
  2816. .show_options = shmem_show_options,
  2817. #endif
  2818. .evict_inode = shmem_evict_inode,
  2819. .drop_inode = generic_delete_inode,
  2820. .put_super = shmem_put_super,
  2821. };
  2822. static const struct vm_operations_struct shmem_vm_ops = {
  2823. .fault = shmem_fault,
  2824. .map_pages = filemap_map_pages,
  2825. #ifdef CONFIG_NUMA
  2826. .set_policy = shmem_set_policy,
  2827. .get_policy = shmem_get_policy,
  2828. #endif
  2829. };
  2830. static struct dentry *shmem_mount(struct file_system_type *fs_type,
  2831. int flags, const char *dev_name, void *data)
  2832. {
  2833. return mount_nodev(fs_type, flags, data, shmem_fill_super);
  2834. }
  2835. static struct file_system_type shmem_fs_type = {
  2836. .owner = THIS_MODULE,
  2837. .name = "tmpfs",
  2838. .mount = shmem_mount,
  2839. .kill_sb = kill_litter_super,
  2840. .fs_flags = FS_USERNS_MOUNT,
  2841. };
  2842. int __init shmem_init(void)
  2843. {
  2844. int error;
  2845. /* If rootfs called this, don't re-init */
  2846. if (shmem_inode_cachep)
  2847. return 0;
  2848. error = shmem_init_inodecache();
  2849. if (error)
  2850. goto out3;
  2851. error = register_filesystem(&shmem_fs_type);
  2852. if (error) {
  2853. printk(KERN_ERR "Could not register tmpfs\n");
  2854. goto out2;
  2855. }
  2856. shm_mnt = kern_mount(&shmem_fs_type);
  2857. if (IS_ERR(shm_mnt)) {
  2858. error = PTR_ERR(shm_mnt);
  2859. printk(KERN_ERR "Could not kern_mount tmpfs\n");
  2860. goto out1;
  2861. }
  2862. return 0;
  2863. out1:
  2864. unregister_filesystem(&shmem_fs_type);
  2865. out2:
  2866. shmem_destroy_inodecache();
  2867. out3:
  2868. shm_mnt = ERR_PTR(error);
  2869. return error;
  2870. }
  2871. #else /* !CONFIG_SHMEM */
  2872. /*
  2873. * tiny-shmem: simple shmemfs and tmpfs using ramfs code
  2874. *
  2875. * This is intended for small system where the benefits of the full
  2876. * shmem code (swap-backed and resource-limited) are outweighed by
  2877. * their complexity. On systems without swap this code should be
  2878. * effectively equivalent, but much lighter weight.
  2879. */
  2880. static struct file_system_type shmem_fs_type = {
  2881. .name = "tmpfs",
  2882. .mount = ramfs_mount,
  2883. .kill_sb = kill_litter_super,
  2884. .fs_flags = FS_USERNS_MOUNT,
  2885. };
  2886. int __init shmem_init(void)
  2887. {
  2888. BUG_ON(register_filesystem(&shmem_fs_type) != 0);
  2889. shm_mnt = kern_mount(&shmem_fs_type);
  2890. BUG_ON(IS_ERR(shm_mnt));
  2891. return 0;
  2892. }
  2893. int shmem_unuse(swp_entry_t swap, struct page *page)
  2894. {
  2895. return 0;
  2896. }
  2897. int shmem_lock(struct file *file, int lock, struct user_struct *user)
  2898. {
  2899. return 0;
  2900. }
  2901. void shmem_unlock_mapping(struct address_space *mapping)
  2902. {
  2903. }
  2904. void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
  2905. {
  2906. truncate_inode_pages_range(inode->i_mapping, lstart, lend);
  2907. }
  2908. EXPORT_SYMBOL_GPL(shmem_truncate_range);
  2909. #define shmem_vm_ops generic_file_vm_ops
  2910. #define shmem_file_operations ramfs_file_operations
  2911. #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
  2912. #define shmem_acct_size(flags, size) 0
  2913. #define shmem_unacct_size(flags, size) do {} while (0)
  2914. #endif /* CONFIG_SHMEM */
  2915. /* common code */
  2916. static struct dentry_operations anon_ops = {
  2917. .d_dname = simple_dname
  2918. };
  2919. static struct file *__shmem_file_setup(const char *name, loff_t size,
  2920. unsigned long flags, unsigned int i_flags)
  2921. {
  2922. struct file *res;
  2923. struct inode *inode;
  2924. struct path path;
  2925. struct super_block *sb;
  2926. struct qstr this;
  2927. if (IS_ERR(shm_mnt))
  2928. return ERR_CAST(shm_mnt);
  2929. if (size < 0 || size > MAX_LFS_FILESIZE)
  2930. return ERR_PTR(-EINVAL);
  2931. if (shmem_acct_size(flags, size))
  2932. return ERR_PTR(-ENOMEM);
  2933. res = ERR_PTR(-ENOMEM);
  2934. this.name = name;
  2935. this.len = strlen(name);
  2936. this.hash = 0; /* will go */
  2937. sb = shm_mnt->mnt_sb;
  2938. path.mnt = mntget(shm_mnt);
  2939. path.dentry = d_alloc_pseudo(sb, &this);
  2940. if (!path.dentry)
  2941. goto put_memory;
  2942. d_set_d_op(path.dentry, &anon_ops);
  2943. res = ERR_PTR(-ENOSPC);
  2944. inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
  2945. if (!inode)
  2946. goto put_memory;
  2947. inode->i_flags |= i_flags;
  2948. d_instantiate(path.dentry, inode);
  2949. inode->i_size = size;
  2950. clear_nlink(inode); /* It is unlinked */
  2951. res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
  2952. if (IS_ERR(res))
  2953. goto put_path;
  2954. res = alloc_file(&path, FMODE_WRITE | FMODE_READ,
  2955. &shmem_file_operations);
  2956. if (IS_ERR(res))
  2957. goto put_path;
  2958. return res;
  2959. put_memory:
  2960. shmem_unacct_size(flags, size);
  2961. put_path:
  2962. path_put(&path);
  2963. return res;
  2964. }
  2965. /**
  2966. * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
  2967. * kernel internal. There will be NO LSM permission checks against the
  2968. * underlying inode. So users of this interface must do LSM checks at a
  2969. * higher layer. The users are the big_key and shm implementations. LSM
  2970. * checks are provided at the key or shm level rather than the inode.
  2971. * @name: name for dentry (to be seen in /proc/<pid>/maps
  2972. * @size: size to be set for the file
  2973. * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
  2974. */
  2975. struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
  2976. {
  2977. return __shmem_file_setup(name, size, flags, S_PRIVATE);
  2978. }
  2979. /**
  2980. * shmem_file_setup - get an unlinked file living in tmpfs
  2981. * @name: name for dentry (to be seen in /proc/<pid>/maps
  2982. * @size: size to be set for the file
  2983. * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
  2984. */
  2985. struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
  2986. {
  2987. return __shmem_file_setup(name, size, flags, 0);
  2988. }
  2989. EXPORT_SYMBOL_GPL(shmem_file_setup);
  2990. /**
  2991. * shmem_zero_setup - setup a shared anonymous mapping
  2992. * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
  2993. */
  2994. int shmem_zero_setup(struct vm_area_struct *vma)
  2995. {
  2996. struct file *file;
  2997. loff_t size = vma->vm_end - vma->vm_start;
  2998. /*
  2999. * Cloning a new file under mmap_sem leads to a lock ordering conflict
  3000. * between XFS directory reading and selinux: since this file is only
  3001. * accessible to the user through its mapping, use S_PRIVATE flag to
  3002. * bypass file security, in the same way as shmem_kernel_file_setup().
  3003. */
  3004. file = __shmem_file_setup("dev/zero", size, vma->vm_flags, S_PRIVATE);
  3005. if (IS_ERR(file))
  3006. return PTR_ERR(file);
  3007. if (vma->vm_file)
  3008. fput(vma->vm_file);
  3009. vma->vm_file = file;
  3010. vma->vm_ops = &shmem_vm_ops;
  3011. return 0;
  3012. }
  3013. /**
  3014. * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
  3015. * @mapping: the page's address_space
  3016. * @index: the page index
  3017. * @gfp: the page allocator flags to use if allocating
  3018. *
  3019. * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
  3020. * with any new page allocations done using the specified allocation flags.
  3021. * But read_cache_page_gfp() uses the ->readpage() method: which does not
  3022. * suit tmpfs, since it may have pages in swapcache, and needs to find those
  3023. * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
  3024. *
  3025. * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
  3026. * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
  3027. */
  3028. struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
  3029. pgoff_t index, gfp_t gfp)
  3030. {
  3031. #ifdef CONFIG_SHMEM
  3032. struct inode *inode = mapping->host;
  3033. struct page *page;
  3034. int error;
  3035. BUG_ON(mapping->a_ops != &shmem_aops);
  3036. error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
  3037. if (error)
  3038. page = ERR_PTR(error);
  3039. else
  3040. unlock_page(page);
  3041. return page;
  3042. #else
  3043. /*
  3044. * The tiny !SHMEM case uses ramfs without swap
  3045. */
  3046. return read_cache_page_gfp(mapping, index, gfp);
  3047. #endif
  3048. }
  3049. EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);