vmalloc.c 68 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693
  1. /*
  2. * linux/mm/vmalloc.c
  3. *
  4. * Copyright (C) 1993 Linus Torvalds
  5. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  6. * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
  7. * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
  8. * Numa awareness, Christoph Lameter, SGI, June 2005
  9. */
  10. #include <linux/vmalloc.h>
  11. #include <linux/mm.h>
  12. #include <linux/module.h>
  13. #include <linux/highmem.h>
  14. #include <linux/sched.h>
  15. #include <linux/slab.h>
  16. #include <linux/spinlock.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/proc_fs.h>
  19. #include <linux/seq_file.h>
  20. #include <linux/debugobjects.h>
  21. #include <linux/kallsyms.h>
  22. #include <linux/list.h>
  23. #include <linux/rbtree.h>
  24. #include <linux/radix-tree.h>
  25. #include <linux/rcupdate.h>
  26. #include <linux/pfn.h>
  27. #include <linux/kmemleak.h>
  28. #include <linux/atomic.h>
  29. #include <linux/compiler.h>
  30. #include <linux/llist.h>
  31. #include <linux/bitops.h>
  32. #include <asm/uaccess.h>
  33. #include <asm/tlbflush.h>
  34. #include <asm/shmparam.h>
  35. #include "internal.h"
  36. struct vfree_deferred {
  37. struct llist_head list;
  38. struct work_struct wq;
  39. };
  40. static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
  41. static void __vunmap(const void *, int);
  42. static void free_work(struct work_struct *w)
  43. {
  44. struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
  45. struct llist_node *llnode = llist_del_all(&p->list);
  46. while (llnode) {
  47. void *p = llnode;
  48. llnode = llist_next(llnode);
  49. __vunmap(p, 1);
  50. }
  51. }
  52. /*** Page table manipulation functions ***/
  53. static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
  54. {
  55. pte_t *pte;
  56. pte = pte_offset_kernel(pmd, addr);
  57. do {
  58. pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
  59. WARN_ON(!pte_none(ptent) && !pte_present(ptent));
  60. } while (pte++, addr += PAGE_SIZE, addr != end);
  61. }
  62. static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
  63. {
  64. pmd_t *pmd;
  65. unsigned long next;
  66. pmd = pmd_offset(pud, addr);
  67. do {
  68. next = pmd_addr_end(addr, end);
  69. if (pmd_clear_huge(pmd))
  70. continue;
  71. if (pmd_none_or_clear_bad(pmd))
  72. continue;
  73. vunmap_pte_range(pmd, addr, next);
  74. } while (pmd++, addr = next, addr != end);
  75. }
  76. static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
  77. {
  78. pud_t *pud;
  79. unsigned long next;
  80. pud = pud_offset(pgd, addr);
  81. do {
  82. next = pud_addr_end(addr, end);
  83. if (pud_clear_huge(pud))
  84. continue;
  85. if (pud_none_or_clear_bad(pud))
  86. continue;
  87. vunmap_pmd_range(pud, addr, next);
  88. } while (pud++, addr = next, addr != end);
  89. }
  90. static void vunmap_page_range(unsigned long addr, unsigned long end)
  91. {
  92. pgd_t *pgd;
  93. unsigned long next;
  94. BUG_ON(addr >= end);
  95. pgd = pgd_offset_k(addr);
  96. do {
  97. next = pgd_addr_end(addr, end);
  98. if (pgd_none_or_clear_bad(pgd))
  99. continue;
  100. vunmap_pud_range(pgd, addr, next);
  101. } while (pgd++, addr = next, addr != end);
  102. }
  103. static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
  104. unsigned long end, pgprot_t prot, struct page **pages, int *nr)
  105. {
  106. pte_t *pte;
  107. /*
  108. * nr is a running index into the array which helps higher level
  109. * callers keep track of where we're up to.
  110. */
  111. pte = pte_alloc_kernel(pmd, addr);
  112. if (!pte)
  113. return -ENOMEM;
  114. do {
  115. struct page *page = pages[*nr];
  116. if (WARN_ON(!pte_none(*pte)))
  117. return -EBUSY;
  118. if (WARN_ON(!page))
  119. return -ENOMEM;
  120. set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
  121. (*nr)++;
  122. } while (pte++, addr += PAGE_SIZE, addr != end);
  123. return 0;
  124. }
  125. static int vmap_pmd_range(pud_t *pud, unsigned long addr,
  126. unsigned long end, pgprot_t prot, struct page **pages, int *nr)
  127. {
  128. pmd_t *pmd;
  129. unsigned long next;
  130. pmd = pmd_alloc(&init_mm, pud, addr);
  131. if (!pmd)
  132. return -ENOMEM;
  133. do {
  134. next = pmd_addr_end(addr, end);
  135. if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
  136. return -ENOMEM;
  137. } while (pmd++, addr = next, addr != end);
  138. return 0;
  139. }
  140. static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
  141. unsigned long end, pgprot_t prot, struct page **pages, int *nr)
  142. {
  143. pud_t *pud;
  144. unsigned long next;
  145. pud = pud_alloc(&init_mm, pgd, addr);
  146. if (!pud)
  147. return -ENOMEM;
  148. do {
  149. next = pud_addr_end(addr, end);
  150. if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
  151. return -ENOMEM;
  152. } while (pud++, addr = next, addr != end);
  153. return 0;
  154. }
  155. /*
  156. * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
  157. * will have pfns corresponding to the "pages" array.
  158. *
  159. * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
  160. */
  161. static int vmap_page_range_noflush(unsigned long start, unsigned long end,
  162. pgprot_t prot, struct page **pages)
  163. {
  164. pgd_t *pgd;
  165. unsigned long next;
  166. unsigned long addr = start;
  167. int err = 0;
  168. int nr = 0;
  169. BUG_ON(addr >= end);
  170. pgd = pgd_offset_k(addr);
  171. do {
  172. next = pgd_addr_end(addr, end);
  173. err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
  174. if (err)
  175. return err;
  176. } while (pgd++, addr = next, addr != end);
  177. return nr;
  178. }
  179. static int vmap_page_range(unsigned long start, unsigned long end,
  180. pgprot_t prot, struct page **pages)
  181. {
  182. int ret;
  183. ret = vmap_page_range_noflush(start, end, prot, pages);
  184. flush_cache_vmap(start, end);
  185. return ret;
  186. }
  187. int is_vmalloc_or_module_addr(const void *x)
  188. {
  189. /*
  190. * ARM, x86-64 and sparc64 put modules in a special place,
  191. * and fall back on vmalloc() if that fails. Others
  192. * just put it in the vmalloc space.
  193. */
  194. #if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
  195. unsigned long addr = (unsigned long)x;
  196. if (addr >= MODULES_VADDR && addr < MODULES_END)
  197. return 1;
  198. #endif
  199. return is_vmalloc_addr(x);
  200. }
  201. /*
  202. * Walk a vmap address to the struct page it maps.
  203. */
  204. struct page *vmalloc_to_page(const void *vmalloc_addr)
  205. {
  206. unsigned long addr = (unsigned long) vmalloc_addr;
  207. struct page *page = NULL;
  208. pgd_t *pgd = pgd_offset_k(addr);
  209. /*
  210. * XXX we might need to change this if we add VIRTUAL_BUG_ON for
  211. * architectures that do not vmalloc module space
  212. */
  213. VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
  214. if (!pgd_none(*pgd)) {
  215. pud_t *pud = pud_offset(pgd, addr);
  216. if (!pud_none(*pud)) {
  217. pmd_t *pmd = pmd_offset(pud, addr);
  218. if (!pmd_none(*pmd)) {
  219. pte_t *ptep, pte;
  220. ptep = pte_offset_map(pmd, addr);
  221. pte = *ptep;
  222. if (pte_present(pte))
  223. page = pte_page(pte);
  224. pte_unmap(ptep);
  225. }
  226. }
  227. }
  228. return page;
  229. }
  230. EXPORT_SYMBOL(vmalloc_to_page);
  231. /*
  232. * Map a vmalloc()-space virtual address to the physical page frame number.
  233. */
  234. unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
  235. {
  236. return page_to_pfn(vmalloc_to_page(vmalloc_addr));
  237. }
  238. EXPORT_SYMBOL(vmalloc_to_pfn);
  239. /*** Global kva allocator ***/
  240. #define VM_LAZY_FREE 0x01
  241. #define VM_LAZY_FREEING 0x02
  242. #define VM_VM_AREA 0x04
  243. static DEFINE_SPINLOCK(vmap_area_lock);
  244. /* Export for kexec only */
  245. LIST_HEAD(vmap_area_list);
  246. static struct rb_root vmap_area_root = RB_ROOT;
  247. /* The vmap cache globals are protected by vmap_area_lock */
  248. static struct rb_node *free_vmap_cache;
  249. static unsigned long cached_hole_size;
  250. static unsigned long cached_vstart;
  251. static unsigned long cached_align;
  252. static unsigned long vmap_area_pcpu_hole;
  253. static struct vmap_area *__find_vmap_area(unsigned long addr)
  254. {
  255. struct rb_node *n = vmap_area_root.rb_node;
  256. while (n) {
  257. struct vmap_area *va;
  258. va = rb_entry(n, struct vmap_area, rb_node);
  259. if (addr < va->va_start)
  260. n = n->rb_left;
  261. else if (addr >= va->va_end)
  262. n = n->rb_right;
  263. else
  264. return va;
  265. }
  266. return NULL;
  267. }
  268. static void __insert_vmap_area(struct vmap_area *va)
  269. {
  270. struct rb_node **p = &vmap_area_root.rb_node;
  271. struct rb_node *parent = NULL;
  272. struct rb_node *tmp;
  273. while (*p) {
  274. struct vmap_area *tmp_va;
  275. parent = *p;
  276. tmp_va = rb_entry(parent, struct vmap_area, rb_node);
  277. if (va->va_start < tmp_va->va_end)
  278. p = &(*p)->rb_left;
  279. else if (va->va_end > tmp_va->va_start)
  280. p = &(*p)->rb_right;
  281. else
  282. BUG();
  283. }
  284. rb_link_node(&va->rb_node, parent, p);
  285. rb_insert_color(&va->rb_node, &vmap_area_root);
  286. /* address-sort this list */
  287. tmp = rb_prev(&va->rb_node);
  288. if (tmp) {
  289. struct vmap_area *prev;
  290. prev = rb_entry(tmp, struct vmap_area, rb_node);
  291. list_add_rcu(&va->list, &prev->list);
  292. } else
  293. list_add_rcu(&va->list, &vmap_area_list);
  294. }
  295. static void purge_vmap_area_lazy(void);
  296. /*
  297. * Allocate a region of KVA of the specified size and alignment, within the
  298. * vstart and vend.
  299. */
  300. static struct vmap_area *alloc_vmap_area(unsigned long size,
  301. unsigned long align,
  302. unsigned long vstart, unsigned long vend,
  303. int node, gfp_t gfp_mask)
  304. {
  305. struct vmap_area *va;
  306. struct rb_node *n;
  307. unsigned long addr;
  308. int purged = 0;
  309. struct vmap_area *first;
  310. BUG_ON(!size);
  311. BUG_ON(offset_in_page(size));
  312. BUG_ON(!is_power_of_2(align));
  313. va = kmalloc_node(sizeof(struct vmap_area),
  314. gfp_mask & GFP_RECLAIM_MASK, node);
  315. if (unlikely(!va))
  316. return ERR_PTR(-ENOMEM);
  317. /*
  318. * Only scan the relevant parts containing pointers to other objects
  319. * to avoid false negatives.
  320. */
  321. kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);
  322. retry:
  323. spin_lock(&vmap_area_lock);
  324. /*
  325. * Invalidate cache if we have more permissive parameters.
  326. * cached_hole_size notes the largest hole noticed _below_
  327. * the vmap_area cached in free_vmap_cache: if size fits
  328. * into that hole, we want to scan from vstart to reuse
  329. * the hole instead of allocating above free_vmap_cache.
  330. * Note that __free_vmap_area may update free_vmap_cache
  331. * without updating cached_hole_size or cached_align.
  332. */
  333. if (!free_vmap_cache ||
  334. size < cached_hole_size ||
  335. vstart < cached_vstart ||
  336. align < cached_align) {
  337. nocache:
  338. cached_hole_size = 0;
  339. free_vmap_cache = NULL;
  340. }
  341. /* record if we encounter less permissive parameters */
  342. cached_vstart = vstart;
  343. cached_align = align;
  344. /* find starting point for our search */
  345. if (free_vmap_cache) {
  346. first = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
  347. addr = ALIGN(first->va_end, align);
  348. if (addr < vstart)
  349. goto nocache;
  350. if (addr + size < addr)
  351. goto overflow;
  352. } else {
  353. addr = ALIGN(vstart, align);
  354. if (addr + size < addr)
  355. goto overflow;
  356. n = vmap_area_root.rb_node;
  357. first = NULL;
  358. while (n) {
  359. struct vmap_area *tmp;
  360. tmp = rb_entry(n, struct vmap_area, rb_node);
  361. if (tmp->va_end >= addr) {
  362. first = tmp;
  363. if (tmp->va_start <= addr)
  364. break;
  365. n = n->rb_left;
  366. } else
  367. n = n->rb_right;
  368. }
  369. if (!first)
  370. goto found;
  371. }
  372. /* from the starting point, walk areas until a suitable hole is found */
  373. while (addr + size > first->va_start && addr + size <= vend) {
  374. if (addr + cached_hole_size < first->va_start)
  375. cached_hole_size = first->va_start - addr;
  376. addr = ALIGN(first->va_end, align);
  377. if (addr + size < addr)
  378. goto overflow;
  379. if (list_is_last(&first->list, &vmap_area_list))
  380. goto found;
  381. first = list_entry(first->list.next,
  382. struct vmap_area, list);
  383. }
  384. found:
  385. if (addr + size > vend)
  386. goto overflow;
  387. va->va_start = addr;
  388. va->va_end = addr + size;
  389. va->flags = 0;
  390. __insert_vmap_area(va);
  391. free_vmap_cache = &va->rb_node;
  392. spin_unlock(&vmap_area_lock);
  393. BUG_ON(va->va_start & (align-1));
  394. BUG_ON(va->va_start < vstart);
  395. BUG_ON(va->va_end > vend);
  396. return va;
  397. overflow:
  398. spin_unlock(&vmap_area_lock);
  399. if (!purged) {
  400. purge_vmap_area_lazy();
  401. purged = 1;
  402. goto retry;
  403. }
  404. if (printk_ratelimit())
  405. pr_warn("vmap allocation for size %lu failed: "
  406. "use vmalloc=<size> to increase size.\n", size);
  407. kfree(va);
  408. return ERR_PTR(-EBUSY);
  409. }
  410. static void __free_vmap_area(struct vmap_area *va)
  411. {
  412. BUG_ON(RB_EMPTY_NODE(&va->rb_node));
  413. if (free_vmap_cache) {
  414. if (va->va_end < cached_vstart) {
  415. free_vmap_cache = NULL;
  416. } else {
  417. struct vmap_area *cache;
  418. cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
  419. if (va->va_start <= cache->va_start) {
  420. free_vmap_cache = rb_prev(&va->rb_node);
  421. /*
  422. * We don't try to update cached_hole_size or
  423. * cached_align, but it won't go very wrong.
  424. */
  425. }
  426. }
  427. }
  428. rb_erase(&va->rb_node, &vmap_area_root);
  429. RB_CLEAR_NODE(&va->rb_node);
  430. list_del_rcu(&va->list);
  431. /*
  432. * Track the highest possible candidate for pcpu area
  433. * allocation. Areas outside of vmalloc area can be returned
  434. * here too, consider only end addresses which fall inside
  435. * vmalloc area proper.
  436. */
  437. if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
  438. vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
  439. kfree_rcu(va, rcu_head);
  440. }
  441. /*
  442. * Free a region of KVA allocated by alloc_vmap_area
  443. */
  444. static void free_vmap_area(struct vmap_area *va)
  445. {
  446. spin_lock(&vmap_area_lock);
  447. __free_vmap_area(va);
  448. spin_unlock(&vmap_area_lock);
  449. }
  450. /*
  451. * Clear the pagetable entries of a given vmap_area
  452. */
  453. static void unmap_vmap_area(struct vmap_area *va)
  454. {
  455. vunmap_page_range(va->va_start, va->va_end);
  456. }
  457. static void vmap_debug_free_range(unsigned long start, unsigned long end)
  458. {
  459. /*
  460. * Unmap page tables and force a TLB flush immediately if
  461. * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free
  462. * bugs similarly to those in linear kernel virtual address
  463. * space after a page has been freed.
  464. *
  465. * All the lazy freeing logic is still retained, in order to
  466. * minimise intrusiveness of this debugging feature.
  467. *
  468. * This is going to be *slow* (linear kernel virtual address
  469. * debugging doesn't do a broadcast TLB flush so it is a lot
  470. * faster).
  471. */
  472. #ifdef CONFIG_DEBUG_PAGEALLOC
  473. vunmap_page_range(start, end);
  474. flush_tlb_kernel_range(start, end);
  475. #endif
  476. }
  477. /*
  478. * lazy_max_pages is the maximum amount of virtual address space we gather up
  479. * before attempting to purge with a TLB flush.
  480. *
  481. * There is a tradeoff here: a larger number will cover more kernel page tables
  482. * and take slightly longer to purge, but it will linearly reduce the number of
  483. * global TLB flushes that must be performed. It would seem natural to scale
  484. * this number up linearly with the number of CPUs (because vmapping activity
  485. * could also scale linearly with the number of CPUs), however it is likely
  486. * that in practice, workloads might be constrained in other ways that mean
  487. * vmap activity will not scale linearly with CPUs. Also, I want to be
  488. * conservative and not introduce a big latency on huge systems, so go with
  489. * a less aggressive log scale. It will still be an improvement over the old
  490. * code, and it will be simple to change the scale factor if we find that it
  491. * becomes a problem on bigger systems.
  492. */
  493. static unsigned long lazy_max_pages(void)
  494. {
  495. unsigned int log;
  496. log = fls(num_online_cpus());
  497. return log * (32UL * 1024 * 1024 / PAGE_SIZE);
  498. }
  499. static atomic_t vmap_lazy_nr = ATOMIC_INIT(0);
  500. /* for per-CPU blocks */
  501. static void purge_fragmented_blocks_allcpus(void);
  502. /*
  503. * called before a call to iounmap() if the caller wants vm_area_struct's
  504. * immediately freed.
  505. */
  506. void set_iounmap_nonlazy(void)
  507. {
  508. atomic_set(&vmap_lazy_nr, lazy_max_pages()+1);
  509. }
  510. /*
  511. * Purges all lazily-freed vmap areas.
  512. *
  513. * If sync is 0 then don't purge if there is already a purge in progress.
  514. * If force_flush is 1, then flush kernel TLBs between *start and *end even
  515. * if we found no lazy vmap areas to unmap (callers can use this to optimise
  516. * their own TLB flushing).
  517. * Returns with *start = min(*start, lowest purged address)
  518. * *end = max(*end, highest purged address)
  519. */
  520. static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
  521. int sync, int force_flush)
  522. {
  523. static DEFINE_SPINLOCK(purge_lock);
  524. LIST_HEAD(valist);
  525. struct vmap_area *va;
  526. struct vmap_area *n_va;
  527. int nr = 0;
  528. /*
  529. * If sync is 0 but force_flush is 1, we'll go sync anyway but callers
  530. * should not expect such behaviour. This just simplifies locking for
  531. * the case that isn't actually used at the moment anyway.
  532. */
  533. if (!sync && !force_flush) {
  534. if (!spin_trylock(&purge_lock))
  535. return;
  536. } else
  537. spin_lock(&purge_lock);
  538. if (sync)
  539. purge_fragmented_blocks_allcpus();
  540. rcu_read_lock();
  541. list_for_each_entry_rcu(va, &vmap_area_list, list) {
  542. if (va->flags & VM_LAZY_FREE) {
  543. if (va->va_start < *start)
  544. *start = va->va_start;
  545. if (va->va_end > *end)
  546. *end = va->va_end;
  547. nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
  548. list_add_tail(&va->purge_list, &valist);
  549. va->flags |= VM_LAZY_FREEING;
  550. va->flags &= ~VM_LAZY_FREE;
  551. }
  552. }
  553. rcu_read_unlock();
  554. if (nr)
  555. atomic_sub(nr, &vmap_lazy_nr);
  556. if (nr || force_flush)
  557. flush_tlb_kernel_range(*start, *end);
  558. if (nr) {
  559. spin_lock(&vmap_area_lock);
  560. list_for_each_entry_safe(va, n_va, &valist, purge_list)
  561. __free_vmap_area(va);
  562. spin_unlock(&vmap_area_lock);
  563. }
  564. spin_unlock(&purge_lock);
  565. }
  566. /*
  567. * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
  568. * is already purging.
  569. */
  570. static void try_purge_vmap_area_lazy(void)
  571. {
  572. unsigned long start = ULONG_MAX, end = 0;
  573. __purge_vmap_area_lazy(&start, &end, 0, 0);
  574. }
  575. /*
  576. * Kick off a purge of the outstanding lazy areas.
  577. */
  578. static void purge_vmap_area_lazy(void)
  579. {
  580. unsigned long start = ULONG_MAX, end = 0;
  581. __purge_vmap_area_lazy(&start, &end, 1, 0);
  582. }
  583. /*
  584. * Free a vmap area, caller ensuring that the area has been unmapped
  585. * and flush_cache_vunmap had been called for the correct range
  586. * previously.
  587. */
  588. static void free_vmap_area_noflush(struct vmap_area *va)
  589. {
  590. va->flags |= VM_LAZY_FREE;
  591. atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr);
  592. if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages()))
  593. try_purge_vmap_area_lazy();
  594. }
  595. /*
  596. * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been
  597. * called for the correct range previously.
  598. */
  599. static void free_unmap_vmap_area_noflush(struct vmap_area *va)
  600. {
  601. unmap_vmap_area(va);
  602. free_vmap_area_noflush(va);
  603. }
  604. /*
  605. * Free and unmap a vmap area
  606. */
  607. static void free_unmap_vmap_area(struct vmap_area *va)
  608. {
  609. flush_cache_vunmap(va->va_start, va->va_end);
  610. free_unmap_vmap_area_noflush(va);
  611. }
  612. static struct vmap_area *find_vmap_area(unsigned long addr)
  613. {
  614. struct vmap_area *va;
  615. spin_lock(&vmap_area_lock);
  616. va = __find_vmap_area(addr);
  617. spin_unlock(&vmap_area_lock);
  618. return va;
  619. }
  620. static void free_unmap_vmap_area_addr(unsigned long addr)
  621. {
  622. struct vmap_area *va;
  623. va = find_vmap_area(addr);
  624. BUG_ON(!va);
  625. free_unmap_vmap_area(va);
  626. }
  627. /*** Per cpu kva allocator ***/
  628. /*
  629. * vmap space is limited especially on 32 bit architectures. Ensure there is
  630. * room for at least 16 percpu vmap blocks per CPU.
  631. */
  632. /*
  633. * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
  634. * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
  635. * instead (we just need a rough idea)
  636. */
  637. #if BITS_PER_LONG == 32
  638. #define VMALLOC_SPACE (128UL*1024*1024)
  639. #else
  640. #define VMALLOC_SPACE (128UL*1024*1024*1024)
  641. #endif
  642. #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
  643. #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
  644. #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
  645. #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
  646. #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
  647. #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
  648. #define VMAP_BBMAP_BITS \
  649. VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
  650. VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
  651. VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
  652. #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
  653. static bool vmap_initialized __read_mostly = false;
  654. struct vmap_block_queue {
  655. spinlock_t lock;
  656. struct list_head free;
  657. };
  658. struct vmap_block {
  659. spinlock_t lock;
  660. struct vmap_area *va;
  661. unsigned long free, dirty;
  662. unsigned long dirty_min, dirty_max; /*< dirty range */
  663. struct list_head free_list;
  664. struct rcu_head rcu_head;
  665. struct list_head purge;
  666. };
  667. /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
  668. static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
  669. /*
  670. * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
  671. * in the free path. Could get rid of this if we change the API to return a
  672. * "cookie" from alloc, to be passed to free. But no big deal yet.
  673. */
  674. static DEFINE_SPINLOCK(vmap_block_tree_lock);
  675. static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
  676. /*
  677. * We should probably have a fallback mechanism to allocate virtual memory
  678. * out of partially filled vmap blocks. However vmap block sizing should be
  679. * fairly reasonable according to the vmalloc size, so it shouldn't be a
  680. * big problem.
  681. */
  682. static unsigned long addr_to_vb_idx(unsigned long addr)
  683. {
  684. addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
  685. addr /= VMAP_BLOCK_SIZE;
  686. return addr;
  687. }
  688. static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
  689. {
  690. unsigned long addr;
  691. addr = va_start + (pages_off << PAGE_SHIFT);
  692. BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
  693. return (void *)addr;
  694. }
  695. /**
  696. * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
  697. * block. Of course pages number can't exceed VMAP_BBMAP_BITS
  698. * @order: how many 2^order pages should be occupied in newly allocated block
  699. * @gfp_mask: flags for the page level allocator
  700. *
  701. * Returns: virtual address in a newly allocated block or ERR_PTR(-errno)
  702. */
  703. static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
  704. {
  705. struct vmap_block_queue *vbq;
  706. struct vmap_block *vb;
  707. struct vmap_area *va;
  708. unsigned long vb_idx;
  709. int node, err;
  710. void *vaddr;
  711. node = numa_node_id();
  712. vb = kmalloc_node(sizeof(struct vmap_block),
  713. gfp_mask & GFP_RECLAIM_MASK, node);
  714. if (unlikely(!vb))
  715. return ERR_PTR(-ENOMEM);
  716. va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
  717. VMALLOC_START, VMALLOC_END,
  718. node, gfp_mask);
  719. if (IS_ERR(va)) {
  720. kfree(vb);
  721. return ERR_CAST(va);
  722. }
  723. err = radix_tree_preload(gfp_mask);
  724. if (unlikely(err)) {
  725. kfree(vb);
  726. free_vmap_area(va);
  727. return ERR_PTR(err);
  728. }
  729. vaddr = vmap_block_vaddr(va->va_start, 0);
  730. spin_lock_init(&vb->lock);
  731. vb->va = va;
  732. /* At least something should be left free */
  733. BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
  734. vb->free = VMAP_BBMAP_BITS - (1UL << order);
  735. vb->dirty = 0;
  736. vb->dirty_min = VMAP_BBMAP_BITS;
  737. vb->dirty_max = 0;
  738. INIT_LIST_HEAD(&vb->free_list);
  739. vb_idx = addr_to_vb_idx(va->va_start);
  740. spin_lock(&vmap_block_tree_lock);
  741. err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
  742. spin_unlock(&vmap_block_tree_lock);
  743. BUG_ON(err);
  744. radix_tree_preload_end();
  745. vbq = &get_cpu_var(vmap_block_queue);
  746. spin_lock(&vbq->lock);
  747. list_add_tail_rcu(&vb->free_list, &vbq->free);
  748. spin_unlock(&vbq->lock);
  749. put_cpu_var(vmap_block_queue);
  750. return vaddr;
  751. }
  752. static void free_vmap_block(struct vmap_block *vb)
  753. {
  754. struct vmap_block *tmp;
  755. unsigned long vb_idx;
  756. vb_idx = addr_to_vb_idx(vb->va->va_start);
  757. spin_lock(&vmap_block_tree_lock);
  758. tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
  759. spin_unlock(&vmap_block_tree_lock);
  760. BUG_ON(tmp != vb);
  761. free_vmap_area_noflush(vb->va);
  762. kfree_rcu(vb, rcu_head);
  763. }
  764. static void purge_fragmented_blocks(int cpu)
  765. {
  766. LIST_HEAD(purge);
  767. struct vmap_block *vb;
  768. struct vmap_block *n_vb;
  769. struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
  770. rcu_read_lock();
  771. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  772. if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
  773. continue;
  774. spin_lock(&vb->lock);
  775. if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
  776. vb->free = 0; /* prevent further allocs after releasing lock */
  777. vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
  778. vb->dirty_min = 0;
  779. vb->dirty_max = VMAP_BBMAP_BITS;
  780. spin_lock(&vbq->lock);
  781. list_del_rcu(&vb->free_list);
  782. spin_unlock(&vbq->lock);
  783. spin_unlock(&vb->lock);
  784. list_add_tail(&vb->purge, &purge);
  785. } else
  786. spin_unlock(&vb->lock);
  787. }
  788. rcu_read_unlock();
  789. list_for_each_entry_safe(vb, n_vb, &purge, purge) {
  790. list_del(&vb->purge);
  791. free_vmap_block(vb);
  792. }
  793. }
  794. static void purge_fragmented_blocks_allcpus(void)
  795. {
  796. int cpu;
  797. for_each_possible_cpu(cpu)
  798. purge_fragmented_blocks(cpu);
  799. }
  800. static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
  801. {
  802. struct vmap_block_queue *vbq;
  803. struct vmap_block *vb;
  804. void *vaddr = NULL;
  805. unsigned int order;
  806. BUG_ON(offset_in_page(size));
  807. BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
  808. if (WARN_ON(size == 0)) {
  809. /*
  810. * Allocating 0 bytes isn't what caller wants since
  811. * get_order(0) returns funny result. Just warn and terminate
  812. * early.
  813. */
  814. return NULL;
  815. }
  816. order = get_order(size);
  817. rcu_read_lock();
  818. vbq = &get_cpu_var(vmap_block_queue);
  819. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  820. unsigned long pages_off;
  821. spin_lock(&vb->lock);
  822. if (vb->free < (1UL << order)) {
  823. spin_unlock(&vb->lock);
  824. continue;
  825. }
  826. pages_off = VMAP_BBMAP_BITS - vb->free;
  827. vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
  828. vb->free -= 1UL << order;
  829. if (vb->free == 0) {
  830. spin_lock(&vbq->lock);
  831. list_del_rcu(&vb->free_list);
  832. spin_unlock(&vbq->lock);
  833. }
  834. spin_unlock(&vb->lock);
  835. break;
  836. }
  837. put_cpu_var(vmap_block_queue);
  838. rcu_read_unlock();
  839. /* Allocate new block if nothing was found */
  840. if (!vaddr)
  841. vaddr = new_vmap_block(order, gfp_mask);
  842. return vaddr;
  843. }
  844. static void vb_free(const void *addr, unsigned long size)
  845. {
  846. unsigned long offset;
  847. unsigned long vb_idx;
  848. unsigned int order;
  849. struct vmap_block *vb;
  850. BUG_ON(offset_in_page(size));
  851. BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
  852. flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
  853. order = get_order(size);
  854. offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
  855. offset >>= PAGE_SHIFT;
  856. vb_idx = addr_to_vb_idx((unsigned long)addr);
  857. rcu_read_lock();
  858. vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
  859. rcu_read_unlock();
  860. BUG_ON(!vb);
  861. vunmap_page_range((unsigned long)addr, (unsigned long)addr + size);
  862. spin_lock(&vb->lock);
  863. /* Expand dirty range */
  864. vb->dirty_min = min(vb->dirty_min, offset);
  865. vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
  866. vb->dirty += 1UL << order;
  867. if (vb->dirty == VMAP_BBMAP_BITS) {
  868. BUG_ON(vb->free);
  869. spin_unlock(&vb->lock);
  870. free_vmap_block(vb);
  871. } else
  872. spin_unlock(&vb->lock);
  873. }
  874. /**
  875. * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
  876. *
  877. * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
  878. * to amortize TLB flushing overheads. What this means is that any page you
  879. * have now, may, in a former life, have been mapped into kernel virtual
  880. * address by the vmap layer and so there might be some CPUs with TLB entries
  881. * still referencing that page (additional to the regular 1:1 kernel mapping).
  882. *
  883. * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
  884. * be sure that none of the pages we have control over will have any aliases
  885. * from the vmap layer.
  886. */
  887. void vm_unmap_aliases(void)
  888. {
  889. unsigned long start = ULONG_MAX, end = 0;
  890. int cpu;
  891. int flush = 0;
  892. if (unlikely(!vmap_initialized))
  893. return;
  894. for_each_possible_cpu(cpu) {
  895. struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
  896. struct vmap_block *vb;
  897. rcu_read_lock();
  898. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  899. spin_lock(&vb->lock);
  900. if (vb->dirty) {
  901. unsigned long va_start = vb->va->va_start;
  902. unsigned long s, e;
  903. s = va_start + (vb->dirty_min << PAGE_SHIFT);
  904. e = va_start + (vb->dirty_max << PAGE_SHIFT);
  905. start = min(s, start);
  906. end = max(e, end);
  907. flush = 1;
  908. }
  909. spin_unlock(&vb->lock);
  910. }
  911. rcu_read_unlock();
  912. }
  913. __purge_vmap_area_lazy(&start, &end, 1, flush);
  914. }
  915. EXPORT_SYMBOL_GPL(vm_unmap_aliases);
  916. /**
  917. * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
  918. * @mem: the pointer returned by vm_map_ram
  919. * @count: the count passed to that vm_map_ram call (cannot unmap partial)
  920. */
  921. void vm_unmap_ram(const void *mem, unsigned int count)
  922. {
  923. unsigned long size = count << PAGE_SHIFT;
  924. unsigned long addr = (unsigned long)mem;
  925. BUG_ON(!addr);
  926. BUG_ON(addr < VMALLOC_START);
  927. BUG_ON(addr > VMALLOC_END);
  928. BUG_ON(addr & (PAGE_SIZE-1));
  929. debug_check_no_locks_freed(mem, size);
  930. vmap_debug_free_range(addr, addr+size);
  931. if (likely(count <= VMAP_MAX_ALLOC))
  932. vb_free(mem, size);
  933. else
  934. free_unmap_vmap_area_addr(addr);
  935. }
  936. EXPORT_SYMBOL(vm_unmap_ram);
  937. /**
  938. * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
  939. * @pages: an array of pointers to the pages to be mapped
  940. * @count: number of pages
  941. * @node: prefer to allocate data structures on this node
  942. * @prot: memory protection to use. PAGE_KERNEL for regular RAM
  943. *
  944. * If you use this function for less than VMAP_MAX_ALLOC pages, it could be
  945. * faster than vmap so it's good. But if you mix long-life and short-life
  946. * objects with vm_map_ram(), it could consume lots of address space through
  947. * fragmentation (especially on a 32bit machine). You could see failures in
  948. * the end. Please use this function for short-lived objects.
  949. *
  950. * Returns: a pointer to the address that has been mapped, or %NULL on failure
  951. */
  952. void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
  953. {
  954. unsigned long size = count << PAGE_SHIFT;
  955. unsigned long addr;
  956. void *mem;
  957. if (likely(count <= VMAP_MAX_ALLOC)) {
  958. mem = vb_alloc(size, GFP_KERNEL);
  959. if (IS_ERR(mem))
  960. return NULL;
  961. addr = (unsigned long)mem;
  962. } else {
  963. struct vmap_area *va;
  964. va = alloc_vmap_area(size, PAGE_SIZE,
  965. VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
  966. if (IS_ERR(va))
  967. return NULL;
  968. addr = va->va_start;
  969. mem = (void *)addr;
  970. }
  971. if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
  972. vm_unmap_ram(mem, count);
  973. return NULL;
  974. }
  975. return mem;
  976. }
  977. EXPORT_SYMBOL(vm_map_ram);
  978. static struct vm_struct *vmlist __initdata;
  979. /**
  980. * vm_area_add_early - add vmap area early during boot
  981. * @vm: vm_struct to add
  982. *
  983. * This function is used to add fixed kernel vm area to vmlist before
  984. * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
  985. * should contain proper values and the other fields should be zero.
  986. *
  987. * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
  988. */
  989. void __init vm_area_add_early(struct vm_struct *vm)
  990. {
  991. struct vm_struct *tmp, **p;
  992. BUG_ON(vmap_initialized);
  993. for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
  994. if (tmp->addr >= vm->addr) {
  995. BUG_ON(tmp->addr < vm->addr + vm->size);
  996. break;
  997. } else
  998. BUG_ON(tmp->addr + tmp->size > vm->addr);
  999. }
  1000. vm->next = *p;
  1001. *p = vm;
  1002. }
  1003. /**
  1004. * vm_area_register_early - register vmap area early during boot
  1005. * @vm: vm_struct to register
  1006. * @align: requested alignment
  1007. *
  1008. * This function is used to register kernel vm area before
  1009. * vmalloc_init() is called. @vm->size and @vm->flags should contain
  1010. * proper values on entry and other fields should be zero. On return,
  1011. * vm->addr contains the allocated address.
  1012. *
  1013. * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
  1014. */
  1015. void __init vm_area_register_early(struct vm_struct *vm, size_t align)
  1016. {
  1017. static size_t vm_init_off __initdata;
  1018. unsigned long addr;
  1019. addr = ALIGN(VMALLOC_START + vm_init_off, align);
  1020. vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
  1021. vm->addr = (void *)addr;
  1022. vm_area_add_early(vm);
  1023. }
  1024. void __init vmalloc_init(void)
  1025. {
  1026. struct vmap_area *va;
  1027. struct vm_struct *tmp;
  1028. int i;
  1029. for_each_possible_cpu(i) {
  1030. struct vmap_block_queue *vbq;
  1031. struct vfree_deferred *p;
  1032. vbq = &per_cpu(vmap_block_queue, i);
  1033. spin_lock_init(&vbq->lock);
  1034. INIT_LIST_HEAD(&vbq->free);
  1035. p = &per_cpu(vfree_deferred, i);
  1036. init_llist_head(&p->list);
  1037. INIT_WORK(&p->wq, free_work);
  1038. }
  1039. /* Import existing vmlist entries. */
  1040. for (tmp = vmlist; tmp; tmp = tmp->next) {
  1041. va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
  1042. va->flags = VM_VM_AREA;
  1043. va->va_start = (unsigned long)tmp->addr;
  1044. va->va_end = va->va_start + tmp->size;
  1045. va->vm = tmp;
  1046. __insert_vmap_area(va);
  1047. }
  1048. vmap_area_pcpu_hole = VMALLOC_END;
  1049. vmap_initialized = true;
  1050. }
  1051. /**
  1052. * map_kernel_range_noflush - map kernel VM area with the specified pages
  1053. * @addr: start of the VM area to map
  1054. * @size: size of the VM area to map
  1055. * @prot: page protection flags to use
  1056. * @pages: pages to map
  1057. *
  1058. * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size
  1059. * specify should have been allocated using get_vm_area() and its
  1060. * friends.
  1061. *
  1062. * NOTE:
  1063. * This function does NOT do any cache flushing. The caller is
  1064. * responsible for calling flush_cache_vmap() on to-be-mapped areas
  1065. * before calling this function.
  1066. *
  1067. * RETURNS:
  1068. * The number of pages mapped on success, -errno on failure.
  1069. */
  1070. int map_kernel_range_noflush(unsigned long addr, unsigned long size,
  1071. pgprot_t prot, struct page **pages)
  1072. {
  1073. return vmap_page_range_noflush(addr, addr + size, prot, pages);
  1074. }
  1075. /**
  1076. * unmap_kernel_range_noflush - unmap kernel VM area
  1077. * @addr: start of the VM area to unmap
  1078. * @size: size of the VM area to unmap
  1079. *
  1080. * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size
  1081. * specify should have been allocated using get_vm_area() and its
  1082. * friends.
  1083. *
  1084. * NOTE:
  1085. * This function does NOT do any cache flushing. The caller is
  1086. * responsible for calling flush_cache_vunmap() on to-be-mapped areas
  1087. * before calling this function and flush_tlb_kernel_range() after.
  1088. */
  1089. void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
  1090. {
  1091. vunmap_page_range(addr, addr + size);
  1092. }
  1093. EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush);
  1094. /**
  1095. * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
  1096. * @addr: start of the VM area to unmap
  1097. * @size: size of the VM area to unmap
  1098. *
  1099. * Similar to unmap_kernel_range_noflush() but flushes vcache before
  1100. * the unmapping and tlb after.
  1101. */
  1102. void unmap_kernel_range(unsigned long addr, unsigned long size)
  1103. {
  1104. unsigned long end = addr + size;
  1105. flush_cache_vunmap(addr, end);
  1106. vunmap_page_range(addr, end);
  1107. flush_tlb_kernel_range(addr, end);
  1108. }
  1109. EXPORT_SYMBOL_GPL(unmap_kernel_range);
  1110. int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages)
  1111. {
  1112. unsigned long addr = (unsigned long)area->addr;
  1113. unsigned long end = addr + get_vm_area_size(area);
  1114. int err;
  1115. err = vmap_page_range(addr, end, prot, pages);
  1116. return err > 0 ? 0 : err;
  1117. }
  1118. EXPORT_SYMBOL_GPL(map_vm_area);
  1119. static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
  1120. unsigned long flags, const void *caller)
  1121. {
  1122. spin_lock(&vmap_area_lock);
  1123. vm->flags = flags;
  1124. vm->addr = (void *)va->va_start;
  1125. vm->size = va->va_end - va->va_start;
  1126. vm->caller = caller;
  1127. va->vm = vm;
  1128. va->flags |= VM_VM_AREA;
  1129. spin_unlock(&vmap_area_lock);
  1130. }
  1131. static void clear_vm_uninitialized_flag(struct vm_struct *vm)
  1132. {
  1133. /*
  1134. * Before removing VM_UNINITIALIZED,
  1135. * we should make sure that vm has proper values.
  1136. * Pair with smp_rmb() in show_numa_info().
  1137. */
  1138. smp_wmb();
  1139. vm->flags &= ~VM_UNINITIALIZED;
  1140. }
  1141. static struct vm_struct *__get_vm_area_node(unsigned long size,
  1142. unsigned long align, unsigned long flags, unsigned long start,
  1143. unsigned long end, int node, gfp_t gfp_mask, const void *caller)
  1144. {
  1145. struct vmap_area *va;
  1146. struct vm_struct *area;
  1147. BUG_ON(in_interrupt());
  1148. if (flags & VM_IOREMAP)
  1149. align = 1ul << clamp_t(int, fls_long(size),
  1150. PAGE_SHIFT, IOREMAP_MAX_ORDER);
  1151. size = PAGE_ALIGN(size);
  1152. if (unlikely(!size))
  1153. return NULL;
  1154. area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
  1155. if (unlikely(!area))
  1156. return NULL;
  1157. if (!(flags & VM_NO_GUARD))
  1158. size += PAGE_SIZE;
  1159. va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
  1160. if (IS_ERR(va)) {
  1161. kfree(area);
  1162. return NULL;
  1163. }
  1164. setup_vmalloc_vm(area, va, flags, caller);
  1165. return area;
  1166. }
  1167. struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
  1168. unsigned long start, unsigned long end)
  1169. {
  1170. return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
  1171. GFP_KERNEL, __builtin_return_address(0));
  1172. }
  1173. EXPORT_SYMBOL_GPL(__get_vm_area);
  1174. struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
  1175. unsigned long start, unsigned long end,
  1176. const void *caller)
  1177. {
  1178. return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
  1179. GFP_KERNEL, caller);
  1180. }
  1181. /**
  1182. * get_vm_area - reserve a contiguous kernel virtual area
  1183. * @size: size of the area
  1184. * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
  1185. *
  1186. * Search an area of @size in the kernel virtual mapping area,
  1187. * and reserved it for out purposes. Returns the area descriptor
  1188. * on success or %NULL on failure.
  1189. */
  1190. struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
  1191. {
  1192. return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
  1193. NUMA_NO_NODE, GFP_KERNEL,
  1194. __builtin_return_address(0));
  1195. }
  1196. struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
  1197. const void *caller)
  1198. {
  1199. return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
  1200. NUMA_NO_NODE, GFP_KERNEL, caller);
  1201. }
  1202. /**
  1203. * find_vm_area - find a continuous kernel virtual area
  1204. * @addr: base address
  1205. *
  1206. * Search for the kernel VM area starting at @addr, and return it.
  1207. * It is up to the caller to do all required locking to keep the returned
  1208. * pointer valid.
  1209. */
  1210. struct vm_struct *find_vm_area(const void *addr)
  1211. {
  1212. struct vmap_area *va;
  1213. va = find_vmap_area((unsigned long)addr);
  1214. if (va && va->flags & VM_VM_AREA)
  1215. return va->vm;
  1216. return NULL;
  1217. }
  1218. /**
  1219. * remove_vm_area - find and remove a continuous kernel virtual area
  1220. * @addr: base address
  1221. *
  1222. * Search for the kernel VM area starting at @addr, and remove it.
  1223. * This function returns the found VM area, but using it is NOT safe
  1224. * on SMP machines, except for its size or flags.
  1225. */
  1226. struct vm_struct *remove_vm_area(const void *addr)
  1227. {
  1228. struct vmap_area *va;
  1229. va = find_vmap_area((unsigned long)addr);
  1230. if (va && va->flags & VM_VM_AREA) {
  1231. struct vm_struct *vm = va->vm;
  1232. spin_lock(&vmap_area_lock);
  1233. va->vm = NULL;
  1234. va->flags &= ~VM_VM_AREA;
  1235. spin_unlock(&vmap_area_lock);
  1236. vmap_debug_free_range(va->va_start, va->va_end);
  1237. kasan_free_shadow(vm);
  1238. free_unmap_vmap_area(va);
  1239. return vm;
  1240. }
  1241. return NULL;
  1242. }
  1243. static void __vunmap(const void *addr, int deallocate_pages)
  1244. {
  1245. struct vm_struct *area;
  1246. if (!addr)
  1247. return;
  1248. if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
  1249. addr))
  1250. return;
  1251. area = remove_vm_area(addr);
  1252. if (unlikely(!area)) {
  1253. WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
  1254. addr);
  1255. return;
  1256. }
  1257. debug_check_no_locks_freed(addr, get_vm_area_size(area));
  1258. debug_check_no_obj_freed(addr, get_vm_area_size(area));
  1259. if (deallocate_pages) {
  1260. int i;
  1261. for (i = 0; i < area->nr_pages; i++) {
  1262. struct page *page = area->pages[i];
  1263. BUG_ON(!page);
  1264. __free_page(page);
  1265. }
  1266. if (area->flags & VM_VPAGES)
  1267. vfree(area->pages);
  1268. else
  1269. kfree(area->pages);
  1270. }
  1271. kfree(area);
  1272. return;
  1273. }
  1274. /**
  1275. * vfree - release memory allocated by vmalloc()
  1276. * @addr: memory base address
  1277. *
  1278. * Free the virtually continuous memory area starting at @addr, as
  1279. * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
  1280. * NULL, no operation is performed.
  1281. *
  1282. * Must not be called in NMI context (strictly speaking, only if we don't
  1283. * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
  1284. * conventions for vfree() arch-depenedent would be a really bad idea)
  1285. *
  1286. * NOTE: assumes that the object at *addr has a size >= sizeof(llist_node)
  1287. */
  1288. void vfree(const void *addr)
  1289. {
  1290. BUG_ON(in_nmi());
  1291. kmemleak_free(addr);
  1292. if (!addr)
  1293. return;
  1294. if (unlikely(in_interrupt())) {
  1295. struct vfree_deferred *p = this_cpu_ptr(&vfree_deferred);
  1296. if (llist_add((struct llist_node *)addr, &p->list))
  1297. schedule_work(&p->wq);
  1298. } else
  1299. __vunmap(addr, 1);
  1300. }
  1301. EXPORT_SYMBOL(vfree);
  1302. /**
  1303. * vunmap - release virtual mapping obtained by vmap()
  1304. * @addr: memory base address
  1305. *
  1306. * Free the virtually contiguous memory area starting at @addr,
  1307. * which was created from the page array passed to vmap().
  1308. *
  1309. * Must not be called in interrupt context.
  1310. */
  1311. void vunmap(const void *addr)
  1312. {
  1313. BUG_ON(in_interrupt());
  1314. might_sleep();
  1315. if (addr)
  1316. __vunmap(addr, 0);
  1317. }
  1318. EXPORT_SYMBOL(vunmap);
  1319. /**
  1320. * vmap - map an array of pages into virtually contiguous space
  1321. * @pages: array of page pointers
  1322. * @count: number of pages to map
  1323. * @flags: vm_area->flags
  1324. * @prot: page protection for the mapping
  1325. *
  1326. * Maps @count pages from @pages into contiguous kernel virtual
  1327. * space.
  1328. */
  1329. void *vmap(struct page **pages, unsigned int count,
  1330. unsigned long flags, pgprot_t prot)
  1331. {
  1332. struct vm_struct *area;
  1333. might_sleep();
  1334. if (count > totalram_pages)
  1335. return NULL;
  1336. area = get_vm_area_caller((count << PAGE_SHIFT), flags,
  1337. __builtin_return_address(0));
  1338. if (!area)
  1339. return NULL;
  1340. if (map_vm_area(area, prot, pages)) {
  1341. vunmap(area->addr);
  1342. return NULL;
  1343. }
  1344. return area->addr;
  1345. }
  1346. EXPORT_SYMBOL(vmap);
  1347. static void *__vmalloc_node(unsigned long size, unsigned long align,
  1348. gfp_t gfp_mask, pgprot_t prot,
  1349. int node, const void *caller);
  1350. static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
  1351. pgprot_t prot, int node)
  1352. {
  1353. const int order = 0;
  1354. struct page **pages;
  1355. unsigned int nr_pages, array_size, i;
  1356. const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
  1357. const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN;
  1358. nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
  1359. array_size = (nr_pages * sizeof(struct page *));
  1360. area->nr_pages = nr_pages;
  1361. /* Please note that the recursion is strictly bounded. */
  1362. if (array_size > PAGE_SIZE) {
  1363. pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM,
  1364. PAGE_KERNEL, node, area->caller);
  1365. area->flags |= VM_VPAGES;
  1366. } else {
  1367. pages = kmalloc_node(array_size, nested_gfp, node);
  1368. }
  1369. area->pages = pages;
  1370. if (!area->pages) {
  1371. remove_vm_area(area->addr);
  1372. kfree(area);
  1373. return NULL;
  1374. }
  1375. for (i = 0; i < area->nr_pages; i++) {
  1376. struct page *page;
  1377. if (node == NUMA_NO_NODE)
  1378. page = alloc_page(alloc_mask);
  1379. else
  1380. page = alloc_pages_node(node, alloc_mask, order);
  1381. if (unlikely(!page)) {
  1382. /* Successfully allocated i pages, free them in __vunmap() */
  1383. area->nr_pages = i;
  1384. goto fail;
  1385. }
  1386. area->pages[i] = page;
  1387. if (gfpflags_allow_blocking(gfp_mask))
  1388. cond_resched();
  1389. }
  1390. if (map_vm_area(area, prot, pages))
  1391. goto fail;
  1392. return area->addr;
  1393. fail:
  1394. warn_alloc_failed(gfp_mask, order,
  1395. "vmalloc: allocation failure, allocated %ld of %ld bytes\n",
  1396. (area->nr_pages*PAGE_SIZE), area->size);
  1397. vfree(area->addr);
  1398. return NULL;
  1399. }
  1400. /**
  1401. * __vmalloc_node_range - allocate virtually contiguous memory
  1402. * @size: allocation size
  1403. * @align: desired alignment
  1404. * @start: vm area range start
  1405. * @end: vm area range end
  1406. * @gfp_mask: flags for the page level allocator
  1407. * @prot: protection mask for the allocated pages
  1408. * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD)
  1409. * @node: node to use for allocation or NUMA_NO_NODE
  1410. * @caller: caller's return address
  1411. *
  1412. * Allocate enough pages to cover @size from the page level
  1413. * allocator with @gfp_mask flags. Map them into contiguous
  1414. * kernel virtual space, using a pagetable protection of @prot.
  1415. */
  1416. void *__vmalloc_node_range(unsigned long size, unsigned long align,
  1417. unsigned long start, unsigned long end, gfp_t gfp_mask,
  1418. pgprot_t prot, unsigned long vm_flags, int node,
  1419. const void *caller)
  1420. {
  1421. struct vm_struct *area;
  1422. void *addr;
  1423. unsigned long real_size = size;
  1424. size = PAGE_ALIGN(size);
  1425. if (!size || (size >> PAGE_SHIFT) > totalram_pages)
  1426. goto fail;
  1427. area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED |
  1428. vm_flags, start, end, node, gfp_mask, caller);
  1429. if (!area)
  1430. goto fail;
  1431. addr = __vmalloc_area_node(area, gfp_mask, prot, node);
  1432. if (!addr)
  1433. return NULL;
  1434. /*
  1435. * In this function, newly allocated vm_struct has VM_UNINITIALIZED
  1436. * flag. It means that vm_struct is not fully initialized.
  1437. * Now, it is fully initialized, so remove this flag here.
  1438. */
  1439. clear_vm_uninitialized_flag(area);
  1440. /*
  1441. * A ref_count = 2 is needed because vm_struct allocated in
  1442. * __get_vm_area_node() contains a reference to the virtual address of
  1443. * the vmalloc'ed block.
  1444. */
  1445. kmemleak_alloc(addr, real_size, 2, gfp_mask);
  1446. return addr;
  1447. fail:
  1448. warn_alloc_failed(gfp_mask, 0,
  1449. "vmalloc: allocation failure: %lu bytes\n",
  1450. real_size);
  1451. return NULL;
  1452. }
  1453. /**
  1454. * __vmalloc_node - allocate virtually contiguous memory
  1455. * @size: allocation size
  1456. * @align: desired alignment
  1457. * @gfp_mask: flags for the page level allocator
  1458. * @prot: protection mask for the allocated pages
  1459. * @node: node to use for allocation or NUMA_NO_NODE
  1460. * @caller: caller's return address
  1461. *
  1462. * Allocate enough pages to cover @size from the page level
  1463. * allocator with @gfp_mask flags. Map them into contiguous
  1464. * kernel virtual space, using a pagetable protection of @prot.
  1465. */
  1466. static void *__vmalloc_node(unsigned long size, unsigned long align,
  1467. gfp_t gfp_mask, pgprot_t prot,
  1468. int node, const void *caller)
  1469. {
  1470. return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
  1471. gfp_mask, prot, 0, node, caller);
  1472. }
  1473. void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
  1474. {
  1475. return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE,
  1476. __builtin_return_address(0));
  1477. }
  1478. EXPORT_SYMBOL(__vmalloc);
  1479. static inline void *__vmalloc_node_flags(unsigned long size,
  1480. int node, gfp_t flags)
  1481. {
  1482. return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
  1483. node, __builtin_return_address(0));
  1484. }
  1485. /**
  1486. * vmalloc - allocate virtually contiguous memory
  1487. * @size: allocation size
  1488. * Allocate enough pages to cover @size from the page level
  1489. * allocator and map them into contiguous kernel virtual space.
  1490. *
  1491. * For tight control over page level allocator and protection flags
  1492. * use __vmalloc() instead.
  1493. */
  1494. void *vmalloc(unsigned long size)
  1495. {
  1496. return __vmalloc_node_flags(size, NUMA_NO_NODE,
  1497. GFP_KERNEL | __GFP_HIGHMEM);
  1498. }
  1499. EXPORT_SYMBOL(vmalloc);
  1500. /**
  1501. * vzalloc - allocate virtually contiguous memory with zero fill
  1502. * @size: allocation size
  1503. * Allocate enough pages to cover @size from the page level
  1504. * allocator and map them into contiguous kernel virtual space.
  1505. * The memory allocated is set to zero.
  1506. *
  1507. * For tight control over page level allocator and protection flags
  1508. * use __vmalloc() instead.
  1509. */
  1510. void *vzalloc(unsigned long size)
  1511. {
  1512. return __vmalloc_node_flags(size, NUMA_NO_NODE,
  1513. GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
  1514. }
  1515. EXPORT_SYMBOL(vzalloc);
  1516. /**
  1517. * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
  1518. * @size: allocation size
  1519. *
  1520. * The resulting memory area is zeroed so it can be mapped to userspace
  1521. * without leaking data.
  1522. */
  1523. void *vmalloc_user(unsigned long size)
  1524. {
  1525. struct vm_struct *area;
  1526. void *ret;
  1527. ret = __vmalloc_node(size, SHMLBA,
  1528. GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
  1529. PAGE_KERNEL, NUMA_NO_NODE,
  1530. __builtin_return_address(0));
  1531. if (ret) {
  1532. area = find_vm_area(ret);
  1533. area->flags |= VM_USERMAP;
  1534. }
  1535. return ret;
  1536. }
  1537. EXPORT_SYMBOL(vmalloc_user);
  1538. /**
  1539. * vmalloc_node - allocate memory on a specific node
  1540. * @size: allocation size
  1541. * @node: numa node
  1542. *
  1543. * Allocate enough pages to cover @size from the page level
  1544. * allocator and map them into contiguous kernel virtual space.
  1545. *
  1546. * For tight control over page level allocator and protection flags
  1547. * use __vmalloc() instead.
  1548. */
  1549. void *vmalloc_node(unsigned long size, int node)
  1550. {
  1551. return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
  1552. node, __builtin_return_address(0));
  1553. }
  1554. EXPORT_SYMBOL(vmalloc_node);
  1555. /**
  1556. * vzalloc_node - allocate memory on a specific node with zero fill
  1557. * @size: allocation size
  1558. * @node: numa node
  1559. *
  1560. * Allocate enough pages to cover @size from the page level
  1561. * allocator and map them into contiguous kernel virtual space.
  1562. * The memory allocated is set to zero.
  1563. *
  1564. * For tight control over page level allocator and protection flags
  1565. * use __vmalloc_node() instead.
  1566. */
  1567. void *vzalloc_node(unsigned long size, int node)
  1568. {
  1569. return __vmalloc_node_flags(size, node,
  1570. GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
  1571. }
  1572. EXPORT_SYMBOL(vzalloc_node);
  1573. #ifndef PAGE_KERNEL_EXEC
  1574. # define PAGE_KERNEL_EXEC PAGE_KERNEL
  1575. #endif
  1576. /**
  1577. * vmalloc_exec - allocate virtually contiguous, executable memory
  1578. * @size: allocation size
  1579. *
  1580. * Kernel-internal function to allocate enough pages to cover @size
  1581. * the page level allocator and map them into contiguous and
  1582. * executable kernel virtual space.
  1583. *
  1584. * For tight control over page level allocator and protection flags
  1585. * use __vmalloc() instead.
  1586. */
  1587. void *vmalloc_exec(unsigned long size)
  1588. {
  1589. return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
  1590. NUMA_NO_NODE, __builtin_return_address(0));
  1591. }
  1592. #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
  1593. #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
  1594. #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
  1595. #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
  1596. #else
  1597. #define GFP_VMALLOC32 GFP_KERNEL
  1598. #endif
  1599. /**
  1600. * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
  1601. * @size: allocation size
  1602. *
  1603. * Allocate enough 32bit PA addressable pages to cover @size from the
  1604. * page level allocator and map them into contiguous kernel virtual space.
  1605. */
  1606. void *vmalloc_32(unsigned long size)
  1607. {
  1608. return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
  1609. NUMA_NO_NODE, __builtin_return_address(0));
  1610. }
  1611. EXPORT_SYMBOL(vmalloc_32);
  1612. /**
  1613. * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
  1614. * @size: allocation size
  1615. *
  1616. * The resulting memory area is 32bit addressable and zeroed so it can be
  1617. * mapped to userspace without leaking data.
  1618. */
  1619. void *vmalloc_32_user(unsigned long size)
  1620. {
  1621. struct vm_struct *area;
  1622. void *ret;
  1623. ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
  1624. NUMA_NO_NODE, __builtin_return_address(0));
  1625. if (ret) {
  1626. area = find_vm_area(ret);
  1627. area->flags |= VM_USERMAP;
  1628. }
  1629. return ret;
  1630. }
  1631. EXPORT_SYMBOL(vmalloc_32_user);
  1632. /*
  1633. * small helper routine , copy contents to buf from addr.
  1634. * If the page is not present, fill zero.
  1635. */
  1636. static int aligned_vread(char *buf, char *addr, unsigned long count)
  1637. {
  1638. struct page *p;
  1639. int copied = 0;
  1640. while (count) {
  1641. unsigned long offset, length;
  1642. offset = offset_in_page(addr);
  1643. length = PAGE_SIZE - offset;
  1644. if (length > count)
  1645. length = count;
  1646. p = vmalloc_to_page(addr);
  1647. /*
  1648. * To do safe access to this _mapped_ area, we need
  1649. * lock. But adding lock here means that we need to add
  1650. * overhead of vmalloc()/vfree() calles for this _debug_
  1651. * interface, rarely used. Instead of that, we'll use
  1652. * kmap() and get small overhead in this access function.
  1653. */
  1654. if (p) {
  1655. /*
  1656. * we can expect USER0 is not used (see vread/vwrite's
  1657. * function description)
  1658. */
  1659. void *map = kmap_atomic(p);
  1660. memcpy(buf, map + offset, length);
  1661. kunmap_atomic(map);
  1662. } else
  1663. memset(buf, 0, length);
  1664. addr += length;
  1665. buf += length;
  1666. copied += length;
  1667. count -= length;
  1668. }
  1669. return copied;
  1670. }
  1671. static int aligned_vwrite(char *buf, char *addr, unsigned long count)
  1672. {
  1673. struct page *p;
  1674. int copied = 0;
  1675. while (count) {
  1676. unsigned long offset, length;
  1677. offset = offset_in_page(addr);
  1678. length = PAGE_SIZE - offset;
  1679. if (length > count)
  1680. length = count;
  1681. p = vmalloc_to_page(addr);
  1682. /*
  1683. * To do safe access to this _mapped_ area, we need
  1684. * lock. But adding lock here means that we need to add
  1685. * overhead of vmalloc()/vfree() calles for this _debug_
  1686. * interface, rarely used. Instead of that, we'll use
  1687. * kmap() and get small overhead in this access function.
  1688. */
  1689. if (p) {
  1690. /*
  1691. * we can expect USER0 is not used (see vread/vwrite's
  1692. * function description)
  1693. */
  1694. void *map = kmap_atomic(p);
  1695. memcpy(map + offset, buf, length);
  1696. kunmap_atomic(map);
  1697. }
  1698. addr += length;
  1699. buf += length;
  1700. copied += length;
  1701. count -= length;
  1702. }
  1703. return copied;
  1704. }
  1705. /**
  1706. * vread() - read vmalloc area in a safe way.
  1707. * @buf: buffer for reading data
  1708. * @addr: vm address.
  1709. * @count: number of bytes to be read.
  1710. *
  1711. * Returns # of bytes which addr and buf should be increased.
  1712. * (same number to @count). Returns 0 if [addr...addr+count) doesn't
  1713. * includes any intersect with alive vmalloc area.
  1714. *
  1715. * This function checks that addr is a valid vmalloc'ed area, and
  1716. * copy data from that area to a given buffer. If the given memory range
  1717. * of [addr...addr+count) includes some valid address, data is copied to
  1718. * proper area of @buf. If there are memory holes, they'll be zero-filled.
  1719. * IOREMAP area is treated as memory hole and no copy is done.
  1720. *
  1721. * If [addr...addr+count) doesn't includes any intersects with alive
  1722. * vm_struct area, returns 0. @buf should be kernel's buffer.
  1723. *
  1724. * Note: In usual ops, vread() is never necessary because the caller
  1725. * should know vmalloc() area is valid and can use memcpy().
  1726. * This is for routines which have to access vmalloc area without
  1727. * any informaion, as /dev/kmem.
  1728. *
  1729. */
  1730. long vread(char *buf, char *addr, unsigned long count)
  1731. {
  1732. struct vmap_area *va;
  1733. struct vm_struct *vm;
  1734. char *vaddr, *buf_start = buf;
  1735. unsigned long buflen = count;
  1736. unsigned long n;
  1737. /* Don't allow overflow */
  1738. if ((unsigned long) addr + count < count)
  1739. count = -(unsigned long) addr;
  1740. spin_lock(&vmap_area_lock);
  1741. list_for_each_entry(va, &vmap_area_list, list) {
  1742. if (!count)
  1743. break;
  1744. if (!(va->flags & VM_VM_AREA))
  1745. continue;
  1746. vm = va->vm;
  1747. vaddr = (char *) vm->addr;
  1748. if (addr >= vaddr + get_vm_area_size(vm))
  1749. continue;
  1750. while (addr < vaddr) {
  1751. if (count == 0)
  1752. goto finished;
  1753. *buf = '\0';
  1754. buf++;
  1755. addr++;
  1756. count--;
  1757. }
  1758. n = vaddr + get_vm_area_size(vm) - addr;
  1759. if (n > count)
  1760. n = count;
  1761. if (!(vm->flags & VM_IOREMAP))
  1762. aligned_vread(buf, addr, n);
  1763. else /* IOREMAP area is treated as memory hole */
  1764. memset(buf, 0, n);
  1765. buf += n;
  1766. addr += n;
  1767. count -= n;
  1768. }
  1769. finished:
  1770. spin_unlock(&vmap_area_lock);
  1771. if (buf == buf_start)
  1772. return 0;
  1773. /* zero-fill memory holes */
  1774. if (buf != buf_start + buflen)
  1775. memset(buf, 0, buflen - (buf - buf_start));
  1776. return buflen;
  1777. }
  1778. /**
  1779. * vwrite() - write vmalloc area in a safe way.
  1780. * @buf: buffer for source data
  1781. * @addr: vm address.
  1782. * @count: number of bytes to be read.
  1783. *
  1784. * Returns # of bytes which addr and buf should be incresed.
  1785. * (same number to @count).
  1786. * If [addr...addr+count) doesn't includes any intersect with valid
  1787. * vmalloc area, returns 0.
  1788. *
  1789. * This function checks that addr is a valid vmalloc'ed area, and
  1790. * copy data from a buffer to the given addr. If specified range of
  1791. * [addr...addr+count) includes some valid address, data is copied from
  1792. * proper area of @buf. If there are memory holes, no copy to hole.
  1793. * IOREMAP area is treated as memory hole and no copy is done.
  1794. *
  1795. * If [addr...addr+count) doesn't includes any intersects with alive
  1796. * vm_struct area, returns 0. @buf should be kernel's buffer.
  1797. *
  1798. * Note: In usual ops, vwrite() is never necessary because the caller
  1799. * should know vmalloc() area is valid and can use memcpy().
  1800. * This is for routines which have to access vmalloc area without
  1801. * any informaion, as /dev/kmem.
  1802. */
  1803. long vwrite(char *buf, char *addr, unsigned long count)
  1804. {
  1805. struct vmap_area *va;
  1806. struct vm_struct *vm;
  1807. char *vaddr;
  1808. unsigned long n, buflen;
  1809. int copied = 0;
  1810. /* Don't allow overflow */
  1811. if ((unsigned long) addr + count < count)
  1812. count = -(unsigned long) addr;
  1813. buflen = count;
  1814. spin_lock(&vmap_area_lock);
  1815. list_for_each_entry(va, &vmap_area_list, list) {
  1816. if (!count)
  1817. break;
  1818. if (!(va->flags & VM_VM_AREA))
  1819. continue;
  1820. vm = va->vm;
  1821. vaddr = (char *) vm->addr;
  1822. if (addr >= vaddr + get_vm_area_size(vm))
  1823. continue;
  1824. while (addr < vaddr) {
  1825. if (count == 0)
  1826. goto finished;
  1827. buf++;
  1828. addr++;
  1829. count--;
  1830. }
  1831. n = vaddr + get_vm_area_size(vm) - addr;
  1832. if (n > count)
  1833. n = count;
  1834. if (!(vm->flags & VM_IOREMAP)) {
  1835. aligned_vwrite(buf, addr, n);
  1836. copied++;
  1837. }
  1838. buf += n;
  1839. addr += n;
  1840. count -= n;
  1841. }
  1842. finished:
  1843. spin_unlock(&vmap_area_lock);
  1844. if (!copied)
  1845. return 0;
  1846. return buflen;
  1847. }
  1848. /**
  1849. * remap_vmalloc_range_partial - map vmalloc pages to userspace
  1850. * @vma: vma to cover
  1851. * @uaddr: target user address to start at
  1852. * @kaddr: virtual address of vmalloc kernel memory
  1853. * @size: size of map area
  1854. *
  1855. * Returns: 0 for success, -Exxx on failure
  1856. *
  1857. * This function checks that @kaddr is a valid vmalloc'ed area,
  1858. * and that it is big enough to cover the range starting at
  1859. * @uaddr in @vma. Will return failure if that criteria isn't
  1860. * met.
  1861. *
  1862. * Similar to remap_pfn_range() (see mm/memory.c)
  1863. */
  1864. int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
  1865. void *kaddr, unsigned long size)
  1866. {
  1867. struct vm_struct *area;
  1868. size = PAGE_ALIGN(size);
  1869. if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
  1870. return -EINVAL;
  1871. area = find_vm_area(kaddr);
  1872. if (!area)
  1873. return -EINVAL;
  1874. if (!(area->flags & VM_USERMAP))
  1875. return -EINVAL;
  1876. if (kaddr + size > area->addr + area->size)
  1877. return -EINVAL;
  1878. do {
  1879. struct page *page = vmalloc_to_page(kaddr);
  1880. int ret;
  1881. ret = vm_insert_page(vma, uaddr, page);
  1882. if (ret)
  1883. return ret;
  1884. uaddr += PAGE_SIZE;
  1885. kaddr += PAGE_SIZE;
  1886. size -= PAGE_SIZE;
  1887. } while (size > 0);
  1888. vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
  1889. return 0;
  1890. }
  1891. EXPORT_SYMBOL(remap_vmalloc_range_partial);
  1892. /**
  1893. * remap_vmalloc_range - map vmalloc pages to userspace
  1894. * @vma: vma to cover (map full range of vma)
  1895. * @addr: vmalloc memory
  1896. * @pgoff: number of pages into addr before first page to map
  1897. *
  1898. * Returns: 0 for success, -Exxx on failure
  1899. *
  1900. * This function checks that addr is a valid vmalloc'ed area, and
  1901. * that it is big enough to cover the vma. Will return failure if
  1902. * that criteria isn't met.
  1903. *
  1904. * Similar to remap_pfn_range() (see mm/memory.c)
  1905. */
  1906. int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
  1907. unsigned long pgoff)
  1908. {
  1909. return remap_vmalloc_range_partial(vma, vma->vm_start,
  1910. addr + (pgoff << PAGE_SHIFT),
  1911. vma->vm_end - vma->vm_start);
  1912. }
  1913. EXPORT_SYMBOL(remap_vmalloc_range);
  1914. /*
  1915. * Implement a stub for vmalloc_sync_all() if the architecture chose not to
  1916. * have one.
  1917. */
  1918. void __weak vmalloc_sync_all(void)
  1919. {
  1920. }
  1921. static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
  1922. {
  1923. pte_t ***p = data;
  1924. if (p) {
  1925. *(*p) = pte;
  1926. (*p)++;
  1927. }
  1928. return 0;
  1929. }
  1930. /**
  1931. * alloc_vm_area - allocate a range of kernel address space
  1932. * @size: size of the area
  1933. * @ptes: returns the PTEs for the address space
  1934. *
  1935. * Returns: NULL on failure, vm_struct on success
  1936. *
  1937. * This function reserves a range of kernel address space, and
  1938. * allocates pagetables to map that range. No actual mappings
  1939. * are created.
  1940. *
  1941. * If @ptes is non-NULL, pointers to the PTEs (in init_mm)
  1942. * allocated for the VM area are returned.
  1943. */
  1944. struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
  1945. {
  1946. struct vm_struct *area;
  1947. area = get_vm_area_caller(size, VM_IOREMAP,
  1948. __builtin_return_address(0));
  1949. if (area == NULL)
  1950. return NULL;
  1951. /*
  1952. * This ensures that page tables are constructed for this region
  1953. * of kernel virtual address space and mapped into init_mm.
  1954. */
  1955. if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
  1956. size, f, ptes ? &ptes : NULL)) {
  1957. free_vm_area(area);
  1958. return NULL;
  1959. }
  1960. return area;
  1961. }
  1962. EXPORT_SYMBOL_GPL(alloc_vm_area);
  1963. void free_vm_area(struct vm_struct *area)
  1964. {
  1965. struct vm_struct *ret;
  1966. ret = remove_vm_area(area->addr);
  1967. BUG_ON(ret != area);
  1968. kfree(area);
  1969. }
  1970. EXPORT_SYMBOL_GPL(free_vm_area);
  1971. #ifdef CONFIG_SMP
  1972. static struct vmap_area *node_to_va(struct rb_node *n)
  1973. {
  1974. return n ? rb_entry(n, struct vmap_area, rb_node) : NULL;
  1975. }
  1976. /**
  1977. * pvm_find_next_prev - find the next and prev vmap_area surrounding @end
  1978. * @end: target address
  1979. * @pnext: out arg for the next vmap_area
  1980. * @pprev: out arg for the previous vmap_area
  1981. *
  1982. * Returns: %true if either or both of next and prev are found,
  1983. * %false if no vmap_area exists
  1984. *
  1985. * Find vmap_areas end addresses of which enclose @end. ie. if not
  1986. * NULL, *pnext->va_end > @end and *pprev->va_end <= @end.
  1987. */
  1988. static bool pvm_find_next_prev(unsigned long end,
  1989. struct vmap_area **pnext,
  1990. struct vmap_area **pprev)
  1991. {
  1992. struct rb_node *n = vmap_area_root.rb_node;
  1993. struct vmap_area *va = NULL;
  1994. while (n) {
  1995. va = rb_entry(n, struct vmap_area, rb_node);
  1996. if (end < va->va_end)
  1997. n = n->rb_left;
  1998. else if (end > va->va_end)
  1999. n = n->rb_right;
  2000. else
  2001. break;
  2002. }
  2003. if (!va)
  2004. return false;
  2005. if (va->va_end > end) {
  2006. *pnext = va;
  2007. *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
  2008. } else {
  2009. *pprev = va;
  2010. *pnext = node_to_va(rb_next(&(*pprev)->rb_node));
  2011. }
  2012. return true;
  2013. }
  2014. /**
  2015. * pvm_determine_end - find the highest aligned address between two vmap_areas
  2016. * @pnext: in/out arg for the next vmap_area
  2017. * @pprev: in/out arg for the previous vmap_area
  2018. * @align: alignment
  2019. *
  2020. * Returns: determined end address
  2021. *
  2022. * Find the highest aligned address between *@pnext and *@pprev below
  2023. * VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned
  2024. * down address is between the end addresses of the two vmap_areas.
  2025. *
  2026. * Please note that the address returned by this function may fall
  2027. * inside *@pnext vmap_area. The caller is responsible for checking
  2028. * that.
  2029. */
  2030. static unsigned long pvm_determine_end(struct vmap_area **pnext,
  2031. struct vmap_area **pprev,
  2032. unsigned long align)
  2033. {
  2034. const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
  2035. unsigned long addr;
  2036. if (*pnext)
  2037. addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
  2038. else
  2039. addr = vmalloc_end;
  2040. while (*pprev && (*pprev)->va_end > addr) {
  2041. *pnext = *pprev;
  2042. *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
  2043. }
  2044. return addr;
  2045. }
  2046. /**
  2047. * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
  2048. * @offsets: array containing offset of each area
  2049. * @sizes: array containing size of each area
  2050. * @nr_vms: the number of areas to allocate
  2051. * @align: alignment, all entries in @offsets and @sizes must be aligned to this
  2052. *
  2053. * Returns: kmalloc'd vm_struct pointer array pointing to allocated
  2054. * vm_structs on success, %NULL on failure
  2055. *
  2056. * Percpu allocator wants to use congruent vm areas so that it can
  2057. * maintain the offsets among percpu areas. This function allocates
  2058. * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
  2059. * be scattered pretty far, distance between two areas easily going up
  2060. * to gigabytes. To avoid interacting with regular vmallocs, these
  2061. * areas are allocated from top.
  2062. *
  2063. * Despite its complicated look, this allocator is rather simple. It
  2064. * does everything top-down and scans areas from the end looking for
  2065. * matching slot. While scanning, if any of the areas overlaps with
  2066. * existing vmap_area, the base address is pulled down to fit the
  2067. * area. Scanning is repeated till all the areas fit and then all
  2068. * necessary data structres are inserted and the result is returned.
  2069. */
  2070. struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
  2071. const size_t *sizes, int nr_vms,
  2072. size_t align)
  2073. {
  2074. const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
  2075. const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
  2076. struct vmap_area **vas, *prev, *next;
  2077. struct vm_struct **vms;
  2078. int area, area2, last_area, term_area;
  2079. unsigned long base, start, end, last_end;
  2080. bool purged = false;
  2081. /* verify parameters and allocate data structures */
  2082. BUG_ON(offset_in_page(align) || !is_power_of_2(align));
  2083. for (last_area = 0, area = 0; area < nr_vms; area++) {
  2084. start = offsets[area];
  2085. end = start + sizes[area];
  2086. /* is everything aligned properly? */
  2087. BUG_ON(!IS_ALIGNED(offsets[area], align));
  2088. BUG_ON(!IS_ALIGNED(sizes[area], align));
  2089. /* detect the area with the highest address */
  2090. if (start > offsets[last_area])
  2091. last_area = area;
  2092. for (area2 = 0; area2 < nr_vms; area2++) {
  2093. unsigned long start2 = offsets[area2];
  2094. unsigned long end2 = start2 + sizes[area2];
  2095. if (area2 == area)
  2096. continue;
  2097. BUG_ON(start2 >= start && start2 < end);
  2098. BUG_ON(end2 <= end && end2 > start);
  2099. }
  2100. }
  2101. last_end = offsets[last_area] + sizes[last_area];
  2102. if (vmalloc_end - vmalloc_start < last_end) {
  2103. WARN_ON(true);
  2104. return NULL;
  2105. }
  2106. vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
  2107. vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
  2108. if (!vas || !vms)
  2109. goto err_free2;
  2110. for (area = 0; area < nr_vms; area++) {
  2111. vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL);
  2112. vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
  2113. if (!vas[area] || !vms[area])
  2114. goto err_free;
  2115. }
  2116. retry:
  2117. spin_lock(&vmap_area_lock);
  2118. /* start scanning - we scan from the top, begin with the last area */
  2119. area = term_area = last_area;
  2120. start = offsets[area];
  2121. end = start + sizes[area];
  2122. if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
  2123. base = vmalloc_end - last_end;
  2124. goto found;
  2125. }
  2126. base = pvm_determine_end(&next, &prev, align) - end;
  2127. while (true) {
  2128. BUG_ON(next && next->va_end <= base + end);
  2129. BUG_ON(prev && prev->va_end > base + end);
  2130. /*
  2131. * base might have underflowed, add last_end before
  2132. * comparing.
  2133. */
  2134. if (base + last_end < vmalloc_start + last_end) {
  2135. spin_unlock(&vmap_area_lock);
  2136. if (!purged) {
  2137. purge_vmap_area_lazy();
  2138. purged = true;
  2139. goto retry;
  2140. }
  2141. goto err_free;
  2142. }
  2143. /*
  2144. * If next overlaps, move base downwards so that it's
  2145. * right below next and then recheck.
  2146. */
  2147. if (next && next->va_start < base + end) {
  2148. base = pvm_determine_end(&next, &prev, align) - end;
  2149. term_area = area;
  2150. continue;
  2151. }
  2152. /*
  2153. * If prev overlaps, shift down next and prev and move
  2154. * base so that it's right below new next and then
  2155. * recheck.
  2156. */
  2157. if (prev && prev->va_end > base + start) {
  2158. next = prev;
  2159. prev = node_to_va(rb_prev(&next->rb_node));
  2160. base = pvm_determine_end(&next, &prev, align) - end;
  2161. term_area = area;
  2162. continue;
  2163. }
  2164. /*
  2165. * This area fits, move on to the previous one. If
  2166. * the previous one is the terminal one, we're done.
  2167. */
  2168. area = (area + nr_vms - 1) % nr_vms;
  2169. if (area == term_area)
  2170. break;
  2171. start = offsets[area];
  2172. end = start + sizes[area];
  2173. pvm_find_next_prev(base + end, &next, &prev);
  2174. }
  2175. found:
  2176. /* we've found a fitting base, insert all va's */
  2177. for (area = 0; area < nr_vms; area++) {
  2178. struct vmap_area *va = vas[area];
  2179. va->va_start = base + offsets[area];
  2180. va->va_end = va->va_start + sizes[area];
  2181. __insert_vmap_area(va);
  2182. }
  2183. vmap_area_pcpu_hole = base + offsets[last_area];
  2184. spin_unlock(&vmap_area_lock);
  2185. /* insert all vm's */
  2186. for (area = 0; area < nr_vms; area++)
  2187. setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
  2188. pcpu_get_vm_areas);
  2189. kfree(vas);
  2190. return vms;
  2191. err_free:
  2192. for (area = 0; area < nr_vms; area++) {
  2193. kfree(vas[area]);
  2194. kfree(vms[area]);
  2195. }
  2196. err_free2:
  2197. kfree(vas);
  2198. kfree(vms);
  2199. return NULL;
  2200. }
  2201. /**
  2202. * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
  2203. * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
  2204. * @nr_vms: the number of allocated areas
  2205. *
  2206. * Free vm_structs and the array allocated by pcpu_get_vm_areas().
  2207. */
  2208. void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
  2209. {
  2210. int i;
  2211. for (i = 0; i < nr_vms; i++)
  2212. free_vm_area(vms[i]);
  2213. kfree(vms);
  2214. }
  2215. #endif /* CONFIG_SMP */
  2216. #ifdef CONFIG_PROC_FS
  2217. static void *s_start(struct seq_file *m, loff_t *pos)
  2218. __acquires(&vmap_area_lock)
  2219. {
  2220. loff_t n = *pos;
  2221. struct vmap_area *va;
  2222. spin_lock(&vmap_area_lock);
  2223. va = list_entry((&vmap_area_list)->next, typeof(*va), list);
  2224. while (n > 0 && &va->list != &vmap_area_list) {
  2225. n--;
  2226. va = list_entry(va->list.next, typeof(*va), list);
  2227. }
  2228. if (!n && &va->list != &vmap_area_list)
  2229. return va;
  2230. return NULL;
  2231. }
  2232. static void *s_next(struct seq_file *m, void *p, loff_t *pos)
  2233. {
  2234. struct vmap_area *va = p, *next;
  2235. ++*pos;
  2236. next = list_entry(va->list.next, typeof(*va), list);
  2237. if (&next->list != &vmap_area_list)
  2238. return next;
  2239. return NULL;
  2240. }
  2241. static void s_stop(struct seq_file *m, void *p)
  2242. __releases(&vmap_area_lock)
  2243. {
  2244. spin_unlock(&vmap_area_lock);
  2245. }
  2246. static void show_numa_info(struct seq_file *m, struct vm_struct *v)
  2247. {
  2248. if (IS_ENABLED(CONFIG_NUMA)) {
  2249. unsigned int nr, *counters = m->private;
  2250. if (!counters)
  2251. return;
  2252. if (v->flags & VM_UNINITIALIZED)
  2253. return;
  2254. /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
  2255. smp_rmb();
  2256. memset(counters, 0, nr_node_ids * sizeof(unsigned int));
  2257. for (nr = 0; nr < v->nr_pages; nr++)
  2258. counters[page_to_nid(v->pages[nr])]++;
  2259. for_each_node_state(nr, N_HIGH_MEMORY)
  2260. if (counters[nr])
  2261. seq_printf(m, " N%u=%u", nr, counters[nr]);
  2262. }
  2263. }
  2264. static int s_show(struct seq_file *m, void *p)
  2265. {
  2266. struct vmap_area *va = p;
  2267. struct vm_struct *v;
  2268. /*
  2269. * s_show can encounter race with remove_vm_area, !VM_VM_AREA on
  2270. * behalf of vmap area is being tear down or vm_map_ram allocation.
  2271. */
  2272. if (!(va->flags & VM_VM_AREA))
  2273. return 0;
  2274. v = va->vm;
  2275. seq_printf(m, "0x%pK-0x%pK %7ld",
  2276. v->addr, v->addr + v->size, v->size);
  2277. if (v->caller)
  2278. seq_printf(m, " %pS", v->caller);
  2279. if (v->nr_pages)
  2280. seq_printf(m, " pages=%d", v->nr_pages);
  2281. if (v->phys_addr)
  2282. seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr);
  2283. if (v->flags & VM_IOREMAP)
  2284. seq_puts(m, " ioremap");
  2285. if (v->flags & VM_ALLOC)
  2286. seq_puts(m, " vmalloc");
  2287. if (v->flags & VM_MAP)
  2288. seq_puts(m, " vmap");
  2289. if (v->flags & VM_USERMAP)
  2290. seq_puts(m, " user");
  2291. if (v->flags & VM_VPAGES)
  2292. seq_puts(m, " vpages");
  2293. show_numa_info(m, v);
  2294. seq_putc(m, '\n');
  2295. return 0;
  2296. }
  2297. static const struct seq_operations vmalloc_op = {
  2298. .start = s_start,
  2299. .next = s_next,
  2300. .stop = s_stop,
  2301. .show = s_show,
  2302. };
  2303. static int vmalloc_open(struct inode *inode, struct file *file)
  2304. {
  2305. if (IS_ENABLED(CONFIG_NUMA))
  2306. return seq_open_private(file, &vmalloc_op,
  2307. nr_node_ids * sizeof(unsigned int));
  2308. else
  2309. return seq_open(file, &vmalloc_op);
  2310. }
  2311. static const struct file_operations proc_vmalloc_operations = {
  2312. .open = vmalloc_open,
  2313. .read = seq_read,
  2314. .llseek = seq_lseek,
  2315. .release = seq_release_private,
  2316. };
  2317. static int __init proc_vmalloc_init(void)
  2318. {
  2319. proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations);
  2320. return 0;
  2321. }
  2322. module_init(proc_vmalloc_init);
  2323. #endif