pageattr.c 45 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928
  1. /*
  2. * Copyright 2002 Andi Kleen, SuSE Labs.
  3. * Thanks to Ben LaHaise for precious feedback.
  4. */
  5. #include <linux/highmem.h>
  6. #include <linux/bootmem.h>
  7. #include <linux/module.h>
  8. #include <linux/sched.h>
  9. #include <linux/mm.h>
  10. #include <linux/interrupt.h>
  11. #include <linux/seq_file.h>
  12. #include <linux/debugfs.h>
  13. #include <linux/pfn.h>
  14. #include <linux/percpu.h>
  15. #include <linux/gfp.h>
  16. #include <linux/pci.h>
  17. #include <asm/e820.h>
  18. #include <asm/processor.h>
  19. #include <asm/tlbflush.h>
  20. #include <asm/sections.h>
  21. #include <asm/setup.h>
  22. #include <asm/uaccess.h>
  23. #include <asm/pgalloc.h>
  24. #include <asm/proto.h>
  25. #include <asm/pat.h>
  26. /*
  27. * The current flushing context - we pass it instead of 5 arguments:
  28. */
  29. struct cpa_data {
  30. unsigned long *vaddr;
  31. pgd_t *pgd;
  32. pgprot_t mask_set;
  33. pgprot_t mask_clr;
  34. int numpages;
  35. int flags;
  36. unsigned long pfn;
  37. unsigned force_split : 1;
  38. int curpage;
  39. struct page **pages;
  40. };
  41. /*
  42. * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
  43. * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
  44. * entries change the page attribute in parallel to some other cpu
  45. * splitting a large page entry along with changing the attribute.
  46. */
  47. static DEFINE_SPINLOCK(cpa_lock);
  48. #define CPA_FLUSHTLB 1
  49. #define CPA_ARRAY 2
  50. #define CPA_PAGES_ARRAY 4
  51. #ifdef CONFIG_PROC_FS
  52. static unsigned long direct_pages_count[PG_LEVEL_NUM];
  53. void update_page_count(int level, unsigned long pages)
  54. {
  55. /* Protect against CPA */
  56. spin_lock(&pgd_lock);
  57. direct_pages_count[level] += pages;
  58. spin_unlock(&pgd_lock);
  59. }
  60. static void split_page_count(int level)
  61. {
  62. direct_pages_count[level]--;
  63. direct_pages_count[level - 1] += PTRS_PER_PTE;
  64. }
  65. void arch_report_meminfo(struct seq_file *m)
  66. {
  67. seq_printf(m, "DirectMap4k: %8lu kB\n",
  68. direct_pages_count[PG_LEVEL_4K] << 2);
  69. #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
  70. seq_printf(m, "DirectMap2M: %8lu kB\n",
  71. direct_pages_count[PG_LEVEL_2M] << 11);
  72. #else
  73. seq_printf(m, "DirectMap4M: %8lu kB\n",
  74. direct_pages_count[PG_LEVEL_2M] << 12);
  75. #endif
  76. #ifdef CONFIG_X86_64
  77. if (direct_gbpages)
  78. seq_printf(m, "DirectMap1G: %8lu kB\n",
  79. direct_pages_count[PG_LEVEL_1G] << 20);
  80. #endif
  81. }
  82. #else
  83. static inline void split_page_count(int level) { }
  84. #endif
  85. #ifdef CONFIG_X86_64
  86. static inline unsigned long highmap_start_pfn(void)
  87. {
  88. return __pa_symbol(_text) >> PAGE_SHIFT;
  89. }
  90. static inline unsigned long highmap_end_pfn(void)
  91. {
  92. return __pa_symbol(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT;
  93. }
  94. #endif
  95. #ifdef CONFIG_DEBUG_PAGEALLOC
  96. # define debug_pagealloc 1
  97. #else
  98. # define debug_pagealloc 0
  99. #endif
  100. static inline int
  101. within(unsigned long addr, unsigned long start, unsigned long end)
  102. {
  103. return addr >= start && addr < end;
  104. }
  105. /*
  106. * Flushing functions
  107. */
  108. /**
  109. * clflush_cache_range - flush a cache range with clflush
  110. * @vaddr: virtual start address
  111. * @size: number of bytes to flush
  112. *
  113. * clflushopt is an unordered instruction which needs fencing with mfence or
  114. * sfence to avoid ordering issues.
  115. */
  116. void clflush_cache_range(void *vaddr, unsigned int size)
  117. {
  118. void *vend = vaddr + size - 1;
  119. mb();
  120. for (; vaddr < vend; vaddr += boot_cpu_data.x86_clflush_size)
  121. clflushopt(vaddr);
  122. /*
  123. * Flush any possible final partial cacheline:
  124. */
  125. clflushopt(vend);
  126. mb();
  127. }
  128. EXPORT_SYMBOL_GPL(clflush_cache_range);
  129. static void __cpa_flush_all(void *arg)
  130. {
  131. unsigned long cache = (unsigned long)arg;
  132. /*
  133. * Flush all to work around Errata in early athlons regarding
  134. * large page flushing.
  135. */
  136. __flush_tlb_all();
  137. if (cache && boot_cpu_data.x86 >= 4)
  138. wbinvd();
  139. }
  140. static void cpa_flush_all(unsigned long cache)
  141. {
  142. BUG_ON(irqs_disabled());
  143. on_each_cpu(__cpa_flush_all, (void *) cache, 1);
  144. }
  145. static void __cpa_flush_range(void *arg)
  146. {
  147. /*
  148. * We could optimize that further and do individual per page
  149. * tlb invalidates for a low number of pages. Caveat: we must
  150. * flush the high aliases on 64bit as well.
  151. */
  152. __flush_tlb_all();
  153. }
  154. static void cpa_flush_range(unsigned long start, int numpages, int cache)
  155. {
  156. unsigned int i, level;
  157. unsigned long addr;
  158. BUG_ON(irqs_disabled());
  159. WARN_ON(PAGE_ALIGN(start) != start);
  160. on_each_cpu(__cpa_flush_range, NULL, 1);
  161. if (!cache)
  162. return;
  163. /*
  164. * We only need to flush on one CPU,
  165. * clflush is a MESI-coherent instruction that
  166. * will cause all other CPUs to flush the same
  167. * cachelines:
  168. */
  169. for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
  170. pte_t *pte = lookup_address(addr, &level);
  171. /*
  172. * Only flush present addresses:
  173. */
  174. if (pte && (pte_val(*pte) & _PAGE_PRESENT))
  175. clflush_cache_range((void *) addr, PAGE_SIZE);
  176. }
  177. }
  178. static void cpa_flush_array(unsigned long *start, int numpages, int cache,
  179. int in_flags, struct page **pages)
  180. {
  181. unsigned int i, level;
  182. unsigned long do_wbinvd = cache && numpages >= 1024; /* 4M threshold */
  183. BUG_ON(irqs_disabled());
  184. on_each_cpu(__cpa_flush_all, (void *) do_wbinvd, 1);
  185. if (!cache || do_wbinvd)
  186. return;
  187. /*
  188. * We only need to flush on one CPU,
  189. * clflush is a MESI-coherent instruction that
  190. * will cause all other CPUs to flush the same
  191. * cachelines:
  192. */
  193. for (i = 0; i < numpages; i++) {
  194. unsigned long addr;
  195. pte_t *pte;
  196. if (in_flags & CPA_PAGES_ARRAY)
  197. addr = (unsigned long)page_address(pages[i]);
  198. else
  199. addr = start[i];
  200. pte = lookup_address(addr, &level);
  201. /*
  202. * Only flush present addresses:
  203. */
  204. if (pte && (pte_val(*pte) & _PAGE_PRESENT))
  205. clflush_cache_range((void *)addr, PAGE_SIZE);
  206. }
  207. }
  208. /*
  209. * Certain areas of memory on x86 require very specific protection flags,
  210. * for example the BIOS area or kernel text. Callers don't always get this
  211. * right (again, ioremap() on BIOS memory is not uncommon) so this function
  212. * checks and fixes these known static required protection bits.
  213. */
  214. static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
  215. unsigned long pfn)
  216. {
  217. pgprot_t forbidden = __pgprot(0);
  218. /*
  219. * The BIOS area between 640k and 1Mb needs to be executable for
  220. * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
  221. */
  222. #ifdef CONFIG_PCI_BIOS
  223. if (pcibios_enabled && within(pfn, BIOS_BEGIN >> PAGE_SHIFT, BIOS_END >> PAGE_SHIFT))
  224. pgprot_val(forbidden) |= _PAGE_NX;
  225. #endif
  226. /*
  227. * The kernel text needs to be executable for obvious reasons
  228. * Does not cover __inittext since that is gone later on. On
  229. * 64bit we do not enforce !NX on the low mapping
  230. */
  231. if (within(address, (unsigned long)_text, (unsigned long)_etext))
  232. pgprot_val(forbidden) |= _PAGE_NX;
  233. /*
  234. * The .rodata section needs to be read-only. Using the pfn
  235. * catches all aliases.
  236. */
  237. if (within(pfn, __pa_symbol(__start_rodata) >> PAGE_SHIFT,
  238. __pa_symbol(__end_rodata) >> PAGE_SHIFT))
  239. pgprot_val(forbidden) |= _PAGE_RW;
  240. #if defined(CONFIG_X86_64) && defined(CONFIG_DEBUG_RODATA)
  241. /*
  242. * Once the kernel maps the text as RO (kernel_set_to_readonly is set),
  243. * kernel text mappings for the large page aligned text, rodata sections
  244. * will be always read-only. For the kernel identity mappings covering
  245. * the holes caused by this alignment can be anything that user asks.
  246. *
  247. * This will preserve the large page mappings for kernel text/data
  248. * at no extra cost.
  249. */
  250. if (kernel_set_to_readonly &&
  251. within(address, (unsigned long)_text,
  252. (unsigned long)__end_rodata_hpage_align)) {
  253. unsigned int level;
  254. /*
  255. * Don't enforce the !RW mapping for the kernel text mapping,
  256. * if the current mapping is already using small page mapping.
  257. * No need to work hard to preserve large page mappings in this
  258. * case.
  259. *
  260. * This also fixes the Linux Xen paravirt guest boot failure
  261. * (because of unexpected read-only mappings for kernel identity
  262. * mappings). In this paravirt guest case, the kernel text
  263. * mapping and the kernel identity mapping share the same
  264. * page-table pages. Thus we can't really use different
  265. * protections for the kernel text and identity mappings. Also,
  266. * these shared mappings are made of small page mappings.
  267. * Thus this don't enforce !RW mapping for small page kernel
  268. * text mapping logic will help Linux Xen parvirt guest boot
  269. * as well.
  270. */
  271. if (lookup_address(address, &level) && (level != PG_LEVEL_4K))
  272. pgprot_val(forbidden) |= _PAGE_RW;
  273. }
  274. #endif
  275. prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
  276. return prot;
  277. }
  278. /*
  279. * Lookup the page table entry for a virtual address in a specific pgd.
  280. * Return a pointer to the entry and the level of the mapping.
  281. */
  282. pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
  283. unsigned int *level)
  284. {
  285. pud_t *pud;
  286. pmd_t *pmd;
  287. *level = PG_LEVEL_NONE;
  288. if (pgd_none(*pgd))
  289. return NULL;
  290. pud = pud_offset(pgd, address);
  291. if (pud_none(*pud))
  292. return NULL;
  293. *level = PG_LEVEL_1G;
  294. if (pud_large(*pud) || !pud_present(*pud))
  295. return (pte_t *)pud;
  296. pmd = pmd_offset(pud, address);
  297. if (pmd_none(*pmd))
  298. return NULL;
  299. *level = PG_LEVEL_2M;
  300. if (pmd_large(*pmd) || !pmd_present(*pmd))
  301. return (pte_t *)pmd;
  302. *level = PG_LEVEL_4K;
  303. return pte_offset_kernel(pmd, address);
  304. }
  305. /*
  306. * Lookup the page table entry for a virtual address. Return a pointer
  307. * to the entry and the level of the mapping.
  308. *
  309. * Note: We return pud and pmd either when the entry is marked large
  310. * or when the present bit is not set. Otherwise we would return a
  311. * pointer to a nonexisting mapping.
  312. */
  313. pte_t *lookup_address(unsigned long address, unsigned int *level)
  314. {
  315. return lookup_address_in_pgd(pgd_offset_k(address), address, level);
  316. }
  317. EXPORT_SYMBOL_GPL(lookup_address);
  318. static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
  319. unsigned int *level)
  320. {
  321. if (cpa->pgd)
  322. return lookup_address_in_pgd(cpa->pgd + pgd_index(address),
  323. address, level);
  324. return lookup_address(address, level);
  325. }
  326. /*
  327. * Lookup the PMD entry for a virtual address. Return a pointer to the entry
  328. * or NULL if not present.
  329. */
  330. pmd_t *lookup_pmd_address(unsigned long address)
  331. {
  332. pgd_t *pgd;
  333. pud_t *pud;
  334. pgd = pgd_offset_k(address);
  335. if (pgd_none(*pgd))
  336. return NULL;
  337. pud = pud_offset(pgd, address);
  338. if (pud_none(*pud) || pud_large(*pud) || !pud_present(*pud))
  339. return NULL;
  340. return pmd_offset(pud, address);
  341. }
  342. /*
  343. * This is necessary because __pa() does not work on some
  344. * kinds of memory, like vmalloc() or the alloc_remap()
  345. * areas on 32-bit NUMA systems. The percpu areas can
  346. * end up in this kind of memory, for instance.
  347. *
  348. * This could be optimized, but it is only intended to be
  349. * used at inititalization time, and keeping it
  350. * unoptimized should increase the testing coverage for
  351. * the more obscure platforms.
  352. */
  353. phys_addr_t slow_virt_to_phys(void *__virt_addr)
  354. {
  355. unsigned long virt_addr = (unsigned long)__virt_addr;
  356. phys_addr_t phys_addr;
  357. unsigned long offset;
  358. enum pg_level level;
  359. unsigned long psize;
  360. unsigned long pmask;
  361. pte_t *pte;
  362. pte = lookup_address(virt_addr, &level);
  363. BUG_ON(!pte);
  364. psize = page_level_size(level);
  365. pmask = page_level_mask(level);
  366. offset = virt_addr & ~pmask;
  367. phys_addr = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
  368. return (phys_addr | offset);
  369. }
  370. EXPORT_SYMBOL_GPL(slow_virt_to_phys);
  371. /*
  372. * Set the new pmd in all the pgds we know about:
  373. */
  374. static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
  375. {
  376. /* change init_mm */
  377. set_pte_atomic(kpte, pte);
  378. #ifdef CONFIG_X86_32
  379. if (!SHARED_KERNEL_PMD) {
  380. struct page *page;
  381. list_for_each_entry(page, &pgd_list, lru) {
  382. pgd_t *pgd;
  383. pud_t *pud;
  384. pmd_t *pmd;
  385. pgd = (pgd_t *)page_address(page) + pgd_index(address);
  386. pud = pud_offset(pgd, address);
  387. pmd = pmd_offset(pud, address);
  388. set_pte_atomic((pte_t *)pmd, pte);
  389. }
  390. }
  391. #endif
  392. }
  393. static int
  394. try_preserve_large_page(pte_t *kpte, unsigned long address,
  395. struct cpa_data *cpa)
  396. {
  397. unsigned long nextpage_addr, numpages, pmask, psize, addr, pfn;
  398. pte_t new_pte, old_pte, *tmp;
  399. pgprot_t old_prot, new_prot, req_prot;
  400. int i, do_split = 1;
  401. enum pg_level level;
  402. if (cpa->force_split)
  403. return 1;
  404. spin_lock(&pgd_lock);
  405. /*
  406. * Check for races, another CPU might have split this page
  407. * up already:
  408. */
  409. tmp = _lookup_address_cpa(cpa, address, &level);
  410. if (tmp != kpte)
  411. goto out_unlock;
  412. switch (level) {
  413. case PG_LEVEL_2M:
  414. #ifdef CONFIG_X86_64
  415. case PG_LEVEL_1G:
  416. #endif
  417. psize = page_level_size(level);
  418. pmask = page_level_mask(level);
  419. break;
  420. default:
  421. do_split = -EINVAL;
  422. goto out_unlock;
  423. }
  424. /*
  425. * Calculate the number of pages, which fit into this large
  426. * page starting at address:
  427. */
  428. nextpage_addr = (address + psize) & pmask;
  429. numpages = (nextpage_addr - address) >> PAGE_SHIFT;
  430. if (numpages < cpa->numpages)
  431. cpa->numpages = numpages;
  432. /*
  433. * We are safe now. Check whether the new pgprot is the same:
  434. * Convert protection attributes to 4k-format, as cpa->mask* are set
  435. * up accordingly.
  436. */
  437. old_pte = *kpte;
  438. old_prot = req_prot = pgprot_large_2_4k(pte_pgprot(old_pte));
  439. pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
  440. pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
  441. /*
  442. * req_prot is in format of 4k pages. It must be converted to large
  443. * page format: the caching mode includes the PAT bit located at
  444. * different bit positions in the two formats.
  445. */
  446. req_prot = pgprot_4k_2_large(req_prot);
  447. /*
  448. * Set the PSE and GLOBAL flags only if the PRESENT flag is
  449. * set otherwise pmd_present/pmd_huge will return true even on
  450. * a non present pmd. The canon_pgprot will clear _PAGE_GLOBAL
  451. * for the ancient hardware that doesn't support it.
  452. */
  453. if (pgprot_val(req_prot) & _PAGE_PRESENT)
  454. pgprot_val(req_prot) |= _PAGE_PSE | _PAGE_GLOBAL;
  455. else
  456. pgprot_val(req_prot) &= ~(_PAGE_PSE | _PAGE_GLOBAL);
  457. req_prot = canon_pgprot(req_prot);
  458. /*
  459. * old_pte points to the large page base address. So we need
  460. * to add the offset of the virtual address:
  461. */
  462. pfn = pte_pfn(old_pte) + ((address & (psize - 1)) >> PAGE_SHIFT);
  463. cpa->pfn = pfn;
  464. new_prot = static_protections(req_prot, address, pfn);
  465. /*
  466. * We need to check the full range, whether
  467. * static_protection() requires a different pgprot for one of
  468. * the pages in the range we try to preserve:
  469. */
  470. addr = address & pmask;
  471. pfn = pte_pfn(old_pte);
  472. for (i = 0; i < (psize >> PAGE_SHIFT); i++, addr += PAGE_SIZE, pfn++) {
  473. pgprot_t chk_prot = static_protections(req_prot, addr, pfn);
  474. if (pgprot_val(chk_prot) != pgprot_val(new_prot))
  475. goto out_unlock;
  476. }
  477. /*
  478. * If there are no changes, return. maxpages has been updated
  479. * above:
  480. */
  481. if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
  482. do_split = 0;
  483. goto out_unlock;
  484. }
  485. /*
  486. * We need to change the attributes. Check, whether we can
  487. * change the large page in one go. We request a split, when
  488. * the address is not aligned and the number of pages is
  489. * smaller than the number of pages in the large page. Note
  490. * that we limited the number of possible pages already to
  491. * the number of pages in the large page.
  492. */
  493. if (address == (address & pmask) && cpa->numpages == (psize >> PAGE_SHIFT)) {
  494. /*
  495. * The address is aligned and the number of pages
  496. * covers the full page.
  497. */
  498. new_pte = pfn_pte(pte_pfn(old_pte), new_prot);
  499. __set_pmd_pte(kpte, address, new_pte);
  500. cpa->flags |= CPA_FLUSHTLB;
  501. do_split = 0;
  502. }
  503. out_unlock:
  504. spin_unlock(&pgd_lock);
  505. return do_split;
  506. }
  507. static int
  508. __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
  509. struct page *base)
  510. {
  511. pte_t *pbase = (pte_t *)page_address(base);
  512. unsigned long pfn, pfninc = 1;
  513. unsigned int i, level;
  514. pte_t *tmp;
  515. pgprot_t ref_prot;
  516. spin_lock(&pgd_lock);
  517. /*
  518. * Check for races, another CPU might have split this page
  519. * up for us already:
  520. */
  521. tmp = _lookup_address_cpa(cpa, address, &level);
  522. if (tmp != kpte) {
  523. spin_unlock(&pgd_lock);
  524. return 1;
  525. }
  526. paravirt_alloc_pte(&init_mm, page_to_pfn(base));
  527. ref_prot = pte_pgprot(pte_clrhuge(*kpte));
  528. /* promote PAT bit to correct position */
  529. if (level == PG_LEVEL_2M)
  530. ref_prot = pgprot_large_2_4k(ref_prot);
  531. #ifdef CONFIG_X86_64
  532. if (level == PG_LEVEL_1G) {
  533. pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
  534. /*
  535. * Set the PSE flags only if the PRESENT flag is set
  536. * otherwise pmd_present/pmd_huge will return true
  537. * even on a non present pmd.
  538. */
  539. if (pgprot_val(ref_prot) & _PAGE_PRESENT)
  540. pgprot_val(ref_prot) |= _PAGE_PSE;
  541. else
  542. pgprot_val(ref_prot) &= ~_PAGE_PSE;
  543. }
  544. #endif
  545. /*
  546. * Set the GLOBAL flags only if the PRESENT flag is set
  547. * otherwise pmd/pte_present will return true even on a non
  548. * present pmd/pte. The canon_pgprot will clear _PAGE_GLOBAL
  549. * for the ancient hardware that doesn't support it.
  550. */
  551. if (pgprot_val(ref_prot) & _PAGE_PRESENT)
  552. pgprot_val(ref_prot) |= _PAGE_GLOBAL;
  553. else
  554. pgprot_val(ref_prot) &= ~_PAGE_GLOBAL;
  555. /*
  556. * Get the target pfn from the original entry:
  557. */
  558. pfn = pte_pfn(*kpte);
  559. for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
  560. set_pte(&pbase[i], pfn_pte(pfn, canon_pgprot(ref_prot)));
  561. if (pfn_range_is_mapped(PFN_DOWN(__pa(address)),
  562. PFN_DOWN(__pa(address)) + 1))
  563. split_page_count(level);
  564. /*
  565. * Install the new, split up pagetable.
  566. *
  567. * We use the standard kernel pagetable protections for the new
  568. * pagetable protections, the actual ptes set above control the
  569. * primary protection behavior:
  570. */
  571. __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));
  572. /*
  573. * Intel Atom errata AAH41 workaround.
  574. *
  575. * The real fix should be in hw or in a microcode update, but
  576. * we also probabilistically try to reduce the window of having
  577. * a large TLB mixed with 4K TLBs while instruction fetches are
  578. * going on.
  579. */
  580. __flush_tlb_all();
  581. spin_unlock(&pgd_lock);
  582. return 0;
  583. }
  584. static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
  585. unsigned long address)
  586. {
  587. struct page *base;
  588. if (!debug_pagealloc)
  589. spin_unlock(&cpa_lock);
  590. base = alloc_pages(GFP_KERNEL | __GFP_NOTRACK, 0);
  591. if (!debug_pagealloc)
  592. spin_lock(&cpa_lock);
  593. if (!base)
  594. return -ENOMEM;
  595. if (__split_large_page(cpa, kpte, address, base))
  596. __free_page(base);
  597. return 0;
  598. }
  599. static bool try_to_free_pte_page(pte_t *pte)
  600. {
  601. int i;
  602. for (i = 0; i < PTRS_PER_PTE; i++)
  603. if (!pte_none(pte[i]))
  604. return false;
  605. free_page((unsigned long)pte);
  606. return true;
  607. }
  608. static bool try_to_free_pmd_page(pmd_t *pmd)
  609. {
  610. int i;
  611. for (i = 0; i < PTRS_PER_PMD; i++)
  612. if (!pmd_none(pmd[i]))
  613. return false;
  614. free_page((unsigned long)pmd);
  615. return true;
  616. }
  617. static bool try_to_free_pud_page(pud_t *pud)
  618. {
  619. int i;
  620. for (i = 0; i < PTRS_PER_PUD; i++)
  621. if (!pud_none(pud[i]))
  622. return false;
  623. free_page((unsigned long)pud);
  624. return true;
  625. }
  626. static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
  627. {
  628. pte_t *pte = pte_offset_kernel(pmd, start);
  629. while (start < end) {
  630. set_pte(pte, __pte(0));
  631. start += PAGE_SIZE;
  632. pte++;
  633. }
  634. if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) {
  635. pmd_clear(pmd);
  636. return true;
  637. }
  638. return false;
  639. }
  640. static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd,
  641. unsigned long start, unsigned long end)
  642. {
  643. if (unmap_pte_range(pmd, start, end))
  644. if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
  645. pud_clear(pud);
  646. }
  647. static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
  648. {
  649. pmd_t *pmd = pmd_offset(pud, start);
  650. /*
  651. * Not on a 2MB page boundary?
  652. */
  653. if (start & (PMD_SIZE - 1)) {
  654. unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
  655. unsigned long pre_end = min_t(unsigned long, end, next_page);
  656. __unmap_pmd_range(pud, pmd, start, pre_end);
  657. start = pre_end;
  658. pmd++;
  659. }
  660. /*
  661. * Try to unmap in 2M chunks.
  662. */
  663. while (end - start >= PMD_SIZE) {
  664. if (pmd_large(*pmd))
  665. pmd_clear(pmd);
  666. else
  667. __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);
  668. start += PMD_SIZE;
  669. pmd++;
  670. }
  671. /*
  672. * 4K leftovers?
  673. */
  674. if (start < end)
  675. return __unmap_pmd_range(pud, pmd, start, end);
  676. /*
  677. * Try again to free the PMD page if haven't succeeded above.
  678. */
  679. if (!pud_none(*pud))
  680. if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
  681. pud_clear(pud);
  682. }
  683. static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end)
  684. {
  685. pud_t *pud = pud_offset(pgd, start);
  686. /*
  687. * Not on a GB page boundary?
  688. */
  689. if (start & (PUD_SIZE - 1)) {
  690. unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
  691. unsigned long pre_end = min_t(unsigned long, end, next_page);
  692. unmap_pmd_range(pud, start, pre_end);
  693. start = pre_end;
  694. pud++;
  695. }
  696. /*
  697. * Try to unmap in 1G chunks?
  698. */
  699. while (end - start >= PUD_SIZE) {
  700. if (pud_large(*pud))
  701. pud_clear(pud);
  702. else
  703. unmap_pmd_range(pud, start, start + PUD_SIZE);
  704. start += PUD_SIZE;
  705. pud++;
  706. }
  707. /*
  708. * 2M leftovers?
  709. */
  710. if (start < end)
  711. unmap_pmd_range(pud, start, end);
  712. /*
  713. * No need to try to free the PUD page because we'll free it in
  714. * populate_pgd's error path
  715. */
  716. }
  717. static void unmap_pgd_range(pgd_t *root, unsigned long addr, unsigned long end)
  718. {
  719. pgd_t *pgd_entry = root + pgd_index(addr);
  720. unmap_pud_range(pgd_entry, addr, end);
  721. if (try_to_free_pud_page((pud_t *)pgd_page_vaddr(*pgd_entry)))
  722. pgd_clear(pgd_entry);
  723. }
  724. static int alloc_pte_page(pmd_t *pmd)
  725. {
  726. pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
  727. if (!pte)
  728. return -1;
  729. set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
  730. return 0;
  731. }
  732. static int alloc_pmd_page(pud_t *pud)
  733. {
  734. pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
  735. if (!pmd)
  736. return -1;
  737. set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
  738. return 0;
  739. }
  740. static void populate_pte(struct cpa_data *cpa,
  741. unsigned long start, unsigned long end,
  742. unsigned num_pages, pmd_t *pmd, pgprot_t pgprot)
  743. {
  744. pte_t *pte;
  745. pte = pte_offset_kernel(pmd, start);
  746. while (num_pages-- && start < end) {
  747. /* deal with the NX bit */
  748. if (!(pgprot_val(pgprot) & _PAGE_NX))
  749. cpa->pfn &= ~_PAGE_NX;
  750. set_pte(pte, pfn_pte(cpa->pfn >> PAGE_SHIFT, pgprot));
  751. start += PAGE_SIZE;
  752. cpa->pfn += PAGE_SIZE;
  753. pte++;
  754. }
  755. }
  756. static int populate_pmd(struct cpa_data *cpa,
  757. unsigned long start, unsigned long end,
  758. unsigned num_pages, pud_t *pud, pgprot_t pgprot)
  759. {
  760. unsigned int cur_pages = 0;
  761. pmd_t *pmd;
  762. pgprot_t pmd_pgprot;
  763. /*
  764. * Not on a 2M boundary?
  765. */
  766. if (start & (PMD_SIZE - 1)) {
  767. unsigned long pre_end = start + (num_pages << PAGE_SHIFT);
  768. unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
  769. pre_end = min_t(unsigned long, pre_end, next_page);
  770. cur_pages = (pre_end - start) >> PAGE_SHIFT;
  771. cur_pages = min_t(unsigned int, num_pages, cur_pages);
  772. /*
  773. * Need a PTE page?
  774. */
  775. pmd = pmd_offset(pud, start);
  776. if (pmd_none(*pmd))
  777. if (alloc_pte_page(pmd))
  778. return -1;
  779. populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);
  780. start = pre_end;
  781. }
  782. /*
  783. * We mapped them all?
  784. */
  785. if (num_pages == cur_pages)
  786. return cur_pages;
  787. pmd_pgprot = pgprot_4k_2_large(pgprot);
  788. while (end - start >= PMD_SIZE) {
  789. /*
  790. * We cannot use a 1G page so allocate a PMD page if needed.
  791. */
  792. if (pud_none(*pud))
  793. if (alloc_pmd_page(pud))
  794. return -1;
  795. pmd = pmd_offset(pud, start);
  796. set_pmd(pmd, __pmd(cpa->pfn | _PAGE_PSE |
  797. massage_pgprot(pmd_pgprot)));
  798. start += PMD_SIZE;
  799. cpa->pfn += PMD_SIZE;
  800. cur_pages += PMD_SIZE >> PAGE_SHIFT;
  801. }
  802. /*
  803. * Map trailing 4K pages.
  804. */
  805. if (start < end) {
  806. pmd = pmd_offset(pud, start);
  807. if (pmd_none(*pmd))
  808. if (alloc_pte_page(pmd))
  809. return -1;
  810. populate_pte(cpa, start, end, num_pages - cur_pages,
  811. pmd, pgprot);
  812. }
  813. return num_pages;
  814. }
  815. static int populate_pud(struct cpa_data *cpa, unsigned long start, pgd_t *pgd,
  816. pgprot_t pgprot)
  817. {
  818. pud_t *pud;
  819. unsigned long end;
  820. int cur_pages = 0;
  821. pgprot_t pud_pgprot;
  822. end = start + (cpa->numpages << PAGE_SHIFT);
  823. /*
  824. * Not on a Gb page boundary? => map everything up to it with
  825. * smaller pages.
  826. */
  827. if (start & (PUD_SIZE - 1)) {
  828. unsigned long pre_end;
  829. unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
  830. pre_end = min_t(unsigned long, end, next_page);
  831. cur_pages = (pre_end - start) >> PAGE_SHIFT;
  832. cur_pages = min_t(int, (int)cpa->numpages, cur_pages);
  833. pud = pud_offset(pgd, start);
  834. /*
  835. * Need a PMD page?
  836. */
  837. if (pud_none(*pud))
  838. if (alloc_pmd_page(pud))
  839. return -1;
  840. cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,
  841. pud, pgprot);
  842. if (cur_pages < 0)
  843. return cur_pages;
  844. start = pre_end;
  845. }
  846. /* We mapped them all? */
  847. if (cpa->numpages == cur_pages)
  848. return cur_pages;
  849. pud = pud_offset(pgd, start);
  850. pud_pgprot = pgprot_4k_2_large(pgprot);
  851. /*
  852. * Map everything starting from the Gb boundary, possibly with 1G pages
  853. */
  854. while (end - start >= PUD_SIZE) {
  855. set_pud(pud, __pud(cpa->pfn | _PAGE_PSE |
  856. massage_pgprot(pud_pgprot)));
  857. start += PUD_SIZE;
  858. cpa->pfn += PUD_SIZE;
  859. cur_pages += PUD_SIZE >> PAGE_SHIFT;
  860. pud++;
  861. }
  862. /* Map trailing leftover */
  863. if (start < end) {
  864. int tmp;
  865. pud = pud_offset(pgd, start);
  866. if (pud_none(*pud))
  867. if (alloc_pmd_page(pud))
  868. return -1;
  869. tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,
  870. pud, pgprot);
  871. if (tmp < 0)
  872. return cur_pages;
  873. cur_pages += tmp;
  874. }
  875. return cur_pages;
  876. }
  877. /*
  878. * Restrictions for kernel page table do not necessarily apply when mapping in
  879. * an alternate PGD.
  880. */
  881. static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
  882. {
  883. pgprot_t pgprot = __pgprot(_KERNPG_TABLE);
  884. pud_t *pud = NULL; /* shut up gcc */
  885. pgd_t *pgd_entry;
  886. int ret;
  887. pgd_entry = cpa->pgd + pgd_index(addr);
  888. /*
  889. * Allocate a PUD page and hand it down for mapping.
  890. */
  891. if (pgd_none(*pgd_entry)) {
  892. pud = (pud_t *)get_zeroed_page(GFP_KERNEL | __GFP_NOTRACK);
  893. if (!pud)
  894. return -1;
  895. set_pgd(pgd_entry, __pgd(__pa(pud) | _KERNPG_TABLE));
  896. }
  897. pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);
  898. pgprot_val(pgprot) |= pgprot_val(cpa->mask_set);
  899. ret = populate_pud(cpa, addr, pgd_entry, pgprot);
  900. if (ret < 0) {
  901. unmap_pgd_range(cpa->pgd, addr,
  902. addr + (cpa->numpages << PAGE_SHIFT));
  903. return ret;
  904. }
  905. cpa->numpages = ret;
  906. return 0;
  907. }
  908. static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
  909. int primary)
  910. {
  911. if (cpa->pgd)
  912. return populate_pgd(cpa, vaddr);
  913. /*
  914. * Ignore all non primary paths.
  915. */
  916. if (!primary)
  917. return 0;
  918. /*
  919. * Ignore the NULL PTE for kernel identity mapping, as it is expected
  920. * to have holes.
  921. * Also set numpages to '1' indicating that we processed cpa req for
  922. * one virtual address page and its pfn. TBD: numpages can be set based
  923. * on the initial value and the level returned by lookup_address().
  924. */
  925. if (within(vaddr, PAGE_OFFSET,
  926. PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
  927. cpa->numpages = 1;
  928. cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
  929. return 0;
  930. } else {
  931. WARN(1, KERN_WARNING "CPA: called for zero pte. "
  932. "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
  933. *cpa->vaddr);
  934. return -EFAULT;
  935. }
  936. }
  937. static int __change_page_attr(struct cpa_data *cpa, int primary)
  938. {
  939. unsigned long address;
  940. int do_split, err;
  941. unsigned int level;
  942. pte_t *kpte, old_pte;
  943. if (cpa->flags & CPA_PAGES_ARRAY) {
  944. struct page *page = cpa->pages[cpa->curpage];
  945. if (unlikely(PageHighMem(page)))
  946. return 0;
  947. address = (unsigned long)page_address(page);
  948. } else if (cpa->flags & CPA_ARRAY)
  949. address = cpa->vaddr[cpa->curpage];
  950. else
  951. address = *cpa->vaddr;
  952. repeat:
  953. kpte = _lookup_address_cpa(cpa, address, &level);
  954. if (!kpte)
  955. return __cpa_process_fault(cpa, address, primary);
  956. old_pte = *kpte;
  957. if (!pte_val(old_pte))
  958. return __cpa_process_fault(cpa, address, primary);
  959. if (level == PG_LEVEL_4K) {
  960. pte_t new_pte;
  961. pgprot_t new_prot = pte_pgprot(old_pte);
  962. unsigned long pfn = pte_pfn(old_pte);
  963. pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
  964. pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
  965. new_prot = static_protections(new_prot, address, pfn);
  966. /*
  967. * Set the GLOBAL flags only if the PRESENT flag is
  968. * set otherwise pte_present will return true even on
  969. * a non present pte. The canon_pgprot will clear
  970. * _PAGE_GLOBAL for the ancient hardware that doesn't
  971. * support it.
  972. */
  973. if (pgprot_val(new_prot) & _PAGE_PRESENT)
  974. pgprot_val(new_prot) |= _PAGE_GLOBAL;
  975. else
  976. pgprot_val(new_prot) &= ~_PAGE_GLOBAL;
  977. /*
  978. * We need to keep the pfn from the existing PTE,
  979. * after all we're only going to change it's attributes
  980. * not the memory it points to
  981. */
  982. new_pte = pfn_pte(pfn, canon_pgprot(new_prot));
  983. cpa->pfn = pfn;
  984. /*
  985. * Do we really change anything ?
  986. */
  987. if (pte_val(old_pte) != pte_val(new_pte)) {
  988. set_pte_atomic(kpte, new_pte);
  989. cpa->flags |= CPA_FLUSHTLB;
  990. }
  991. cpa->numpages = 1;
  992. return 0;
  993. }
  994. /*
  995. * Check, whether we can keep the large page intact
  996. * and just change the pte:
  997. */
  998. do_split = try_preserve_large_page(kpte, address, cpa);
  999. /*
  1000. * When the range fits into the existing large page,
  1001. * return. cp->numpages and cpa->tlbflush have been updated in
  1002. * try_large_page:
  1003. */
  1004. if (do_split <= 0)
  1005. return do_split;
  1006. /*
  1007. * We have to split the large page:
  1008. */
  1009. err = split_large_page(cpa, kpte, address);
  1010. if (!err) {
  1011. /*
  1012. * Do a global flush tlb after splitting the large page
  1013. * and before we do the actual change page attribute in the PTE.
  1014. *
  1015. * With out this, we violate the TLB application note, that says
  1016. * "The TLBs may contain both ordinary and large-page
  1017. * translations for a 4-KByte range of linear addresses. This
  1018. * may occur if software modifies the paging structures so that
  1019. * the page size used for the address range changes. If the two
  1020. * translations differ with respect to page frame or attributes
  1021. * (e.g., permissions), processor behavior is undefined and may
  1022. * be implementation-specific."
  1023. *
  1024. * We do this global tlb flush inside the cpa_lock, so that we
  1025. * don't allow any other cpu, with stale tlb entries change the
  1026. * page attribute in parallel, that also falls into the
  1027. * just split large page entry.
  1028. */
  1029. flush_tlb_all();
  1030. goto repeat;
  1031. }
  1032. return err;
  1033. }
  1034. static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
  1035. static int cpa_process_alias(struct cpa_data *cpa)
  1036. {
  1037. struct cpa_data alias_cpa;
  1038. unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
  1039. unsigned long vaddr;
  1040. int ret;
  1041. if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
  1042. return 0;
  1043. /*
  1044. * No need to redo, when the primary call touched the direct
  1045. * mapping already:
  1046. */
  1047. if (cpa->flags & CPA_PAGES_ARRAY) {
  1048. struct page *page = cpa->pages[cpa->curpage];
  1049. if (unlikely(PageHighMem(page)))
  1050. return 0;
  1051. vaddr = (unsigned long)page_address(page);
  1052. } else if (cpa->flags & CPA_ARRAY)
  1053. vaddr = cpa->vaddr[cpa->curpage];
  1054. else
  1055. vaddr = *cpa->vaddr;
  1056. if (!(within(vaddr, PAGE_OFFSET,
  1057. PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
  1058. alias_cpa = *cpa;
  1059. alias_cpa.vaddr = &laddr;
  1060. alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
  1061. ret = __change_page_attr_set_clr(&alias_cpa, 0);
  1062. if (ret)
  1063. return ret;
  1064. }
  1065. #ifdef CONFIG_X86_64
  1066. /*
  1067. * If the primary call didn't touch the high mapping already
  1068. * and the physical address is inside the kernel map, we need
  1069. * to touch the high mapped kernel as well:
  1070. */
  1071. if (!within(vaddr, (unsigned long)_text, _brk_end) &&
  1072. within(cpa->pfn, highmap_start_pfn(), highmap_end_pfn())) {
  1073. unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
  1074. __START_KERNEL_map - phys_base;
  1075. alias_cpa = *cpa;
  1076. alias_cpa.vaddr = &temp_cpa_vaddr;
  1077. alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
  1078. /*
  1079. * The high mapping range is imprecise, so ignore the
  1080. * return value.
  1081. */
  1082. __change_page_attr_set_clr(&alias_cpa, 0);
  1083. }
  1084. #endif
  1085. return 0;
  1086. }
  1087. static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
  1088. {
  1089. int ret, numpages = cpa->numpages;
  1090. while (numpages) {
  1091. /*
  1092. * Store the remaining nr of pages for the large page
  1093. * preservation check.
  1094. */
  1095. cpa->numpages = numpages;
  1096. /* for array changes, we can't use large page */
  1097. if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
  1098. cpa->numpages = 1;
  1099. if (!debug_pagealloc)
  1100. spin_lock(&cpa_lock);
  1101. ret = __change_page_attr(cpa, checkalias);
  1102. if (!debug_pagealloc)
  1103. spin_unlock(&cpa_lock);
  1104. if (ret)
  1105. return ret;
  1106. if (checkalias) {
  1107. ret = cpa_process_alias(cpa);
  1108. if (ret)
  1109. return ret;
  1110. }
  1111. /*
  1112. * Adjust the number of pages with the result of the
  1113. * CPA operation. Either a large page has been
  1114. * preserved or a single page update happened.
  1115. */
  1116. BUG_ON(cpa->numpages > numpages);
  1117. numpages -= cpa->numpages;
  1118. if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
  1119. cpa->curpage++;
  1120. else
  1121. *cpa->vaddr += cpa->numpages * PAGE_SIZE;
  1122. }
  1123. return 0;
  1124. }
  1125. static int change_page_attr_set_clr(unsigned long *addr, int numpages,
  1126. pgprot_t mask_set, pgprot_t mask_clr,
  1127. int force_split, int in_flag,
  1128. struct page **pages)
  1129. {
  1130. struct cpa_data cpa;
  1131. int ret, cache, checkalias;
  1132. unsigned long baddr = 0;
  1133. memset(&cpa, 0, sizeof(cpa));
  1134. /*
  1135. * Check, if we are requested to change a not supported
  1136. * feature:
  1137. */
  1138. mask_set = canon_pgprot(mask_set);
  1139. mask_clr = canon_pgprot(mask_clr);
  1140. if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
  1141. return 0;
  1142. /* Ensure we are PAGE_SIZE aligned */
  1143. if (in_flag & CPA_ARRAY) {
  1144. int i;
  1145. for (i = 0; i < numpages; i++) {
  1146. if (addr[i] & ~PAGE_MASK) {
  1147. addr[i] &= PAGE_MASK;
  1148. WARN_ON_ONCE(1);
  1149. }
  1150. }
  1151. } else if (!(in_flag & CPA_PAGES_ARRAY)) {
  1152. /*
  1153. * in_flag of CPA_PAGES_ARRAY implies it is aligned.
  1154. * No need to cehck in that case
  1155. */
  1156. if (*addr & ~PAGE_MASK) {
  1157. *addr &= PAGE_MASK;
  1158. /*
  1159. * People should not be passing in unaligned addresses:
  1160. */
  1161. WARN_ON_ONCE(1);
  1162. }
  1163. /*
  1164. * Save address for cache flush. *addr is modified in the call
  1165. * to __change_page_attr_set_clr() below.
  1166. */
  1167. baddr = *addr;
  1168. }
  1169. /* Must avoid aliasing mappings in the highmem code */
  1170. kmap_flush_unused();
  1171. vm_unmap_aliases();
  1172. cpa.vaddr = addr;
  1173. cpa.pages = pages;
  1174. cpa.numpages = numpages;
  1175. cpa.mask_set = mask_set;
  1176. cpa.mask_clr = mask_clr;
  1177. cpa.flags = 0;
  1178. cpa.curpage = 0;
  1179. cpa.force_split = force_split;
  1180. if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
  1181. cpa.flags |= in_flag;
  1182. /* No alias checking for _NX bit modifications */
  1183. checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
  1184. ret = __change_page_attr_set_clr(&cpa, checkalias);
  1185. /*
  1186. * Check whether we really changed something:
  1187. */
  1188. if (!(cpa.flags & CPA_FLUSHTLB))
  1189. goto out;
  1190. /*
  1191. * No need to flush, when we did not set any of the caching
  1192. * attributes:
  1193. */
  1194. cache = !!pgprot2cachemode(mask_set);
  1195. /*
  1196. * On success we use CLFLUSH, when the CPU supports it to
  1197. * avoid the WBINVD. If the CPU does not support it and in the
  1198. * error case we fall back to cpa_flush_all (which uses
  1199. * WBINVD):
  1200. */
  1201. if (!ret && cpu_has_clflush) {
  1202. if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
  1203. cpa_flush_array(addr, numpages, cache,
  1204. cpa.flags, pages);
  1205. } else
  1206. cpa_flush_range(baddr, numpages, cache);
  1207. } else
  1208. cpa_flush_all(cache);
  1209. out:
  1210. return ret;
  1211. }
  1212. static inline int change_page_attr_set(unsigned long *addr, int numpages,
  1213. pgprot_t mask, int array)
  1214. {
  1215. return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
  1216. (array ? CPA_ARRAY : 0), NULL);
  1217. }
  1218. static inline int change_page_attr_clear(unsigned long *addr, int numpages,
  1219. pgprot_t mask, int array)
  1220. {
  1221. return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
  1222. (array ? CPA_ARRAY : 0), NULL);
  1223. }
  1224. static inline int cpa_set_pages_array(struct page **pages, int numpages,
  1225. pgprot_t mask)
  1226. {
  1227. return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
  1228. CPA_PAGES_ARRAY, pages);
  1229. }
  1230. static inline int cpa_clear_pages_array(struct page **pages, int numpages,
  1231. pgprot_t mask)
  1232. {
  1233. return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
  1234. CPA_PAGES_ARRAY, pages);
  1235. }
  1236. int _set_memory_uc(unsigned long addr, int numpages)
  1237. {
  1238. /*
  1239. * for now UC MINUS. see comments in ioremap_nocache()
  1240. */
  1241. return change_page_attr_set(&addr, numpages,
  1242. cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
  1243. 0);
  1244. }
  1245. int set_memory_uc(unsigned long addr, int numpages)
  1246. {
  1247. int ret;
  1248. /*
  1249. * for now UC MINUS. see comments in ioremap_nocache()
  1250. */
  1251. ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
  1252. _PAGE_CACHE_MODE_UC_MINUS, NULL);
  1253. if (ret)
  1254. goto out_err;
  1255. ret = _set_memory_uc(addr, numpages);
  1256. if (ret)
  1257. goto out_free;
  1258. return 0;
  1259. out_free:
  1260. free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
  1261. out_err:
  1262. return ret;
  1263. }
  1264. EXPORT_SYMBOL(set_memory_uc);
  1265. static int _set_memory_array(unsigned long *addr, int addrinarray,
  1266. enum page_cache_mode new_type)
  1267. {
  1268. int i, j;
  1269. int ret;
  1270. /*
  1271. * for now UC MINUS. see comments in ioremap_nocache()
  1272. */
  1273. for (i = 0; i < addrinarray; i++) {
  1274. ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,
  1275. new_type, NULL);
  1276. if (ret)
  1277. goto out_free;
  1278. }
  1279. ret = change_page_attr_set(addr, addrinarray,
  1280. cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
  1281. 1);
  1282. if (!ret && new_type == _PAGE_CACHE_MODE_WC)
  1283. ret = change_page_attr_set_clr(addr, addrinarray,
  1284. cachemode2pgprot(
  1285. _PAGE_CACHE_MODE_WC),
  1286. __pgprot(_PAGE_CACHE_MASK),
  1287. 0, CPA_ARRAY, NULL);
  1288. if (ret)
  1289. goto out_free;
  1290. return 0;
  1291. out_free:
  1292. for (j = 0; j < i; j++)
  1293. free_memtype(__pa(addr[j]), __pa(addr[j]) + PAGE_SIZE);
  1294. return ret;
  1295. }
  1296. int set_memory_array_uc(unsigned long *addr, int addrinarray)
  1297. {
  1298. return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
  1299. }
  1300. EXPORT_SYMBOL(set_memory_array_uc);
  1301. int set_memory_array_wc(unsigned long *addr, int addrinarray)
  1302. {
  1303. return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WC);
  1304. }
  1305. EXPORT_SYMBOL(set_memory_array_wc);
  1306. int _set_memory_wc(unsigned long addr, int numpages)
  1307. {
  1308. int ret;
  1309. unsigned long addr_copy = addr;
  1310. ret = change_page_attr_set(&addr, numpages,
  1311. cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
  1312. 0);
  1313. if (!ret) {
  1314. ret = change_page_attr_set_clr(&addr_copy, numpages,
  1315. cachemode2pgprot(
  1316. _PAGE_CACHE_MODE_WC),
  1317. __pgprot(_PAGE_CACHE_MASK),
  1318. 0, 0, NULL);
  1319. }
  1320. return ret;
  1321. }
  1322. int set_memory_wc(unsigned long addr, int numpages)
  1323. {
  1324. int ret;
  1325. if (!pat_enabled)
  1326. return set_memory_uc(addr, numpages);
  1327. ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
  1328. _PAGE_CACHE_MODE_WC, NULL);
  1329. if (ret)
  1330. goto out_err;
  1331. ret = _set_memory_wc(addr, numpages);
  1332. if (ret)
  1333. goto out_free;
  1334. return 0;
  1335. out_free:
  1336. free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
  1337. out_err:
  1338. return ret;
  1339. }
  1340. EXPORT_SYMBOL(set_memory_wc);
  1341. int _set_memory_wb(unsigned long addr, int numpages)
  1342. {
  1343. /* WB cache mode is hard wired to all cache attribute bits being 0 */
  1344. return change_page_attr_clear(&addr, numpages,
  1345. __pgprot(_PAGE_CACHE_MASK), 0);
  1346. }
  1347. int set_memory_wb(unsigned long addr, int numpages)
  1348. {
  1349. int ret;
  1350. ret = _set_memory_wb(addr, numpages);
  1351. if (ret)
  1352. return ret;
  1353. free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
  1354. return 0;
  1355. }
  1356. EXPORT_SYMBOL(set_memory_wb);
  1357. int set_memory_array_wb(unsigned long *addr, int addrinarray)
  1358. {
  1359. int i;
  1360. int ret;
  1361. /* WB cache mode is hard wired to all cache attribute bits being 0 */
  1362. ret = change_page_attr_clear(addr, addrinarray,
  1363. __pgprot(_PAGE_CACHE_MASK), 1);
  1364. if (ret)
  1365. return ret;
  1366. for (i = 0; i < addrinarray; i++)
  1367. free_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE);
  1368. return 0;
  1369. }
  1370. EXPORT_SYMBOL(set_memory_array_wb);
  1371. int set_memory_x(unsigned long addr, int numpages)
  1372. {
  1373. if (!(__supported_pte_mask & _PAGE_NX))
  1374. return 0;
  1375. return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
  1376. }
  1377. EXPORT_SYMBOL(set_memory_x);
  1378. int set_memory_nx(unsigned long addr, int numpages)
  1379. {
  1380. if (!(__supported_pte_mask & _PAGE_NX))
  1381. return 0;
  1382. return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
  1383. }
  1384. EXPORT_SYMBOL(set_memory_nx);
  1385. int set_memory_ro(unsigned long addr, int numpages)
  1386. {
  1387. return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
  1388. }
  1389. EXPORT_SYMBOL_GPL(set_memory_ro);
  1390. int set_memory_rw(unsigned long addr, int numpages)
  1391. {
  1392. return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
  1393. }
  1394. EXPORT_SYMBOL_GPL(set_memory_rw);
  1395. int set_memory_np(unsigned long addr, int numpages)
  1396. {
  1397. return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
  1398. }
  1399. int set_memory_4k(unsigned long addr, int numpages)
  1400. {
  1401. return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
  1402. __pgprot(0), 1, 0, NULL);
  1403. }
  1404. int set_pages_uc(struct page *page, int numpages)
  1405. {
  1406. unsigned long addr = (unsigned long)page_address(page);
  1407. return set_memory_uc(addr, numpages);
  1408. }
  1409. EXPORT_SYMBOL(set_pages_uc);
  1410. static int _set_pages_array(struct page **pages, int addrinarray,
  1411. enum page_cache_mode new_type)
  1412. {
  1413. unsigned long start;
  1414. unsigned long end;
  1415. int i;
  1416. int free_idx;
  1417. int ret;
  1418. for (i = 0; i < addrinarray; i++) {
  1419. if (PageHighMem(pages[i]))
  1420. continue;
  1421. start = page_to_pfn(pages[i]) << PAGE_SHIFT;
  1422. end = start + PAGE_SIZE;
  1423. if (reserve_memtype(start, end, new_type, NULL))
  1424. goto err_out;
  1425. }
  1426. ret = cpa_set_pages_array(pages, addrinarray,
  1427. cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS));
  1428. if (!ret && new_type == _PAGE_CACHE_MODE_WC)
  1429. ret = change_page_attr_set_clr(NULL, addrinarray,
  1430. cachemode2pgprot(
  1431. _PAGE_CACHE_MODE_WC),
  1432. __pgprot(_PAGE_CACHE_MASK),
  1433. 0, CPA_PAGES_ARRAY, pages);
  1434. if (ret)
  1435. goto err_out;
  1436. return 0; /* Success */
  1437. err_out:
  1438. free_idx = i;
  1439. for (i = 0; i < free_idx; i++) {
  1440. if (PageHighMem(pages[i]))
  1441. continue;
  1442. start = page_to_pfn(pages[i]) << PAGE_SHIFT;
  1443. end = start + PAGE_SIZE;
  1444. free_memtype(start, end);
  1445. }
  1446. return -EINVAL;
  1447. }
  1448. int set_pages_array_uc(struct page **pages, int addrinarray)
  1449. {
  1450. return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
  1451. }
  1452. EXPORT_SYMBOL(set_pages_array_uc);
  1453. int set_pages_array_wc(struct page **pages, int addrinarray)
  1454. {
  1455. return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WC);
  1456. }
  1457. EXPORT_SYMBOL(set_pages_array_wc);
  1458. int set_pages_wb(struct page *page, int numpages)
  1459. {
  1460. unsigned long addr = (unsigned long)page_address(page);
  1461. return set_memory_wb(addr, numpages);
  1462. }
  1463. EXPORT_SYMBOL(set_pages_wb);
  1464. int set_pages_array_wb(struct page **pages, int addrinarray)
  1465. {
  1466. int retval;
  1467. unsigned long start;
  1468. unsigned long end;
  1469. int i;
  1470. /* WB cache mode is hard wired to all cache attribute bits being 0 */
  1471. retval = cpa_clear_pages_array(pages, addrinarray,
  1472. __pgprot(_PAGE_CACHE_MASK));
  1473. if (retval)
  1474. return retval;
  1475. for (i = 0; i < addrinarray; i++) {
  1476. if (PageHighMem(pages[i]))
  1477. continue;
  1478. start = page_to_pfn(pages[i]) << PAGE_SHIFT;
  1479. end = start + PAGE_SIZE;
  1480. free_memtype(start, end);
  1481. }
  1482. return 0;
  1483. }
  1484. EXPORT_SYMBOL(set_pages_array_wb);
  1485. int set_pages_x(struct page *page, int numpages)
  1486. {
  1487. unsigned long addr = (unsigned long)page_address(page);
  1488. return set_memory_x(addr, numpages);
  1489. }
  1490. EXPORT_SYMBOL(set_pages_x);
  1491. int set_pages_nx(struct page *page, int numpages)
  1492. {
  1493. unsigned long addr = (unsigned long)page_address(page);
  1494. return set_memory_nx(addr, numpages);
  1495. }
  1496. EXPORT_SYMBOL(set_pages_nx);
  1497. int set_pages_ro(struct page *page, int numpages)
  1498. {
  1499. unsigned long addr = (unsigned long)page_address(page);
  1500. return set_memory_ro(addr, numpages);
  1501. }
  1502. int set_pages_rw(struct page *page, int numpages)
  1503. {
  1504. unsigned long addr = (unsigned long)page_address(page);
  1505. return set_memory_rw(addr, numpages);
  1506. }
  1507. #ifdef CONFIG_DEBUG_PAGEALLOC
  1508. static int __set_pages_p(struct page *page, int numpages)
  1509. {
  1510. unsigned long tempaddr = (unsigned long) page_address(page);
  1511. struct cpa_data cpa = { .vaddr = &tempaddr,
  1512. .pgd = NULL,
  1513. .numpages = numpages,
  1514. .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
  1515. .mask_clr = __pgprot(0),
  1516. .flags = 0};
  1517. /*
  1518. * No alias checking needed for setting present flag. otherwise,
  1519. * we may need to break large pages for 64-bit kernel text
  1520. * mappings (this adds to complexity if we want to do this from
  1521. * atomic context especially). Let's keep it simple!
  1522. */
  1523. return __change_page_attr_set_clr(&cpa, 0);
  1524. }
  1525. static int __set_pages_np(struct page *page, int numpages)
  1526. {
  1527. unsigned long tempaddr = (unsigned long) page_address(page);
  1528. struct cpa_data cpa = { .vaddr = &tempaddr,
  1529. .pgd = NULL,
  1530. .numpages = numpages,
  1531. .mask_set = __pgprot(0),
  1532. .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
  1533. .flags = 0};
  1534. /*
  1535. * No alias checking needed for setting not present flag. otherwise,
  1536. * we may need to break large pages for 64-bit kernel text
  1537. * mappings (this adds to complexity if we want to do this from
  1538. * atomic context especially). Let's keep it simple!
  1539. */
  1540. return __change_page_attr_set_clr(&cpa, 0);
  1541. }
  1542. void __kernel_map_pages(struct page *page, int numpages, int enable)
  1543. {
  1544. if (PageHighMem(page))
  1545. return;
  1546. if (!enable) {
  1547. debug_check_no_locks_freed(page_address(page),
  1548. numpages * PAGE_SIZE);
  1549. }
  1550. /*
  1551. * The return value is ignored as the calls cannot fail.
  1552. * Large pages for identity mappings are not used at boot time
  1553. * and hence no memory allocations during large page split.
  1554. */
  1555. if (enable)
  1556. __set_pages_p(page, numpages);
  1557. else
  1558. __set_pages_np(page, numpages);
  1559. /*
  1560. * We should perform an IPI and flush all tlbs,
  1561. * but that can deadlock->flush only current cpu:
  1562. */
  1563. __flush_tlb_all();
  1564. arch_flush_lazy_mmu_mode();
  1565. }
  1566. #ifdef CONFIG_HIBERNATION
  1567. bool kernel_page_present(struct page *page)
  1568. {
  1569. unsigned int level;
  1570. pte_t *pte;
  1571. if (PageHighMem(page))
  1572. return false;
  1573. pte = lookup_address((unsigned long)page_address(page), &level);
  1574. return (pte_val(*pte) & _PAGE_PRESENT);
  1575. }
  1576. #endif /* CONFIG_HIBERNATION */
  1577. #endif /* CONFIG_DEBUG_PAGEALLOC */
  1578. int kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address,
  1579. unsigned numpages, unsigned long page_flags)
  1580. {
  1581. int retval = -EINVAL;
  1582. struct cpa_data cpa = {
  1583. .vaddr = &address,
  1584. .pfn = pfn,
  1585. .pgd = pgd,
  1586. .numpages = numpages,
  1587. .mask_set = __pgprot(0),
  1588. .mask_clr = __pgprot(0),
  1589. .flags = 0,
  1590. };
  1591. if (!(__supported_pte_mask & _PAGE_NX))
  1592. goto out;
  1593. if (!(page_flags & _PAGE_NX))
  1594. cpa.mask_clr = __pgprot(_PAGE_NX);
  1595. cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);
  1596. retval = __change_page_attr_set_clr(&cpa, 0);
  1597. __flush_tlb_all();
  1598. out:
  1599. return retval;
  1600. }
  1601. void kernel_unmap_pages_in_pgd(pgd_t *root, unsigned long address,
  1602. unsigned numpages)
  1603. {
  1604. unmap_pgd_range(root, address, address + (numpages << PAGE_SHIFT));
  1605. }
  1606. /*
  1607. * The testcases use internal knowledge of the implementation that shouldn't
  1608. * be exposed to the rest of the kernel. Include these directly here.
  1609. */
  1610. #ifdef CONFIG_CPA_DEBUG
  1611. #include "pageattr-test.c"
  1612. #endif