pat.c 28 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136
  1. /*
  2. * Handle caching attributes in page tables (PAT)
  3. *
  4. * Authors: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
  5. * Suresh B Siddha <suresh.b.siddha@intel.com>
  6. *
  7. * Loosely based on earlier PAT patchset from Eric Biederman and Andi Kleen.
  8. */
  9. #include <linux/seq_file.h>
  10. #include <linux/bootmem.h>
  11. #include <linux/debugfs.h>
  12. #include <linux/ioport.h>
  13. #include <linux/kernel.h>
  14. #include <linux/pfn_t.h>
  15. #include <linux/slab.h>
  16. #include <linux/mm.h>
  17. #include <linux/fs.h>
  18. #include <linux/rbtree.h>
  19. #include <asm/cacheflush.h>
  20. #include <asm/processor.h>
  21. #include <asm/tlbflush.h>
  22. #include <asm/x86_init.h>
  23. #include <asm/pgtable.h>
  24. #include <asm/fcntl.h>
  25. #include <asm/e820/api.h>
  26. #include <asm/mtrr.h>
  27. #include <asm/page.h>
  28. #include <asm/msr.h>
  29. #include <asm/pat.h>
  30. #include <asm/io.h>
  31. #include "pat_internal.h"
  32. #include "mm_internal.h"
  33. #undef pr_fmt
  34. #define pr_fmt(fmt) "" fmt
  35. static bool __read_mostly boot_cpu_done;
  36. static bool __read_mostly pat_disabled = !IS_ENABLED(CONFIG_X86_PAT);
  37. static bool __read_mostly pat_initialized;
  38. static bool __read_mostly init_cm_done;
  39. void pat_disable(const char *reason)
  40. {
  41. if (pat_disabled)
  42. return;
  43. if (boot_cpu_done) {
  44. WARN_ONCE(1, "x86/PAT: PAT cannot be disabled after initialization\n");
  45. return;
  46. }
  47. pat_disabled = true;
  48. pr_info("x86/PAT: %s\n", reason);
  49. }
  50. static int __init nopat(char *str)
  51. {
  52. pat_disable("PAT support disabled.");
  53. return 0;
  54. }
  55. early_param("nopat", nopat);
  56. bool pat_enabled(void)
  57. {
  58. return pat_initialized;
  59. }
  60. EXPORT_SYMBOL_GPL(pat_enabled);
  61. int pat_debug_enable;
  62. static int __init pat_debug_setup(char *str)
  63. {
  64. pat_debug_enable = 1;
  65. return 0;
  66. }
  67. __setup("debugpat", pat_debug_setup);
  68. #ifdef CONFIG_X86_PAT
  69. /*
  70. * X86 PAT uses page flags arch_1 and uncached together to keep track of
  71. * memory type of pages that have backing page struct.
  72. *
  73. * X86 PAT supports 4 different memory types:
  74. * - _PAGE_CACHE_MODE_WB
  75. * - _PAGE_CACHE_MODE_WC
  76. * - _PAGE_CACHE_MODE_UC_MINUS
  77. * - _PAGE_CACHE_MODE_WT
  78. *
  79. * _PAGE_CACHE_MODE_WB is the default type.
  80. */
  81. #define _PGMT_WB 0
  82. #define _PGMT_WC (1UL << PG_arch_1)
  83. #define _PGMT_UC_MINUS (1UL << PG_uncached)
  84. #define _PGMT_WT (1UL << PG_uncached | 1UL << PG_arch_1)
  85. #define _PGMT_MASK (1UL << PG_uncached | 1UL << PG_arch_1)
  86. #define _PGMT_CLEAR_MASK (~_PGMT_MASK)
  87. static inline enum page_cache_mode get_page_memtype(struct page *pg)
  88. {
  89. unsigned long pg_flags = pg->flags & _PGMT_MASK;
  90. if (pg_flags == _PGMT_WB)
  91. return _PAGE_CACHE_MODE_WB;
  92. else if (pg_flags == _PGMT_WC)
  93. return _PAGE_CACHE_MODE_WC;
  94. else if (pg_flags == _PGMT_UC_MINUS)
  95. return _PAGE_CACHE_MODE_UC_MINUS;
  96. else
  97. return _PAGE_CACHE_MODE_WT;
  98. }
  99. static inline void set_page_memtype(struct page *pg,
  100. enum page_cache_mode memtype)
  101. {
  102. unsigned long memtype_flags;
  103. unsigned long old_flags;
  104. unsigned long new_flags;
  105. switch (memtype) {
  106. case _PAGE_CACHE_MODE_WC:
  107. memtype_flags = _PGMT_WC;
  108. break;
  109. case _PAGE_CACHE_MODE_UC_MINUS:
  110. memtype_flags = _PGMT_UC_MINUS;
  111. break;
  112. case _PAGE_CACHE_MODE_WT:
  113. memtype_flags = _PGMT_WT;
  114. break;
  115. case _PAGE_CACHE_MODE_WB:
  116. default:
  117. memtype_flags = _PGMT_WB;
  118. break;
  119. }
  120. do {
  121. old_flags = pg->flags;
  122. new_flags = (old_flags & _PGMT_CLEAR_MASK) | memtype_flags;
  123. } while (cmpxchg(&pg->flags, old_flags, new_flags) != old_flags);
  124. }
  125. #else
  126. static inline enum page_cache_mode get_page_memtype(struct page *pg)
  127. {
  128. return -1;
  129. }
  130. static inline void set_page_memtype(struct page *pg,
  131. enum page_cache_mode memtype)
  132. {
  133. }
  134. #endif
  135. enum {
  136. PAT_UC = 0, /* uncached */
  137. PAT_WC = 1, /* Write combining */
  138. PAT_WT = 4, /* Write Through */
  139. PAT_WP = 5, /* Write Protected */
  140. PAT_WB = 6, /* Write Back (default) */
  141. PAT_UC_MINUS = 7, /* UC, but can be overridden by MTRR */
  142. };
  143. #define CM(c) (_PAGE_CACHE_MODE_ ## c)
  144. static enum page_cache_mode pat_get_cache_mode(unsigned pat_val, char *msg)
  145. {
  146. enum page_cache_mode cache;
  147. char *cache_mode;
  148. switch (pat_val) {
  149. case PAT_UC: cache = CM(UC); cache_mode = "UC "; break;
  150. case PAT_WC: cache = CM(WC); cache_mode = "WC "; break;
  151. case PAT_WT: cache = CM(WT); cache_mode = "WT "; break;
  152. case PAT_WP: cache = CM(WP); cache_mode = "WP "; break;
  153. case PAT_WB: cache = CM(WB); cache_mode = "WB "; break;
  154. case PAT_UC_MINUS: cache = CM(UC_MINUS); cache_mode = "UC- "; break;
  155. default: cache = CM(WB); cache_mode = "WB "; break;
  156. }
  157. memcpy(msg, cache_mode, 4);
  158. return cache;
  159. }
  160. #undef CM
  161. /*
  162. * Update the cache mode to pgprot translation tables according to PAT
  163. * configuration.
  164. * Using lower indices is preferred, so we start with highest index.
  165. */
  166. static void __init_cache_modes(u64 pat)
  167. {
  168. enum page_cache_mode cache;
  169. char pat_msg[33];
  170. int i;
  171. pat_msg[32] = 0;
  172. for (i = 7; i >= 0; i--) {
  173. cache = pat_get_cache_mode((pat >> (i * 8)) & 7,
  174. pat_msg + 4 * i);
  175. update_cache_mode_entry(i, cache);
  176. }
  177. pr_info("x86/PAT: Configuration [0-7]: %s\n", pat_msg);
  178. init_cm_done = true;
  179. }
  180. #define PAT(x, y) ((u64)PAT_ ## y << ((x)*8))
  181. static void pat_bsp_init(u64 pat)
  182. {
  183. u64 tmp_pat;
  184. if (!boot_cpu_has(X86_FEATURE_PAT)) {
  185. pat_disable("PAT not supported by CPU.");
  186. return;
  187. }
  188. rdmsrl(MSR_IA32_CR_PAT, tmp_pat);
  189. if (!tmp_pat) {
  190. pat_disable("PAT MSR is 0, disabled.");
  191. return;
  192. }
  193. wrmsrl(MSR_IA32_CR_PAT, pat);
  194. pat_initialized = true;
  195. __init_cache_modes(pat);
  196. }
  197. static void pat_ap_init(u64 pat)
  198. {
  199. if (!boot_cpu_has(X86_FEATURE_PAT)) {
  200. /*
  201. * If this happens we are on a secondary CPU, but switched to
  202. * PAT on the boot CPU. We have no way to undo PAT.
  203. */
  204. panic("x86/PAT: PAT enabled, but not supported by secondary CPU\n");
  205. }
  206. wrmsrl(MSR_IA32_CR_PAT, pat);
  207. }
  208. void init_cache_modes(void)
  209. {
  210. u64 pat = 0;
  211. if (init_cm_done)
  212. return;
  213. if (boot_cpu_has(X86_FEATURE_PAT)) {
  214. /*
  215. * CPU supports PAT. Set PAT table to be consistent with
  216. * PAT MSR. This case supports "nopat" boot option, and
  217. * virtual machine environments which support PAT without
  218. * MTRRs. In specific, Xen has unique setup to PAT MSR.
  219. *
  220. * If PAT MSR returns 0, it is considered invalid and emulates
  221. * as No PAT.
  222. */
  223. rdmsrl(MSR_IA32_CR_PAT, pat);
  224. }
  225. if (!pat) {
  226. /*
  227. * No PAT. Emulate the PAT table that corresponds to the two
  228. * cache bits, PWT (Write Through) and PCD (Cache Disable).
  229. * This setup is also the same as the BIOS default setup.
  230. *
  231. * PTE encoding:
  232. *
  233. * PCD
  234. * |PWT PAT
  235. * || slot
  236. * 00 0 WB : _PAGE_CACHE_MODE_WB
  237. * 01 1 WT : _PAGE_CACHE_MODE_WT
  238. * 10 2 UC-: _PAGE_CACHE_MODE_UC_MINUS
  239. * 11 3 UC : _PAGE_CACHE_MODE_UC
  240. *
  241. * NOTE: When WC or WP is used, it is redirected to UC- per
  242. * the default setup in __cachemode2pte_tbl[].
  243. */
  244. pat = PAT(0, WB) | PAT(1, WT) | PAT(2, UC_MINUS) | PAT(3, UC) |
  245. PAT(4, WB) | PAT(5, WT) | PAT(6, UC_MINUS) | PAT(7, UC);
  246. }
  247. __init_cache_modes(pat);
  248. }
  249. /**
  250. * pat_init - Initialize PAT MSR and PAT table
  251. *
  252. * This function initializes PAT MSR and PAT table with an OS-defined value
  253. * to enable additional cache attributes, WC and WT.
  254. *
  255. * This function must be called on all CPUs using the specific sequence of
  256. * operations defined in Intel SDM. mtrr_rendezvous_handler() provides this
  257. * procedure for PAT.
  258. */
  259. void pat_init(void)
  260. {
  261. u64 pat;
  262. struct cpuinfo_x86 *c = &boot_cpu_data;
  263. if (pat_disabled)
  264. return;
  265. if ((c->x86_vendor == X86_VENDOR_INTEL) &&
  266. (((c->x86 == 0x6) && (c->x86_model <= 0xd)) ||
  267. ((c->x86 == 0xf) && (c->x86_model <= 0x6)))) {
  268. /*
  269. * PAT support with the lower four entries. Intel Pentium 2,
  270. * 3, M, and 4 are affected by PAT errata, which makes the
  271. * upper four entries unusable. To be on the safe side, we don't
  272. * use those.
  273. *
  274. * PTE encoding:
  275. * PAT
  276. * |PCD
  277. * ||PWT PAT
  278. * ||| slot
  279. * 000 0 WB : _PAGE_CACHE_MODE_WB
  280. * 001 1 WC : _PAGE_CACHE_MODE_WC
  281. * 010 2 UC-: _PAGE_CACHE_MODE_UC_MINUS
  282. * 011 3 UC : _PAGE_CACHE_MODE_UC
  283. * PAT bit unused
  284. *
  285. * NOTE: When WT or WP is used, it is redirected to UC- per
  286. * the default setup in __cachemode2pte_tbl[].
  287. */
  288. pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
  289. PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, UC);
  290. } else {
  291. /*
  292. * Full PAT support. We put WT in slot 7 to improve
  293. * robustness in the presence of errata that might cause
  294. * the high PAT bit to be ignored. This way, a buggy slot 7
  295. * access will hit slot 3, and slot 3 is UC, so at worst
  296. * we lose performance without causing a correctness issue.
  297. * Pentium 4 erratum N46 is an example for such an erratum,
  298. * although we try not to use PAT at all on affected CPUs.
  299. *
  300. * PTE encoding:
  301. * PAT
  302. * |PCD
  303. * ||PWT PAT
  304. * ||| slot
  305. * 000 0 WB : _PAGE_CACHE_MODE_WB
  306. * 001 1 WC : _PAGE_CACHE_MODE_WC
  307. * 010 2 UC-: _PAGE_CACHE_MODE_UC_MINUS
  308. * 011 3 UC : _PAGE_CACHE_MODE_UC
  309. * 100 4 WB : Reserved
  310. * 101 5 WC : Reserved
  311. * 110 6 UC-: Reserved
  312. * 111 7 WT : _PAGE_CACHE_MODE_WT
  313. *
  314. * The reserved slots are unused, but mapped to their
  315. * corresponding types in the presence of PAT errata.
  316. */
  317. pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
  318. PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, WT);
  319. }
  320. if (!boot_cpu_done) {
  321. pat_bsp_init(pat);
  322. boot_cpu_done = true;
  323. } else {
  324. pat_ap_init(pat);
  325. }
  326. }
  327. #undef PAT
  328. static DEFINE_SPINLOCK(memtype_lock); /* protects memtype accesses */
  329. /*
  330. * Does intersection of PAT memory type and MTRR memory type and returns
  331. * the resulting memory type as PAT understands it.
  332. * (Type in pat and mtrr will not have same value)
  333. * The intersection is based on "Effective Memory Type" tables in IA-32
  334. * SDM vol 3a
  335. */
  336. static unsigned long pat_x_mtrr_type(u64 start, u64 end,
  337. enum page_cache_mode req_type)
  338. {
  339. /*
  340. * Look for MTRR hint to get the effective type in case where PAT
  341. * request is for WB.
  342. */
  343. if (req_type == _PAGE_CACHE_MODE_WB) {
  344. u8 mtrr_type, uniform;
  345. mtrr_type = mtrr_type_lookup(start, end, &uniform);
  346. if (mtrr_type != MTRR_TYPE_WRBACK)
  347. return _PAGE_CACHE_MODE_UC_MINUS;
  348. return _PAGE_CACHE_MODE_WB;
  349. }
  350. return req_type;
  351. }
  352. struct pagerange_state {
  353. unsigned long cur_pfn;
  354. int ram;
  355. int not_ram;
  356. };
  357. static int
  358. pagerange_is_ram_callback(unsigned long initial_pfn, unsigned long total_nr_pages, void *arg)
  359. {
  360. struct pagerange_state *state = arg;
  361. state->not_ram |= initial_pfn > state->cur_pfn;
  362. state->ram |= total_nr_pages > 0;
  363. state->cur_pfn = initial_pfn + total_nr_pages;
  364. return state->ram && state->not_ram;
  365. }
  366. static int pat_pagerange_is_ram(resource_size_t start, resource_size_t end)
  367. {
  368. int ret = 0;
  369. unsigned long start_pfn = start >> PAGE_SHIFT;
  370. unsigned long end_pfn = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
  371. struct pagerange_state state = {start_pfn, 0, 0};
  372. /*
  373. * For legacy reasons, physical address range in the legacy ISA
  374. * region is tracked as non-RAM. This will allow users of
  375. * /dev/mem to map portions of legacy ISA region, even when
  376. * some of those portions are listed(or not even listed) with
  377. * different e820 types(RAM/reserved/..)
  378. */
  379. if (start_pfn < ISA_END_ADDRESS >> PAGE_SHIFT)
  380. start_pfn = ISA_END_ADDRESS >> PAGE_SHIFT;
  381. if (start_pfn < end_pfn) {
  382. ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn,
  383. &state, pagerange_is_ram_callback);
  384. }
  385. return (ret > 0) ? -1 : (state.ram ? 1 : 0);
  386. }
  387. /*
  388. * For RAM pages, we use page flags to mark the pages with appropriate type.
  389. * The page flags are limited to four types, WB (default), WC, WT and UC-.
  390. * WP request fails with -EINVAL, and UC gets redirected to UC-. Setting
  391. * a new memory type is only allowed for a page mapped with the default WB
  392. * type.
  393. *
  394. * Here we do two passes:
  395. * - Find the memtype of all the pages in the range, look for any conflicts.
  396. * - In case of no conflicts, set the new memtype for pages in the range.
  397. */
  398. static int reserve_ram_pages_type(u64 start, u64 end,
  399. enum page_cache_mode req_type,
  400. enum page_cache_mode *new_type)
  401. {
  402. struct page *page;
  403. u64 pfn;
  404. if (req_type == _PAGE_CACHE_MODE_WP) {
  405. if (new_type)
  406. *new_type = _PAGE_CACHE_MODE_UC_MINUS;
  407. return -EINVAL;
  408. }
  409. if (req_type == _PAGE_CACHE_MODE_UC) {
  410. /* We do not support strong UC */
  411. WARN_ON_ONCE(1);
  412. req_type = _PAGE_CACHE_MODE_UC_MINUS;
  413. }
  414. for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
  415. enum page_cache_mode type;
  416. page = pfn_to_page(pfn);
  417. type = get_page_memtype(page);
  418. if (type != _PAGE_CACHE_MODE_WB) {
  419. pr_info("x86/PAT: reserve_ram_pages_type failed [mem %#010Lx-%#010Lx], track 0x%x, req 0x%x\n",
  420. start, end - 1, type, req_type);
  421. if (new_type)
  422. *new_type = type;
  423. return -EBUSY;
  424. }
  425. }
  426. if (new_type)
  427. *new_type = req_type;
  428. for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
  429. page = pfn_to_page(pfn);
  430. set_page_memtype(page, req_type);
  431. }
  432. return 0;
  433. }
  434. static int free_ram_pages_type(u64 start, u64 end)
  435. {
  436. struct page *page;
  437. u64 pfn;
  438. for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
  439. page = pfn_to_page(pfn);
  440. set_page_memtype(page, _PAGE_CACHE_MODE_WB);
  441. }
  442. return 0;
  443. }
  444. /*
  445. * req_type typically has one of the:
  446. * - _PAGE_CACHE_MODE_WB
  447. * - _PAGE_CACHE_MODE_WC
  448. * - _PAGE_CACHE_MODE_UC_MINUS
  449. * - _PAGE_CACHE_MODE_UC
  450. * - _PAGE_CACHE_MODE_WT
  451. *
  452. * If new_type is NULL, function will return an error if it cannot reserve the
  453. * region with req_type. If new_type is non-NULL, function will return
  454. * available type in new_type in case of no error. In case of any error
  455. * it will return a negative return value.
  456. */
  457. int reserve_memtype(u64 start, u64 end, enum page_cache_mode req_type,
  458. enum page_cache_mode *new_type)
  459. {
  460. struct memtype *new;
  461. enum page_cache_mode actual_type;
  462. int is_range_ram;
  463. int err = 0;
  464. BUG_ON(start >= end); /* end is exclusive */
  465. if (!pat_enabled()) {
  466. /* This is identical to page table setting without PAT */
  467. if (new_type)
  468. *new_type = req_type;
  469. return 0;
  470. }
  471. /* Low ISA region is always mapped WB in page table. No need to track */
  472. if (x86_platform.is_untracked_pat_range(start, end)) {
  473. if (new_type)
  474. *new_type = _PAGE_CACHE_MODE_WB;
  475. return 0;
  476. }
  477. /*
  478. * Call mtrr_lookup to get the type hint. This is an
  479. * optimization for /dev/mem mmap'ers into WB memory (BIOS
  480. * tools and ACPI tools). Use WB request for WB memory and use
  481. * UC_MINUS otherwise.
  482. */
  483. actual_type = pat_x_mtrr_type(start, end, req_type);
  484. if (new_type)
  485. *new_type = actual_type;
  486. is_range_ram = pat_pagerange_is_ram(start, end);
  487. if (is_range_ram == 1) {
  488. err = reserve_ram_pages_type(start, end, req_type, new_type);
  489. return err;
  490. } else if (is_range_ram < 0) {
  491. return -EINVAL;
  492. }
  493. new = kzalloc(sizeof(struct memtype), GFP_KERNEL);
  494. if (!new)
  495. return -ENOMEM;
  496. new->start = start;
  497. new->end = end;
  498. new->type = actual_type;
  499. spin_lock(&memtype_lock);
  500. err = rbt_memtype_check_insert(new, new_type);
  501. if (err) {
  502. pr_info("x86/PAT: reserve_memtype failed [mem %#010Lx-%#010Lx], track %s, req %s\n",
  503. start, end - 1,
  504. cattr_name(new->type), cattr_name(req_type));
  505. kfree(new);
  506. spin_unlock(&memtype_lock);
  507. return err;
  508. }
  509. spin_unlock(&memtype_lock);
  510. dprintk("reserve_memtype added [mem %#010Lx-%#010Lx], track %s, req %s, ret %s\n",
  511. start, end - 1, cattr_name(new->type), cattr_name(req_type),
  512. new_type ? cattr_name(*new_type) : "-");
  513. return err;
  514. }
  515. int free_memtype(u64 start, u64 end)
  516. {
  517. int err = -EINVAL;
  518. int is_range_ram;
  519. struct memtype *entry;
  520. if (!pat_enabled())
  521. return 0;
  522. /* Low ISA region is always mapped WB. No need to track */
  523. if (x86_platform.is_untracked_pat_range(start, end))
  524. return 0;
  525. is_range_ram = pat_pagerange_is_ram(start, end);
  526. if (is_range_ram == 1) {
  527. err = free_ram_pages_type(start, end);
  528. return err;
  529. } else if (is_range_ram < 0) {
  530. return -EINVAL;
  531. }
  532. spin_lock(&memtype_lock);
  533. entry = rbt_memtype_erase(start, end);
  534. spin_unlock(&memtype_lock);
  535. if (IS_ERR(entry)) {
  536. pr_info("x86/PAT: %s:%d freeing invalid memtype [mem %#010Lx-%#010Lx]\n",
  537. current->comm, current->pid, start, end - 1);
  538. return -EINVAL;
  539. }
  540. kfree(entry);
  541. dprintk("free_memtype request [mem %#010Lx-%#010Lx]\n", start, end - 1);
  542. return 0;
  543. }
  544. /**
  545. * lookup_memtype - Looksup the memory type for a physical address
  546. * @paddr: physical address of which memory type needs to be looked up
  547. *
  548. * Only to be called when PAT is enabled
  549. *
  550. * Returns _PAGE_CACHE_MODE_WB, _PAGE_CACHE_MODE_WC, _PAGE_CACHE_MODE_UC_MINUS
  551. * or _PAGE_CACHE_MODE_WT.
  552. */
  553. static enum page_cache_mode lookup_memtype(u64 paddr)
  554. {
  555. enum page_cache_mode rettype = _PAGE_CACHE_MODE_WB;
  556. struct memtype *entry;
  557. if (x86_platform.is_untracked_pat_range(paddr, paddr + PAGE_SIZE))
  558. return rettype;
  559. if (pat_pagerange_is_ram(paddr, paddr + PAGE_SIZE)) {
  560. struct page *page;
  561. page = pfn_to_page(paddr >> PAGE_SHIFT);
  562. return get_page_memtype(page);
  563. }
  564. spin_lock(&memtype_lock);
  565. entry = rbt_memtype_lookup(paddr);
  566. if (entry != NULL)
  567. rettype = entry->type;
  568. else
  569. rettype = _PAGE_CACHE_MODE_UC_MINUS;
  570. spin_unlock(&memtype_lock);
  571. return rettype;
  572. }
  573. /**
  574. * io_reserve_memtype - Request a memory type mapping for a region of memory
  575. * @start: start (physical address) of the region
  576. * @end: end (physical address) of the region
  577. * @type: A pointer to memtype, with requested type. On success, requested
  578. * or any other compatible type that was available for the region is returned
  579. *
  580. * On success, returns 0
  581. * On failure, returns non-zero
  582. */
  583. int io_reserve_memtype(resource_size_t start, resource_size_t end,
  584. enum page_cache_mode *type)
  585. {
  586. resource_size_t size = end - start;
  587. enum page_cache_mode req_type = *type;
  588. enum page_cache_mode new_type;
  589. int ret;
  590. WARN_ON_ONCE(iomem_map_sanity_check(start, size));
  591. ret = reserve_memtype(start, end, req_type, &new_type);
  592. if (ret)
  593. goto out_err;
  594. if (!is_new_memtype_allowed(start, size, req_type, new_type))
  595. goto out_free;
  596. if (kernel_map_sync_memtype(start, size, new_type) < 0)
  597. goto out_free;
  598. *type = new_type;
  599. return 0;
  600. out_free:
  601. free_memtype(start, end);
  602. ret = -EBUSY;
  603. out_err:
  604. return ret;
  605. }
  606. /**
  607. * io_free_memtype - Release a memory type mapping for a region of memory
  608. * @start: start (physical address) of the region
  609. * @end: end (physical address) of the region
  610. */
  611. void io_free_memtype(resource_size_t start, resource_size_t end)
  612. {
  613. free_memtype(start, end);
  614. }
  615. int arch_io_reserve_memtype_wc(resource_size_t start, resource_size_t size)
  616. {
  617. enum page_cache_mode type = _PAGE_CACHE_MODE_WC;
  618. return io_reserve_memtype(start, start + size, &type);
  619. }
  620. EXPORT_SYMBOL(arch_io_reserve_memtype_wc);
  621. void arch_io_free_memtype_wc(resource_size_t start, resource_size_t size)
  622. {
  623. io_free_memtype(start, start + size);
  624. }
  625. EXPORT_SYMBOL(arch_io_free_memtype_wc);
  626. pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
  627. unsigned long size, pgprot_t vma_prot)
  628. {
  629. return vma_prot;
  630. }
  631. #ifdef CONFIG_STRICT_DEVMEM
  632. /* This check is done in drivers/char/mem.c in case of STRICT_DEVMEM */
  633. static inline int range_is_allowed(unsigned long pfn, unsigned long size)
  634. {
  635. return 1;
  636. }
  637. #else
  638. /* This check is needed to avoid cache aliasing when PAT is enabled */
  639. static inline int range_is_allowed(unsigned long pfn, unsigned long size)
  640. {
  641. u64 from = ((u64)pfn) << PAGE_SHIFT;
  642. u64 to = from + size;
  643. u64 cursor = from;
  644. if (!pat_enabled())
  645. return 1;
  646. while (cursor < to) {
  647. if (!devmem_is_allowed(pfn))
  648. return 0;
  649. cursor += PAGE_SIZE;
  650. pfn++;
  651. }
  652. return 1;
  653. }
  654. #endif /* CONFIG_STRICT_DEVMEM */
  655. int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
  656. unsigned long size, pgprot_t *vma_prot)
  657. {
  658. enum page_cache_mode pcm = _PAGE_CACHE_MODE_WB;
  659. if (!range_is_allowed(pfn, size))
  660. return 0;
  661. if (file->f_flags & O_DSYNC)
  662. pcm = _PAGE_CACHE_MODE_UC_MINUS;
  663. *vma_prot = __pgprot((pgprot_val(*vma_prot) & ~_PAGE_CACHE_MASK) |
  664. cachemode2protval(pcm));
  665. return 1;
  666. }
  667. /*
  668. * Change the memory type for the physial address range in kernel identity
  669. * mapping space if that range is a part of identity map.
  670. */
  671. int kernel_map_sync_memtype(u64 base, unsigned long size,
  672. enum page_cache_mode pcm)
  673. {
  674. unsigned long id_sz;
  675. if (base > __pa(high_memory-1))
  676. return 0;
  677. /*
  678. * some areas in the middle of the kernel identity range
  679. * are not mapped, like the PCI space.
  680. */
  681. if (!page_is_ram(base >> PAGE_SHIFT))
  682. return 0;
  683. id_sz = (__pa(high_memory-1) <= base + size) ?
  684. __pa(high_memory) - base :
  685. size;
  686. if (ioremap_change_attr((unsigned long)__va(base), id_sz, pcm) < 0) {
  687. pr_info("x86/PAT: %s:%d ioremap_change_attr failed %s for [mem %#010Lx-%#010Lx]\n",
  688. current->comm, current->pid,
  689. cattr_name(pcm),
  690. base, (unsigned long long)(base + size-1));
  691. return -EINVAL;
  692. }
  693. return 0;
  694. }
  695. /*
  696. * Internal interface to reserve a range of physical memory with prot.
  697. * Reserved non RAM regions only and after successful reserve_memtype,
  698. * this func also keeps identity mapping (if any) in sync with this new prot.
  699. */
  700. static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot,
  701. int strict_prot)
  702. {
  703. int is_ram = 0;
  704. int ret;
  705. enum page_cache_mode want_pcm = pgprot2cachemode(*vma_prot);
  706. enum page_cache_mode pcm = want_pcm;
  707. is_ram = pat_pagerange_is_ram(paddr, paddr + size);
  708. /*
  709. * reserve_pfn_range() for RAM pages. We do not refcount to keep
  710. * track of number of mappings of RAM pages. We can assert that
  711. * the type requested matches the type of first page in the range.
  712. */
  713. if (is_ram) {
  714. if (!pat_enabled())
  715. return 0;
  716. pcm = lookup_memtype(paddr);
  717. if (want_pcm != pcm) {
  718. pr_warn("x86/PAT: %s:%d map pfn RAM range req %s for [mem %#010Lx-%#010Lx], got %s\n",
  719. current->comm, current->pid,
  720. cattr_name(want_pcm),
  721. (unsigned long long)paddr,
  722. (unsigned long long)(paddr + size - 1),
  723. cattr_name(pcm));
  724. *vma_prot = __pgprot((pgprot_val(*vma_prot) &
  725. (~_PAGE_CACHE_MASK)) |
  726. cachemode2protval(pcm));
  727. }
  728. return 0;
  729. }
  730. ret = reserve_memtype(paddr, paddr + size, want_pcm, &pcm);
  731. if (ret)
  732. return ret;
  733. if (pcm != want_pcm) {
  734. if (strict_prot ||
  735. !is_new_memtype_allowed(paddr, size, want_pcm, pcm)) {
  736. free_memtype(paddr, paddr + size);
  737. pr_err("x86/PAT: %s:%d map pfn expected mapping type %s for [mem %#010Lx-%#010Lx], got %s\n",
  738. current->comm, current->pid,
  739. cattr_name(want_pcm),
  740. (unsigned long long)paddr,
  741. (unsigned long long)(paddr + size - 1),
  742. cattr_name(pcm));
  743. return -EINVAL;
  744. }
  745. /*
  746. * We allow returning different type than the one requested in
  747. * non strict case.
  748. */
  749. *vma_prot = __pgprot((pgprot_val(*vma_prot) &
  750. (~_PAGE_CACHE_MASK)) |
  751. cachemode2protval(pcm));
  752. }
  753. if (kernel_map_sync_memtype(paddr, size, pcm) < 0) {
  754. free_memtype(paddr, paddr + size);
  755. return -EINVAL;
  756. }
  757. return 0;
  758. }
  759. /*
  760. * Internal interface to free a range of physical memory.
  761. * Frees non RAM regions only.
  762. */
  763. static void free_pfn_range(u64 paddr, unsigned long size)
  764. {
  765. int is_ram;
  766. is_ram = pat_pagerange_is_ram(paddr, paddr + size);
  767. if (is_ram == 0)
  768. free_memtype(paddr, paddr + size);
  769. }
  770. /*
  771. * track_pfn_copy is called when vma that is covering the pfnmap gets
  772. * copied through copy_page_range().
  773. *
  774. * If the vma has a linear pfn mapping for the entire range, we get the prot
  775. * from pte and reserve the entire vma range with single reserve_pfn_range call.
  776. */
  777. int track_pfn_copy(struct vm_area_struct *vma)
  778. {
  779. resource_size_t paddr;
  780. unsigned long prot;
  781. unsigned long vma_size = vma->vm_end - vma->vm_start;
  782. pgprot_t pgprot;
  783. if (vma->vm_flags & VM_PAT) {
  784. /*
  785. * reserve the whole chunk covered by vma. We need the
  786. * starting address and protection from pte.
  787. */
  788. if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) {
  789. WARN_ON_ONCE(1);
  790. return -EINVAL;
  791. }
  792. pgprot = __pgprot(prot);
  793. return reserve_pfn_range(paddr, vma_size, &pgprot, 1);
  794. }
  795. return 0;
  796. }
  797. /*
  798. * prot is passed in as a parameter for the new mapping. If the vma has
  799. * a linear pfn mapping for the entire range, or no vma is provided,
  800. * reserve the entire pfn + size range with single reserve_pfn_range
  801. * call.
  802. */
  803. int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
  804. unsigned long pfn, unsigned long addr, unsigned long size)
  805. {
  806. resource_size_t paddr = (resource_size_t)pfn << PAGE_SHIFT;
  807. enum page_cache_mode pcm;
  808. /* reserve the whole chunk starting from paddr */
  809. if (!vma || (addr == vma->vm_start
  810. && size == (vma->vm_end - vma->vm_start))) {
  811. int ret;
  812. ret = reserve_pfn_range(paddr, size, prot, 0);
  813. if (ret == 0 && vma)
  814. vma->vm_flags |= VM_PAT;
  815. return ret;
  816. }
  817. if (!pat_enabled())
  818. return 0;
  819. /*
  820. * For anything smaller than the vma size we set prot based on the
  821. * lookup.
  822. */
  823. pcm = lookup_memtype(paddr);
  824. /* Check memtype for the remaining pages */
  825. while (size > PAGE_SIZE) {
  826. size -= PAGE_SIZE;
  827. paddr += PAGE_SIZE;
  828. if (pcm != lookup_memtype(paddr))
  829. return -EINVAL;
  830. }
  831. *prot = __pgprot((pgprot_val(*prot) & (~_PAGE_CACHE_MASK)) |
  832. cachemode2protval(pcm));
  833. return 0;
  834. }
  835. void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, pfn_t pfn)
  836. {
  837. enum page_cache_mode pcm;
  838. if (!pat_enabled())
  839. return;
  840. /* Set prot based on lookup */
  841. pcm = lookup_memtype(pfn_t_to_phys(pfn));
  842. *prot = __pgprot((pgprot_val(*prot) & (~_PAGE_CACHE_MASK)) |
  843. cachemode2protval(pcm));
  844. }
  845. /*
  846. * untrack_pfn is called while unmapping a pfnmap for a region.
  847. * untrack can be called for a specific region indicated by pfn and size or
  848. * can be for the entire vma (in which case pfn, size are zero).
  849. */
  850. void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
  851. unsigned long size)
  852. {
  853. resource_size_t paddr;
  854. unsigned long prot;
  855. if (vma && !(vma->vm_flags & VM_PAT))
  856. return;
  857. /* free the chunk starting from pfn or the whole chunk */
  858. paddr = (resource_size_t)pfn << PAGE_SHIFT;
  859. if (!paddr && !size) {
  860. if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) {
  861. WARN_ON_ONCE(1);
  862. return;
  863. }
  864. size = vma->vm_end - vma->vm_start;
  865. }
  866. free_pfn_range(paddr, size);
  867. if (vma)
  868. vma->vm_flags &= ~VM_PAT;
  869. }
  870. /*
  871. * untrack_pfn_moved is called, while mremapping a pfnmap for a new region,
  872. * with the old vma after its pfnmap page table has been removed. The new
  873. * vma has a new pfnmap to the same pfn & cache type with VM_PAT set.
  874. */
  875. void untrack_pfn_moved(struct vm_area_struct *vma)
  876. {
  877. vma->vm_flags &= ~VM_PAT;
  878. }
  879. pgprot_t pgprot_writecombine(pgprot_t prot)
  880. {
  881. return __pgprot(pgprot_val(prot) |
  882. cachemode2protval(_PAGE_CACHE_MODE_WC));
  883. }
  884. EXPORT_SYMBOL_GPL(pgprot_writecombine);
  885. pgprot_t pgprot_writethrough(pgprot_t prot)
  886. {
  887. return __pgprot(pgprot_val(prot) |
  888. cachemode2protval(_PAGE_CACHE_MODE_WT));
  889. }
  890. EXPORT_SYMBOL_GPL(pgprot_writethrough);
  891. #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_X86_PAT)
  892. static struct memtype *memtype_get_idx(loff_t pos)
  893. {
  894. struct memtype *print_entry;
  895. int ret;
  896. print_entry = kzalloc(sizeof(struct memtype), GFP_KERNEL);
  897. if (!print_entry)
  898. return NULL;
  899. spin_lock(&memtype_lock);
  900. ret = rbt_memtype_copy_nth_element(print_entry, pos);
  901. spin_unlock(&memtype_lock);
  902. if (!ret) {
  903. return print_entry;
  904. } else {
  905. kfree(print_entry);
  906. return NULL;
  907. }
  908. }
  909. static void *memtype_seq_start(struct seq_file *seq, loff_t *pos)
  910. {
  911. if (*pos == 0) {
  912. ++*pos;
  913. seq_puts(seq, "PAT memtype list:\n");
  914. }
  915. return memtype_get_idx(*pos);
  916. }
  917. static void *memtype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
  918. {
  919. ++*pos;
  920. return memtype_get_idx(*pos);
  921. }
  922. static void memtype_seq_stop(struct seq_file *seq, void *v)
  923. {
  924. }
  925. static int memtype_seq_show(struct seq_file *seq, void *v)
  926. {
  927. struct memtype *print_entry = (struct memtype *)v;
  928. seq_printf(seq, "%s @ 0x%Lx-0x%Lx\n", cattr_name(print_entry->type),
  929. print_entry->start, print_entry->end);
  930. kfree(print_entry);
  931. return 0;
  932. }
  933. static const struct seq_operations memtype_seq_ops = {
  934. .start = memtype_seq_start,
  935. .next = memtype_seq_next,
  936. .stop = memtype_seq_stop,
  937. .show = memtype_seq_show,
  938. };
  939. static int memtype_seq_open(struct inode *inode, struct file *file)
  940. {
  941. return seq_open(file, &memtype_seq_ops);
  942. }
  943. static const struct file_operations memtype_fops = {
  944. .open = memtype_seq_open,
  945. .read = seq_read,
  946. .llseek = seq_lseek,
  947. .release = seq_release,
  948. };
  949. static int __init pat_memtype_list_init(void)
  950. {
  951. if (pat_enabled()) {
  952. debugfs_create_file("pat_memtype_list", S_IRUSR,
  953. arch_debugfs_dir, NULL, &memtype_fops);
  954. }
  955. return 0;
  956. }
  957. late_initcall(pat_memtype_list_init);
  958. #endif /* CONFIG_DEBUG_FS && CONFIG_X86_PAT */