vmem.c 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426
  1. /*
  2. * Copyright IBM Corp. 2006
  3. * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
  4. */
  5. #include <linux/bootmem.h>
  6. #include <linux/pfn.h>
  7. #include <linux/mm.h>
  8. #include <linux/module.h>
  9. #include <linux/list.h>
  10. #include <linux/hugetlb.h>
  11. #include <linux/slab.h>
  12. #include <asm/pgalloc.h>
  13. #include <asm/pgtable.h>
  14. #include <asm/setup.h>
  15. #include <asm/tlbflush.h>
  16. #include <asm/sections.h>
  17. static DEFINE_MUTEX(vmem_mutex);
  18. struct memory_segment {
  19. struct list_head list;
  20. unsigned long start;
  21. unsigned long size;
  22. };
  23. static LIST_HEAD(mem_segs);
  24. static void __ref *vmem_alloc_pages(unsigned int order)
  25. {
  26. if (slab_is_available())
  27. return (void *)__get_free_pages(GFP_KERNEL, order);
  28. return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
  29. }
  30. static inline pud_t *vmem_pud_alloc(void)
  31. {
  32. pud_t *pud = NULL;
  33. #ifdef CONFIG_64BIT
  34. pud = vmem_alloc_pages(2);
  35. if (!pud)
  36. return NULL;
  37. clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
  38. #endif
  39. return pud;
  40. }
  41. static inline pmd_t *vmem_pmd_alloc(void)
  42. {
  43. pmd_t *pmd = NULL;
  44. #ifdef CONFIG_64BIT
  45. pmd = vmem_alloc_pages(2);
  46. if (!pmd)
  47. return NULL;
  48. clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
  49. #endif
  50. return pmd;
  51. }
  52. static pte_t __ref *vmem_pte_alloc(unsigned long address)
  53. {
  54. pte_t *pte;
  55. if (slab_is_available())
  56. pte = (pte_t *) page_table_alloc(&init_mm, address);
  57. else
  58. pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t));
  59. if (!pte)
  60. return NULL;
  61. clear_table((unsigned long *) pte, _PAGE_INVALID,
  62. PTRS_PER_PTE * sizeof(pte_t));
  63. return pte;
  64. }
  65. /*
  66. * Add a physical memory range to the 1:1 mapping.
  67. */
  68. static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
  69. {
  70. unsigned long end = start + size;
  71. unsigned long address = start;
  72. pgd_t *pg_dir;
  73. pud_t *pu_dir;
  74. pmd_t *pm_dir;
  75. pte_t *pt_dir;
  76. int ret = -ENOMEM;
  77. while (address < end) {
  78. pg_dir = pgd_offset_k(address);
  79. if (pgd_none(*pg_dir)) {
  80. pu_dir = vmem_pud_alloc();
  81. if (!pu_dir)
  82. goto out;
  83. pgd_populate(&init_mm, pg_dir, pu_dir);
  84. }
  85. pu_dir = pud_offset(pg_dir, address);
  86. #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC)
  87. if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address &&
  88. !(address & ~PUD_MASK) && (address + PUD_SIZE <= end)) {
  89. pud_val(*pu_dir) = __pa(address) |
  90. _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE |
  91. (ro ? _REGION_ENTRY_PROTECT : 0);
  92. address += PUD_SIZE;
  93. continue;
  94. }
  95. #endif
  96. if (pud_none(*pu_dir)) {
  97. pm_dir = vmem_pmd_alloc();
  98. if (!pm_dir)
  99. goto out;
  100. pud_populate(&init_mm, pu_dir, pm_dir);
  101. }
  102. pm_dir = pmd_offset(pu_dir, address);
  103. #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC)
  104. if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address &&
  105. !(address & ~PMD_MASK) && (address + PMD_SIZE <= end)) {
  106. pmd_val(*pm_dir) = __pa(address) |
  107. _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE |
  108. _SEGMENT_ENTRY_YOUNG |
  109. (ro ? _SEGMENT_ENTRY_PROTECT : 0);
  110. address += PMD_SIZE;
  111. continue;
  112. }
  113. #endif
  114. if (pmd_none(*pm_dir)) {
  115. pt_dir = vmem_pte_alloc(address);
  116. if (!pt_dir)
  117. goto out;
  118. pmd_populate(&init_mm, pm_dir, pt_dir);
  119. }
  120. pt_dir = pte_offset_kernel(pm_dir, address);
  121. pte_val(*pt_dir) = __pa(address) |
  122. pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
  123. address += PAGE_SIZE;
  124. }
  125. ret = 0;
  126. out:
  127. return ret;
  128. }
  129. /*
  130. * Remove a physical memory range from the 1:1 mapping.
  131. * Currently only invalidates page table entries.
  132. */
  133. static void vmem_remove_range(unsigned long start, unsigned long size)
  134. {
  135. unsigned long end = start + size;
  136. unsigned long address = start;
  137. pgd_t *pg_dir;
  138. pud_t *pu_dir;
  139. pmd_t *pm_dir;
  140. pte_t *pt_dir;
  141. pte_t pte;
  142. pte_val(pte) = _PAGE_INVALID;
  143. while (address < end) {
  144. pg_dir = pgd_offset_k(address);
  145. if (pgd_none(*pg_dir)) {
  146. address += PGDIR_SIZE;
  147. continue;
  148. }
  149. pu_dir = pud_offset(pg_dir, address);
  150. if (pud_none(*pu_dir)) {
  151. address += PUD_SIZE;
  152. continue;
  153. }
  154. if (pud_large(*pu_dir)) {
  155. pud_clear(pu_dir);
  156. address += PUD_SIZE;
  157. continue;
  158. }
  159. pm_dir = pmd_offset(pu_dir, address);
  160. if (pmd_none(*pm_dir)) {
  161. address += PMD_SIZE;
  162. continue;
  163. }
  164. if (pmd_large(*pm_dir)) {
  165. pmd_clear(pm_dir);
  166. address += PMD_SIZE;
  167. continue;
  168. }
  169. pt_dir = pte_offset_kernel(pm_dir, address);
  170. *pt_dir = pte;
  171. address += PAGE_SIZE;
  172. }
  173. flush_tlb_kernel_range(start, end);
  174. }
  175. /*
  176. * Add a backed mem_map array to the virtual mem_map array.
  177. */
  178. int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
  179. {
  180. unsigned long address = start;
  181. pgd_t *pg_dir;
  182. pud_t *pu_dir;
  183. pmd_t *pm_dir;
  184. pte_t *pt_dir;
  185. int ret = -ENOMEM;
  186. for (address = start; address < end;) {
  187. pg_dir = pgd_offset_k(address);
  188. if (pgd_none(*pg_dir)) {
  189. pu_dir = vmem_pud_alloc();
  190. if (!pu_dir)
  191. goto out;
  192. pgd_populate(&init_mm, pg_dir, pu_dir);
  193. }
  194. pu_dir = pud_offset(pg_dir, address);
  195. if (pud_none(*pu_dir)) {
  196. pm_dir = vmem_pmd_alloc();
  197. if (!pm_dir)
  198. goto out;
  199. pud_populate(&init_mm, pu_dir, pm_dir);
  200. }
  201. pm_dir = pmd_offset(pu_dir, address);
  202. if (pmd_none(*pm_dir)) {
  203. #ifdef CONFIG_64BIT
  204. /* Use 1MB frames for vmemmap if available. We always
  205. * use large frames even if they are only partially
  206. * used.
  207. * Otherwise we would have also page tables since
  208. * vmemmap_populate gets called for each section
  209. * separately. */
  210. if (MACHINE_HAS_EDAT1) {
  211. void *new_page;
  212. new_page = vmemmap_alloc_block(PMD_SIZE, node);
  213. if (!new_page)
  214. goto out;
  215. pmd_val(*pm_dir) = __pa(new_page) |
  216. _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE |
  217. _SEGMENT_ENTRY_CO;
  218. address = (address + PMD_SIZE) & PMD_MASK;
  219. continue;
  220. }
  221. #endif
  222. pt_dir = vmem_pte_alloc(address);
  223. if (!pt_dir)
  224. goto out;
  225. pmd_populate(&init_mm, pm_dir, pt_dir);
  226. } else if (pmd_large(*pm_dir)) {
  227. address = (address + PMD_SIZE) & PMD_MASK;
  228. continue;
  229. }
  230. pt_dir = pte_offset_kernel(pm_dir, address);
  231. if (pte_none(*pt_dir)) {
  232. unsigned long new_page;
  233. new_page =__pa(vmem_alloc_pages(0));
  234. if (!new_page)
  235. goto out;
  236. pte_val(*pt_dir) =
  237. __pa(new_page) | pgprot_val(PAGE_KERNEL);
  238. }
  239. address += PAGE_SIZE;
  240. }
  241. memset((void *)start, 0, end - start);
  242. ret = 0;
  243. out:
  244. return ret;
  245. }
  246. void vmemmap_free(unsigned long start, unsigned long end)
  247. {
  248. }
  249. /*
  250. * Add memory segment to the segment list if it doesn't overlap with
  251. * an already present segment.
  252. */
  253. static int insert_memory_segment(struct memory_segment *seg)
  254. {
  255. struct memory_segment *tmp;
  256. if (seg->start + seg->size > VMEM_MAX_PHYS ||
  257. seg->start + seg->size < seg->start)
  258. return -ERANGE;
  259. list_for_each_entry(tmp, &mem_segs, list) {
  260. if (seg->start >= tmp->start + tmp->size)
  261. continue;
  262. if (seg->start + seg->size <= tmp->start)
  263. continue;
  264. return -ENOSPC;
  265. }
  266. list_add(&seg->list, &mem_segs);
  267. return 0;
  268. }
  269. /*
  270. * Remove memory segment from the segment list.
  271. */
  272. static void remove_memory_segment(struct memory_segment *seg)
  273. {
  274. list_del(&seg->list);
  275. }
  276. static void __remove_shared_memory(struct memory_segment *seg)
  277. {
  278. remove_memory_segment(seg);
  279. vmem_remove_range(seg->start, seg->size);
  280. }
  281. int vmem_remove_mapping(unsigned long start, unsigned long size)
  282. {
  283. struct memory_segment *seg;
  284. int ret;
  285. mutex_lock(&vmem_mutex);
  286. ret = -ENOENT;
  287. list_for_each_entry(seg, &mem_segs, list) {
  288. if (seg->start == start && seg->size == size)
  289. break;
  290. }
  291. if (seg->start != start || seg->size != size)
  292. goto out;
  293. ret = 0;
  294. __remove_shared_memory(seg);
  295. kfree(seg);
  296. out:
  297. mutex_unlock(&vmem_mutex);
  298. return ret;
  299. }
  300. int vmem_add_mapping(unsigned long start, unsigned long size)
  301. {
  302. struct memory_segment *seg;
  303. int ret;
  304. mutex_lock(&vmem_mutex);
  305. ret = -ENOMEM;
  306. seg = kzalloc(sizeof(*seg), GFP_KERNEL);
  307. if (!seg)
  308. goto out;
  309. seg->start = start;
  310. seg->size = size;
  311. ret = insert_memory_segment(seg);
  312. if (ret)
  313. goto out_free;
  314. ret = vmem_add_mem(start, size, 0);
  315. if (ret)
  316. goto out_remove;
  317. goto out;
  318. out_remove:
  319. __remove_shared_memory(seg);
  320. out_free:
  321. kfree(seg);
  322. out:
  323. mutex_unlock(&vmem_mutex);
  324. return ret;
  325. }
  326. /*
  327. * map whole physical memory to virtual memory (identity mapping)
  328. * we reserve enough space in the vmalloc area for vmemmap to hotplug
  329. * additional memory segments.
  330. */
  331. void __init vmem_map_init(void)
  332. {
  333. unsigned long ro_start, ro_end;
  334. unsigned long start, end;
  335. int i;
  336. ro_start = PFN_ALIGN((unsigned long)&_stext);
  337. ro_end = (unsigned long)&_eshared & PAGE_MASK;
  338. for (i = 0; i < MEMORY_CHUNKS; i++) {
  339. if (!memory_chunk[i].size)
  340. continue;
  341. start = memory_chunk[i].addr;
  342. end = memory_chunk[i].addr + memory_chunk[i].size;
  343. if (start >= ro_end || end <= ro_start)
  344. vmem_add_mem(start, end - start, 0);
  345. else if (start >= ro_start && end <= ro_end)
  346. vmem_add_mem(start, end - start, 1);
  347. else if (start >= ro_start) {
  348. vmem_add_mem(start, ro_end - start, 1);
  349. vmem_add_mem(ro_end, end - ro_end, 0);
  350. } else if (end < ro_end) {
  351. vmem_add_mem(start, ro_start - start, 0);
  352. vmem_add_mem(ro_start, end - ro_start, 1);
  353. } else {
  354. vmem_add_mem(start, ro_start - start, 0);
  355. vmem_add_mem(ro_start, ro_end - ro_start, 1);
  356. vmem_add_mem(ro_end, end - ro_end, 0);
  357. }
  358. }
  359. }
  360. /*
  361. * Convert memory chunk array to a memory segment list so there is a single
  362. * list that contains both r/w memory and shared memory segments.
  363. */
  364. static int __init vmem_convert_memory_chunk(void)
  365. {
  366. struct memory_segment *seg;
  367. int i;
  368. mutex_lock(&vmem_mutex);
  369. for (i = 0; i < MEMORY_CHUNKS; i++) {
  370. if (!memory_chunk[i].size)
  371. continue;
  372. seg = kzalloc(sizeof(*seg), GFP_KERNEL);
  373. if (!seg)
  374. panic("Out of memory...\n");
  375. seg->start = memory_chunk[i].addr;
  376. seg->size = memory_chunk[i].size;
  377. insert_memory_segment(seg);
  378. }
  379. mutex_unlock(&vmem_mutex);
  380. return 0;
  381. }
  382. core_initcall(vmem_convert_memory_chunk);