init.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803
  1. /*
  2. * linux/arch/arm/mm/init.c
  3. *
  4. * Copyright (C) 1995-2005 Russell King
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License version 2 as
  8. * published by the Free Software Foundation.
  9. */
  10. #include <linux/kernel.h>
  11. #include <linux/errno.h>
  12. #include <linux/swap.h>
  13. #include <linux/init.h>
  14. #include <linux/bootmem.h>
  15. #include <linux/mman.h>
  16. #include <linux/sched/signal.h>
  17. #include <linux/sched/task.h>
  18. #include <linux/export.h>
  19. #include <linux/nodemask.h>
  20. #include <linux/initrd.h>
  21. #include <linux/of_fdt.h>
  22. #include <linux/highmem.h>
  23. #include <linux/gfp.h>
  24. #include <linux/memblock.h>
  25. #include <linux/dma-contiguous.h>
  26. #include <linux/sizes.h>
  27. #include <linux/stop_machine.h>
  28. #include <asm/cp15.h>
  29. #include <asm/mach-types.h>
  30. #include <asm/memblock.h>
  31. #include <asm/memory.h>
  32. #include <asm/prom.h>
  33. #include <asm/sections.h>
  34. #include <asm/setup.h>
  35. #include <asm/system_info.h>
  36. #include <asm/tlb.h>
  37. #include <asm/fixmap.h>
  38. #include <asm/ptdump.h>
  39. #include <asm/mach/arch.h>
  40. #include <asm/mach/map.h>
  41. #include "mm.h"
  42. #ifdef CONFIG_CPU_CP15_MMU
  43. unsigned long __init __clear_cr(unsigned long mask)
  44. {
  45. cr_alignment = cr_alignment & ~mask;
  46. return cr_alignment;
  47. }
  48. #endif
  49. static phys_addr_t phys_initrd_start __initdata = 0;
  50. static unsigned long phys_initrd_size __initdata = 0;
  51. static int __init early_initrd(char *p)
  52. {
  53. phys_addr_t start;
  54. unsigned long size;
  55. char *endp;
  56. start = memparse(p, &endp);
  57. if (*endp == ',') {
  58. size = memparse(endp + 1, NULL);
  59. phys_initrd_start = start;
  60. phys_initrd_size = size;
  61. }
  62. return 0;
  63. }
  64. early_param("initrd", early_initrd);
  65. static int __init parse_tag_initrd(const struct tag *tag)
  66. {
  67. pr_warn("ATAG_INITRD is deprecated; "
  68. "please update your bootloader.\n");
  69. phys_initrd_start = __virt_to_phys(tag->u.initrd.start);
  70. phys_initrd_size = tag->u.initrd.size;
  71. return 0;
  72. }
  73. __tagtable(ATAG_INITRD, parse_tag_initrd);
  74. static int __init parse_tag_initrd2(const struct tag *tag)
  75. {
  76. phys_initrd_start = tag->u.initrd.start;
  77. phys_initrd_size = tag->u.initrd.size;
  78. return 0;
  79. }
  80. __tagtable(ATAG_INITRD2, parse_tag_initrd2);
  81. static void __init find_limits(unsigned long *min, unsigned long *max_low,
  82. unsigned long *max_high)
  83. {
  84. *max_low = PFN_DOWN(memblock_get_current_limit());
  85. *min = PFN_UP(memblock_start_of_DRAM());
  86. *max_high = PFN_DOWN(memblock_end_of_DRAM());
  87. }
  88. #ifdef CONFIG_ZONE_DMA
  89. phys_addr_t arm_dma_zone_size __read_mostly;
  90. EXPORT_SYMBOL(arm_dma_zone_size);
  91. /*
  92. * The DMA mask corresponding to the maximum bus address allocatable
  93. * using GFP_DMA. The default here places no restriction on DMA
  94. * allocations. This must be the smallest DMA mask in the system,
  95. * so a successful GFP_DMA allocation will always satisfy this.
  96. */
  97. phys_addr_t arm_dma_limit;
  98. unsigned long arm_dma_pfn_limit;
  99. static void __init arm_adjust_dma_zone(unsigned long *size, unsigned long *hole,
  100. unsigned long dma_size)
  101. {
  102. if (size[0] <= dma_size)
  103. return;
  104. size[ZONE_NORMAL] = size[0] - dma_size;
  105. size[ZONE_DMA] = dma_size;
  106. hole[ZONE_NORMAL] = hole[0];
  107. hole[ZONE_DMA] = 0;
  108. }
  109. #endif
  110. void __init setup_dma_zone(const struct machine_desc *mdesc)
  111. {
  112. #ifdef CONFIG_ZONE_DMA
  113. if (mdesc->dma_zone_size) {
  114. arm_dma_zone_size = mdesc->dma_zone_size;
  115. arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1;
  116. } else
  117. arm_dma_limit = 0xffffffff;
  118. arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT;
  119. #endif
  120. }
  121. static void __init zone_sizes_init(unsigned long min, unsigned long max_low,
  122. unsigned long max_high)
  123. {
  124. unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
  125. struct memblock_region *reg;
  126. /*
  127. * initialise the zones.
  128. */
  129. memset(zone_size, 0, sizeof(zone_size));
  130. /*
  131. * The memory size has already been determined. If we need
  132. * to do anything fancy with the allocation of this memory
  133. * to the zones, now is the time to do it.
  134. */
  135. zone_size[0] = max_low - min;
  136. #ifdef CONFIG_HIGHMEM
  137. zone_size[ZONE_HIGHMEM] = max_high - max_low;
  138. #endif
  139. /*
  140. * Calculate the size of the holes.
  141. * holes = node_size - sum(bank_sizes)
  142. */
  143. memcpy(zhole_size, zone_size, sizeof(zhole_size));
  144. for_each_memblock(memory, reg) {
  145. unsigned long start = memblock_region_memory_base_pfn(reg);
  146. unsigned long end = memblock_region_memory_end_pfn(reg);
  147. if (start < max_low) {
  148. unsigned long low_end = min(end, max_low);
  149. zhole_size[0] -= low_end - start;
  150. }
  151. #ifdef CONFIG_HIGHMEM
  152. if (end > max_low) {
  153. unsigned long high_start = max(start, max_low);
  154. zhole_size[ZONE_HIGHMEM] -= end - high_start;
  155. }
  156. #endif
  157. }
  158. #ifdef CONFIG_ZONE_DMA
  159. /*
  160. * Adjust the sizes according to any special requirements for
  161. * this machine type.
  162. */
  163. if (arm_dma_zone_size)
  164. arm_adjust_dma_zone(zone_size, zhole_size,
  165. arm_dma_zone_size >> PAGE_SHIFT);
  166. #endif
  167. free_area_init_node(0, zone_size, min, zhole_size);
  168. }
  169. #ifdef CONFIG_HAVE_ARCH_PFN_VALID
  170. int pfn_valid(unsigned long pfn)
  171. {
  172. return memblock_is_map_memory(__pfn_to_phys(pfn));
  173. }
  174. EXPORT_SYMBOL(pfn_valid);
  175. #endif
  176. #ifndef CONFIG_SPARSEMEM
  177. static void __init arm_memory_present(void)
  178. {
  179. }
  180. #else
  181. static void __init arm_memory_present(void)
  182. {
  183. struct memblock_region *reg;
  184. for_each_memblock(memory, reg)
  185. memory_present(0, memblock_region_memory_base_pfn(reg),
  186. memblock_region_memory_end_pfn(reg));
  187. }
  188. #endif
  189. static bool arm_memblock_steal_permitted = true;
  190. phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align)
  191. {
  192. phys_addr_t phys;
  193. BUG_ON(!arm_memblock_steal_permitted);
  194. phys = memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ANYWHERE);
  195. memblock_free(phys, size);
  196. memblock_remove(phys, size);
  197. return phys;
  198. }
  199. static void __init arm_initrd_init(void)
  200. {
  201. #ifdef CONFIG_BLK_DEV_INITRD
  202. phys_addr_t start;
  203. unsigned long size;
  204. /* FDT scan will populate initrd_start */
  205. if (initrd_start && !phys_initrd_size) {
  206. phys_initrd_start = __virt_to_phys(initrd_start);
  207. phys_initrd_size = initrd_end - initrd_start;
  208. }
  209. initrd_start = initrd_end = 0;
  210. if (!phys_initrd_size)
  211. return;
  212. /*
  213. * Round the memory region to page boundaries as per free_initrd_mem()
  214. * This allows us to detect whether the pages overlapping the initrd
  215. * are in use, but more importantly, reserves the entire set of pages
  216. * as we don't want these pages allocated for other purposes.
  217. */
  218. start = round_down(phys_initrd_start, PAGE_SIZE);
  219. size = phys_initrd_size + (phys_initrd_start - start);
  220. size = round_up(size, PAGE_SIZE);
  221. if (!memblock_is_region_memory(start, size)) {
  222. pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n",
  223. (u64)start, size);
  224. return;
  225. }
  226. if (memblock_is_region_reserved(start, size)) {
  227. pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n",
  228. (u64)start, size);
  229. return;
  230. }
  231. memblock_reserve(start, size);
  232. /* Now convert initrd to virtual addresses */
  233. initrd_start = __phys_to_virt(phys_initrd_start);
  234. initrd_end = initrd_start + phys_initrd_size;
  235. #endif
  236. }
  237. void __init arm_memblock_init(const struct machine_desc *mdesc)
  238. {
  239. /* Register the kernel text, kernel data and initrd with memblock. */
  240. memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START);
  241. arm_initrd_init();
  242. arm_mm_memblock_reserve();
  243. /* reserve any platform specific memblock areas */
  244. if (mdesc->reserve)
  245. mdesc->reserve();
  246. early_init_fdt_reserve_self();
  247. early_init_fdt_scan_reserved_mem();
  248. /* reserve memory for DMA contiguous allocations */
  249. dma_contiguous_reserve(arm_dma_limit);
  250. arm_memblock_steal_permitted = false;
  251. memblock_dump_all();
  252. }
  253. void __init bootmem_init(void)
  254. {
  255. unsigned long min, max_low, max_high;
  256. memblock_allow_resize();
  257. max_low = max_high = 0;
  258. find_limits(&min, &max_low, &max_high);
  259. early_memtest((phys_addr_t)min << PAGE_SHIFT,
  260. (phys_addr_t)max_low << PAGE_SHIFT);
  261. /*
  262. * Sparsemem tries to allocate bootmem in memory_present(),
  263. * so must be done after the fixed reservations
  264. */
  265. arm_memory_present();
  266. /*
  267. * sparse_init() needs the bootmem allocator up and running.
  268. */
  269. sparse_init();
  270. /*
  271. * Now free the memory - free_area_init_node needs
  272. * the sparse mem_map arrays initialized by sparse_init()
  273. * for memmap_init_zone(), otherwise all PFNs are invalid.
  274. */
  275. zone_sizes_init(min, max_low, max_high);
  276. /*
  277. * This doesn't seem to be used by the Linux memory manager any
  278. * more, but is used by ll_rw_block. If we can get rid of it, we
  279. * also get rid of some of the stuff above as well.
  280. */
  281. min_low_pfn = min;
  282. max_low_pfn = max_low;
  283. max_pfn = max_high;
  284. }
  285. /*
  286. * Poison init memory with an undefined instruction (ARM) or a branch to an
  287. * undefined instruction (Thumb).
  288. */
  289. static inline void poison_init_mem(void *s, size_t count)
  290. {
  291. u32 *p = (u32 *)s;
  292. for (; count != 0; count -= 4)
  293. *p++ = 0xe7fddef0;
  294. }
  295. static inline void
  296. free_memmap(unsigned long start_pfn, unsigned long end_pfn)
  297. {
  298. struct page *start_pg, *end_pg;
  299. phys_addr_t pg, pgend;
  300. /*
  301. * Convert start_pfn/end_pfn to a struct page pointer.
  302. */
  303. start_pg = pfn_to_page(start_pfn - 1) + 1;
  304. end_pg = pfn_to_page(end_pfn - 1) + 1;
  305. /*
  306. * Convert to physical addresses, and
  307. * round start upwards and end downwards.
  308. */
  309. pg = PAGE_ALIGN(__pa(start_pg));
  310. pgend = __pa(end_pg) & PAGE_MASK;
  311. /*
  312. * If there are free pages between these,
  313. * free the section of the memmap array.
  314. */
  315. if (pg < pgend)
  316. memblock_free_early(pg, pgend - pg);
  317. }
  318. /*
  319. * The mem_map array can get very big. Free the unused area of the memory map.
  320. */
  321. static void __init free_unused_memmap(void)
  322. {
  323. unsigned long start, prev_end = 0;
  324. struct memblock_region *reg;
  325. /*
  326. * This relies on each bank being in address order.
  327. * The banks are sorted previously in bootmem_init().
  328. */
  329. for_each_memblock(memory, reg) {
  330. start = memblock_region_memory_base_pfn(reg);
  331. #ifdef CONFIG_SPARSEMEM
  332. /*
  333. * Take care not to free memmap entries that don't exist
  334. * due to SPARSEMEM sections which aren't present.
  335. */
  336. start = min(start,
  337. ALIGN(prev_end, PAGES_PER_SECTION));
  338. #else
  339. /*
  340. * Align down here since the VM subsystem insists that the
  341. * memmap entries are valid from the bank start aligned to
  342. * MAX_ORDER_NR_PAGES.
  343. */
  344. start = round_down(start, MAX_ORDER_NR_PAGES);
  345. #endif
  346. /*
  347. * If we had a previous bank, and there is a space
  348. * between the current bank and the previous, free it.
  349. */
  350. if (prev_end && prev_end < start)
  351. free_memmap(prev_end, start);
  352. /*
  353. * Align up here since the VM subsystem insists that the
  354. * memmap entries are valid from the bank end aligned to
  355. * MAX_ORDER_NR_PAGES.
  356. */
  357. prev_end = ALIGN(memblock_region_memory_end_pfn(reg),
  358. MAX_ORDER_NR_PAGES);
  359. }
  360. #ifdef CONFIG_SPARSEMEM
  361. if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
  362. free_memmap(prev_end,
  363. ALIGN(prev_end, PAGES_PER_SECTION));
  364. #endif
  365. }
  366. #ifdef CONFIG_HIGHMEM
  367. static inline void free_area_high(unsigned long pfn, unsigned long end)
  368. {
  369. for (; pfn < end; pfn++)
  370. free_highmem_page(pfn_to_page(pfn));
  371. }
  372. #endif
  373. static void __init free_highpages(void)
  374. {
  375. #ifdef CONFIG_HIGHMEM
  376. unsigned long max_low = max_low_pfn;
  377. struct memblock_region *mem, *res;
  378. /* set highmem page free */
  379. for_each_memblock(memory, mem) {
  380. unsigned long start = memblock_region_memory_base_pfn(mem);
  381. unsigned long end = memblock_region_memory_end_pfn(mem);
  382. /* Ignore complete lowmem entries */
  383. if (end <= max_low)
  384. continue;
  385. if (memblock_is_nomap(mem))
  386. continue;
  387. /* Truncate partial highmem entries */
  388. if (start < max_low)
  389. start = max_low;
  390. /* Find and exclude any reserved regions */
  391. for_each_memblock(reserved, res) {
  392. unsigned long res_start, res_end;
  393. res_start = memblock_region_reserved_base_pfn(res);
  394. res_end = memblock_region_reserved_end_pfn(res);
  395. if (res_end < start)
  396. continue;
  397. if (res_start < start)
  398. res_start = start;
  399. if (res_start > end)
  400. res_start = end;
  401. if (res_end > end)
  402. res_end = end;
  403. if (res_start != start)
  404. free_area_high(start, res_start);
  405. start = res_end;
  406. if (start == end)
  407. break;
  408. }
  409. /* And now free anything which remains */
  410. if (start < end)
  411. free_area_high(start, end);
  412. }
  413. #endif
  414. }
  415. /*
  416. * mem_init() marks the free areas in the mem_map and tells us how much
  417. * memory is free. This is done after various parts of the system have
  418. * claimed their memory after the kernel image.
  419. */
  420. void __init mem_init(void)
  421. {
  422. #ifdef CONFIG_HAVE_TCM
  423. /* These pointers are filled in on TCM detection */
  424. extern u32 dtcm_end;
  425. extern u32 itcm_end;
  426. #endif
  427. set_max_mapnr(pfn_to_page(max_pfn) - mem_map);
  428. /* this will put all unused low memory onto the freelists */
  429. free_unused_memmap();
  430. free_all_bootmem();
  431. #ifdef CONFIG_SA1111
  432. /* now that our DMA memory is actually so designated, we can free it */
  433. free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL);
  434. #endif
  435. free_highpages();
  436. mem_init_print_info(NULL);
  437. #define MLK(b, t) b, t, ((t) - (b)) >> 10
  438. #define MLM(b, t) b, t, ((t) - (b)) >> 20
  439. #define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
  440. pr_notice("Virtual kernel memory layout:\n"
  441. " vector : 0x%08lx - 0x%08lx (%4ld kB)\n"
  442. #ifdef CONFIG_HAVE_TCM
  443. " DTCM : 0x%08lx - 0x%08lx (%4ld kB)\n"
  444. " ITCM : 0x%08lx - 0x%08lx (%4ld kB)\n"
  445. #endif
  446. " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
  447. " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n"
  448. " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n"
  449. #ifdef CONFIG_HIGHMEM
  450. " pkmap : 0x%08lx - 0x%08lx (%4ld MB)\n"
  451. #endif
  452. #ifdef CONFIG_MODULES
  453. " modules : 0x%08lx - 0x%08lx (%4ld MB)\n"
  454. #endif
  455. " .text : 0x%p" " - 0x%p" " (%4td kB)\n"
  456. " .init : 0x%p" " - 0x%p" " (%4td kB)\n"
  457. " .data : 0x%p" " - 0x%p" " (%4td kB)\n"
  458. " .bss : 0x%p" " - 0x%p" " (%4td kB)\n",
  459. MLK(VECTORS_BASE, VECTORS_BASE + PAGE_SIZE),
  460. #ifdef CONFIG_HAVE_TCM
  461. MLK(DTCM_OFFSET, (unsigned long) dtcm_end),
  462. MLK(ITCM_OFFSET, (unsigned long) itcm_end),
  463. #endif
  464. MLK(FIXADDR_START, FIXADDR_END),
  465. MLM(VMALLOC_START, VMALLOC_END),
  466. MLM(PAGE_OFFSET, (unsigned long)high_memory),
  467. #ifdef CONFIG_HIGHMEM
  468. MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) *
  469. (PAGE_SIZE)),
  470. #endif
  471. #ifdef CONFIG_MODULES
  472. MLM(MODULES_VADDR, MODULES_END),
  473. #endif
  474. MLK_ROUNDUP(_text, _etext),
  475. MLK_ROUNDUP(__init_begin, __init_end),
  476. MLK_ROUNDUP(_sdata, _edata),
  477. MLK_ROUNDUP(__bss_start, __bss_stop));
  478. #undef MLK
  479. #undef MLM
  480. #undef MLK_ROUNDUP
  481. /*
  482. * Check boundaries twice: Some fundamental inconsistencies can
  483. * be detected at build time already.
  484. */
  485. #ifdef CONFIG_MMU
  486. BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR);
  487. BUG_ON(TASK_SIZE > MODULES_VADDR);
  488. #endif
  489. #ifdef CONFIG_HIGHMEM
  490. BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
  491. BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
  492. #endif
  493. }
  494. #ifdef CONFIG_STRICT_KERNEL_RWX
  495. struct section_perm {
  496. const char *name;
  497. unsigned long start;
  498. unsigned long end;
  499. pmdval_t mask;
  500. pmdval_t prot;
  501. pmdval_t clear;
  502. };
  503. /* First section-aligned location at or after __start_rodata. */
  504. extern char __start_rodata_section_aligned[];
  505. static struct section_perm nx_perms[] = {
  506. /* Make pages tables, etc before _stext RW (set NX). */
  507. {
  508. .name = "pre-text NX",
  509. .start = PAGE_OFFSET,
  510. .end = (unsigned long)_stext,
  511. .mask = ~PMD_SECT_XN,
  512. .prot = PMD_SECT_XN,
  513. },
  514. /* Make init RW (set NX). */
  515. {
  516. .name = "init NX",
  517. .start = (unsigned long)__init_begin,
  518. .end = (unsigned long)_sdata,
  519. .mask = ~PMD_SECT_XN,
  520. .prot = PMD_SECT_XN,
  521. },
  522. /* Make rodata NX (set RO in ro_perms below). */
  523. {
  524. .name = "rodata NX",
  525. .start = (unsigned long)__start_rodata_section_aligned,
  526. .end = (unsigned long)__init_begin,
  527. .mask = ~PMD_SECT_XN,
  528. .prot = PMD_SECT_XN,
  529. },
  530. };
  531. static struct section_perm ro_perms[] = {
  532. /* Make kernel code and rodata RX (set RO). */
  533. {
  534. .name = "text/rodata RO",
  535. .start = (unsigned long)_stext,
  536. .end = (unsigned long)__init_begin,
  537. #ifdef CONFIG_ARM_LPAE
  538. .mask = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2),
  539. .prot = L_PMD_SECT_RDONLY | PMD_SECT_AP2,
  540. #else
  541. .mask = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE),
  542. .prot = PMD_SECT_APX | PMD_SECT_AP_WRITE,
  543. .clear = PMD_SECT_AP_WRITE,
  544. #endif
  545. },
  546. };
  547. /*
  548. * Updates section permissions only for the current mm (sections are
  549. * copied into each mm). During startup, this is the init_mm. Is only
  550. * safe to be called with preemption disabled, as under stop_machine().
  551. */
  552. static inline void section_update(unsigned long addr, pmdval_t mask,
  553. pmdval_t prot, struct mm_struct *mm)
  554. {
  555. pmd_t *pmd;
  556. pmd = pmd_offset(pud_offset(pgd_offset(mm, addr), addr), addr);
  557. #ifdef CONFIG_ARM_LPAE
  558. pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
  559. #else
  560. if (addr & SECTION_SIZE)
  561. pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot);
  562. else
  563. pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
  564. #endif
  565. flush_pmd_entry(pmd);
  566. local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE);
  567. }
  568. /* Make sure extended page tables are in use. */
  569. static inline bool arch_has_strict_perms(void)
  570. {
  571. if (cpu_architecture() < CPU_ARCH_ARMv6)
  572. return false;
  573. return !!(get_cr() & CR_XP);
  574. }
  575. void set_section_perms(struct section_perm *perms, int n, bool set,
  576. struct mm_struct *mm)
  577. {
  578. size_t i;
  579. unsigned long addr;
  580. if (!arch_has_strict_perms())
  581. return;
  582. for (i = 0; i < n; i++) {
  583. if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) ||
  584. !IS_ALIGNED(perms[i].end, SECTION_SIZE)) {
  585. pr_err("BUG: %s section %lx-%lx not aligned to %lx\n",
  586. perms[i].name, perms[i].start, perms[i].end,
  587. SECTION_SIZE);
  588. continue;
  589. }
  590. for (addr = perms[i].start;
  591. addr < perms[i].end;
  592. addr += SECTION_SIZE)
  593. section_update(addr, perms[i].mask,
  594. set ? perms[i].prot : perms[i].clear, mm);
  595. }
  596. }
  597. /**
  598. * update_sections_early intended to be called only through stop_machine
  599. * framework and executed by only one CPU while all other CPUs will spin and
  600. * wait, so no locking is required in this function.
  601. */
  602. static void update_sections_early(struct section_perm perms[], int n)
  603. {
  604. struct task_struct *t, *s;
  605. for_each_process(t) {
  606. if (t->flags & PF_KTHREAD)
  607. continue;
  608. for_each_thread(t, s)
  609. set_section_perms(perms, n, true, s->mm);
  610. }
  611. set_section_perms(perms, n, true, current->active_mm);
  612. set_section_perms(perms, n, true, &init_mm);
  613. }
  614. static int __fix_kernmem_perms(void *unused)
  615. {
  616. update_sections_early(nx_perms, ARRAY_SIZE(nx_perms));
  617. return 0;
  618. }
  619. static void fix_kernmem_perms(void)
  620. {
  621. stop_machine(__fix_kernmem_perms, NULL, NULL);
  622. }
  623. static int __mark_rodata_ro(void *unused)
  624. {
  625. update_sections_early(ro_perms, ARRAY_SIZE(ro_perms));
  626. return 0;
  627. }
  628. static int kernel_set_to_readonly __read_mostly;
  629. void mark_rodata_ro(void)
  630. {
  631. kernel_set_to_readonly = 1;
  632. stop_machine(__mark_rodata_ro, NULL, NULL);
  633. debug_checkwx();
  634. }
  635. void set_kernel_text_rw(void)
  636. {
  637. if (!kernel_set_to_readonly)
  638. return;
  639. set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), false,
  640. current->active_mm);
  641. }
  642. void set_kernel_text_ro(void)
  643. {
  644. if (!kernel_set_to_readonly)
  645. return;
  646. set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true,
  647. current->active_mm);
  648. }
  649. #else
  650. static inline void fix_kernmem_perms(void) { }
  651. #endif /* CONFIG_STRICT_KERNEL_RWX */
  652. void free_initmem(void)
  653. {
  654. fix_kernmem_perms();
  655. poison_init_mem(__init_begin, __init_end - __init_begin);
  656. if (!machine_is_integrator() && !machine_is_cintegrator())
  657. free_initmem_default(-1);
  658. }
  659. #ifdef CONFIG_BLK_DEV_INITRD
  660. static int keep_initrd;
  661. void free_initrd_mem(unsigned long start, unsigned long end)
  662. {
  663. if (!keep_initrd) {
  664. if (start == initrd_start)
  665. start = round_down(start, PAGE_SIZE);
  666. if (end == initrd_end)
  667. end = round_up(end, PAGE_SIZE);
  668. poison_init_mem((void *)start, PAGE_ALIGN(end) - start);
  669. free_reserved_area((void *)start, (void *)end, -1, "initrd");
  670. }
  671. }
  672. static int __init keepinitrd_setup(char *__unused)
  673. {
  674. keep_initrd = 1;
  675. return 1;
  676. }
  677. __setup("keepinitrd", keepinitrd_setup);
  678. #endif