hibernate.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493
  1. /*:
  2. * Hibernate support specific for ARM64
  3. *
  4. * Derived from work on ARM hibernation support by:
  5. *
  6. * Ubuntu project, hibernation support for mach-dove
  7. * Copyright (C) 2010 Nokia Corporation (Hiroshi Doyu)
  8. * Copyright (C) 2010 Texas Instruments, Inc. (Teerth Reddy et al.)
  9. * https://lkml.org/lkml/2010/6/18/4
  10. * https://lists.linux-foundation.org/pipermail/linux-pm/2010-June/027422.html
  11. * https://patchwork.kernel.org/patch/96442/
  12. *
  13. * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
  14. *
  15. * License terms: GNU General Public License (GPL) version 2
  16. */
  17. #define pr_fmt(x) "hibernate: " x
  18. #include <linux/kvm_host.h>
  19. #include <linux/mm.h>
  20. #include <linux/notifier.h>
  21. #include <linux/pm.h>
  22. #include <linux/sched.h>
  23. #include <linux/suspend.h>
  24. #include <linux/utsname.h>
  25. #include <linux/version.h>
  26. #include <asm/barrier.h>
  27. #include <asm/cacheflush.h>
  28. #include <asm/irqflags.h>
  29. #include <asm/memory.h>
  30. #include <asm/mmu_context.h>
  31. #include <asm/pgalloc.h>
  32. #include <asm/pgtable.h>
  33. #include <asm/pgtable-hwdef.h>
  34. #include <asm/sections.h>
  35. #include <asm/smp.h>
  36. #include <asm/suspend.h>
  37. #include <asm/virt.h>
  38. /*
  39. * Hibernate core relies on this value being 0 on resume, and marks it
  40. * __nosavedata assuming it will keep the resume kernel's '0' value. This
  41. * doesn't happen with either KASLR.
  42. *
  43. * defined as "__visible int in_suspend __nosavedata" in
  44. * kernel/power/hibernate.c
  45. */
  46. extern int in_suspend;
  47. /* Find a symbols alias in the linear map */
  48. #define LMADDR(x) phys_to_virt(virt_to_phys(x))
  49. /* Do we need to reset el2? */
  50. #define el2_reset_needed() (is_hyp_mode_available() && !is_kernel_in_hyp_mode())
  51. /*
  52. * Start/end of the hibernate exit code, this must be copied to a 'safe'
  53. * location in memory, and executed from there.
  54. */
  55. extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[];
  56. /* temporary el2 vectors in the __hibernate_exit_text section. */
  57. extern char hibernate_el2_vectors[];
  58. /* hyp-stub vectors, used to restore el2 during resume from hibernate. */
  59. extern char __hyp_stub_vectors[];
  60. /*
  61. * Values that may not change over hibernate/resume. We put the build number
  62. * and date in here so that we guarantee not to resume with a different
  63. * kernel.
  64. */
  65. struct arch_hibernate_hdr_invariants {
  66. char uts_version[__NEW_UTS_LEN + 1];
  67. };
  68. /* These values need to be know across a hibernate/restore. */
  69. static struct arch_hibernate_hdr {
  70. struct arch_hibernate_hdr_invariants invariants;
  71. /* These are needed to find the relocated kernel if built with kaslr */
  72. phys_addr_t ttbr1_el1;
  73. void (*reenter_kernel)(void);
  74. /*
  75. * We need to know where the __hyp_stub_vectors are after restore to
  76. * re-configure el2.
  77. */
  78. phys_addr_t __hyp_stub_vectors;
  79. } resume_hdr;
  80. static inline void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i)
  81. {
  82. memset(i, 0, sizeof(*i));
  83. memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version));
  84. }
  85. int pfn_is_nosave(unsigned long pfn)
  86. {
  87. unsigned long nosave_begin_pfn = virt_to_pfn(&__nosave_begin);
  88. unsigned long nosave_end_pfn = virt_to_pfn(&__nosave_end - 1);
  89. return (pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn);
  90. }
  91. void notrace save_processor_state(void)
  92. {
  93. WARN_ON(num_online_cpus() != 1);
  94. }
  95. void notrace restore_processor_state(void)
  96. {
  97. }
  98. int arch_hibernation_header_save(void *addr, unsigned int max_size)
  99. {
  100. struct arch_hibernate_hdr *hdr = addr;
  101. if (max_size < sizeof(*hdr))
  102. return -EOVERFLOW;
  103. arch_hdr_invariants(&hdr->invariants);
  104. hdr->ttbr1_el1 = virt_to_phys(swapper_pg_dir);
  105. hdr->reenter_kernel = _cpu_resume;
  106. /* We can't use __hyp_get_vectors() because kvm may still be loaded */
  107. if (el2_reset_needed())
  108. hdr->__hyp_stub_vectors = virt_to_phys(__hyp_stub_vectors);
  109. else
  110. hdr->__hyp_stub_vectors = 0;
  111. return 0;
  112. }
  113. EXPORT_SYMBOL(arch_hibernation_header_save);
  114. int arch_hibernation_header_restore(void *addr)
  115. {
  116. struct arch_hibernate_hdr_invariants invariants;
  117. struct arch_hibernate_hdr *hdr = addr;
  118. arch_hdr_invariants(&invariants);
  119. if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) {
  120. pr_crit("Hibernate image not generated by this kernel!\n");
  121. return -EINVAL;
  122. }
  123. resume_hdr = *hdr;
  124. return 0;
  125. }
  126. EXPORT_SYMBOL(arch_hibernation_header_restore);
  127. /*
  128. * Copies length bytes, starting at src_start into an new page,
  129. * perform cache maintentance, then maps it at the specified address low
  130. * address as executable.
  131. *
  132. * This is used by hibernate to copy the code it needs to execute when
  133. * overwriting the kernel text. This function generates a new set of page
  134. * tables, which it loads into ttbr0.
  135. *
  136. * Length is provided as we probably only want 4K of data, even on a 64K
  137. * page system.
  138. */
  139. static int create_safe_exec_page(void *src_start, size_t length,
  140. unsigned long dst_addr,
  141. phys_addr_t *phys_dst_addr,
  142. void *(*allocator)(gfp_t mask),
  143. gfp_t mask)
  144. {
  145. int rc = 0;
  146. pgd_t *pgd;
  147. pud_t *pud;
  148. pmd_t *pmd;
  149. pte_t *pte;
  150. unsigned long dst = (unsigned long)allocator(mask);
  151. if (!dst) {
  152. rc = -ENOMEM;
  153. goto out;
  154. }
  155. memcpy((void *)dst, src_start, length);
  156. flush_icache_range(dst, dst + length);
  157. pgd = pgd_offset_raw(allocator(mask), dst_addr);
  158. if (pgd_none(*pgd)) {
  159. pud = allocator(mask);
  160. if (!pud) {
  161. rc = -ENOMEM;
  162. goto out;
  163. }
  164. pgd_populate(&init_mm, pgd, pud);
  165. }
  166. pud = pud_offset(pgd, dst_addr);
  167. if (pud_none(*pud)) {
  168. pmd = allocator(mask);
  169. if (!pmd) {
  170. rc = -ENOMEM;
  171. goto out;
  172. }
  173. pud_populate(&init_mm, pud, pmd);
  174. }
  175. pmd = pmd_offset(pud, dst_addr);
  176. if (pmd_none(*pmd)) {
  177. pte = allocator(mask);
  178. if (!pte) {
  179. rc = -ENOMEM;
  180. goto out;
  181. }
  182. pmd_populate_kernel(&init_mm, pmd, pte);
  183. }
  184. pte = pte_offset_kernel(pmd, dst_addr);
  185. set_pte(pte, __pte(virt_to_phys((void *)dst) |
  186. pgprot_val(PAGE_KERNEL_EXEC)));
  187. /* Load our new page tables */
  188. asm volatile("msr ttbr0_el1, %0;"
  189. "isb;"
  190. "tlbi vmalle1is;"
  191. "dsb ish;"
  192. "isb" : : "r"(virt_to_phys(pgd)));
  193. *phys_dst_addr = virt_to_phys((void *)dst);
  194. out:
  195. return rc;
  196. }
  197. int swsusp_arch_suspend(void)
  198. {
  199. int ret = 0;
  200. unsigned long flags;
  201. struct sleep_stack_data state;
  202. if (cpus_are_stuck_in_kernel()) {
  203. pr_err("Can't hibernate: no mechanism to offline secondary CPUs.\n");
  204. return -EBUSY;
  205. }
  206. local_dbg_save(flags);
  207. if (__cpu_suspend_enter(&state)) {
  208. ret = swsusp_save();
  209. } else {
  210. /* Clean kernel to PoC for secondary core startup */
  211. __flush_dcache_area(LMADDR(KERNEL_START), KERNEL_END - KERNEL_START);
  212. /*
  213. * Tell the hibernation core that we've just restored
  214. * the memory
  215. */
  216. in_suspend = 0;
  217. __cpu_suspend_exit();
  218. }
  219. local_dbg_restore(flags);
  220. return ret;
  221. }
  222. static int copy_pte(pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long start,
  223. unsigned long end)
  224. {
  225. pte_t *src_pte;
  226. pte_t *dst_pte;
  227. unsigned long addr = start;
  228. dst_pte = (pte_t *)get_safe_page(GFP_ATOMIC);
  229. if (!dst_pte)
  230. return -ENOMEM;
  231. pmd_populate_kernel(&init_mm, dst_pmd, dst_pte);
  232. dst_pte = pte_offset_kernel(dst_pmd, start);
  233. src_pte = pte_offset_kernel(src_pmd, start);
  234. do {
  235. if (!pte_none(*src_pte))
  236. /*
  237. * Resume will overwrite areas that may be marked
  238. * read only (code, rodata). Clear the RDONLY bit from
  239. * the temporary mappings we use during restore.
  240. */
  241. set_pte(dst_pte, __pte(pte_val(*src_pte) & ~PTE_RDONLY));
  242. } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
  243. return 0;
  244. }
  245. static int copy_pmd(pud_t *dst_pud, pud_t *src_pud, unsigned long start,
  246. unsigned long end)
  247. {
  248. pmd_t *src_pmd;
  249. pmd_t *dst_pmd;
  250. unsigned long next;
  251. unsigned long addr = start;
  252. if (pud_none(*dst_pud)) {
  253. dst_pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
  254. if (!dst_pmd)
  255. return -ENOMEM;
  256. pud_populate(&init_mm, dst_pud, dst_pmd);
  257. }
  258. dst_pmd = pmd_offset(dst_pud, start);
  259. src_pmd = pmd_offset(src_pud, start);
  260. do {
  261. next = pmd_addr_end(addr, end);
  262. if (pmd_none(*src_pmd))
  263. continue;
  264. if (pmd_table(*src_pmd)) {
  265. if (copy_pte(dst_pmd, src_pmd, addr, next))
  266. return -ENOMEM;
  267. } else {
  268. set_pmd(dst_pmd,
  269. __pmd(pmd_val(*src_pmd) & ~PMD_SECT_RDONLY));
  270. }
  271. } while (dst_pmd++, src_pmd++, addr = next, addr != end);
  272. return 0;
  273. }
  274. static int copy_pud(pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long start,
  275. unsigned long end)
  276. {
  277. pud_t *dst_pud;
  278. pud_t *src_pud;
  279. unsigned long next;
  280. unsigned long addr = start;
  281. if (pgd_none(*dst_pgd)) {
  282. dst_pud = (pud_t *)get_safe_page(GFP_ATOMIC);
  283. if (!dst_pud)
  284. return -ENOMEM;
  285. pgd_populate(&init_mm, dst_pgd, dst_pud);
  286. }
  287. dst_pud = pud_offset(dst_pgd, start);
  288. src_pud = pud_offset(src_pgd, start);
  289. do {
  290. next = pud_addr_end(addr, end);
  291. if (pud_none(*src_pud))
  292. continue;
  293. if (pud_table(*(src_pud))) {
  294. if (copy_pmd(dst_pud, src_pud, addr, next))
  295. return -ENOMEM;
  296. } else {
  297. set_pud(dst_pud,
  298. __pud(pud_val(*src_pud) & ~PMD_SECT_RDONLY));
  299. }
  300. } while (dst_pud++, src_pud++, addr = next, addr != end);
  301. return 0;
  302. }
  303. static int copy_page_tables(pgd_t *dst_pgd, unsigned long start,
  304. unsigned long end)
  305. {
  306. unsigned long next;
  307. unsigned long addr = start;
  308. pgd_t *src_pgd = pgd_offset_k(start);
  309. dst_pgd = pgd_offset_raw(dst_pgd, start);
  310. do {
  311. next = pgd_addr_end(addr, end);
  312. if (pgd_none(*src_pgd))
  313. continue;
  314. if (copy_pud(dst_pgd, src_pgd, addr, next))
  315. return -ENOMEM;
  316. } while (dst_pgd++, src_pgd++, addr = next, addr != end);
  317. return 0;
  318. }
  319. /*
  320. * Setup then Resume from the hibernate image using swsusp_arch_suspend_exit().
  321. *
  322. * Memory allocated by get_safe_page() will be dealt with by the hibernate code,
  323. * we don't need to free it here.
  324. */
  325. int swsusp_arch_resume(void)
  326. {
  327. int rc = 0;
  328. void *zero_page;
  329. size_t exit_size;
  330. pgd_t *tmp_pg_dir;
  331. void *lm_restore_pblist;
  332. phys_addr_t phys_hibernate_exit;
  333. void __noreturn (*hibernate_exit)(phys_addr_t, phys_addr_t, void *,
  334. void *, phys_addr_t, phys_addr_t);
  335. /*
  336. * Locate the exit code in the bottom-but-one page, so that *NULL
  337. * still has disastrous affects.
  338. */
  339. hibernate_exit = (void *)PAGE_SIZE;
  340. exit_size = __hibernate_exit_text_end - __hibernate_exit_text_start;
  341. /*
  342. * Copy swsusp_arch_suspend_exit() to a safe page. This will generate
  343. * a new set of ttbr0 page tables and load them.
  344. */
  345. rc = create_safe_exec_page(__hibernate_exit_text_start, exit_size,
  346. (unsigned long)hibernate_exit,
  347. &phys_hibernate_exit,
  348. (void *)get_safe_page, GFP_ATOMIC);
  349. if (rc) {
  350. pr_err("Failed to create safe executable page for hibernate_exit code.");
  351. goto out;
  352. }
  353. /*
  354. * The hibernate exit text contains a set of el2 vectors, that will
  355. * be executed at el2 with the mmu off in order to reload hyp-stub.
  356. */
  357. __flush_dcache_area(hibernate_exit, exit_size);
  358. /*
  359. * Restoring the memory image will overwrite the ttbr1 page tables.
  360. * Create a second copy of just the linear map, and use this when
  361. * restoring.
  362. */
  363. tmp_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC);
  364. if (!tmp_pg_dir) {
  365. pr_err("Failed to allocate memory for temporary page tables.");
  366. rc = -ENOMEM;
  367. goto out;
  368. }
  369. rc = copy_page_tables(tmp_pg_dir, PAGE_OFFSET, 0);
  370. if (rc)
  371. goto out;
  372. /*
  373. * Since we only copied the linear map, we need to find restore_pblist's
  374. * linear map address.
  375. */
  376. lm_restore_pblist = LMADDR(restore_pblist);
  377. /*
  378. * KASLR will cause the el2 vectors to be in a different location in
  379. * the resumed kernel. Load hibernate's temporary copy into el2.
  380. *
  381. * We can skip this step if we booted at EL1, or are running with VHE.
  382. */
  383. if (el2_reset_needed()) {
  384. phys_addr_t el2_vectors = phys_hibernate_exit; /* base */
  385. el2_vectors += hibernate_el2_vectors -
  386. __hibernate_exit_text_start; /* offset */
  387. __hyp_set_vectors(el2_vectors);
  388. }
  389. /*
  390. * We need a zero page that is zero before & after resume in order to
  391. * to break before make on the ttbr1 page tables.
  392. */
  393. zero_page = (void *)get_safe_page(GFP_ATOMIC);
  394. hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1,
  395. resume_hdr.reenter_kernel, lm_restore_pblist,
  396. resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page));
  397. out:
  398. return rc;
  399. }
  400. static int check_boot_cpu_online_pm_callback(struct notifier_block *nb,
  401. unsigned long action, void *ptr)
  402. {
  403. if (action == PM_HIBERNATION_PREPARE &&
  404. cpumask_first(cpu_online_mask) != 0) {
  405. pr_warn("CPU0 is offline.\n");
  406. return notifier_from_errno(-ENODEV);
  407. }
  408. return NOTIFY_OK;
  409. }
  410. static int __init check_boot_cpu_online_init(void)
  411. {
  412. /*
  413. * Set this pm_notifier callback with a lower priority than
  414. * cpu_hotplug_pm_callback, so that cpu_hotplug_pm_callback will be
  415. * called earlier to disable cpu hotplug before the cpu online check.
  416. */
  417. pm_notifier(check_boot_cpu_online_pm_callback, -INT_MAX);
  418. return 0;
  419. }
  420. core_initcall(check_boot_cpu_online_init);