head.S 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709
  1. /*
  2. * Low-level CPU initialisation
  3. * Based on arch/arm/kernel/head.S
  4. *
  5. * Copyright (C) 1994-2002 Russell King
  6. * Copyright (C) 2003-2012 ARM Ltd.
  7. * Authors: Catalin Marinas <catalin.marinas@arm.com>
  8. * Will Deacon <will.deacon@arm.com>
  9. *
  10. * This program is free software; you can redistribute it and/or modify
  11. * it under the terms of the GNU General Public License version 2 as
  12. * published by the Free Software Foundation.
  13. *
  14. * This program is distributed in the hope that it will be useful,
  15. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. * GNU General Public License for more details.
  18. *
  19. * You should have received a copy of the GNU General Public License
  20. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  21. */
  22. #include <linux/linkage.h>
  23. #include <linux/init.h>
  24. #include <asm/assembler.h>
  25. #include <asm/ptrace.h>
  26. #include <asm/asm-offsets.h>
  27. #include <asm/cache.h>
  28. #include <asm/cputype.h>
  29. #include <asm/memory.h>
  30. #include <asm/thread_info.h>
  31. #include <asm/pgtable-hwdef.h>
  32. #include <asm/pgtable.h>
  33. #include <asm/page.h>
  34. #include <asm/virt.h>
  35. /*
  36. * swapper_pg_dir is the virtual address of the initial page table. We place
  37. * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has
  38. * 2 pages and is placed below swapper_pg_dir.
  39. */
  40. #define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET)
  41. #if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000
  42. #error KERNEL_RAM_VADDR must start at 0xXXX80000
  43. #endif
  44. #define SWAPPER_DIR_SIZE (3 * PAGE_SIZE)
  45. #define IDMAP_DIR_SIZE (2 * PAGE_SIZE)
  46. .globl swapper_pg_dir
  47. .equ swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE
  48. .globl idmap_pg_dir
  49. .equ idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE
  50. .macro pgtbl, ttb0, ttb1, phys
  51. add \ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE
  52. sub \ttb0, \ttb1, #IDMAP_DIR_SIZE
  53. .endm
  54. #ifdef CONFIG_ARM64_64K_PAGES
  55. #define BLOCK_SHIFT PAGE_SHIFT
  56. #define BLOCK_SIZE PAGE_SIZE
  57. #else
  58. #define BLOCK_SHIFT SECTION_SHIFT
  59. #define BLOCK_SIZE SECTION_SIZE
  60. #endif
  61. #define KERNEL_START KERNEL_RAM_VADDR
  62. #define KERNEL_END _end
  63. /*
  64. * Initial memory map attributes.
  65. */
  66. #ifndef CONFIG_SMP
  67. #define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF
  68. #define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF
  69. #else
  70. #define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF | PTE_SHARED
  71. #define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S
  72. #endif
  73. #ifdef CONFIG_ARM64_64K_PAGES
  74. #define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS
  75. #else
  76. #define MM_MMUFLAGS PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS
  77. #endif
  78. /*
  79. * Kernel startup entry point.
  80. * ---------------------------
  81. *
  82. * The requirements are:
  83. * MMU = off, D-cache = off, I-cache = on or off,
  84. * x0 = physical address to the FDT blob.
  85. *
  86. * This code is mostly position independent so you call this at
  87. * __pa(PAGE_OFFSET + TEXT_OFFSET).
  88. *
  89. * Note that the callee-saved registers are used for storing variables
  90. * that are useful before the MMU is enabled. The allocations are described
  91. * in the entry routines.
  92. */
  93. __HEAD
  94. /*
  95. * DO NOT MODIFY. Image header expected by Linux boot-loaders.
  96. */
  97. #ifdef CONFIG_EFI
  98. efi_head:
  99. /*
  100. * This add instruction has no meaningful effect except that
  101. * its opcode forms the magic "MZ" signature required by UEFI.
  102. */
  103. add x13, x18, #0x16
  104. b stext
  105. #else
  106. b stext // branch to kernel start, magic
  107. .long 0 // reserved
  108. #endif
  109. .quad TEXT_OFFSET // Image load offset from start of RAM
  110. .quad 0 // reserved
  111. .quad 0 // reserved
  112. .quad 0 // reserved
  113. .quad 0 // reserved
  114. .quad 0 // reserved
  115. .byte 0x41 // Magic number, "ARM\x64"
  116. .byte 0x52
  117. .byte 0x4d
  118. .byte 0x64
  119. #ifdef CONFIG_EFI
  120. .long pe_header - efi_head // Offset to the PE header.
  121. #else
  122. .word 0 // reserved
  123. #endif
  124. #ifdef CONFIG_EFI
  125. .align 3
  126. pe_header:
  127. .ascii "PE"
  128. .short 0
  129. coff_header:
  130. .short 0xaa64 // AArch64
  131. .short 2 // nr_sections
  132. .long 0 // TimeDateStamp
  133. .long 0 // PointerToSymbolTable
  134. .long 1 // NumberOfSymbols
  135. .short section_table - optional_header // SizeOfOptionalHeader
  136. .short 0x206 // Characteristics.
  137. // IMAGE_FILE_DEBUG_STRIPPED |
  138. // IMAGE_FILE_EXECUTABLE_IMAGE |
  139. // IMAGE_FILE_LINE_NUMS_STRIPPED
  140. optional_header:
  141. .short 0x20b // PE32+ format
  142. .byte 0x02 // MajorLinkerVersion
  143. .byte 0x14 // MinorLinkerVersion
  144. .long _edata - stext // SizeOfCode
  145. .long 0 // SizeOfInitializedData
  146. .long 0 // SizeOfUninitializedData
  147. .long efi_stub_entry - efi_head // AddressOfEntryPoint
  148. .long stext - efi_head // BaseOfCode
  149. extra_header_fields:
  150. .quad 0 // ImageBase
  151. .long 0x20 // SectionAlignment
  152. .long 0x8 // FileAlignment
  153. .short 0 // MajorOperatingSystemVersion
  154. .short 0 // MinorOperatingSystemVersion
  155. .short 0 // MajorImageVersion
  156. .short 0 // MinorImageVersion
  157. .short 0 // MajorSubsystemVersion
  158. .short 0 // MinorSubsystemVersion
  159. .long 0 // Win32VersionValue
  160. .long _edata - efi_head // SizeOfImage
  161. // Everything before the kernel image is considered part of the header
  162. .long stext - efi_head // SizeOfHeaders
  163. .long 0 // CheckSum
  164. .short 0xa // Subsystem (EFI application)
  165. .short 0 // DllCharacteristics
  166. .quad 0 // SizeOfStackReserve
  167. .quad 0 // SizeOfStackCommit
  168. .quad 0 // SizeOfHeapReserve
  169. .quad 0 // SizeOfHeapCommit
  170. .long 0 // LoaderFlags
  171. .long 0x6 // NumberOfRvaAndSizes
  172. .quad 0 // ExportTable
  173. .quad 0 // ImportTable
  174. .quad 0 // ResourceTable
  175. .quad 0 // ExceptionTable
  176. .quad 0 // CertificationTable
  177. .quad 0 // BaseRelocationTable
  178. // Section table
  179. section_table:
  180. /*
  181. * The EFI application loader requires a relocation section
  182. * because EFI applications must be relocatable. This is a
  183. * dummy section as far as we are concerned.
  184. */
  185. .ascii ".reloc"
  186. .byte 0
  187. .byte 0 // end of 0 padding of section name
  188. .long 0
  189. .long 0
  190. .long 0 // SizeOfRawData
  191. .long 0 // PointerToRawData
  192. .long 0 // PointerToRelocations
  193. .long 0 // PointerToLineNumbers
  194. .short 0 // NumberOfRelocations
  195. .short 0 // NumberOfLineNumbers
  196. .long 0x42100040 // Characteristics (section flags)
  197. .ascii ".text"
  198. .byte 0
  199. .byte 0
  200. .byte 0 // end of 0 padding of section name
  201. .long _edata - stext // VirtualSize
  202. .long stext - efi_head // VirtualAddress
  203. .long _edata - stext // SizeOfRawData
  204. .long stext - efi_head // PointerToRawData
  205. .long 0 // PointerToRelocations (0 for executables)
  206. .long 0 // PointerToLineNumbers (0 for executables)
  207. .short 0 // NumberOfRelocations (0 for executables)
  208. .short 0 // NumberOfLineNumbers (0 for executables)
  209. .long 0xe0500020 // Characteristics (section flags)
  210. .align 5
  211. #endif
  212. ENTRY(stext)
  213. mov x21, x0 // x21=FDT
  214. bl el2_setup // Drop to EL1, w20=cpu_boot_mode
  215. bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
  216. bl set_cpu_boot_mode_flag
  217. mrs x22, midr_el1 // x22=cpuid
  218. mov x0, x22
  219. bl lookup_processor_type
  220. mov x23, x0 // x23=current cpu_table
  221. cbz x23, __error_p // invalid processor (x23=0)?
  222. bl __vet_fdt
  223. bl __create_page_tables // x25=TTBR0, x26=TTBR1
  224. /*
  225. * The following calls CPU specific code in a position independent
  226. * manner. See arch/arm64/mm/proc.S for details. x23 = base of
  227. * cpu_info structure selected by lookup_processor_type above.
  228. * On return, the CPU will be ready for the MMU to be turned on and
  229. * the TCR will have been set.
  230. */
  231. ldr x27, __switch_data // address to jump to after
  232. // MMU has been enabled
  233. adr lr, __enable_mmu // return (PIC) address
  234. ldr x12, [x23, #CPU_INFO_SETUP]
  235. add x12, x12, x28 // __virt_to_phys
  236. br x12 // initialise processor
  237. ENDPROC(stext)
  238. /*
  239. * If we're fortunate enough to boot at EL2, ensure that the world is
  240. * sane before dropping to EL1.
  241. *
  242. * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x20 if
  243. * booted in EL1 or EL2 respectively.
  244. */
  245. ENTRY(el2_setup)
  246. mrs x0, CurrentEL
  247. cmp x0, #CurrentEL_EL2
  248. b.ne 1f
  249. mrs x0, sctlr_el2
  250. CPU_BE( orr x0, x0, #(1 << 25) ) // Set the EE bit for EL2
  251. CPU_LE( bic x0, x0, #(1 << 25) ) // Clear the EE bit for EL2
  252. msr sctlr_el2, x0
  253. b 2f
  254. 1: mrs x0, sctlr_el1
  255. CPU_BE( orr x0, x0, #(3 << 24) ) // Set the EE and E0E bits for EL1
  256. CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1
  257. msr sctlr_el1, x0
  258. mov w20, #BOOT_CPU_MODE_EL1 // This cpu booted in EL1
  259. isb
  260. ret
  261. /* Hyp configuration. */
  262. 2: mov x0, #(1 << 31) // 64-bit EL1
  263. msr hcr_el2, x0
  264. /* Generic timers. */
  265. mrs x0, cnthctl_el2
  266. orr x0, x0, #3 // Enable EL1 physical timers
  267. msr cnthctl_el2, x0
  268. msr cntvoff_el2, xzr // Clear virtual offset
  269. /* Populate ID registers. */
  270. mrs x0, midr_el1
  271. mrs x1, mpidr_el1
  272. msr vpidr_el2, x0
  273. msr vmpidr_el2, x1
  274. /* sctlr_el1 */
  275. mov x0, #0x0800 // Set/clear RES{1,0} bits
  276. CPU_BE( movk x0, #0x33d0, lsl #16 ) // Set EE and E0E on BE systems
  277. CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems
  278. msr sctlr_el1, x0
  279. /* Coprocessor traps. */
  280. mov x0, #0x33ff
  281. msr cptr_el2, x0 // Disable copro. traps to EL2
  282. #ifdef CONFIG_COMPAT
  283. msr hstr_el2, xzr // Disable CP15 traps to EL2
  284. #endif
  285. /* Stage-2 translation */
  286. msr vttbr_el2, xzr
  287. /* Hypervisor stub */
  288. adr x0, __hyp_stub_vectors
  289. msr vbar_el2, x0
  290. /* spsr */
  291. mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
  292. PSR_MODE_EL1h)
  293. msr spsr_el2, x0
  294. msr elr_el2, lr
  295. mov w20, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2
  296. eret
  297. ENDPROC(el2_setup)
  298. /*
  299. * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed
  300. * in x20. See arch/arm64/include/asm/virt.h for more info.
  301. */
  302. ENTRY(set_cpu_boot_mode_flag)
  303. ldr x1, =__boot_cpu_mode // Compute __boot_cpu_mode
  304. add x1, x1, x28
  305. cmp w20, #BOOT_CPU_MODE_EL2
  306. b.ne 1f
  307. add x1, x1, #4
  308. 1: str w20, [x1] // This CPU has booted in EL1
  309. dmb sy
  310. dc ivac, x1 // Invalidate potentially stale cache line
  311. ret
  312. ENDPROC(set_cpu_boot_mode_flag)
  313. /*
  314. * We need to find out the CPU boot mode long after boot, so we need to
  315. * store it in a writable variable.
  316. *
  317. * This is not in .bss, because we set it sufficiently early that the boot-time
  318. * zeroing of .bss would clobber it.
  319. */
  320. .pushsection .data..cacheline_aligned
  321. ENTRY(__boot_cpu_mode)
  322. .align L1_CACHE_SHIFT
  323. .long BOOT_CPU_MODE_EL2
  324. .long 0
  325. .popsection
  326. .align 3
  327. 2: .quad .
  328. .quad PAGE_OFFSET
  329. #ifdef CONFIG_SMP
  330. .align 3
  331. 1: .quad .
  332. .quad secondary_holding_pen_release
  333. /*
  334. * This provides a "holding pen" for platforms to hold all secondary
  335. * cores are held until we're ready for them to initialise.
  336. */
  337. ENTRY(secondary_holding_pen)
  338. bl el2_setup // Drop to EL1, w20=cpu_boot_mode
  339. bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
  340. bl set_cpu_boot_mode_flag
  341. mrs x0, mpidr_el1
  342. ldr x1, =MPIDR_HWID_BITMASK
  343. and x0, x0, x1
  344. adr x1, 1b
  345. ldp x2, x3, [x1]
  346. sub x1, x1, x2
  347. add x3, x3, x1
  348. pen: ldr x4, [x3]
  349. cmp x4, x0
  350. b.eq secondary_startup
  351. wfe
  352. b pen
  353. ENDPROC(secondary_holding_pen)
  354. /*
  355. * Secondary entry point that jumps straight into the kernel. Only to
  356. * be used where CPUs are brought online dynamically by the kernel.
  357. */
  358. ENTRY(secondary_entry)
  359. bl el2_setup // Drop to EL1
  360. bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
  361. bl set_cpu_boot_mode_flag
  362. b secondary_startup
  363. ENDPROC(secondary_entry)
  364. ENTRY(secondary_startup)
  365. /*
  366. * Common entry point for secondary CPUs.
  367. */
  368. mrs x22, midr_el1 // x22=cpuid
  369. mov x0, x22
  370. bl lookup_processor_type
  371. mov x23, x0 // x23=current cpu_table
  372. cbz x23, __error_p // invalid processor (x23=0)?
  373. pgtbl x25, x26, x24 // x25=TTBR0, x26=TTBR1
  374. ldr x12, [x23, #CPU_INFO_SETUP]
  375. add x12, x12, x28 // __virt_to_phys
  376. blr x12 // initialise processor
  377. ldr x21, =secondary_data
  378. ldr x27, =__secondary_switched // address to jump to after enabling the MMU
  379. b __enable_mmu
  380. ENDPROC(secondary_startup)
  381. ENTRY(__secondary_switched)
  382. ldr x0, [x21] // get secondary_data.stack
  383. mov sp, x0
  384. mov x29, #0
  385. b secondary_start_kernel
  386. ENDPROC(__secondary_switched)
  387. #endif /* CONFIG_SMP */
  388. /*
  389. * Setup common bits before finally enabling the MMU. Essentially this is just
  390. * loading the page table pointer and vector base registers.
  391. *
  392. * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on
  393. * the MMU.
  394. */
  395. __enable_mmu:
  396. ldr x5, =vectors
  397. msr vbar_el1, x5
  398. msr ttbr0_el1, x25 // load TTBR0
  399. msr ttbr1_el1, x26 // load TTBR1
  400. isb
  401. b __turn_mmu_on
  402. ENDPROC(__enable_mmu)
  403. /*
  404. * Enable the MMU. This completely changes the structure of the visible memory
  405. * space. You will not be able to trace execution through this.
  406. *
  407. * x0 = system control register
  408. * x27 = *virtual* address to jump to upon completion
  409. *
  410. * other registers depend on the function called upon completion
  411. */
  412. .align 6
  413. __turn_mmu_on:
  414. msr sctlr_el1, x0
  415. isb
  416. br x27
  417. ENDPROC(__turn_mmu_on)
  418. /*
  419. * Calculate the start of physical memory.
  420. */
  421. __calc_phys_offset:
  422. adr x0, 1f
  423. ldp x1, x2, [x0]
  424. sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET
  425. add x24, x2, x28 // x24 = PHYS_OFFSET
  426. ret
  427. ENDPROC(__calc_phys_offset)
  428. .align 3
  429. 1: .quad .
  430. .quad PAGE_OFFSET
  431. /*
  432. * Macro to populate the PGD for the corresponding block entry in the next
  433. * level (tbl) for the given virtual address.
  434. *
  435. * Preserves: pgd, tbl, virt
  436. * Corrupts: tmp1, tmp2
  437. */
  438. .macro create_pgd_entry, pgd, tbl, virt, tmp1, tmp2
  439. lsr \tmp1, \virt, #PGDIR_SHIFT
  440. and \tmp1, \tmp1, #PTRS_PER_PGD - 1 // PGD index
  441. orr \tmp2, \tbl, #3 // PGD entry table type
  442. str \tmp2, [\pgd, \tmp1, lsl #3]
  443. .endm
  444. /*
  445. * Macro to populate block entries in the page table for the start..end
  446. * virtual range (inclusive).
  447. *
  448. * Preserves: tbl, flags
  449. * Corrupts: phys, start, end, pstate
  450. */
  451. .macro create_block_map, tbl, flags, phys, start, end
  452. lsr \phys, \phys, #BLOCK_SHIFT
  453. lsr \start, \start, #BLOCK_SHIFT
  454. and \start, \start, #PTRS_PER_PTE - 1 // table index
  455. orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry
  456. lsr \end, \end, #BLOCK_SHIFT
  457. and \end, \end, #PTRS_PER_PTE - 1 // table end index
  458. 9999: str \phys, [\tbl, \start, lsl #3] // store the entry
  459. add \start, \start, #1 // next entry
  460. add \phys, \phys, #BLOCK_SIZE // next block
  461. cmp \start, \end
  462. b.ls 9999b
  463. .endm
  464. /*
  465. * Setup the initial page tables. We only setup the barest amount which is
  466. * required to get the kernel running. The following sections are required:
  467. * - identity mapping to enable the MMU (low address, TTBR0)
  468. * - first few MB of the kernel linear mapping to jump to once the MMU has
  469. * been enabled, including the FDT blob (TTBR1)
  470. * - pgd entry for fixed mappings (TTBR1)
  471. */
  472. __create_page_tables:
  473. pgtbl x25, x26, x24 // idmap_pg_dir and swapper_pg_dir addresses
  474. mov x27, lr
  475. /*
  476. * Invalidate the idmap and swapper page tables to avoid potential
  477. * dirty cache lines being evicted.
  478. */
  479. mov x0, x25
  480. add x1, x26, #SWAPPER_DIR_SIZE
  481. bl __inval_cache_range
  482. /*
  483. * Clear the idmap and swapper page tables.
  484. */
  485. mov x0, x25
  486. add x6, x26, #SWAPPER_DIR_SIZE
  487. 1: stp xzr, xzr, [x0], #16
  488. stp xzr, xzr, [x0], #16
  489. stp xzr, xzr, [x0], #16
  490. stp xzr, xzr, [x0], #16
  491. cmp x0, x6
  492. b.lo 1b
  493. ldr x7, =MM_MMUFLAGS
  494. /*
  495. * Create the identity mapping.
  496. */
  497. add x0, x25, #PAGE_SIZE // section table address
  498. ldr x3, =KERNEL_START
  499. add x3, x3, x28 // __pa(KERNEL_START)
  500. create_pgd_entry x25, x0, x3, x5, x6
  501. ldr x6, =KERNEL_END
  502. mov x5, x3 // __pa(KERNEL_START)
  503. add x6, x6, x28 // __pa(KERNEL_END)
  504. create_block_map x0, x7, x3, x5, x6
  505. /*
  506. * Map the kernel image (starting with PHYS_OFFSET).
  507. */
  508. add x0, x26, #PAGE_SIZE // section table address
  509. mov x5, #PAGE_OFFSET
  510. create_pgd_entry x26, x0, x5, x3, x6
  511. ldr x6, =KERNEL_END
  512. mov x3, x24 // phys offset
  513. create_block_map x0, x7, x3, x5, x6
  514. /*
  515. * Map the FDT blob (maximum 2MB; must be within 512MB of
  516. * PHYS_OFFSET).
  517. */
  518. mov x3, x21 // FDT phys address
  519. and x3, x3, #~((1 << 21) - 1) // 2MB aligned
  520. mov x6, #PAGE_OFFSET
  521. sub x5, x3, x24 // subtract PHYS_OFFSET
  522. tst x5, #~((1 << 29) - 1) // within 512MB?
  523. csel x21, xzr, x21, ne // zero the FDT pointer
  524. b.ne 1f
  525. add x5, x5, x6 // __va(FDT blob)
  526. add x6, x5, #1 << 21 // 2MB for the FDT blob
  527. sub x6, x6, #1 // inclusive range
  528. create_block_map x0, x7, x3, x5, x6
  529. 1:
  530. /*
  531. * Create the pgd entry for the fixed mappings.
  532. */
  533. ldr x5, =FIXADDR_TOP // Fixed mapping virtual address
  534. add x0, x26, #2 * PAGE_SIZE // section table address
  535. create_pgd_entry x26, x0, x5, x6, x7
  536. /*
  537. * Since the page tables have been populated with non-cacheable
  538. * accesses (MMU disabled), invalidate the idmap and swapper page
  539. * tables again to remove any speculatively loaded cache lines.
  540. */
  541. mov x0, x25
  542. add x1, x26, #SWAPPER_DIR_SIZE
  543. bl __inval_cache_range
  544. mov lr, x27
  545. ret
  546. ENDPROC(__create_page_tables)
  547. .ltorg
  548. .align 3
  549. .type __switch_data, %object
  550. __switch_data:
  551. .quad __mmap_switched
  552. .quad __bss_start // x6
  553. .quad _end // x7
  554. .quad processor_id // x4
  555. .quad __fdt_pointer // x5
  556. .quad memstart_addr // x6
  557. .quad init_thread_union + THREAD_START_SP // sp
  558. /*
  559. * The following fragment of code is executed with the MMU on in MMU mode, and
  560. * uses absolute addresses; this is not position independent.
  561. */
  562. __mmap_switched:
  563. adr x3, __switch_data + 8
  564. ldp x6, x7, [x3], #16
  565. 1: cmp x6, x7
  566. b.hs 2f
  567. str xzr, [x6], #8 // Clear BSS
  568. b 1b
  569. 2:
  570. ldp x4, x5, [x3], #16
  571. ldr x6, [x3], #8
  572. ldr x16, [x3]
  573. mov sp, x16
  574. str x22, [x4] // Save processor ID
  575. str x21, [x5] // Save FDT pointer
  576. str x24, [x6] // Save PHYS_OFFSET
  577. mov x29, #0
  578. b start_kernel
  579. ENDPROC(__mmap_switched)
  580. /*
  581. * Exception handling. Something went wrong and we can't proceed. We ought to
  582. * tell the user, but since we don't have any guarantee that we're even
  583. * running on the right architecture, we do virtually nothing.
  584. */
  585. __error_p:
  586. ENDPROC(__error_p)
  587. __error:
  588. 1: nop
  589. b 1b
  590. ENDPROC(__error)
  591. /*
  592. * This function gets the processor ID in w0 and searches the cpu_table[] for
  593. * a match. It returns a pointer to the struct cpu_info it found. The
  594. * cpu_table[] must end with an empty (all zeros) structure.
  595. *
  596. * This routine can be called via C code and it needs to work with the MMU
  597. * both disabled and enabled (the offset is calculated automatically).
  598. */
  599. ENTRY(lookup_processor_type)
  600. adr x1, __lookup_processor_type_data
  601. ldp x2, x3, [x1]
  602. sub x1, x1, x2 // get offset between VA and PA
  603. add x3, x3, x1 // convert VA to PA
  604. 1:
  605. ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask
  606. cbz w5, 2f // end of list?
  607. and w6, w6, w0
  608. cmp w5, w6
  609. b.eq 3f
  610. add x3, x3, #CPU_INFO_SZ
  611. b 1b
  612. 2:
  613. mov x3, #0 // unknown processor
  614. 3:
  615. mov x0, x3
  616. ret
  617. ENDPROC(lookup_processor_type)
  618. .align 3
  619. .type __lookup_processor_type_data, %object
  620. __lookup_processor_type_data:
  621. .quad .
  622. .quad cpu_table
  623. .size __lookup_processor_type_data, . - __lookup_processor_type_data
  624. /*
  625. * Determine validity of the x21 FDT pointer.
  626. * The dtb must be 8-byte aligned and live in the first 512M of memory.
  627. */
  628. __vet_fdt:
  629. tst x21, #0x7
  630. b.ne 1f
  631. cmp x21, x24
  632. b.lt 1f
  633. mov x0, #(1 << 29)
  634. add x0, x0, x24
  635. cmp x21, x0
  636. b.ge 1f
  637. ret
  638. 1:
  639. mov x21, #0
  640. ret
  641. ENDPROC(__vet_fdt)