|
@@ -31,6 +31,7 @@
|
|
|
#include <asm/asm-offsets.h>
|
|
|
#include <asm/ptrace.h>
|
|
|
#include <asm/export.h>
|
|
|
+#include <asm/code-patching-asm.h>
|
|
|
|
|
|
#if CONFIG_TASK_SIZE <= 0x80000000 && CONFIG_PAGE_OFFSET >= 0x80000000
|
|
|
/* By simply checking Address >= 0x80000000, we know if its a kernel address */
|
|
@@ -318,8 +319,8 @@ InstructionTLBMiss:
|
|
|
cmpli cr0, r11, PAGE_OFFSET@h
|
|
|
#ifndef CONFIG_PIN_TLB_TEXT
|
|
|
/* It is assumed that kernel code fits into the first 8M page */
|
|
|
-_ENTRY(ITLBMiss_cmp)
|
|
|
- cmpli cr7, r11, (PAGE_OFFSET + 0x0800000)@h
|
|
|
+0: cmpli cr7, r11, (PAGE_OFFSET + 0x0800000)@h
|
|
|
+ patch_site 0b, patch__itlbmiss_linmem_top
|
|
|
#endif
|
|
|
#endif
|
|
|
#endif
|
|
@@ -353,13 +354,14 @@ _ENTRY(ITLBMiss_cmp)
|
|
|
#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
|
|
|
mtcr r12
|
|
|
#endif
|
|
|
-
|
|
|
-#ifdef CONFIG_SWAP
|
|
|
- rlwinm r11, r10, 31, _PAGE_ACCESSED >> 1
|
|
|
-#endif
|
|
|
/* Load the MI_TWC with the attributes for this "segment." */
|
|
|
mtspr SPRN_MI_TWC, r11 /* Set segment attributes */
|
|
|
|
|
|
+#ifdef CONFIG_SWAP
|
|
|
+ rlwinm r11, r10, 32-5, _PAGE_PRESENT
|
|
|
+ and r11, r11, r10
|
|
|
+ rlwimi r10, r11, 0, _PAGE_PRESENT
|
|
|
+#endif
|
|
|
li r11, RPN_PATTERN | 0x200
|
|
|
/* The Linux PTE won't go exactly into the MMU TLB.
|
|
|
* Software indicator bits 20 and 23 must be clear.
|
|
@@ -372,16 +374,17 @@ _ENTRY(ITLBMiss_cmp)
|
|
|
mtspr SPRN_MI_RPN, r10 /* Update TLB entry */
|
|
|
|
|
|
/* Restore registers */
|
|
|
-_ENTRY(itlb_miss_exit_1)
|
|
|
- mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
+0: mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
mfspr r11, SPRN_SPRG_SCRATCH1
|
|
|
#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
|
|
|
mfspr r12, SPRN_SPRG_SCRATCH2
|
|
|
#endif
|
|
|
rfi
|
|
|
+ patch_site 0b, patch__itlbmiss_exit_1
|
|
|
+
|
|
|
#ifdef CONFIG_PERF_EVENTS
|
|
|
-_ENTRY(itlb_miss_perf)
|
|
|
- lis r10, (itlb_miss_counter - PAGE_OFFSET)@ha
|
|
|
+ patch_site 0f, patch__itlbmiss_perf
|
|
|
+0: lis r10, (itlb_miss_counter - PAGE_OFFSET)@ha
|
|
|
lwz r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10)
|
|
|
addi r11, r11, 1
|
|
|
stw r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10)
|
|
@@ -435,11 +438,11 @@ DataStoreTLBMiss:
|
|
|
#ifndef CONFIG_PIN_TLB_IMMR
|
|
|
cmpli cr0, r11, VIRT_IMMR_BASE@h
|
|
|
#endif
|
|
|
-_ENTRY(DTLBMiss_cmp)
|
|
|
- cmpli cr7, r11, (PAGE_OFFSET + 0x1800000)@h
|
|
|
+0: cmpli cr7, r11, (PAGE_OFFSET + 0x1800000)@h
|
|
|
+ patch_site 0b, patch__dtlbmiss_linmem_top
|
|
|
#ifndef CONFIG_PIN_TLB_IMMR
|
|
|
-_ENTRY(DTLBMiss_jmp)
|
|
|
- beq- DTLBMissIMMR
|
|
|
+0: beq- DTLBMissIMMR
|
|
|
+ patch_site 0b, patch__dtlbmiss_immr_jmp
|
|
|
#endif
|
|
|
blt cr7, DTLBMissLinear
|
|
|
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
|
|
@@ -470,14 +473,22 @@ _ENTRY(DTLBMiss_jmp)
|
|
|
* above.
|
|
|
*/
|
|
|
rlwimi r11, r10, 0, _PAGE_GUARDED
|
|
|
-#ifdef CONFIG_SWAP
|
|
|
- /* _PAGE_ACCESSED has to be set. We use second APG bit for that, 0
|
|
|
- * on that bit will represent a Non Access group
|
|
|
- */
|
|
|
- rlwinm r11, r10, 31, _PAGE_ACCESSED >> 1
|
|
|
-#endif
|
|
|
mtspr SPRN_MD_TWC, r11
|
|
|
|
|
|
+ /* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set.
|
|
|
+ * We also need to know if the insn is a load/store, so:
|
|
|
+ * Clear _PAGE_PRESENT and load that which will
|
|
|
+ * trap into DTLB Error with store bit set accordinly.
|
|
|
+ */
|
|
|
+ /* PRESENT=0x1, ACCESSED=0x20
|
|
|
+ * r11 = ((r10 & PRESENT) & ((r10 & ACCESSED) >> 5));
|
|
|
+ * r10 = (r10 & ~PRESENT) | r11;
|
|
|
+ */
|
|
|
+#ifdef CONFIG_SWAP
|
|
|
+ rlwinm r11, r10, 32-5, _PAGE_PRESENT
|
|
|
+ and r11, r11, r10
|
|
|
+ rlwimi r10, r11, 0, _PAGE_PRESENT
|
|
|
+#endif
|
|
|
/* The Linux PTE won't go exactly into the MMU TLB.
|
|
|
* Software indicator bits 24, 25, 26, and 27 must be
|
|
|
* set. All other Linux PTE bits control the behavior
|
|
@@ -489,14 +500,16 @@ _ENTRY(DTLBMiss_jmp)
|
|
|
|
|
|
/* Restore registers */
|
|
|
mtspr SPRN_DAR, r11 /* Tag DAR */
|
|
|
-_ENTRY(dtlb_miss_exit_1)
|
|
|
- mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
+
|
|
|
+0: mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
mfspr r11, SPRN_SPRG_SCRATCH1
|
|
|
mfspr r12, SPRN_SPRG_SCRATCH2
|
|
|
rfi
|
|
|
+ patch_site 0b, patch__dtlbmiss_exit_1
|
|
|
+
|
|
|
#ifdef CONFIG_PERF_EVENTS
|
|
|
-_ENTRY(dtlb_miss_perf)
|
|
|
- lis r10, (dtlb_miss_counter - PAGE_OFFSET)@ha
|
|
|
+ patch_site 0f, patch__dtlbmiss_perf
|
|
|
+0: lis r10, (dtlb_miss_counter - PAGE_OFFSET)@ha
|
|
|
lwz r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10)
|
|
|
addi r11, r11, 1
|
|
|
stw r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10)
|
|
@@ -637,8 +650,8 @@ InstructionBreakpoint:
|
|
|
*/
|
|
|
DTLBMissIMMR:
|
|
|
mtcr r12
|
|
|
- /* Set 512k byte guarded page and mark it valid and accessed */
|
|
|
- li r10, MD_PS512K | MD_GUARDED | MD_SVALID | M_APG2
|
|
|
+ /* Set 512k byte guarded page and mark it valid */
|
|
|
+ li r10, MD_PS512K | MD_GUARDED | MD_SVALID
|
|
|
mtspr SPRN_MD_TWC, r10
|
|
|
mfspr r10, SPRN_IMMR /* Get current IMMR */
|
|
|
rlwinm r10, r10, 0, 0xfff80000 /* Get 512 kbytes boundary */
|
|
@@ -648,16 +661,17 @@ DTLBMissIMMR:
|
|
|
|
|
|
li r11, RPN_PATTERN
|
|
|
mtspr SPRN_DAR, r11 /* Tag DAR */
|
|
|
-_ENTRY(dtlb_miss_exit_2)
|
|
|
- mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
+
|
|
|
+0: mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
mfspr r11, SPRN_SPRG_SCRATCH1
|
|
|
mfspr r12, SPRN_SPRG_SCRATCH2
|
|
|
rfi
|
|
|
+ patch_site 0b, patch__dtlbmiss_exit_2
|
|
|
|
|
|
DTLBMissLinear:
|
|
|
mtcr r12
|
|
|
- /* Set 8M byte page and mark it valid and accessed */
|
|
|
- li r11, MD_PS8MEG | MD_SVALID | M_APG2
|
|
|
+ /* Set 8M byte page and mark it valid */
|
|
|
+ li r11, MD_PS8MEG | MD_SVALID
|
|
|
mtspr SPRN_MD_TWC, r11
|
|
|
rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */
|
|
|
ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
|
|
@@ -666,28 +680,29 @@ DTLBMissLinear:
|
|
|
|
|
|
li r11, RPN_PATTERN
|
|
|
mtspr SPRN_DAR, r11 /* Tag DAR */
|
|
|
-_ENTRY(dtlb_miss_exit_3)
|
|
|
- mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
+
|
|
|
+0: mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
mfspr r11, SPRN_SPRG_SCRATCH1
|
|
|
mfspr r12, SPRN_SPRG_SCRATCH2
|
|
|
rfi
|
|
|
+ patch_site 0b, patch__dtlbmiss_exit_3
|
|
|
|
|
|
#ifndef CONFIG_PIN_TLB_TEXT
|
|
|
ITLBMissLinear:
|
|
|
mtcr r12
|
|
|
- /* Set 8M byte page and mark it valid,accessed */
|
|
|
- li r11, MI_PS8MEG | MI_SVALID | M_APG2
|
|
|
+ /* Set 8M byte page and mark it valid */
|
|
|
+ li r11, MI_PS8MEG | MI_SVALID
|
|
|
mtspr SPRN_MI_TWC, r11
|
|
|
rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */
|
|
|
ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_SH | _PAGE_DIRTY | \
|
|
|
_PAGE_PRESENT
|
|
|
mtspr SPRN_MI_RPN, r10 /* Update TLB entry */
|
|
|
|
|
|
-_ENTRY(itlb_miss_exit_2)
|
|
|
- mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
+0: mfspr r10, SPRN_SPRG_SCRATCH0
|
|
|
mfspr r11, SPRN_SPRG_SCRATCH1
|
|
|
mfspr r12, SPRN_SPRG_SCRATCH2
|
|
|
rfi
|
|
|
+ patch_site 0b, patch__itlbmiss_exit_2
|
|
|
#endif
|
|
|
|
|
|
/* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
|
|
@@ -705,8 +720,10 @@ FixupDAR:/* Entry point for dcbx workaround. */
|
|
|
mfspr r11, SPRN_M_TW /* Get level 1 table */
|
|
|
blt+ 3f
|
|
|
rlwinm r11, r10, 16, 0xfff8
|
|
|
-_ENTRY(FixupDAR_cmp)
|
|
|
- cmpli cr7, r11, (PAGE_OFFSET + 0x1800000)@h
|
|
|
+
|
|
|
+0: cmpli cr7, r11, (PAGE_OFFSET + 0x1800000)@h
|
|
|
+ patch_site 0b, patch__fixupdar_linmem_top
|
|
|
+
|
|
|
/* create physical page address from effective address */
|
|
|
tophys(r11, r10)
|
|
|
blt- cr7, 201f
|
|
@@ -960,7 +977,7 @@ initial_mmu:
|
|
|
ori r8, r8, MI_EVALID /* Mark it valid */
|
|
|
mtspr SPRN_MI_EPN, r8
|
|
|
li r8, MI_PS8MEG /* Set 8M byte page */
|
|
|
- ori r8, r8, MI_SVALID | M_APG2 /* Make it valid, APG 2 */
|
|
|
+ ori r8, r8, MI_SVALID /* Make it valid */
|
|
|
mtspr SPRN_MI_TWC, r8
|
|
|
li r8, MI_BOOTINIT /* Create RPN for address 0 */
|
|
|
mtspr SPRN_MI_RPN, r8 /* Store TLB entry */
|
|
@@ -987,7 +1004,7 @@ initial_mmu:
|
|
|
ori r8, r8, MD_EVALID /* Mark it valid */
|
|
|
mtspr SPRN_MD_EPN, r8
|
|
|
li r8, MD_PS512K | MD_GUARDED /* Set 512k byte page */
|
|
|
- ori r8, r8, MD_SVALID | M_APG2 /* Make it valid and accessed */
|
|
|
+ ori r8, r8, MD_SVALID /* Make it valid */
|
|
|
mtspr SPRN_MD_TWC, r8
|
|
|
mr r8, r9 /* Create paddr for TLB */
|
|
|
ori r8, r8, MI_BOOTINIT|0x2 /* Inhibit cache -- Cort */
|