|
@@ -48,6 +48,19 @@
|
|
|
mtspr spr, reg
|
|
mtspr spr, reg
|
|
|
#endif
|
|
#endif
|
|
|
|
|
|
|
|
|
|
+/* Macro to test if an address is a kernel address */
|
|
|
|
|
+#if CONFIG_TASK_SIZE <= 0x80000000 && CONFIG_PAGE_OFFSET >= 0x80000000
|
|
|
|
|
+#define IS_KERNEL(tmp, addr) \
|
|
|
|
|
+ andis. tmp, addr, 0x8000 /* Address >= 0x80000000 */
|
|
|
|
|
+#define BRANCH_UNLESS_KERNEL(label) beq label
|
|
|
|
|
+#else
|
|
|
|
|
+#define IS_KERNEL(tmp, addr) \
|
|
|
|
|
+ rlwinm tmp, addr, 16, 16, 31; \
|
|
|
|
|
+ cmpli cr0, tmp, PAGE_OFFSET >> 16
|
|
|
|
|
+#define BRANCH_UNLESS_KERNEL(label) blt label
|
|
|
|
|
+#endif
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
/*
|
|
/*
|
|
|
* Value for the bits that have fixed value in RPN entries.
|
|
* Value for the bits that have fixed value in RPN entries.
|
|
|
* Also used for tagging DAR for DTLBerror.
|
|
* Also used for tagging DAR for DTLBerror.
|
|
@@ -116,13 +129,13 @@ turn_on_mmu:
|
|
|
*/
|
|
*/
|
|
|
#define EXCEPTION_PROLOG \
|
|
#define EXCEPTION_PROLOG \
|
|
|
EXCEPTION_PROLOG_0; \
|
|
EXCEPTION_PROLOG_0; \
|
|
|
|
|
+ mfcr r10; \
|
|
|
EXCEPTION_PROLOG_1; \
|
|
EXCEPTION_PROLOG_1; \
|
|
|
EXCEPTION_PROLOG_2
|
|
EXCEPTION_PROLOG_2
|
|
|
|
|
|
|
|
#define EXCEPTION_PROLOG_0 \
|
|
#define EXCEPTION_PROLOG_0 \
|
|
|
mtspr SPRN_SPRG_SCRATCH0,r10; \
|
|
mtspr SPRN_SPRG_SCRATCH0,r10; \
|
|
|
- mtspr SPRN_SPRG_SCRATCH1,r11; \
|
|
|
|
|
- mfcr r10
|
|
|
|
|
|
|
+ mtspr SPRN_SPRG_SCRATCH1,r11
|
|
|
|
|
|
|
|
#define EXCEPTION_PROLOG_1 \
|
|
#define EXCEPTION_PROLOG_1 \
|
|
|
mfspr r11,SPRN_SRR1; /* check whether user or kernel */ \
|
|
mfspr r11,SPRN_SRR1; /* check whether user or kernel */ \
|
|
@@ -162,7 +175,6 @@ turn_on_mmu:
|
|
|
* Exception exit code.
|
|
* Exception exit code.
|
|
|
*/
|
|
*/
|
|
|
#define EXCEPTION_EPILOG_0 \
|
|
#define EXCEPTION_EPILOG_0 \
|
|
|
- mtcr r10; \
|
|
|
|
|
mfspr r10,SPRN_SPRG_SCRATCH0; \
|
|
mfspr r10,SPRN_SPRG_SCRATCH0; \
|
|
|
mfspr r11,SPRN_SPRG_SCRATCH1
|
|
mfspr r11,SPRN_SPRG_SCRATCH1
|
|
|
|
|
|
|
@@ -297,19 +309,22 @@ SystemCall:
|
|
|
* We have to use the MD_xxx registers for the tablewalk because the
|
|
* We have to use the MD_xxx registers for the tablewalk because the
|
|
|
* equivalent MI_xxx registers only perform the attribute functions.
|
|
* equivalent MI_xxx registers only perform the attribute functions.
|
|
|
*/
|
|
*/
|
|
|
|
|
+
|
|
|
|
|
+#ifdef CONFIG_8xx_CPU15
|
|
|
|
|
+#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr) \
|
|
|
|
|
+ addi tmp, addr, PAGE_SIZE; \
|
|
|
|
|
+ tlbie tmp; \
|
|
|
|
|
+ addi tmp, addr, -PAGE_SIZE; \
|
|
|
|
|
+ tlbie tmp
|
|
|
|
|
+#else
|
|
|
|
|
+#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr)
|
|
|
|
|
+#endif
|
|
|
|
|
+
|
|
|
InstructionTLBMiss:
|
|
InstructionTLBMiss:
|
|
|
#ifdef CONFIG_8xx_CPU6
|
|
#ifdef CONFIG_8xx_CPU6
|
|
|
- mtspr SPRN_DAR, r3
|
|
|
|
|
|
|
+ mtspr SPRN_SPRG_SCRATCH2, r3
|
|
|
#endif
|
|
#endif
|
|
|
EXCEPTION_PROLOG_0
|
|
EXCEPTION_PROLOG_0
|
|
|
- mtspr SPRN_SPRG_SCRATCH2, r10
|
|
|
|
|
- mfspr r10, SPRN_SRR0 /* Get effective address of fault */
|
|
|
|
|
-#ifdef CONFIG_8xx_CPU15
|
|
|
|
|
- addi r11, r10, PAGE_SIZE
|
|
|
|
|
- tlbie r11
|
|
|
|
|
- addi r11, r10, -PAGE_SIZE
|
|
|
|
|
- tlbie r11
|
|
|
|
|
-#endif
|
|
|
|
|
|
|
|
|
|
/* If we are faulting a kernel address, we have to use the
|
|
/* If we are faulting a kernel address, we have to use the
|
|
|
* kernel page tables.
|
|
* kernel page tables.
|
|
@@ -317,24 +332,34 @@ InstructionTLBMiss:
|
|
|
#ifdef CONFIG_MODULES
|
|
#ifdef CONFIG_MODULES
|
|
|
/* Only modules will cause ITLB Misses as we always
|
|
/* Only modules will cause ITLB Misses as we always
|
|
|
* pin the first 8MB of kernel memory */
|
|
* pin the first 8MB of kernel memory */
|
|
|
- andis. r11, r10, 0x8000 /* Address >= 0x80000000 */
|
|
|
|
|
-#endif
|
|
|
|
|
|
|
+ mfspr r11, SPRN_SRR0 /* Get effective address of fault */
|
|
|
|
|
+ INVALIDATE_ADJACENT_PAGES_CPU15(r10, r11)
|
|
|
|
|
+ mfcr r10
|
|
|
|
|
+ IS_KERNEL(r11, r11)
|
|
|
mfspr r11, SPRN_M_TW /* Get level 1 table */
|
|
mfspr r11, SPRN_M_TW /* Get level 1 table */
|
|
|
-#ifdef CONFIG_MODULES
|
|
|
|
|
- beq 3f
|
|
|
|
|
|
|
+ BRANCH_UNLESS_KERNEL(3f)
|
|
|
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
|
|
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
|
|
|
3:
|
|
3:
|
|
|
|
|
+ mtcr r10
|
|
|
|
|
+ mfspr r10, SPRN_SRR0 /* Get effective address of fault */
|
|
|
|
|
+#else
|
|
|
|
|
+ mfspr r10, SPRN_SRR0 /* Get effective address of fault */
|
|
|
|
|
+ INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
|
|
|
|
|
+ mfspr r11, SPRN_M_TW /* Get level 1 table base address */
|
|
|
#endif
|
|
#endif
|
|
|
/* Insert level 1 index */
|
|
/* Insert level 1 index */
|
|
|
rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
|
|
rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
|
|
|
lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
|
|
lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
|
|
|
|
|
|
|
|
- /* Load the MI_TWC with the attributes for this "segment." */
|
|
|
|
|
- MTSPR_CPU6(SPRN_MI_TWC, r11, r3) /* Set segment attributes */
|
|
|
|
|
- rlwinm r11, r11,0,0,19 /* Extract page descriptor page address */
|
|
|
|
|
/* Extract level 2 index */
|
|
/* Extract level 2 index */
|
|
|
rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
|
|
rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
|
|
|
- lwzx r10, r10, r11 /* Get the pte */
|
|
|
|
|
|
|
+ rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
|
|
|
|
|
+ lwz r10, 0(r10) /* Get the pte */
|
|
|
|
|
+
|
|
|
|
|
+ /* Insert the APG into the TWC from the Linux PTE. */
|
|
|
|
|
+ rlwimi r11, r10, 0, 25, 26
|
|
|
|
|
+ /* Load the MI_TWC with the attributes for this "segment." */
|
|
|
|
|
+ MTSPR_CPU6(SPRN_MI_TWC, r11, r3) /* Set segment attributes */
|
|
|
|
|
|
|
|
#ifdef CONFIG_SWAP
|
|
#ifdef CONFIG_SWAP
|
|
|
rlwinm r11, r10, 32-5, _PAGE_PRESENT
|
|
rlwinm r11, r10, 32-5, _PAGE_PRESENT
|
|
@@ -343,40 +368,41 @@ InstructionTLBMiss:
|
|
|
#endif
|
|
#endif
|
|
|
li r11, RPN_PATTERN
|
|
li r11, RPN_PATTERN
|
|
|
/* The Linux PTE won't go exactly into the MMU TLB.
|
|
/* The Linux PTE won't go exactly into the MMU TLB.
|
|
|
- * Software indicator bits 21 and 28 must be clear.
|
|
|
|
|
|
|
+ * Software indicator bits 20-23 and 28 must be clear.
|
|
|
* Software indicator bits 24, 25, 26, and 27 must be
|
|
* Software indicator bits 24, 25, 26, and 27 must be
|
|
|
* set. All other Linux PTE bits control the behavior
|
|
* set. All other Linux PTE bits control the behavior
|
|
|
* of the MMU.
|
|
* of the MMU.
|
|
|
*/
|
|
*/
|
|
|
- rlwimi r10, r11, 0, 0x07f8 /* Set 24-27, clear 21-23,28 */
|
|
|
|
|
|
|
+ rlwimi r10, r11, 0, 0x0ff8 /* Set 24-27, clear 20-23,28 */
|
|
|
MTSPR_CPU6(SPRN_MI_RPN, r10, r3) /* Update TLB entry */
|
|
MTSPR_CPU6(SPRN_MI_RPN, r10, r3) /* Update TLB entry */
|
|
|
|
|
|
|
|
/* Restore registers */
|
|
/* Restore registers */
|
|
|
#ifdef CONFIG_8xx_CPU6
|
|
#ifdef CONFIG_8xx_CPU6
|
|
|
- mfspr r3, SPRN_DAR
|
|
|
|
|
- mtspr SPRN_DAR, r11 /* Tag DAR */
|
|
|
|
|
|
|
+ mfspr r3, SPRN_SPRG_SCRATCH2
|
|
|
#endif
|
|
#endif
|
|
|
- mfspr r10, SPRN_SPRG_SCRATCH2
|
|
|
|
|
EXCEPTION_EPILOG_0
|
|
EXCEPTION_EPILOG_0
|
|
|
rfi
|
|
rfi
|
|
|
|
|
|
|
|
. = 0x1200
|
|
. = 0x1200
|
|
|
DataStoreTLBMiss:
|
|
DataStoreTLBMiss:
|
|
|
#ifdef CONFIG_8xx_CPU6
|
|
#ifdef CONFIG_8xx_CPU6
|
|
|
- mtspr SPRN_DAR, r3
|
|
|
|
|
|
|
+ mtspr SPRN_SPRG_SCRATCH2, r3
|
|
|
#endif
|
|
#endif
|
|
|
EXCEPTION_PROLOG_0
|
|
EXCEPTION_PROLOG_0
|
|
|
- mtspr SPRN_SPRG_SCRATCH2, r10
|
|
|
|
|
- mfspr r10, SPRN_MD_EPN
|
|
|
|
|
|
|
+ mfcr r10
|
|
|
|
|
|
|
|
/* If we are faulting a kernel address, we have to use the
|
|
/* If we are faulting a kernel address, we have to use the
|
|
|
* kernel page tables.
|
|
* kernel page tables.
|
|
|
*/
|
|
*/
|
|
|
- andis. r11, r10, 0x8000
|
|
|
|
|
|
|
+ mfspr r11, SPRN_MD_EPN
|
|
|
|
|
+ IS_KERNEL(r11, r11)
|
|
|
mfspr r11, SPRN_M_TW /* Get level 1 table */
|
|
mfspr r11, SPRN_M_TW /* Get level 1 table */
|
|
|
- beq 3f
|
|
|
|
|
|
|
+ BRANCH_UNLESS_KERNEL(3f)
|
|
|
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
|
|
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
|
|
|
3:
|
|
3:
|
|
|
|
|
+ mtcr r10
|
|
|
|
|
+ mfspr r10, SPRN_MD_EPN
|
|
|
|
|
+
|
|
|
/* Insert level 1 index */
|
|
/* Insert level 1 index */
|
|
|
rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
|
|
rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
|
|
|
lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
|
|
lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
|
|
@@ -388,13 +414,13 @@ DataStoreTLBMiss:
|
|
|
rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
|
|
rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
|
|
|
lwz r10, 0(r10) /* Get the pte */
|
|
lwz r10, 0(r10) /* Get the pte */
|
|
|
|
|
|
|
|
- /* Insert the Guarded flag into the TWC from the Linux PTE.
|
|
|
|
|
- * It is bit 27 of both the Linux PTE and the TWC (at least
|
|
|
|
|
|
|
+ /* Insert the Guarded flag and APG into the TWC from the Linux PTE.
|
|
|
|
|
+ * It is bit 26-27 of both the Linux PTE and the TWC (at least
|
|
|
* I got that right :-). It will be better when we can put
|
|
* I got that right :-). It will be better when we can put
|
|
|
* this into the Linux pgd/pmd and load it in the operation
|
|
* this into the Linux pgd/pmd and load it in the operation
|
|
|
* above.
|
|
* above.
|
|
|
*/
|
|
*/
|
|
|
- rlwimi r11, r10, 0, 27, 27
|
|
|
|
|
|
|
+ rlwimi r11, r10, 0, 26, 27
|
|
|
/* Insert the WriteThru flag into the TWC from the Linux PTE.
|
|
/* Insert the WriteThru flag into the TWC from the Linux PTE.
|
|
|
* It is bit 25 in the Linux PTE and bit 30 in the TWC
|
|
* It is bit 25 in the Linux PTE and bit 30 in the TWC
|
|
|
*/
|
|
*/
|
|
@@ -423,14 +449,14 @@ DataStoreTLBMiss:
|
|
|
*/
|
|
*/
|
|
|
li r11, RPN_PATTERN
|
|
li r11, RPN_PATTERN
|
|
|
rlwimi r10, r11, 0, 24, 28 /* Set 24-27, clear 28 */
|
|
rlwimi r10, r11, 0, 24, 28 /* Set 24-27, clear 28 */
|
|
|
|
|
+ rlwimi r10, r11, 0, 20, 20 /* clear 20 */
|
|
|
MTSPR_CPU6(SPRN_MD_RPN, r10, r3) /* Update TLB entry */
|
|
MTSPR_CPU6(SPRN_MD_RPN, r10, r3) /* Update TLB entry */
|
|
|
|
|
|
|
|
/* Restore registers */
|
|
/* Restore registers */
|
|
|
#ifdef CONFIG_8xx_CPU6
|
|
#ifdef CONFIG_8xx_CPU6
|
|
|
- mfspr r3, SPRN_DAR
|
|
|
|
|
|
|
+ mfspr r3, SPRN_SPRG_SCRATCH2
|
|
|
#endif
|
|
#endif
|
|
|
mtspr SPRN_DAR, r11 /* Tag DAR */
|
|
mtspr SPRN_DAR, r11 /* Tag DAR */
|
|
|
- mfspr r10, SPRN_SPRG_SCRATCH2
|
|
|
|
|
EXCEPTION_EPILOG_0
|
|
EXCEPTION_EPILOG_0
|
|
|
rfi
|
|
rfi
|
|
|
|
|
|
|
@@ -456,6 +482,7 @@ InstructionTLBError:
|
|
|
. = 0x1400
|
|
. = 0x1400
|
|
|
DataTLBError:
|
|
DataTLBError:
|
|
|
EXCEPTION_PROLOG_0
|
|
EXCEPTION_PROLOG_0
|
|
|
|
|
+ mfcr r10
|
|
|
|
|
|
|
|
mfspr r11, SPRN_DAR
|
|
mfspr r11, SPRN_DAR
|
|
|
cmpwi cr0, r11, RPN_PATTERN
|
|
cmpwi cr0, r11, RPN_PATTERN
|
|
@@ -503,9 +530,9 @@ FixupDAR:/* Entry point for dcbx workaround. */
|
|
|
mtspr SPRN_SPRG_SCRATCH2, r10
|
|
mtspr SPRN_SPRG_SCRATCH2, r10
|
|
|
/* fetch instruction from memory. */
|
|
/* fetch instruction from memory. */
|
|
|
mfspr r10, SPRN_SRR0
|
|
mfspr r10, SPRN_SRR0
|
|
|
- andis. r11, r10, 0x8000 /* Address >= 0x80000000 */
|
|
|
|
|
|
|
+ IS_KERNEL(r11, r10)
|
|
|
mfspr r11, SPRN_M_TW /* Get level 1 table */
|
|
mfspr r11, SPRN_M_TW /* Get level 1 table */
|
|
|
- beq 3f
|
|
|
|
|
|
|
+ BRANCH_UNLESS_KERNEL(3f)
|
|
|
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
|
|
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
|
|
|
/* Insert level 1 index */
|
|
/* Insert level 1 index */
|
|
|
3: rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
|
|
3: rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
|
|
@@ -743,15 +770,20 @@ initial_mmu:
|
|
|
ori r8, r8, MI_EVALID /* Mark it valid */
|
|
ori r8, r8, MI_EVALID /* Mark it valid */
|
|
|
mtspr SPRN_MI_EPN, r8
|
|
mtspr SPRN_MI_EPN, r8
|
|
|
mtspr SPRN_MD_EPN, r8
|
|
mtspr SPRN_MD_EPN, r8
|
|
|
- li r8, MI_PS8MEG /* Set 8M byte page */
|
|
|
|
|
|
|
+ li r8, MI_PS8MEG | (2 << 5) /* Set 8M byte page, APG 2 */
|
|
|
ori r8, r8, MI_SVALID /* Make it valid */
|
|
ori r8, r8, MI_SVALID /* Make it valid */
|
|
|
mtspr SPRN_MI_TWC, r8
|
|
mtspr SPRN_MI_TWC, r8
|
|
|
|
|
+ li r8, MI_PS8MEG /* Set 8M byte page, APG 0 */
|
|
|
|
|
+ ori r8, r8, MI_SVALID /* Make it valid */
|
|
|
mtspr SPRN_MD_TWC, r8
|
|
mtspr SPRN_MD_TWC, r8
|
|
|
li r8, MI_BOOTINIT /* Create RPN for address 0 */
|
|
li r8, MI_BOOTINIT /* Create RPN for address 0 */
|
|
|
mtspr SPRN_MI_RPN, r8 /* Store TLB entry */
|
|
mtspr SPRN_MI_RPN, r8 /* Store TLB entry */
|
|
|
mtspr SPRN_MD_RPN, r8
|
|
mtspr SPRN_MD_RPN, r8
|
|
|
- lis r8, MI_Kp@h /* Set the protection mode */
|
|
|
|
|
|
|
+ lis r8, MI_APG_INIT@h /* Set protection modes */
|
|
|
|
|
+ ori r8, r8, MI_APG_INIT@l
|
|
|
mtspr SPRN_MI_AP, r8
|
|
mtspr SPRN_MI_AP, r8
|
|
|
|
|
+ lis r8, MD_APG_INIT@h
|
|
|
|
|
+ ori r8, r8, MD_APG_INIT@l
|
|
|
mtspr SPRN_MD_AP, r8
|
|
mtspr SPRN_MD_AP, r8
|
|
|
|
|
|
|
|
/* Map another 8 MByte at the IMMR to get the processor
|
|
/* Map another 8 MByte at the IMMR to get the processor
|