|
@@ -256,7 +256,7 @@ BEGIN_FTR_SECTION
|
|
LOAD_HANDLER(r12, machine_check_handle_early)
|
|
LOAD_HANDLER(r12, machine_check_handle_early)
|
|
1: mtspr SPRN_SRR0,r12
|
|
1: mtspr SPRN_SRR0,r12
|
|
mtspr SPRN_SRR1,r11
|
|
mtspr SPRN_SRR1,r11
|
|
- rfid
|
|
|
|
|
|
+ RFI_TO_KERNEL
|
|
b . /* prevent speculative execution */
|
|
b . /* prevent speculative execution */
|
|
2:
|
|
2:
|
|
/* Stack overflow. Stay on emergency stack and panic.
|
|
/* Stack overflow. Stay on emergency stack and panic.
|
|
@@ -445,7 +445,7 @@ EXC_COMMON_BEGIN(machine_check_handle_early)
|
|
li r3,MSR_ME
|
|
li r3,MSR_ME
|
|
andc r10,r10,r3 /* Turn off MSR_ME */
|
|
andc r10,r10,r3 /* Turn off MSR_ME */
|
|
mtspr SPRN_SRR1,r10
|
|
mtspr SPRN_SRR1,r10
|
|
- rfid
|
|
|
|
|
|
+ RFI_TO_KERNEL
|
|
b .
|
|
b .
|
|
2:
|
|
2:
|
|
/*
|
|
/*
|
|
@@ -463,7 +463,7 @@ EXC_COMMON_BEGIN(machine_check_handle_early)
|
|
*/
|
|
*/
|
|
bl machine_check_queue_event
|
|
bl machine_check_queue_event
|
|
MACHINE_CHECK_HANDLER_WINDUP
|
|
MACHINE_CHECK_HANDLER_WINDUP
|
|
- rfid
|
|
|
|
|
|
+ RFI_TO_USER_OR_KERNEL
|
|
9:
|
|
9:
|
|
/* Deliver the machine check to host kernel in V mode. */
|
|
/* Deliver the machine check to host kernel in V mode. */
|
|
MACHINE_CHECK_HANDLER_WINDUP
|
|
MACHINE_CHECK_HANDLER_WINDUP
|
|
@@ -598,6 +598,9 @@ EXC_COMMON_BEGIN(slb_miss_common)
|
|
stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
|
|
stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
|
|
std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
|
|
std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
|
|
|
|
|
|
|
|
+ andi. r9,r11,MSR_PR // Check for exception from userspace
|
|
|
|
+ cmpdi cr4,r9,MSR_PR // And save the result in CR4 for later
|
|
|
|
+
|
|
/*
|
|
/*
|
|
* Test MSR_RI before calling slb_allocate_realmode, because the
|
|
* Test MSR_RI before calling slb_allocate_realmode, because the
|
|
* MSR in r11 gets clobbered. However we still want to allocate
|
|
* MSR in r11 gets clobbered. However we still want to allocate
|
|
@@ -624,9 +627,12 @@ END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
|
|
|
|
|
|
/* All done -- return from exception. */
|
|
/* All done -- return from exception. */
|
|
|
|
|
|
|
|
+ bne cr4,1f /* returning to kernel */
|
|
|
|
+
|
|
.machine push
|
|
.machine push
|
|
.machine "power4"
|
|
.machine "power4"
|
|
mtcrf 0x80,r9
|
|
mtcrf 0x80,r9
|
|
|
|
+ mtcrf 0x08,r9 /* MSR[PR] indication is in cr4 */
|
|
mtcrf 0x04,r9 /* MSR[RI] indication is in cr5 */
|
|
mtcrf 0x04,r9 /* MSR[RI] indication is in cr5 */
|
|
mtcrf 0x02,r9 /* I/D indication is in cr6 */
|
|
mtcrf 0x02,r9 /* I/D indication is in cr6 */
|
|
mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
|
|
mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
|
|
@@ -640,9 +646,30 @@ END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
|
|
ld r11,PACA_EXSLB+EX_R11(r13)
|
|
ld r11,PACA_EXSLB+EX_R11(r13)
|
|
ld r12,PACA_EXSLB+EX_R12(r13)
|
|
ld r12,PACA_EXSLB+EX_R12(r13)
|
|
ld r13,PACA_EXSLB+EX_R13(r13)
|
|
ld r13,PACA_EXSLB+EX_R13(r13)
|
|
- rfid
|
|
|
|
|
|
+ RFI_TO_USER
|
|
|
|
+ b . /* prevent speculative execution */
|
|
|
|
+1:
|
|
|
|
+.machine push
|
|
|
|
+.machine "power4"
|
|
|
|
+ mtcrf 0x80,r9
|
|
|
|
+ mtcrf 0x08,r9 /* MSR[PR] indication is in cr4 */
|
|
|
|
+ mtcrf 0x04,r9 /* MSR[RI] indication is in cr5 */
|
|
|
|
+ mtcrf 0x02,r9 /* I/D indication is in cr6 */
|
|
|
|
+ mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
|
|
|
|
+.machine pop
|
|
|
|
+
|
|
|
|
+ RESTORE_CTR(r9, PACA_EXSLB)
|
|
|
|
+ RESTORE_PPR_PACA(PACA_EXSLB, r9)
|
|
|
|
+ mr r3,r12
|
|
|
|
+ ld r9,PACA_EXSLB+EX_R9(r13)
|
|
|
|
+ ld r10,PACA_EXSLB+EX_R10(r13)
|
|
|
|
+ ld r11,PACA_EXSLB+EX_R11(r13)
|
|
|
|
+ ld r12,PACA_EXSLB+EX_R12(r13)
|
|
|
|
+ ld r13,PACA_EXSLB+EX_R13(r13)
|
|
|
|
+ RFI_TO_KERNEL
|
|
b . /* prevent speculative execution */
|
|
b . /* prevent speculative execution */
|
|
|
|
|
|
|
|
+
|
|
2: std r3,PACA_EXSLB+EX_DAR(r13)
|
|
2: std r3,PACA_EXSLB+EX_DAR(r13)
|
|
mr r3,r12
|
|
mr r3,r12
|
|
mfspr r11,SPRN_SRR0
|
|
mfspr r11,SPRN_SRR0
|
|
@@ -651,7 +678,7 @@ END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
|
|
mtspr SPRN_SRR0,r10
|
|
mtspr SPRN_SRR0,r10
|
|
ld r10,PACAKMSR(r13)
|
|
ld r10,PACAKMSR(r13)
|
|
mtspr SPRN_SRR1,r10
|
|
mtspr SPRN_SRR1,r10
|
|
- rfid
|
|
|
|
|
|
+ RFI_TO_KERNEL
|
|
b .
|
|
b .
|
|
|
|
|
|
8: std r3,PACA_EXSLB+EX_DAR(r13)
|
|
8: std r3,PACA_EXSLB+EX_DAR(r13)
|
|
@@ -662,7 +689,7 @@ END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
|
|
mtspr SPRN_SRR0,r10
|
|
mtspr SPRN_SRR0,r10
|
|
ld r10,PACAKMSR(r13)
|
|
ld r10,PACAKMSR(r13)
|
|
mtspr SPRN_SRR1,r10
|
|
mtspr SPRN_SRR1,r10
|
|
- rfid
|
|
|
|
|
|
+ RFI_TO_KERNEL
|
|
b .
|
|
b .
|
|
|
|
|
|
EXC_COMMON_BEGIN(unrecov_slb)
|
|
EXC_COMMON_BEGIN(unrecov_slb)
|
|
@@ -901,7 +928,7 @@ EXC_COMMON(trap_0b_common, 0xb00, unknown_exception)
|
|
mtspr SPRN_SRR0,r10 ; \
|
|
mtspr SPRN_SRR0,r10 ; \
|
|
ld r10,PACAKMSR(r13) ; \
|
|
ld r10,PACAKMSR(r13) ; \
|
|
mtspr SPRN_SRR1,r10 ; \
|
|
mtspr SPRN_SRR1,r10 ; \
|
|
- rfid ; \
|
|
|
|
|
|
+ RFI_TO_KERNEL ; \
|
|
b . ; /* prevent speculative execution */
|
|
b . ; /* prevent speculative execution */
|
|
|
|
|
|
#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
|
|
#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
|
|
@@ -917,7 +944,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \
|
|
xori r12,r12,MSR_LE ; \
|
|
xori r12,r12,MSR_LE ; \
|
|
mtspr SPRN_SRR1,r12 ; \
|
|
mtspr SPRN_SRR1,r12 ; \
|
|
mr r13,r9 ; \
|
|
mr r13,r9 ; \
|
|
- rfid ; /* return to userspace */ \
|
|
|
|
|
|
+ RFI_TO_USER ; /* return to userspace */ \
|
|
b . ; /* prevent speculative execution */
|
|
b . ; /* prevent speculative execution */
|
|
#else
|
|
#else
|
|
#define SYSCALL_FASTENDIAN_TEST
|
|
#define SYSCALL_FASTENDIAN_TEST
|
|
@@ -1063,7 +1090,7 @@ TRAMP_REAL_BEGIN(hmi_exception_early)
|
|
mtcr r11
|
|
mtcr r11
|
|
REST_GPR(11, r1)
|
|
REST_GPR(11, r1)
|
|
ld r1,GPR1(r1)
|
|
ld r1,GPR1(r1)
|
|
- hrfid
|
|
|
|
|
|
+ HRFI_TO_USER_OR_KERNEL
|
|
|
|
|
|
1: mtcr r11
|
|
1: mtcr r11
|
|
REST_GPR(11, r1)
|
|
REST_GPR(11, r1)
|
|
@@ -1314,7 +1341,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
|
|
ld r11,PACA_EXGEN+EX_R11(r13)
|
|
ld r11,PACA_EXGEN+EX_R11(r13)
|
|
ld r12,PACA_EXGEN+EX_R12(r13)
|
|
ld r12,PACA_EXGEN+EX_R12(r13)
|
|
ld r13,PACA_EXGEN+EX_R13(r13)
|
|
ld r13,PACA_EXGEN+EX_R13(r13)
|
|
- HRFID
|
|
|
|
|
|
+ HRFI_TO_UNKNOWN
|
|
b .
|
|
b .
|
|
#endif
|
|
#endif
|
|
|
|
|
|
@@ -1418,10 +1445,94 @@ masked_##_H##interrupt: \
|
|
ld r10,PACA_EXGEN+EX_R10(r13); \
|
|
ld r10,PACA_EXGEN+EX_R10(r13); \
|
|
ld r11,PACA_EXGEN+EX_R11(r13); \
|
|
ld r11,PACA_EXGEN+EX_R11(r13); \
|
|
/* returns to kernel where r13 must be set up, so don't restore it */ \
|
|
/* returns to kernel where r13 must be set up, so don't restore it */ \
|
|
- ##_H##rfid; \
|
|
|
|
|
|
+ ##_H##RFI_TO_KERNEL; \
|
|
b .; \
|
|
b .; \
|
|
MASKED_DEC_HANDLER(_H)
|
|
MASKED_DEC_HANDLER(_H)
|
|
|
|
|
|
|
|
+TRAMP_REAL_BEGIN(rfi_flush_fallback)
|
|
|
|
+ SET_SCRATCH0(r13);
|
|
|
|
+ GET_PACA(r13);
|
|
|
|
+ std r9,PACA_EXRFI+EX_R9(r13)
|
|
|
|
+ std r10,PACA_EXRFI+EX_R10(r13)
|
|
|
|
+ std r11,PACA_EXRFI+EX_R11(r13)
|
|
|
|
+ std r12,PACA_EXRFI+EX_R12(r13)
|
|
|
|
+ std r8,PACA_EXRFI+EX_R13(r13)
|
|
|
|
+ mfctr r9
|
|
|
|
+ ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
|
|
|
|
+ ld r11,PACA_L1D_FLUSH_SETS(r13)
|
|
|
|
+ ld r12,PACA_L1D_FLUSH_CONGRUENCE(r13)
|
|
|
|
+ /*
|
|
|
|
+ * The load adresses are at staggered offsets within cachelines,
|
|
|
|
+ * which suits some pipelines better (on others it should not
|
|
|
|
+ * hurt).
|
|
|
|
+ */
|
|
|
|
+ addi r12,r12,8
|
|
|
|
+ mtctr r11
|
|
|
|
+ DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
|
|
|
|
+
|
|
|
|
+ /* order ld/st prior to dcbt stop all streams with flushing */
|
|
|
|
+ sync
|
|
|
|
+1: li r8,0
|
|
|
|
+ .rept 8 /* 8-way set associative */
|
|
|
|
+ ldx r11,r10,r8
|
|
|
|
+ add r8,r8,r12
|
|
|
|
+ xor r11,r11,r11 // Ensure r11 is 0 even if fallback area is not
|
|
|
|
+ add r8,r8,r11 // Add 0, this creates a dependency on the ldx
|
|
|
|
+ .endr
|
|
|
|
+ addi r10,r10,128 /* 128 byte cache line */
|
|
|
|
+ bdnz 1b
|
|
|
|
+
|
|
|
|
+ mtctr r9
|
|
|
|
+ ld r9,PACA_EXRFI+EX_R9(r13)
|
|
|
|
+ ld r10,PACA_EXRFI+EX_R10(r13)
|
|
|
|
+ ld r11,PACA_EXRFI+EX_R11(r13)
|
|
|
|
+ ld r12,PACA_EXRFI+EX_R12(r13)
|
|
|
|
+ ld r8,PACA_EXRFI+EX_R13(r13)
|
|
|
|
+ GET_SCRATCH0(r13);
|
|
|
|
+ rfid
|
|
|
|
+
|
|
|
|
+TRAMP_REAL_BEGIN(hrfi_flush_fallback)
|
|
|
|
+ SET_SCRATCH0(r13);
|
|
|
|
+ GET_PACA(r13);
|
|
|
|
+ std r9,PACA_EXRFI+EX_R9(r13)
|
|
|
|
+ std r10,PACA_EXRFI+EX_R10(r13)
|
|
|
|
+ std r11,PACA_EXRFI+EX_R11(r13)
|
|
|
|
+ std r12,PACA_EXRFI+EX_R12(r13)
|
|
|
|
+ std r8,PACA_EXRFI+EX_R13(r13)
|
|
|
|
+ mfctr r9
|
|
|
|
+ ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
|
|
|
|
+ ld r11,PACA_L1D_FLUSH_SETS(r13)
|
|
|
|
+ ld r12,PACA_L1D_FLUSH_CONGRUENCE(r13)
|
|
|
|
+ /*
|
|
|
|
+ * The load adresses are at staggered offsets within cachelines,
|
|
|
|
+ * which suits some pipelines better (on others it should not
|
|
|
|
+ * hurt).
|
|
|
|
+ */
|
|
|
|
+ addi r12,r12,8
|
|
|
|
+ mtctr r11
|
|
|
|
+ DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
|
|
|
|
+
|
|
|
|
+ /* order ld/st prior to dcbt stop all streams with flushing */
|
|
|
|
+ sync
|
|
|
|
+1: li r8,0
|
|
|
|
+ .rept 8 /* 8-way set associative */
|
|
|
|
+ ldx r11,r10,r8
|
|
|
|
+ add r8,r8,r12
|
|
|
|
+ xor r11,r11,r11 // Ensure r11 is 0 even if fallback area is not
|
|
|
|
+ add r8,r8,r11 // Add 0, this creates a dependency on the ldx
|
|
|
|
+ .endr
|
|
|
|
+ addi r10,r10,128 /* 128 byte cache line */
|
|
|
|
+ bdnz 1b
|
|
|
|
+
|
|
|
|
+ mtctr r9
|
|
|
|
+ ld r9,PACA_EXRFI+EX_R9(r13)
|
|
|
|
+ ld r10,PACA_EXRFI+EX_R10(r13)
|
|
|
|
+ ld r11,PACA_EXRFI+EX_R11(r13)
|
|
|
|
+ ld r12,PACA_EXRFI+EX_R12(r13)
|
|
|
|
+ ld r8,PACA_EXRFI+EX_R13(r13)
|
|
|
|
+ GET_SCRATCH0(r13);
|
|
|
|
+ hrfid
|
|
|
|
+
|
|
/*
|
|
/*
|
|
* Real mode exceptions actually use this too, but alternate
|
|
* Real mode exceptions actually use this too, but alternate
|
|
* instruction code patches (which end up in the common .text area)
|
|
* instruction code patches (which end up in the common .text area)
|
|
@@ -1441,7 +1552,7 @@ TRAMP_REAL_BEGIN(kvmppc_skip_interrupt)
|
|
addi r13, r13, 4
|
|
addi r13, r13, 4
|
|
mtspr SPRN_SRR0, r13
|
|
mtspr SPRN_SRR0, r13
|
|
GET_SCRATCH0(r13)
|
|
GET_SCRATCH0(r13)
|
|
- rfid
|
|
|
|
|
|
+ RFI_TO_KERNEL
|
|
b .
|
|
b .
|
|
|
|
|
|
TRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt)
|
|
TRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt)
|
|
@@ -1453,7 +1564,7 @@ TRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt)
|
|
addi r13, r13, 4
|
|
addi r13, r13, 4
|
|
mtspr SPRN_HSRR0, r13
|
|
mtspr SPRN_HSRR0, r13
|
|
GET_SCRATCH0(r13)
|
|
GET_SCRATCH0(r13)
|
|
- hrfid
|
|
|
|
|
|
+ HRFI_TO_KERNEL
|
|
b .
|
|
b .
|
|
#endif
|
|
#endif
|
|
|
|
|