|
@@ -78,7 +78,7 @@ static u32 get_ccsidr(u32 csselr)
|
|
|
* See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
|
|
|
*/
|
|
|
static bool access_dcsw(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *r)
|
|
|
{
|
|
|
if (!p->is_write)
|
|
@@ -94,21 +94,19 @@ static bool access_dcsw(struct kvm_vcpu *vcpu,
|
|
|
* sys_regs and leave it in complete control of the caches.
|
|
|
*/
|
|
|
static bool access_vm_reg(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *r)
|
|
|
{
|
|
|
- unsigned long val;
|
|
|
bool was_enabled = vcpu_has_cache_enabled(vcpu);
|
|
|
|
|
|
BUG_ON(!p->is_write);
|
|
|
|
|
|
- val = *vcpu_reg(vcpu, p->Rt);
|
|
|
if (!p->is_aarch32) {
|
|
|
- vcpu_sys_reg(vcpu, r->reg) = val;
|
|
|
+ vcpu_sys_reg(vcpu, r->reg) = p->regval;
|
|
|
} else {
|
|
|
if (!p->is_32bit)
|
|
|
- vcpu_cp15_64_high(vcpu, r->reg) = val >> 32;
|
|
|
- vcpu_cp15_64_low(vcpu, r->reg) = val & 0xffffffffUL;
|
|
|
+ vcpu_cp15_64_high(vcpu, r->reg) = upper_32_bits(p->regval);
|
|
|
+ vcpu_cp15_64_low(vcpu, r->reg) = lower_32_bits(p->regval);
|
|
|
}
|
|
|
|
|
|
kvm_toggle_cache(vcpu, was_enabled);
|
|
@@ -122,22 +120,19 @@ static bool access_vm_reg(struct kvm_vcpu *vcpu,
|
|
|
* for both AArch64 and AArch32 accesses.
|
|
|
*/
|
|
|
static bool access_gic_sgi(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *r)
|
|
|
{
|
|
|
- u64 val;
|
|
|
-
|
|
|
if (!p->is_write)
|
|
|
return read_from_write_only(vcpu, p);
|
|
|
|
|
|
- val = *vcpu_reg(vcpu, p->Rt);
|
|
|
- vgic_v3_dispatch_sgi(vcpu, val);
|
|
|
+ vgic_v3_dispatch_sgi(vcpu, p->regval);
|
|
|
|
|
|
return true;
|
|
|
}
|
|
|
|
|
|
static bool trap_raz_wi(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *r)
|
|
|
{
|
|
|
if (p->is_write)
|
|
@@ -147,19 +142,19 @@ static bool trap_raz_wi(struct kvm_vcpu *vcpu,
|
|
|
}
|
|
|
|
|
|
static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *r)
|
|
|
{
|
|
|
if (p->is_write) {
|
|
|
return ignore_write(vcpu, p);
|
|
|
} else {
|
|
|
- *vcpu_reg(vcpu, p->Rt) = (1 << 3);
|
|
|
+ p->regval = (1 << 3);
|
|
|
return true;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *r)
|
|
|
{
|
|
|
if (p->is_write) {
|
|
@@ -167,7 +162,7 @@ static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
|
|
|
} else {
|
|
|
u32 val;
|
|
|
asm volatile("mrs %0, dbgauthstatus_el1" : "=r" (val));
|
|
|
- *vcpu_reg(vcpu, p->Rt) = val;
|
|
|
+ p->regval = val;
|
|
|
return true;
|
|
|
}
|
|
|
}
|
|
@@ -200,17 +195,17 @@ static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
|
|
|
* now use the debug registers.
|
|
|
*/
|
|
|
static bool trap_debug_regs(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *r)
|
|
|
{
|
|
|
if (p->is_write) {
|
|
|
- vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
|
|
|
+ vcpu_sys_reg(vcpu, r->reg) = p->regval;
|
|
|
vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
|
|
|
} else {
|
|
|
- *vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
|
|
|
+ p->regval = vcpu_sys_reg(vcpu, r->reg);
|
|
|
}
|
|
|
|
|
|
- trace_trap_reg(__func__, r->reg, p->is_write, *vcpu_reg(vcpu, p->Rt));
|
|
|
+ trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
|
|
|
|
|
|
return true;
|
|
|
}
|
|
@@ -225,10 +220,10 @@ static bool trap_debug_regs(struct kvm_vcpu *vcpu,
|
|
|
* hyp.S code switches between host and guest values in future.
|
|
|
*/
|
|
|
static inline void reg_to_dbg(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
u64 *dbg_reg)
|
|
|
{
|
|
|
- u64 val = *vcpu_reg(vcpu, p->Rt);
|
|
|
+ u64 val = p->regval;
|
|
|
|
|
|
if (p->is_32bit) {
|
|
|
val &= 0xffffffffUL;
|
|
@@ -240,19 +235,16 @@ static inline void reg_to_dbg(struct kvm_vcpu *vcpu,
|
|
|
}
|
|
|
|
|
|
static inline void dbg_to_reg(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
u64 *dbg_reg)
|
|
|
{
|
|
|
- u64 val = *dbg_reg;
|
|
|
-
|
|
|
+ p->regval = *dbg_reg;
|
|
|
if (p->is_32bit)
|
|
|
- val &= 0xffffffffUL;
|
|
|
-
|
|
|
- *vcpu_reg(vcpu, p->Rt) = val;
|
|
|
+ p->regval &= 0xffffffffUL;
|
|
|
}
|
|
|
|
|
|
static inline bool trap_bvr(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *rd)
|
|
|
{
|
|
|
u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
|
|
@@ -294,7 +286,7 @@ static inline void reset_bvr(struct kvm_vcpu *vcpu,
|
|
|
}
|
|
|
|
|
|
static inline bool trap_bcr(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *rd)
|
|
|
{
|
|
|
u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
|
|
@@ -337,7 +329,7 @@ static inline void reset_bcr(struct kvm_vcpu *vcpu,
|
|
|
}
|
|
|
|
|
|
static inline bool trap_wvr(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *rd)
|
|
|
{
|
|
|
u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
|
|
@@ -380,7 +372,7 @@ static inline void reset_wvr(struct kvm_vcpu *vcpu,
|
|
|
}
|
|
|
|
|
|
static inline bool trap_wcr(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *rd)
|
|
|
{
|
|
|
u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
|
|
@@ -687,7 +679,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
|
|
|
};
|
|
|
|
|
|
static bool trap_dbgidr(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *r)
|
|
|
{
|
|
|
if (p->is_write) {
|
|
@@ -697,23 +689,23 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu,
|
|
|
u64 pfr = read_system_reg(SYS_ID_AA64PFR0_EL1);
|
|
|
u32 el3 = !!cpuid_feature_extract_field(pfr, ID_AA64PFR0_EL3_SHIFT);
|
|
|
|
|
|
- *vcpu_reg(vcpu, p->Rt) = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
|
|
|
- (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
|
|
|
- (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20) |
|
|
|
- (6 << 16) | (el3 << 14) | (el3 << 12));
|
|
|
+ p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
|
|
|
+ (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
|
|
|
+ (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
|
|
|
+ | (6 << 16) | (el3 << 14) | (el3 << 12));
|
|
|
return true;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
static bool trap_debug32(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *r)
|
|
|
{
|
|
|
if (p->is_write) {
|
|
|
- vcpu_cp14(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
|
|
|
+ vcpu_cp14(vcpu, r->reg) = p->regval;
|
|
|
vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
|
|
|
} else {
|
|
|
- *vcpu_reg(vcpu, p->Rt) = vcpu_cp14(vcpu, r->reg);
|
|
|
+ p->regval = vcpu_cp14(vcpu, r->reg);
|
|
|
}
|
|
|
|
|
|
return true;
|
|
@@ -731,7 +723,7 @@ static bool trap_debug32(struct kvm_vcpu *vcpu,
|
|
|
*/
|
|
|
|
|
|
static inline bool trap_xvr(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *p,
|
|
|
+ struct sys_reg_params *p,
|
|
|
const struct sys_reg_desc *rd)
|
|
|
{
|
|
|
u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
|
|
@@ -740,12 +732,12 @@ static inline bool trap_xvr(struct kvm_vcpu *vcpu,
|
|
|
u64 val = *dbg_reg;
|
|
|
|
|
|
val &= 0xffffffffUL;
|
|
|
- val |= *vcpu_reg(vcpu, p->Rt) << 32;
|
|
|
+ val |= p->regval << 32;
|
|
|
*dbg_reg = val;
|
|
|
|
|
|
vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
|
|
|
} else {
|
|
|
- *vcpu_reg(vcpu, p->Rt) = *dbg_reg >> 32;
|
|
|
+ p->regval = *dbg_reg >> 32;
|
|
|
}
|
|
|
|
|
|
trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
|
|
@@ -991,7 +983,7 @@ int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
|
|
* Return 0 if the access has been handled, and -1 if not.
|
|
|
*/
|
|
|
static int emulate_cp(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *params,
|
|
|
+ struct sys_reg_params *params,
|
|
|
const struct sys_reg_desc *table,
|
|
|
size_t num)
|
|
|
{
|
|
@@ -1062,12 +1054,12 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
|
|
|
{
|
|
|
struct sys_reg_params params;
|
|
|
u32 hsr = kvm_vcpu_get_hsr(vcpu);
|
|
|
+ int Rt = (hsr >> 5) & 0xf;
|
|
|
int Rt2 = (hsr >> 10) & 0xf;
|
|
|
|
|
|
params.is_aarch32 = true;
|
|
|
params.is_32bit = false;
|
|
|
params.CRm = (hsr >> 1) & 0xf;
|
|
|
- params.Rt = (hsr >> 5) & 0xf;
|
|
|
params.is_write = ((hsr & 1) == 0);
|
|
|
|
|
|
params.Op0 = 0;
|
|
@@ -1076,15 +1068,12 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
|
|
|
params.CRn = 0;
|
|
|
|
|
|
/*
|
|
|
- * Massive hack here. Store Rt2 in the top 32bits so we only
|
|
|
- * have one register to deal with. As we use the same trap
|
|
|
+ * Make a 64-bit value out of Rt and Rt2. As we use the same trap
|
|
|
* backends between AArch32 and AArch64, we get away with it.
|
|
|
*/
|
|
|
if (params.is_write) {
|
|
|
- u64 val = *vcpu_reg(vcpu, params.Rt);
|
|
|
- val &= 0xffffffff;
|
|
|
- val |= *vcpu_reg(vcpu, Rt2) << 32;
|
|
|
- *vcpu_reg(vcpu, params.Rt) = val;
|
|
|
+ params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
|
|
|
+ params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
|
|
|
}
|
|
|
|
|
|
if (!emulate_cp(vcpu, ¶ms, target_specific, nr_specific))
|
|
@@ -1095,11 +1084,10 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
|
|
|
unhandled_cp_access(vcpu, ¶ms);
|
|
|
|
|
|
out:
|
|
|
- /* Do the opposite hack for the read side */
|
|
|
+ /* Split up the value between registers for the read side */
|
|
|
if (!params.is_write) {
|
|
|
- u64 val = *vcpu_reg(vcpu, params.Rt);
|
|
|
- val >>= 32;
|
|
|
- *vcpu_reg(vcpu, Rt2) = val;
|
|
|
+ vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
|
|
|
+ vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
|
|
|
}
|
|
|
|
|
|
return 1;
|
|
@@ -1118,21 +1106,24 @@ static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
|
|
|
{
|
|
|
struct sys_reg_params params;
|
|
|
u32 hsr = kvm_vcpu_get_hsr(vcpu);
|
|
|
+ int Rt = (hsr >> 5) & 0xf;
|
|
|
|
|
|
params.is_aarch32 = true;
|
|
|
params.is_32bit = true;
|
|
|
params.CRm = (hsr >> 1) & 0xf;
|
|
|
- params.Rt = (hsr >> 5) & 0xf;
|
|
|
+ params.regval = vcpu_get_reg(vcpu, Rt);
|
|
|
params.is_write = ((hsr & 1) == 0);
|
|
|
params.CRn = (hsr >> 10) & 0xf;
|
|
|
params.Op0 = 0;
|
|
|
params.Op1 = (hsr >> 14) & 0x7;
|
|
|
params.Op2 = (hsr >> 17) & 0x7;
|
|
|
|
|
|
- if (!emulate_cp(vcpu, ¶ms, target_specific, nr_specific))
|
|
|
- return 1;
|
|
|
- if (!emulate_cp(vcpu, ¶ms, global, nr_global))
|
|
|
+ if (!emulate_cp(vcpu, ¶ms, target_specific, nr_specific) ||
|
|
|
+ !emulate_cp(vcpu, ¶ms, global, nr_global)) {
|
|
|
+ if (!params.is_write)
|
|
|
+ vcpu_set_reg(vcpu, Rt, params.regval);
|
|
|
return 1;
|
|
|
+ }
|
|
|
|
|
|
unhandled_cp_access(vcpu, ¶ms);
|
|
|
return 1;
|
|
@@ -1175,7 +1166,7 @@ int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
|
|
}
|
|
|
|
|
|
static int emulate_sys_reg(struct kvm_vcpu *vcpu,
|
|
|
- const struct sys_reg_params *params)
|
|
|
+ struct sys_reg_params *params)
|
|
|
{
|
|
|
size_t num;
|
|
|
const struct sys_reg_desc *table, *r;
|
|
@@ -1230,6 +1221,8 @@ int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
|
|
{
|
|
|
struct sys_reg_params params;
|
|
|
unsigned long esr = kvm_vcpu_get_hsr(vcpu);
|
|
|
+ int Rt = (esr >> 5) & 0x1f;
|
|
|
+ int ret;
|
|
|
|
|
|
trace_kvm_handle_sys_reg(esr);
|
|
|
|
|
@@ -1240,10 +1233,14 @@ int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
|
|
params.CRn = (esr >> 10) & 0xf;
|
|
|
params.CRm = (esr >> 1) & 0xf;
|
|
|
params.Op2 = (esr >> 17) & 0x7;
|
|
|
- params.Rt = (esr >> 5) & 0x1f;
|
|
|
+ params.regval = vcpu_get_reg(vcpu, Rt);
|
|
|
params.is_write = !(esr & 1);
|
|
|
|
|
|
- return emulate_sys_reg(vcpu, ¶ms);
|
|
|
+ ret = emulate_sys_reg(vcpu, ¶ms);
|
|
|
+
|
|
|
+ if (!params.is_write)
|
|
|
+ vcpu_set_reg(vcpu, Rt, params.regval);
|
|
|
+ return ret;
|
|
|
}
|
|
|
|
|
|
/******************************************************************************
|