|
@@ -114,7 +114,12 @@
|
|
|
* returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
|
|
|
* whatever is in the FPSIMD registers is not saved to memory, but discarded.
|
|
|
*/
|
|
|
-static DEFINE_PER_CPU(struct fpsimd_state *, fpsimd_last_state);
|
|
|
+struct fpsimd_last_state_struct {
|
|
|
+ struct fpsimd_state *st;
|
|
|
+ bool sve_in_use;
|
|
|
+};
|
|
|
+
|
|
|
+static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
|
|
|
|
|
|
/* Default VL for tasks that don't set it explicitly: */
|
|
|
static int sve_default_vl = -1;
|
|
@@ -905,7 +910,7 @@ void fpsimd_thread_switch(struct task_struct *next)
|
|
|
*/
|
|
|
struct fpsimd_state *st = &next->thread.fpsimd_state;
|
|
|
|
|
|
- if (__this_cpu_read(fpsimd_last_state) == st
|
|
|
+ if (__this_cpu_read(fpsimd_last_state.st) == st
|
|
|
&& st->cpu == smp_processor_id())
|
|
|
clear_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
|
|
|
else
|
|
@@ -991,6 +996,21 @@ void fpsimd_signal_preserve_current_state(void)
|
|
|
sve_to_fpsimd(current);
|
|
|
}
|
|
|
|
|
|
+/*
|
|
|
+ * Associate current's FPSIMD context with this cpu
|
|
|
+ * Preemption must be disabled when calling this function.
|
|
|
+ */
|
|
|
+static void fpsimd_bind_to_cpu(void)
|
|
|
+{
|
|
|
+ struct fpsimd_last_state_struct *last =
|
|
|
+ this_cpu_ptr(&fpsimd_last_state);
|
|
|
+ struct fpsimd_state *st = ¤t->thread.fpsimd_state;
|
|
|
+
|
|
|
+ last->st = st;
|
|
|
+ last->sve_in_use = test_thread_flag(TIF_SVE);
|
|
|
+ st->cpu = smp_processor_id();
|
|
|
+}
|
|
|
+
|
|
|
/*
|
|
|
* Load the userland FPSIMD state of 'current' from memory, but only if the
|
|
|
* FPSIMD state already held in the registers is /not/ the most recent FPSIMD
|
|
@@ -1004,11 +1024,8 @@ void fpsimd_restore_current_state(void)
|
|
|
local_bh_disable();
|
|
|
|
|
|
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
|
|
|
- struct fpsimd_state *st = ¤t->thread.fpsimd_state;
|
|
|
-
|
|
|
task_fpsimd_load();
|
|
|
- __this_cpu_write(fpsimd_last_state, st);
|
|
|
- st->cpu = smp_processor_id();
|
|
|
+ fpsimd_bind_to_cpu();
|
|
|
}
|
|
|
|
|
|
local_bh_enable();
|
|
@@ -1032,12 +1049,8 @@ void fpsimd_update_current_state(struct fpsimd_state *state)
|
|
|
|
|
|
task_fpsimd_load();
|
|
|
|
|
|
- if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
|
|
|
- struct fpsimd_state *st = ¤t->thread.fpsimd_state;
|
|
|
-
|
|
|
- __this_cpu_write(fpsimd_last_state, st);
|
|
|
- st->cpu = smp_processor_id();
|
|
|
- }
|
|
|
+ if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE))
|
|
|
+ fpsimd_bind_to_cpu();
|
|
|
|
|
|
local_bh_enable();
|
|
|
}
|
|
@@ -1052,7 +1065,7 @@ void fpsimd_flush_task_state(struct task_struct *t)
|
|
|
|
|
|
static inline void fpsimd_flush_cpu_state(void)
|
|
|
{
|
|
|
- __this_cpu_write(fpsimd_last_state, NULL);
|
|
|
+ __this_cpu_write(fpsimd_last_state.st, NULL);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -1065,14 +1078,10 @@ static inline void fpsimd_flush_cpu_state(void)
|
|
|
#ifdef CONFIG_ARM64_SVE
|
|
|
void sve_flush_cpu_state(void)
|
|
|
{
|
|
|
- struct fpsimd_state *const fpstate = __this_cpu_read(fpsimd_last_state);
|
|
|
- struct task_struct *tsk;
|
|
|
-
|
|
|
- if (!fpstate)
|
|
|
- return;
|
|
|
+ struct fpsimd_last_state_struct const *last =
|
|
|
+ this_cpu_ptr(&fpsimd_last_state);
|
|
|
|
|
|
- tsk = container_of(fpstate, struct task_struct, thread.fpsimd_state);
|
|
|
- if (test_tsk_thread_flag(tsk, TIF_SVE))
|
|
|
+ if (last->st && last->sve_in_use)
|
|
|
fpsimd_flush_cpu_state();
|
|
|
}
|
|
|
#endif /* CONFIG_ARM64_SVE */
|
|
@@ -1267,7 +1276,7 @@ static inline void fpsimd_pm_init(void) { }
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
static int fpsimd_cpu_dead(unsigned int cpu)
|
|
|
{
|
|
|
- per_cpu(fpsimd_last_state, cpu) = NULL;
|
|
|
+ per_cpu(fpsimd_last_state.st, cpu) = NULL;
|
|
|
return 0;
|
|
|
}
|
|
|
|