|
@@ -100,7 +100,7 @@ void __kernel_fpu_begin(void)
|
|
|
|
|
|
|
|
kernel_fpu_disable();
|
|
kernel_fpu_disable();
|
|
|
|
|
|
|
|
- if (fpu->fpregs_active) {
|
|
|
|
|
|
|
+ if (fpu->initialized) {
|
|
|
/*
|
|
/*
|
|
|
* Ignore return value -- we don't care if reg state
|
|
* Ignore return value -- we don't care if reg state
|
|
|
* is clobbered.
|
|
* is clobbered.
|
|
@@ -116,7 +116,7 @@ void __kernel_fpu_end(void)
|
|
|
{
|
|
{
|
|
|
struct fpu *fpu = ¤t->thread.fpu;
|
|
struct fpu *fpu = ¤t->thread.fpu;
|
|
|
|
|
|
|
|
- if (fpu->fpregs_active)
|
|
|
|
|
|
|
+ if (fpu->initialized)
|
|
|
copy_kernel_to_fpregs(&fpu->state);
|
|
copy_kernel_to_fpregs(&fpu->state);
|
|
|
|
|
|
|
|
kernel_fpu_enable();
|
|
kernel_fpu_enable();
|
|
@@ -148,7 +148,7 @@ void fpu__save(struct fpu *fpu)
|
|
|
|
|
|
|
|
preempt_disable();
|
|
preempt_disable();
|
|
|
trace_x86_fpu_before_save(fpu);
|
|
trace_x86_fpu_before_save(fpu);
|
|
|
- if (fpu->fpregs_active) {
|
|
|
|
|
|
|
+ if (fpu->initialized) {
|
|
|
if (!copy_fpregs_to_fpstate(fpu)) {
|
|
if (!copy_fpregs_to_fpstate(fpu)) {
|
|
|
copy_kernel_to_fpregs(&fpu->state);
|
|
copy_kernel_to_fpregs(&fpu->state);
|
|
|
}
|
|
}
|
|
@@ -189,10 +189,9 @@ EXPORT_SYMBOL_GPL(fpstate_init);
|
|
|
|
|
|
|
|
int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
|
|
int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
|
|
|
{
|
|
{
|
|
|
- dst_fpu->fpregs_active = 0;
|
|
|
|
|
dst_fpu->last_cpu = -1;
|
|
dst_fpu->last_cpu = -1;
|
|
|
|
|
|
|
|
- if (!src_fpu->fpstate_active || !static_cpu_has(X86_FEATURE_FPU))
|
|
|
|
|
|
|
+ if (!src_fpu->initialized || !static_cpu_has(X86_FEATURE_FPU))
|
|
|
return 0;
|
|
return 0;
|
|
|
|
|
|
|
|
WARN_ON_FPU(src_fpu != ¤t->thread.fpu);
|
|
WARN_ON_FPU(src_fpu != ¤t->thread.fpu);
|
|
@@ -206,26 +205,14 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
|
|
|
/*
|
|
/*
|
|
|
* Save current FPU registers directly into the child
|
|
* Save current FPU registers directly into the child
|
|
|
* FPU context, without any memory-to-memory copying.
|
|
* FPU context, without any memory-to-memory copying.
|
|
|
- * In lazy mode, if the FPU context isn't loaded into
|
|
|
|
|
- * fpregs, CR0.TS will be set and do_device_not_available
|
|
|
|
|
- * will load the FPU context.
|
|
|
|
|
*
|
|
*
|
|
|
- * We have to do all this with preemption disabled,
|
|
|
|
|
- * mostly because of the FNSAVE case, because in that
|
|
|
|
|
- * case we must not allow preemption in the window
|
|
|
|
|
- * between the FNSAVE and us marking the context lazy.
|
|
|
|
|
- *
|
|
|
|
|
- * It shouldn't be an issue as even FNSAVE is plenty
|
|
|
|
|
- * fast in terms of critical section length.
|
|
|
|
|
|
|
+ * ( The function 'fails' in the FNSAVE case, which destroys
|
|
|
|
|
+ * register contents so we have to copy them back. )
|
|
|
*/
|
|
*/
|
|
|
- preempt_disable();
|
|
|
|
|
if (!copy_fpregs_to_fpstate(dst_fpu)) {
|
|
if (!copy_fpregs_to_fpstate(dst_fpu)) {
|
|
|
- memcpy(&src_fpu->state, &dst_fpu->state,
|
|
|
|
|
- fpu_kernel_xstate_size);
|
|
|
|
|
-
|
|
|
|
|
|
|
+ memcpy(&src_fpu->state, &dst_fpu->state, fpu_kernel_xstate_size);
|
|
|
copy_kernel_to_fpregs(&src_fpu->state);
|
|
copy_kernel_to_fpregs(&src_fpu->state);
|
|
|
}
|
|
}
|
|
|
- preempt_enable();
|
|
|
|
|
|
|
|
|
|
trace_x86_fpu_copy_src(src_fpu);
|
|
trace_x86_fpu_copy_src(src_fpu);
|
|
|
trace_x86_fpu_copy_dst(dst_fpu);
|
|
trace_x86_fpu_copy_dst(dst_fpu);
|
|
@@ -237,45 +224,48 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
|
|
|
* Activate the current task's in-memory FPU context,
|
|
* Activate the current task's in-memory FPU context,
|
|
|
* if it has not been used before:
|
|
* if it has not been used before:
|
|
|
*/
|
|
*/
|
|
|
-void fpu__activate_curr(struct fpu *fpu)
|
|
|
|
|
|
|
+void fpu__initialize(struct fpu *fpu)
|
|
|
{
|
|
{
|
|
|
WARN_ON_FPU(fpu != ¤t->thread.fpu);
|
|
WARN_ON_FPU(fpu != ¤t->thread.fpu);
|
|
|
|
|
|
|
|
- if (!fpu->fpstate_active) {
|
|
|
|
|
|
|
+ if (!fpu->initialized) {
|
|
|
fpstate_init(&fpu->state);
|
|
fpstate_init(&fpu->state);
|
|
|
trace_x86_fpu_init_state(fpu);
|
|
trace_x86_fpu_init_state(fpu);
|
|
|
|
|
|
|
|
trace_x86_fpu_activate_state(fpu);
|
|
trace_x86_fpu_activate_state(fpu);
|
|
|
/* Safe to do for the current task: */
|
|
/* Safe to do for the current task: */
|
|
|
- fpu->fpstate_active = 1;
|
|
|
|
|
|
|
+ fpu->initialized = 1;
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
-EXPORT_SYMBOL_GPL(fpu__activate_curr);
|
|
|
|
|
|
|
+EXPORT_SYMBOL_GPL(fpu__initialize);
|
|
|
|
|
|
|
|
/*
|
|
/*
|
|
|
* This function must be called before we read a task's fpstate.
|
|
* This function must be called before we read a task's fpstate.
|
|
|
*
|
|
*
|
|
|
- * If the task has not used the FPU before then initialize its
|
|
|
|
|
- * fpstate.
|
|
|
|
|
|
|
+ * There's two cases where this gets called:
|
|
|
|
|
+ *
|
|
|
|
|
+ * - for the current task (when coredumping), in which case we have
|
|
|
|
|
+ * to save the latest FPU registers into the fpstate,
|
|
|
|
|
+ *
|
|
|
|
|
+ * - or it's called for stopped tasks (ptrace), in which case the
|
|
|
|
|
+ * registers were already saved by the context-switch code when
|
|
|
|
|
+ * the task scheduled out - we only have to initialize the registers
|
|
|
|
|
+ * if they've never been initialized.
|
|
|
*
|
|
*
|
|
|
* If the task has used the FPU before then save it.
|
|
* If the task has used the FPU before then save it.
|
|
|
*/
|
|
*/
|
|
|
-void fpu__activate_fpstate_read(struct fpu *fpu)
|
|
|
|
|
|
|
+void fpu__prepare_read(struct fpu *fpu)
|
|
|
{
|
|
{
|
|
|
- /*
|
|
|
|
|
- * If fpregs are active (in the current CPU), then
|
|
|
|
|
- * copy them to the fpstate:
|
|
|
|
|
- */
|
|
|
|
|
- if (fpu->fpregs_active) {
|
|
|
|
|
|
|
+ if (fpu == ¤t->thread.fpu) {
|
|
|
fpu__save(fpu);
|
|
fpu__save(fpu);
|
|
|
} else {
|
|
} else {
|
|
|
- if (!fpu->fpstate_active) {
|
|
|
|
|
|
|
+ if (!fpu->initialized) {
|
|
|
fpstate_init(&fpu->state);
|
|
fpstate_init(&fpu->state);
|
|
|
trace_x86_fpu_init_state(fpu);
|
|
trace_x86_fpu_init_state(fpu);
|
|
|
|
|
|
|
|
trace_x86_fpu_activate_state(fpu);
|
|
trace_x86_fpu_activate_state(fpu);
|
|
|
/* Safe to do for current and for stopped child tasks: */
|
|
/* Safe to do for current and for stopped child tasks: */
|
|
|
- fpu->fpstate_active = 1;
|
|
|
|
|
|
|
+ fpu->initialized = 1;
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
@@ -283,17 +273,17 @@ void fpu__activate_fpstate_read(struct fpu *fpu)
|
|
|
/*
|
|
/*
|
|
|
* This function must be called before we write a task's fpstate.
|
|
* This function must be called before we write a task's fpstate.
|
|
|
*
|
|
*
|
|
|
- * If the task has used the FPU before then unlazy it.
|
|
|
|
|
|
|
+ * If the task has used the FPU before then invalidate any cached FPU registers.
|
|
|
* If the task has not used the FPU before then initialize its fpstate.
|
|
* If the task has not used the FPU before then initialize its fpstate.
|
|
|
*
|
|
*
|
|
|
* After this function call, after registers in the fpstate are
|
|
* After this function call, after registers in the fpstate are
|
|
|
* modified and the child task has woken up, the child task will
|
|
* modified and the child task has woken up, the child task will
|
|
|
* restore the modified FPU state from the modified context. If we
|
|
* restore the modified FPU state from the modified context. If we
|
|
|
- * didn't clear its lazy status here then the lazy in-registers
|
|
|
|
|
|
|
+ * didn't clear its cached status here then the cached in-registers
|
|
|
* state pending on its former CPU could be restored, corrupting
|
|
* state pending on its former CPU could be restored, corrupting
|
|
|
* the modifications.
|
|
* the modifications.
|
|
|
*/
|
|
*/
|
|
|
-void fpu__activate_fpstate_write(struct fpu *fpu)
|
|
|
|
|
|
|
+void fpu__prepare_write(struct fpu *fpu)
|
|
|
{
|
|
{
|
|
|
/*
|
|
/*
|
|
|
* Only stopped child tasks can be used to modify the FPU
|
|
* Only stopped child tasks can be used to modify the FPU
|
|
@@ -301,8 +291,8 @@ void fpu__activate_fpstate_write(struct fpu *fpu)
|
|
|
*/
|
|
*/
|
|
|
WARN_ON_FPU(fpu == ¤t->thread.fpu);
|
|
WARN_ON_FPU(fpu == ¤t->thread.fpu);
|
|
|
|
|
|
|
|
- if (fpu->fpstate_active) {
|
|
|
|
|
- /* Invalidate any lazy state: */
|
|
|
|
|
|
|
+ if (fpu->initialized) {
|
|
|
|
|
+ /* Invalidate any cached state: */
|
|
|
__fpu_invalidate_fpregs_state(fpu);
|
|
__fpu_invalidate_fpregs_state(fpu);
|
|
|
} else {
|
|
} else {
|
|
|
fpstate_init(&fpu->state);
|
|
fpstate_init(&fpu->state);
|
|
@@ -310,73 +300,10 @@ void fpu__activate_fpstate_write(struct fpu *fpu)
|
|
|
|
|
|
|
|
trace_x86_fpu_activate_state(fpu);
|
|
trace_x86_fpu_activate_state(fpu);
|
|
|
/* Safe to do for stopped child tasks: */
|
|
/* Safe to do for stopped child tasks: */
|
|
|
- fpu->fpstate_active = 1;
|
|
|
|
|
|
|
+ fpu->initialized = 1;
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-/*
|
|
|
|
|
- * This function must be called before we write the current
|
|
|
|
|
- * task's fpstate.
|
|
|
|
|
- *
|
|
|
|
|
- * This call gets the current FPU register state and moves
|
|
|
|
|
- * it in to the 'fpstate'. Preemption is disabled so that
|
|
|
|
|
- * no writes to the 'fpstate' can occur from context
|
|
|
|
|
- * swiches.
|
|
|
|
|
- *
|
|
|
|
|
- * Must be followed by a fpu__current_fpstate_write_end().
|
|
|
|
|
- */
|
|
|
|
|
-void fpu__current_fpstate_write_begin(void)
|
|
|
|
|
-{
|
|
|
|
|
- struct fpu *fpu = ¤t->thread.fpu;
|
|
|
|
|
-
|
|
|
|
|
- /*
|
|
|
|
|
- * Ensure that the context-switching code does not write
|
|
|
|
|
- * over the fpstate while we are doing our update.
|
|
|
|
|
- */
|
|
|
|
|
- preempt_disable();
|
|
|
|
|
-
|
|
|
|
|
- /*
|
|
|
|
|
- * Move the fpregs in to the fpu's 'fpstate'.
|
|
|
|
|
- */
|
|
|
|
|
- fpu__activate_fpstate_read(fpu);
|
|
|
|
|
-
|
|
|
|
|
- /*
|
|
|
|
|
- * The caller is about to write to 'fpu'. Ensure that no
|
|
|
|
|
- * CPU thinks that its fpregs match the fpstate. This
|
|
|
|
|
- * ensures we will not be lazy and skip a XRSTOR in the
|
|
|
|
|
- * future.
|
|
|
|
|
- */
|
|
|
|
|
- __fpu_invalidate_fpregs_state(fpu);
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
-/*
|
|
|
|
|
- * This function must be paired with fpu__current_fpstate_write_begin()
|
|
|
|
|
- *
|
|
|
|
|
- * This will ensure that the modified fpstate gets placed back in
|
|
|
|
|
- * the fpregs if necessary.
|
|
|
|
|
- *
|
|
|
|
|
- * Note: This function may be called whether or not an _actual_
|
|
|
|
|
- * write to the fpstate occurred.
|
|
|
|
|
- */
|
|
|
|
|
-void fpu__current_fpstate_write_end(void)
|
|
|
|
|
-{
|
|
|
|
|
- struct fpu *fpu = ¤t->thread.fpu;
|
|
|
|
|
-
|
|
|
|
|
- /*
|
|
|
|
|
- * 'fpu' now has an updated copy of the state, but the
|
|
|
|
|
- * registers may still be out of date. Update them with
|
|
|
|
|
- * an XRSTOR if they are active.
|
|
|
|
|
- */
|
|
|
|
|
- if (fpregs_active())
|
|
|
|
|
- copy_kernel_to_fpregs(&fpu->state);
|
|
|
|
|
-
|
|
|
|
|
- /*
|
|
|
|
|
- * Our update is done and the fpregs/fpstate are in sync
|
|
|
|
|
- * if necessary. Context switches can happen again.
|
|
|
|
|
- */
|
|
|
|
|
- preempt_enable();
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
/*
|
|
/*
|
|
|
* 'fpu__restore()' is called to copy FPU registers from
|
|
* 'fpu__restore()' is called to copy FPU registers from
|
|
|
* the FPU fpstate to the live hw registers and to activate
|
|
* the FPU fpstate to the live hw registers and to activate
|
|
@@ -389,7 +316,7 @@ void fpu__current_fpstate_write_end(void)
|
|
|
*/
|
|
*/
|
|
|
void fpu__restore(struct fpu *fpu)
|
|
void fpu__restore(struct fpu *fpu)
|
|
|
{
|
|
{
|
|
|
- fpu__activate_curr(fpu);
|
|
|
|
|
|
|
+ fpu__initialize(fpu);
|
|
|
|
|
|
|
|
/* Avoid __kernel_fpu_begin() right after fpregs_activate() */
|
|
/* Avoid __kernel_fpu_begin() right after fpregs_activate() */
|
|
|
kernel_fpu_disable();
|
|
kernel_fpu_disable();
|
|
@@ -414,15 +341,17 @@ void fpu__drop(struct fpu *fpu)
|
|
|
{
|
|
{
|
|
|
preempt_disable();
|
|
preempt_disable();
|
|
|
|
|
|
|
|
- if (fpu->fpregs_active) {
|
|
|
|
|
- /* Ignore delayed exceptions from user space */
|
|
|
|
|
- asm volatile("1: fwait\n"
|
|
|
|
|
- "2:\n"
|
|
|
|
|
- _ASM_EXTABLE(1b, 2b));
|
|
|
|
|
- fpregs_deactivate(fpu);
|
|
|
|
|
|
|
+ if (fpu == ¤t->thread.fpu) {
|
|
|
|
|
+ if (fpu->initialized) {
|
|
|
|
|
+ /* Ignore delayed exceptions from user space */
|
|
|
|
|
+ asm volatile("1: fwait\n"
|
|
|
|
|
+ "2:\n"
|
|
|
|
|
+ _ASM_EXTABLE(1b, 2b));
|
|
|
|
|
+ fpregs_deactivate(fpu);
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- fpu->fpstate_active = 0;
|
|
|
|
|
|
|
+ fpu->initialized = 0;
|
|
|
|
|
|
|
|
trace_x86_fpu_dropped(fpu);
|
|
trace_x86_fpu_dropped(fpu);
|
|
|
|
|
|
|
@@ -462,9 +391,11 @@ void fpu__clear(struct fpu *fpu)
|
|
|
* Make sure fpstate is cleared and initialized.
|
|
* Make sure fpstate is cleared and initialized.
|
|
|
*/
|
|
*/
|
|
|
if (static_cpu_has(X86_FEATURE_FPU)) {
|
|
if (static_cpu_has(X86_FEATURE_FPU)) {
|
|
|
- fpu__activate_curr(fpu);
|
|
|
|
|
|
|
+ preempt_disable();
|
|
|
|
|
+ fpu__initialize(fpu);
|
|
|
user_fpu_begin();
|
|
user_fpu_begin();
|
|
|
copy_init_fpstate_to_fpregs();
|
|
copy_init_fpstate_to_fpregs();
|
|
|
|
|
+ preempt_enable();
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|