|
@@ -77,8 +77,7 @@ within(unsigned long addr, unsigned long start, unsigned long end)
|
|
|
return addr >= start && addr < end;
|
|
|
}
|
|
|
|
|
|
-static int
|
|
|
-do_ftrace_mod_code(unsigned long ip, const void *new_code)
|
|
|
+static unsigned long text_ip_addr(unsigned long ip)
|
|
|
{
|
|
|
/*
|
|
|
* On x86_64, kernel text mappings are mapped read-only with
|
|
@@ -91,7 +90,7 @@ do_ftrace_mod_code(unsigned long ip, const void *new_code)
|
|
|
if (within(ip, (unsigned long)_text, (unsigned long)_etext))
|
|
|
ip = (unsigned long)__va(__pa_symbol(ip));
|
|
|
|
|
|
- return probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE);
|
|
|
+ return ip;
|
|
|
}
|
|
|
|
|
|
static const unsigned char *ftrace_nop_replace(void)
|
|
@@ -123,8 +122,10 @@ ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
|
|
|
if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
|
|
|
return -EINVAL;
|
|
|
|
|
|
+ ip = text_ip_addr(ip);
|
|
|
+
|
|
|
/* replace the text with the new text */
|
|
|
- if (do_ftrace_mod_code(ip, new_code))
|
|
|
+ if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
|
|
|
return -EPERM;
|
|
|
|
|
|
sync_core();
|
|
@@ -221,37 +222,51 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
|
|
|
return -EINVAL;
|
|
|
}
|
|
|
|
|
|
-int ftrace_update_ftrace_func(ftrace_func_t func)
|
|
|
+static unsigned long ftrace_update_func;
|
|
|
+
|
|
|
+static int update_ftrace_func(unsigned long ip, void *new)
|
|
|
{
|
|
|
- unsigned long ip = (unsigned long)(&ftrace_call);
|
|
|
- unsigned char old[MCOUNT_INSN_SIZE], *new;
|
|
|
+ unsigned char old[MCOUNT_INSN_SIZE];
|
|
|
int ret;
|
|
|
|
|
|
- memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
|
|
|
- new = ftrace_call_replace(ip, (unsigned long)func);
|
|
|
+ memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
|
|
|
+
|
|
|
+ ftrace_update_func = ip;
|
|
|
+ /* Make sure the breakpoints see the ftrace_update_func update */
|
|
|
+ smp_wmb();
|
|
|
|
|
|
/* See comment above by declaration of modifying_ftrace_code */
|
|
|
atomic_inc(&modifying_ftrace_code);
|
|
|
|
|
|
ret = ftrace_modify_code(ip, old, new);
|
|
|
|
|
|
+ atomic_dec(&modifying_ftrace_code);
|
|
|
+
|
|
|
+ return ret;
|
|
|
+}
|
|
|
+
|
|
|
+int ftrace_update_ftrace_func(ftrace_func_t func)
|
|
|
+{
|
|
|
+ unsigned long ip = (unsigned long)(&ftrace_call);
|
|
|
+ unsigned char *new;
|
|
|
+ int ret;
|
|
|
+
|
|
|
+ new = ftrace_call_replace(ip, (unsigned long)func);
|
|
|
+ ret = update_ftrace_func(ip, new);
|
|
|
+
|
|
|
/* Also update the regs callback function */
|
|
|
if (!ret) {
|
|
|
ip = (unsigned long)(&ftrace_regs_call);
|
|
|
- memcpy(old, &ftrace_regs_call, MCOUNT_INSN_SIZE);
|
|
|
new = ftrace_call_replace(ip, (unsigned long)func);
|
|
|
- ret = ftrace_modify_code(ip, old, new);
|
|
|
+ ret = update_ftrace_func(ip, new);
|
|
|
}
|
|
|
|
|
|
- atomic_dec(&modifying_ftrace_code);
|
|
|
-
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
static int is_ftrace_caller(unsigned long ip)
|
|
|
{
|
|
|
- if (ip == (unsigned long)(&ftrace_call) ||
|
|
|
- ip == (unsigned long)(&ftrace_regs_call))
|
|
|
+ if (ip == ftrace_update_func)
|
|
|
return 1;
|
|
|
|
|
|
return 0;
|
|
@@ -677,45 +692,41 @@ int __init ftrace_dyn_arch_init(void *data)
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
|
|
extern void ftrace_graph_call(void);
|
|
|
|
|
|
-static int ftrace_mod_jmp(unsigned long ip,
|
|
|
- int old_offset, int new_offset)
|
|
|
+static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
|
|
|
{
|
|
|
- unsigned char code[MCOUNT_INSN_SIZE];
|
|
|
+ static union ftrace_code_union calc;
|
|
|
|
|
|
- if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
|
|
|
- return -EFAULT;
|
|
|
+ /* Jmp not a call (ignore the .e8) */
|
|
|
+ calc.e8 = 0xe9;
|
|
|
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
|
|
|
|
|
|
- if (code[0] != 0xe9 || old_offset != *(int *)(&code[1]))
|
|
|
- return -EINVAL;
|
|
|
+ /*
|
|
|
+ * ftrace external locks synchronize the access to the static variable.
|
|
|
+ */
|
|
|
+ return calc.code;
|
|
|
+}
|
|
|
|
|
|
- *(int *)(&code[1]) = new_offset;
|
|
|
+static int ftrace_mod_jmp(unsigned long ip, void *func)
|
|
|
+{
|
|
|
+ unsigned char *new;
|
|
|
|
|
|
- if (do_ftrace_mod_code(ip, &code))
|
|
|
- return -EPERM;
|
|
|
+ new = ftrace_jmp_replace(ip, (unsigned long)func);
|
|
|
|
|
|
- return 0;
|
|
|
+ return update_ftrace_func(ip, new);
|
|
|
}
|
|
|
|
|
|
int ftrace_enable_ftrace_graph_caller(void)
|
|
|
{
|
|
|
unsigned long ip = (unsigned long)(&ftrace_graph_call);
|
|
|
- int old_offset, new_offset;
|
|
|
|
|
|
- old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
|
|
|
- new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
|
|
|
-
|
|
|
- return ftrace_mod_jmp(ip, old_offset, new_offset);
|
|
|
+ return ftrace_mod_jmp(ip, &ftrace_graph_caller);
|
|
|
}
|
|
|
|
|
|
int ftrace_disable_ftrace_graph_caller(void)
|
|
|
{
|
|
|
unsigned long ip = (unsigned long)(&ftrace_graph_call);
|
|
|
- int old_offset, new_offset;
|
|
|
-
|
|
|
- old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
|
|
|
- new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
|
|
|
|
|
|
- return ftrace_mod_jmp(ip, old_offset, new_offset);
|
|
|
+ return ftrace_mod_jmp(ip, &ftrace_stub);
|
|
|
}
|
|
|
|
|
|
#endif /* !CONFIG_DYNAMIC_FTRACE */
|