tracing/ftrace: Add a better way to pass data via the probe functions

With the redesign of the registration and execution of the function probes
(triggers), data can now be passed from the setup of the probe to the probe
callers that are specific to the trace_array it is on. Although, all probes
still only affect the toplevel trace array, this change will allow for
instances to have their own probes separated from other instances and the
top array.

That is, something like the stacktrace probe can be set to trace only in an
instance and not the toplevel trace array. This isn't implement yet, but
this change sets the ground work for the change.

When a probe callback is triggered (someone writes the probe format into
set_ftrace_filter), it calls register_ftrace_function_probe() passing in
init_data that will be used to initialize the probe. Then for every matching
function, register_ftrace_function_probe() will call the probe_ops->init()
function with the init data that was passed to it, as well as an address to
a place holder that is associated with the probe and the instance. The first
occurrence will have a NULL in the pointer. The init() function will then
initialize it. If other probes are added, or more functions are part of the
probe, the place holder will be passed to the init() function with the place
holder data that it was initialized to the last time.

Then this place_holder is passed to each of the other probe_ops functions,
where it can be used in the function callback. When the probe_ops free()
function is called, it can be called either with the rip of the function
that is being removed from the probe, or zero, indicating that there are no
more functions attached to the probe, and the place holder is about to be
freed. This gives the probe_ops a way to free the data it assigned to the
place holder if it was allocade during the first init call.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>

kernel/trace/ftrace.c

@@ -1106,6 +1106,7 @@ struct ftrace_func_probe {
 	struct ftrace_ops	ops;
 	struct trace_array	*tr;
 	struct list_head	list;
+	void			*data;
 	int			ref;
 };
 
@@ -3187,7 +3188,7 @@ t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
 	probe_ops = probe->probe_ops;
 
 	if (probe_ops->print)
-		return probe_ops->print(m, probe_entry->ip, probe_ops, NULL);
+		return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
 
 	seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
 		   (void *)probe_ops->func);
@@ -3814,7 +3815,7 @@ static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
 	 * on the hash. rcu_read_lock is too dangerous here.
 	 */
 	preempt_disable_notrace();
-	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, NULL);
+	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
 	preempt_enable_notrace();
 }
 
@@ -3972,6 +3973,12 @@ static void release_probe(struct ftrace_func_probe *probe)
 
 	if (!probe->ref) {
 		probe_ops = probe->probe_ops;
+		/*
+		 * Sending zero as ip tells probe_ops to free
+		 * the probe->data itself
+		 */
+		if (probe_ops->free)
+			probe_ops->free(probe_ops, probe->tr, 0, probe->data);
 		list_del(&probe->list);
 		kfree(probe);
 	}
@@ -4060,9 +4067,15 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr,
 			 */
 			if (probe_ops->init) {
 				ret = probe_ops->init(probe_ops, tr,
-						      entry->ip, data);
-				if (ret < 0)
+						      entry->ip, data,
+						      &probe->data);
+				if (ret < 0) {
+					if (probe_ops->free && count)
+						probe_ops->free(probe_ops, tr,
+								0, probe->data);
+					probe->data = NULL;
 					goto out;
+				}
 			}
 			count++;
 		}
@@ -4109,7 +4122,7 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr,
 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
 			if (ftrace_lookup_ip(old_hash, entry->ip))
 				continue;
-			probe_ops->free(probe_ops, tr, entry->ip, NULL);
+			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
 		}
 	}
 	goto out_unlock;
@@ -4227,7 +4240,7 @@ unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
 	hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
 		hlist_del(&entry->hlist);
 		if (probe_ops->free)
-			probe_ops->free(probe_ops, tr, entry->ip, NULL);
+			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
 		kfree(entry);
 	}
 	mutex_unlock(&ftrace_lock);

kernel/trace/trace.c

@@ -6737,7 +6737,7 @@ static const struct file_operations tracing_dyn_info_fops = {
 static void
 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
 		struct trace_array *tr, struct ftrace_probe_ops *ops,
-		void **data)
+		void *data)
 {
 	tracing_snapshot();
 }
@@ -6745,9 +6745,9 @@ ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
 static void
 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
 		      struct trace_array *tr, struct ftrace_probe_ops *ops,
-		      void **data)
+		      void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = data;
 	long *count = NULL;
 
 	if (mapper)
@@ -6768,7 +6768,7 @@ static int
 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
 		      struct ftrace_probe_ops *ops, void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = data;
 	long *count = NULL;
 
 	seq_printf(m, "%ps:", (void *)ip);
@@ -6788,18 +6788,32 @@ ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
 
 static int
 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
-		     unsigned long ip, void *data)
+		     unsigned long ip, void *init_data, void **data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = *data;
+
+	if (!mapper) {
+		mapper = allocate_ftrace_func_mapper();
+		if (!mapper)
+			return -ENOMEM;
+		*data = mapper;
+	}
 
-	return ftrace_func_mapper_add_ip(mapper, ip, data);
+	return ftrace_func_mapper_add_ip(mapper, ip, init_data);
 }
 
 static void
 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
-		     unsigned long ip, void **data)
+		     unsigned long ip, void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = data;
+
+	if (!ip) {
+		if (!mapper)
+			return;
+		free_ftrace_func_mapper(mapper, NULL);
+		return;
+	}
 
 	ftrace_func_mapper_remove_ip(mapper, ip);
 }
@@ -6842,12 +6856,6 @@ ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
 	if (!strlen(number))
 		goto out_reg;
 
-	if (!ops->private_data) {
-		ops->private_data = allocate_ftrace_func_mapper();
-		if (!ops->private_data)
-			return -ENOMEM;
-	}
-
 	/*
 	 * We use the callback data field (which is a pointer)
 	 * as our counter.

kernel/trace/trace.h

@@ -943,18 +943,18 @@ struct ftrace_probe_ops {
 					unsigned long parent_ip,
 					struct trace_array *tr,
 					struct ftrace_probe_ops *ops,
-					void **data);
+					void *data);
 	int			(*init)(struct ftrace_probe_ops *ops,
 					struct trace_array *tr,
-					unsigned long ip, void *data);
+					unsigned long ip, void *init_data,
+					void **data);
 	void			(*free)(struct ftrace_probe_ops *ops,
 					struct trace_array *tr,
-					unsigned long ip, void **data);
+					unsigned long ip, void *data);
 	int			(*print)(struct seq_file *m,
 					 unsigned long ip,
 					 struct ftrace_probe_ops *ops,
 					 void *data);
-	void			*private_data;
 };
 
 struct ftrace_func_mapper;

kernel/trace/trace_events.c

@@ -2471,54 +2471,54 @@ static void update_event_probe(struct event_probe_data *data)
 static void
 event_enable_probe(unsigned long ip, unsigned long parent_ip,
 		   struct trace_array *tr, struct ftrace_probe_ops *ops,
-		   void **_data)
+		   void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
-	struct event_probe_data *data;
+	struct ftrace_func_mapper *mapper = data;
+	struct event_probe_data *edata;
 	void **pdata;
 
 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
 	if (!pdata || !*pdata)
 		return;
 
-	data = *pdata;
-	update_event_probe(data);
+	edata = *pdata;
+	update_event_probe(edata);
 }
 
 static void
 event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
 			 struct trace_array *tr, struct ftrace_probe_ops *ops,
-			 void **_data)
+			 void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
-	struct event_probe_data *data;
+	struct ftrace_func_mapper *mapper = data;
+	struct event_probe_data *edata;
 	void **pdata;
 
 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
 	if (!pdata || !*pdata)
 		return;
 
-	data = *pdata;
+	edata = *pdata;
 
-	if (!data->count)
+	if (!edata->count)
 		return;
 
 	/* Skip if the event is in a state we want to switch to */
-	if (data->enable == !(data->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
+	if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
 		return;
 
-	if (data->count != -1)
-		(data->count)--;
+	if (edata->count != -1)
+		(edata->count)--;
 
-	update_event_probe(data);
+	update_event_probe(edata);
 }
 
 static int
 event_enable_print(struct seq_file *m, unsigned long ip,
-		   struct ftrace_probe_ops *ops, void *_data)
+		   struct ftrace_probe_ops *ops, void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
-	struct event_probe_data *data;
+	struct ftrace_func_mapper *mapper = data;
+	struct event_probe_data *edata;
 	void **pdata;
 
 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
@@ -2526,62 +2526,84 @@ event_enable_print(struct seq_file *m, unsigned long ip,
 	if (WARN_ON_ONCE(!pdata || !*pdata))
 		return 0;
 
-	data = *pdata;
+	edata = *pdata;
 
 	seq_printf(m, "%ps:", (void *)ip);
 
 	seq_printf(m, "%s:%s:%s",
-		   data->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
-		   data->file->event_call->class->system,
-		   trace_event_name(data->file->event_call));
+		   edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
+		   edata->file->event_call->class->system,
+		   trace_event_name(edata->file->event_call));
 
-	if (data->count == -1)
+	if (edata->count == -1)
 		seq_puts(m, ":unlimited\n");
 	else
-		seq_printf(m, ":count=%ld\n", data->count);
+		seq_printf(m, ":count=%ld\n", edata->count);
 
 	return 0;
 }
 
 static int
 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
-		  unsigned long ip, void *_data)
+		  unsigned long ip, void *init_data, void **data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
-	struct event_probe_data *data = _data;
+	struct ftrace_func_mapper *mapper = *data;
+	struct event_probe_data *edata = init_data;
 	int ret;
 
-	ret = ftrace_func_mapper_add_ip(mapper, ip, data);
+	if (!mapper) {
+		mapper = allocate_ftrace_func_mapper();
+		if (!mapper)
+			return -ENODEV;
+		*data = mapper;
+	}
+
+	ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
 	if (ret < 0)
 		return ret;
 
-	data->ref++;
+	edata->ref++;
+
+	return 0;
+}
+
+static int free_probe_data(void *data)
+{
+	struct event_probe_data *edata = data;
 
+	edata->ref--;
+	if (!edata->ref) {
+		/* Remove the SOFT_MODE flag */
+		__ftrace_event_enable_disable(edata->file, 0, 1);
+		module_put(edata->file->event_call->mod);
+		kfree(edata);
+	}
 	return 0;
 }
 
 static void
 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
-		  unsigned long ip, void **_data)
+		  unsigned long ip, void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
-	struct event_probe_data *data;
+	struct ftrace_func_mapper *mapper = data;
+	struct event_probe_data *edata;
+
+	if (!ip) {
+		if (!mapper)
+			return;
+		free_ftrace_func_mapper(mapper, free_probe_data);
+		return;
+	}
 
-	data = ftrace_func_mapper_remove_ip(mapper, ip);
+	edata = ftrace_func_mapper_remove_ip(mapper, ip);
 
-	if (WARN_ON_ONCE(!data))
+	if (WARN_ON_ONCE(!edata))
 		return;
 
-	if (WARN_ON_ONCE(data->ref <= 0))
+	if (WARN_ON_ONCE(edata->ref <= 0))
 		return;
 
-	data->ref--;
-	if (!data->ref) {
-		/* Remove the SOFT_MODE flag */
-		__ftrace_event_enable_disable(data->file, 0, 1);
-		module_put(data->file->event_call->mod);
-		kfree(data);
-	}
+	free_probe_data(edata);
 }
 
 static struct ftrace_probe_ops event_enable_probe_ops = {
@@ -2659,12 +2681,6 @@ event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
 
 	ret = -ENOMEM;
 
-	if (!ops->private_data) {
-		ops->private_data = allocate_ftrace_func_mapper();
-		if (!ops->private_data)
-			goto out;
-	}
-
 	data = kzalloc(sizeof(*data), GFP_KERNEL);
 	if (!data)
 		goto out;

kernel/trace/trace_functions.c

@@ -268,9 +268,10 @@ static struct tracer function_trace __tracer_data =
 
 #ifdef CONFIG_DYNAMIC_FTRACE
 static void update_traceon_count(struct ftrace_probe_ops *ops,
-				 unsigned long ip, bool on)
+				 unsigned long ip, bool on,
+				 void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = data;
 	long *count;
 	long old_count;
 
@@ -329,23 +330,23 @@ static void update_traceon_count(struct ftrace_probe_ops *ops,
 static void
 ftrace_traceon_count(unsigned long ip, unsigned long parent_ip,
 		     struct trace_array *tr, struct ftrace_probe_ops *ops,
-		     void **data)
+		     void *data)
 {
-	update_traceon_count(ops, ip, 1);
+	update_traceon_count(ops, ip, 1, data);
 }
 
 static void
 ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip,
 		      struct trace_array *tr, struct ftrace_probe_ops *ops,
-		      void **data)
+		      void *data)
 {
-	update_traceon_count(ops, ip, 0);
+	update_traceon_count(ops, ip, 0, data);
 }
 
 static void
 ftrace_traceon(unsigned long ip, unsigned long parent_ip,
 	       struct trace_array *tr, struct ftrace_probe_ops *ops,
-	       void **data)
+	       void *data)
 {
 	if (tracing_is_on())
 		return;
@@ -356,7 +357,7 @@ ftrace_traceon(unsigned long ip, unsigned long parent_ip,
 static void
 ftrace_traceoff(unsigned long ip, unsigned long parent_ip,
 		struct trace_array *tr, struct ftrace_probe_ops *ops,
-		void **data)
+		void *data)
 {
 	if (!tracing_is_on())
 		return;
@@ -376,7 +377,7 @@ ftrace_traceoff(unsigned long ip, unsigned long parent_ip,
 static void
 ftrace_stacktrace(unsigned long ip, unsigned long parent_ip,
 		  struct trace_array *tr, struct ftrace_probe_ops *ops,
-		  void **data)
+		  void *data)
 {
 	trace_dump_stack(STACK_SKIP);
 }
@@ -384,9 +385,9 @@ ftrace_stacktrace(unsigned long ip, unsigned long parent_ip,
 static void
 ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip,
 			struct trace_array *tr, struct ftrace_probe_ops *ops,
-			void **data)
+			void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = data;
 	long *count;
 	long old_count;
 	long new_count;
@@ -423,9 +424,10 @@ ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip,
 	} while (new_count != old_count);
 }
 
-static int update_count(struct ftrace_probe_ops *ops, unsigned long ip)
+static int update_count(struct ftrace_probe_ops *ops, unsigned long ip,
+			void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = data;
 	long *count = NULL;
 
 	if (mapper)
@@ -443,9 +445,9 @@ static int update_count(struct ftrace_probe_ops *ops, unsigned long ip)
 static void
 ftrace_dump_probe(unsigned long ip, unsigned long parent_ip,
 		  struct trace_array *tr, struct ftrace_probe_ops *ops,
-		  void **data)
+		  void *data)
 {
-	if (update_count(ops, ip))
+	if (update_count(ops, ip, data))
 		ftrace_dump(DUMP_ALL);
 }
 
@@ -453,17 +455,18 @@ ftrace_dump_probe(unsigned long ip, unsigned long parent_ip,
 static void
 ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip,
 		     struct trace_array *tr, struct ftrace_probe_ops *ops,
-		     void **data)
+		     void *data)
 {
-	if (update_count(ops, ip))
+	if (update_count(ops, ip, data))
 		ftrace_dump(DUMP_ORIG);
 }
 
 static int
 ftrace_probe_print(const char *name, struct seq_file *m,
-		   unsigned long ip, struct ftrace_probe_ops *ops)
+		   unsigned long ip, struct ftrace_probe_ops *ops,
+		   void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = data;
 	long *count = NULL;
 
 	seq_printf(m, "%ps:%s", (void *)ip, name);
@@ -484,52 +487,64 @@ ftrace_traceon_print(struct seq_file *m, unsigned long ip,
 		     struct ftrace_probe_ops *ops,
 		     void *data)
 {
-	return ftrace_probe_print("traceon", m, ip, ops);
+	return ftrace_probe_print("traceon", m, ip, ops, data);
 }
 
 static int
 ftrace_traceoff_print(struct seq_file *m, unsigned long ip,
 			 struct ftrace_probe_ops *ops, void *data)
 {
-	return ftrace_probe_print("traceoff", m, ip, ops);
+	return ftrace_probe_print("traceoff", m, ip, ops, data);
 }
 
 static int
 ftrace_stacktrace_print(struct seq_file *m, unsigned long ip,
 			struct ftrace_probe_ops *ops, void *data)
 {
-	return ftrace_probe_print("stacktrace", m, ip, ops);
+	return ftrace_probe_print("stacktrace", m, ip, ops, data);
 }
 
 static int
 ftrace_dump_print(struct seq_file *m, unsigned long ip,
 			struct ftrace_probe_ops *ops, void *data)
 {
-	return ftrace_probe_print("dump", m, ip, ops);
+	return ftrace_probe_print("dump", m, ip, ops, data);
 }
 
 static int
 ftrace_cpudump_print(struct seq_file *m, unsigned long ip,
 			struct ftrace_probe_ops *ops, void *data)
 {
-	return ftrace_probe_print("cpudump", m, ip, ops);
+	return ftrace_probe_print("cpudump", m, ip, ops, data);
 }
 
 
 static int
 ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
-		  unsigned long ip, void *data)
+		  unsigned long ip, void *init_data, void **data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = *data;
+
+	if (!mapper) {
+		mapper = allocate_ftrace_func_mapper();
+		if (!mapper)
+			return -ENOMEM;
+		*data = mapper;
+	}
 
-	return ftrace_func_mapper_add_ip(mapper, ip, data);
+	return ftrace_func_mapper_add_ip(mapper, ip, init_data);
 }
 
 static void
 ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
-		  unsigned long ip, void **_data)
+		  unsigned long ip, void *data)
 {
-	struct ftrace_func_mapper *mapper = ops->private_data;
+	struct ftrace_func_mapper *mapper = data;
+
+	if (!ip) {
+		free_ftrace_func_mapper(mapper, NULL);
+		return;
+	}
 
 	ftrace_func_mapper_remove_ip(mapper, ip);
 }
@@ -607,12 +622,6 @@ ftrace_trace_probe_callback(struct trace_array *tr,
 	if (!strlen(number))
 		goto out_reg;
 
-	if (!ops->private_data) {
-		ops->private_data = allocate_ftrace_func_mapper();
-		if (!ops->private_data)
-			return -ENOMEM;
-	}
-
 	/*
 	 * We use the callback data field (which is a pointer)
 	 * as our counter.