|
@@ -38,11 +38,23 @@
|
|
|
|
|
|
#include "rcu.h"
|
|
#include "rcu.h"
|
|
|
|
|
|
-/* Forward declarations for tiny_plugin.h. */
|
|
|
|
-struct rcu_ctrlblk;
|
|
|
|
-static void __call_rcu(struct rcu_head *head,
|
|
|
|
- rcu_callback_t func,
|
|
|
|
- struct rcu_ctrlblk *rcp);
|
|
|
|
|
|
+/* Global control variables for rcupdate callback mechanism. */
|
|
|
|
+struct rcu_ctrlblk {
|
|
|
|
+ struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */
|
|
|
|
+ struct rcu_head **donetail; /* ->next pointer of last "done" CB. */
|
|
|
|
+ struct rcu_head **curtail; /* ->next pointer of last CB. */
|
|
|
|
+};
|
|
|
|
+
|
|
|
|
+/* Definition for rcupdate control block. */
|
|
|
|
+static struct rcu_ctrlblk rcu_sched_ctrlblk = {
|
|
|
|
+ .donetail = &rcu_sched_ctrlblk.rcucblist,
|
|
|
|
+ .curtail = &rcu_sched_ctrlblk.rcucblist,
|
|
|
|
+};
|
|
|
|
+
|
|
|
|
+static struct rcu_ctrlblk rcu_bh_ctrlblk = {
|
|
|
|
+ .donetail = &rcu_bh_ctrlblk.rcucblist,
|
|
|
|
+ .curtail = &rcu_bh_ctrlblk.rcucblist,
|
|
|
|
+};
|
|
|
|
|
|
#include "tiny_plugin.h"
|
|
#include "tiny_plugin.h"
|
|
|
|
|
|
@@ -65,7 +77,6 @@ EXPORT_SYMBOL(rcu_barrier_sched);
|
|
*/
|
|
*/
|
|
static int rcu_qsctr_help(struct rcu_ctrlblk *rcp)
|
|
static int rcu_qsctr_help(struct rcu_ctrlblk *rcp)
|
|
{
|
|
{
|
|
- RCU_TRACE(reset_cpu_stall_ticks(rcp);)
|
|
|
|
if (rcp->donetail != rcp->curtail) {
|
|
if (rcp->donetail != rcp->curtail) {
|
|
rcp->donetail = rcp->curtail;
|
|
rcp->donetail = rcp->curtail;
|
|
return 1;
|
|
return 1;
|
|
@@ -111,7 +122,6 @@ void rcu_bh_qs(void)
|
|
*/
|
|
*/
|
|
void rcu_check_callbacks(int user)
|
|
void rcu_check_callbacks(int user)
|
|
{
|
|
{
|
|
- RCU_TRACE(check_cpu_stalls();)
|
|
|
|
if (user)
|
|
if (user)
|
|
rcu_sched_qs();
|
|
rcu_sched_qs();
|
|
else if (!in_softirq())
|
|
else if (!in_softirq())
|
|
@@ -126,10 +136,8 @@ void rcu_check_callbacks(int user)
|
|
*/
|
|
*/
|
|
static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
|
|
static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
|
|
{
|
|
{
|
|
- const char *rn = NULL;
|
|
|
|
struct rcu_head *next, *list;
|
|
struct rcu_head *next, *list;
|
|
unsigned long flags;
|
|
unsigned long flags;
|
|
- RCU_TRACE(int cb_count = 0;)
|
|
|
|
|
|
|
|
/* Move the ready-to-invoke callbacks to a local list. */
|
|
/* Move the ready-to-invoke callbacks to a local list. */
|
|
local_irq_save(flags);
|
|
local_irq_save(flags);
|
|
@@ -147,18 +155,15 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
|
|
local_irq_restore(flags);
|
|
local_irq_restore(flags);
|
|
|
|
|
|
/* Invoke the callbacks on the local list. */
|
|
/* Invoke the callbacks on the local list. */
|
|
- RCU_TRACE(rn = rcp->name;)
|
|
|
|
while (list) {
|
|
while (list) {
|
|
next = list->next;
|
|
next = list->next;
|
|
prefetch(next);
|
|
prefetch(next);
|
|
debug_rcu_head_unqueue(list);
|
|
debug_rcu_head_unqueue(list);
|
|
local_bh_disable();
|
|
local_bh_disable();
|
|
- __rcu_reclaim(rn, list);
|
|
|
|
|
|
+ __rcu_reclaim("", list);
|
|
local_bh_enable();
|
|
local_bh_enable();
|
|
list = next;
|
|
list = next;
|
|
- RCU_TRACE(cb_count++;)
|
|
|
|
}
|
|
}
|
|
- RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count);)
|
|
|
|
}
|
|
}
|
|
|
|
|
|
static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
|
|
static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
|
|
@@ -202,7 +207,6 @@ static void __call_rcu(struct rcu_head *head,
|
|
local_irq_save(flags);
|
|
local_irq_save(flags);
|
|
*rcp->curtail = head;
|
|
*rcp->curtail = head;
|
|
rcp->curtail = &head->next;
|
|
rcp->curtail = &head->next;
|
|
- RCU_TRACE(rcp->qlen++;)
|
|
|
|
local_irq_restore(flags);
|
|
local_irq_restore(flags);
|
|
|
|
|
|
if (unlikely(is_idle_task(current))) {
|
|
if (unlikely(is_idle_task(current))) {
|
|
@@ -235,8 +239,5 @@ EXPORT_SYMBOL_GPL(call_rcu_bh);
|
|
void __init rcu_init(void)
|
|
void __init rcu_init(void)
|
|
{
|
|
{
|
|
open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
|
|
open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
|
|
- RCU_TRACE(reset_cpu_stall_ticks(&rcu_sched_ctrlblk);)
|
|
|
|
- RCU_TRACE(reset_cpu_stall_ticks(&rcu_bh_ctrlblk);)
|
|
|
|
-
|
|
|
|
rcu_early_boot_tests();
|
|
rcu_early_boot_tests();
|
|
}
|
|
}
|