|
|
@@ -219,6 +219,9 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
|
|
|
#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
|
|
|
};
|
|
|
|
|
|
+DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
|
|
|
+EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
|
|
|
+
|
|
|
/*
|
|
|
* Let the RCU core know that this CPU has gone through the scheduler,
|
|
|
* which is a quiescent state. This is called when the need for a
|
|
|
@@ -288,6 +291,22 @@ void rcu_note_context_switch(void)
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(rcu_note_context_switch);
|
|
|
|
|
|
+/*
|
|
|
+ * Register a quiesecent state for all RCU flavors. If there is an
|
|
|
+ * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
|
|
|
+ * dyntick-idle quiescent state visible to other CPUs (but only for those
|
|
|
+ * RCU flavors in desparate need of a quiescent state, which will normally
|
|
|
+ * be none of them). Either way, do a lightweight quiescent state for
|
|
|
+ * all RCU flavors.
|
|
|
+ */
|
|
|
+void rcu_all_qs(void)
|
|
|
+{
|
|
|
+ if (unlikely(raw_cpu_read(rcu_sched_qs_mask)))
|
|
|
+ rcu_momentary_dyntick_idle();
|
|
|
+ this_cpu_inc(rcu_qs_ctr);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(rcu_all_qs);
|
|
|
+
|
|
|
static long blimit = 10; /* Maximum callbacks per rcu_do_batch. */
|
|
|
static long qhimark = 10000; /* If this many pending, ignore blimit. */
|
|
|
static long qlowmark = 100; /* Once only this many pending, use blimit. */
|
|
|
@@ -1609,6 +1628,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
|
|
|
rdp->gpnum = rnp->gpnum;
|
|
|
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
|
|
|
rdp->passed_quiesce = 0;
|
|
|
+ rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
|
|
|
rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
|
|
|
zero_cpu_stall_ticks(rdp);
|
|
|
ACCESS_ONCE(rdp->gpwrap) = false;
|
|
|
@@ -2075,8 +2095,10 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
|
|
|
rnp = rdp->mynode;
|
|
|
raw_spin_lock_irqsave(&rnp->lock, flags);
|
|
|
smp_mb__after_unlock_lock();
|
|
|
- if (rdp->passed_quiesce == 0 || rdp->gpnum != rnp->gpnum ||
|
|
|
- rnp->completed == rnp->gpnum || rdp->gpwrap) {
|
|
|
+ if ((rdp->passed_quiesce == 0 &&
|
|
|
+ rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
|
|
|
+ rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
|
|
|
+ rdp->gpwrap) {
|
|
|
|
|
|
/*
|
|
|
* The grace period in which this quiescent state was
|
|
|
@@ -2085,6 +2107,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
|
|
|
* within the current grace period.
|
|
|
*/
|
|
|
rdp->passed_quiesce = 0; /* need qs for new gp. */
|
|
|
+ rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
|
|
|
raw_spin_unlock_irqrestore(&rnp->lock, flags);
|
|
|
return;
|
|
|
}
|
|
|
@@ -2129,7 +2152,8 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
|
|
|
* Was there a quiescent state since the beginning of the grace
|
|
|
* period? If no, then exit and wait for the next call.
|
|
|
*/
|
|
|
- if (!rdp->passed_quiesce)
|
|
|
+ if (!rdp->passed_quiesce &&
|
|
|
+ rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
|
|
|
return;
|
|
|
|
|
|
/*
|
|
|
@@ -3174,9 +3198,12 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
|
|
|
|
|
|
/* Is the RCU core waiting for a quiescent state from this CPU? */
|
|
|
if (rcu_scheduler_fully_active &&
|
|
|
- rdp->qs_pending && !rdp->passed_quiesce) {
|
|
|
+ rdp->qs_pending && !rdp->passed_quiesce &&
|
|
|
+ rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
|
|
|
rdp->n_rp_qs_pending++;
|
|
|
- } else if (rdp->qs_pending && rdp->passed_quiesce) {
|
|
|
+ } else if (rdp->qs_pending &&
|
|
|
+ (rdp->passed_quiesce ||
|
|
|
+ rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
|
|
|
rdp->n_rp_report_qs++;
|
|
|
return 1;
|
|
|
}
|
|
|
@@ -3510,6 +3537,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
|
|
|
rdp->gpnum = rnp->completed;
|
|
|
rdp->completed = rnp->completed;
|
|
|
rdp->passed_quiesce = 0;
|
|
|
+ rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
|
|
|
rdp->qs_pending = 0;
|
|
|
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
|
|
|
}
|
Paul E. McKenney