|
@@ -611,112 +611,103 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ if ((long)delta_jiffies <= 1) {
|
|
|
+ if (!ts->tick_stopped)
|
|
|
+ goto out;
|
|
|
+ if (delta_jiffies == 0) {
|
|
|
+ /* Tick is stopped, but required now. Enforce it */
|
|
|
+ tick_nohz_restart(ts, now);
|
|
|
+ goto out;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
/*
|
|
|
- * Do not stop the tick, if we are only one off (or less)
|
|
|
- * or if the cpu is required for RCU:
|
|
|
+ * If this cpu is the one which updates jiffies, then give up
|
|
|
+ * the assignment and let it be taken by the cpu which runs
|
|
|
+ * the tick timer next, which might be this cpu as well. If we
|
|
|
+ * don't drop this here the jiffies might be stale and
|
|
|
+ * do_timer() never invoked. Keep track of the fact that it
|
|
|
+ * was the one which had the do_timer() duty last. If this cpu
|
|
|
+ * is the one which had the do_timer() duty last, we limit the
|
|
|
+ * sleep time to the timekeeping max_deferement value which we
|
|
|
+ * retrieved above. Otherwise we can sleep as long as we want.
|
|
|
*/
|
|
|
- if (!ts->tick_stopped && delta_jiffies <= 1)
|
|
|
- goto out;
|
|
|
-
|
|
|
- /* Schedule the tick, if we are at least one jiffie off */
|
|
|
- if ((long)delta_jiffies >= 1) {
|
|
|
-
|
|
|
- /*
|
|
|
- * If this cpu is the one which updates jiffies, then
|
|
|
- * give up the assignment and let it be taken by the
|
|
|
- * cpu which runs the tick timer next, which might be
|
|
|
- * this cpu as well. If we don't drop this here the
|
|
|
- * jiffies might be stale and do_timer() never
|
|
|
- * invoked. Keep track of the fact that it was the one
|
|
|
- * which had the do_timer() duty last. If this cpu is
|
|
|
- * the one which had the do_timer() duty last, we
|
|
|
- * limit the sleep time to the timekeeping
|
|
|
- * max_deferement value which we retrieved
|
|
|
- * above. Otherwise we can sleep as long as we want.
|
|
|
- */
|
|
|
- if (cpu == tick_do_timer_cpu) {
|
|
|
- tick_do_timer_cpu = TICK_DO_TIMER_NONE;
|
|
|
- ts->do_timer_last = 1;
|
|
|
- } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
|
|
|
- time_delta = KTIME_MAX;
|
|
|
- ts->do_timer_last = 0;
|
|
|
- } else if (!ts->do_timer_last) {
|
|
|
- time_delta = KTIME_MAX;
|
|
|
- }
|
|
|
+ if (cpu == tick_do_timer_cpu) {
|
|
|
+ tick_do_timer_cpu = TICK_DO_TIMER_NONE;
|
|
|
+ ts->do_timer_last = 1;
|
|
|
+ } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
|
|
|
+ time_delta = KTIME_MAX;
|
|
|
+ ts->do_timer_last = 0;
|
|
|
+ } else if (!ts->do_timer_last) {
|
|
|
+ time_delta = KTIME_MAX;
|
|
|
+ }
|
|
|
|
|
|
#ifdef CONFIG_NO_HZ_FULL
|
|
|
- if (!ts->inidle) {
|
|
|
- time_delta = min(time_delta,
|
|
|
- scheduler_tick_max_deferment());
|
|
|
- }
|
|
|
+ if (!ts->inidle)
|
|
|
+ time_delta = min(time_delta, scheduler_tick_max_deferment());
|
|
|
#endif
|
|
|
|
|
|
+ /*
|
|
|
+ * calculate the expiry time for the next timer wheel
|
|
|
+ * timer. delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that
|
|
|
+ * there is no timer pending or at least extremely far into
|
|
|
+ * the future (12 days for HZ=1000). In this case we set the
|
|
|
+ * expiry to the end of time.
|
|
|
+ */
|
|
|
+ if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) {
|
|
|
/*
|
|
|
- * calculate the expiry time for the next timer wheel
|
|
|
- * timer. delta_jiffies >= NEXT_TIMER_MAX_DELTA signals
|
|
|
- * that there is no timer pending or at least extremely
|
|
|
- * far into the future (12 days for HZ=1000). In this
|
|
|
- * case we set the expiry to the end of time.
|
|
|
+ * Calculate the time delta for the next timer event.
|
|
|
+ * If the time delta exceeds the maximum time delta
|
|
|
+ * permitted by the current clocksource then adjust
|
|
|
+ * the time delta accordingly to ensure the
|
|
|
+ * clocksource does not wrap.
|
|
|
*/
|
|
|
- if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) {
|
|
|
- /*
|
|
|
- * Calculate the time delta for the next timer event.
|
|
|
- * If the time delta exceeds the maximum time delta
|
|
|
- * permitted by the current clocksource then adjust
|
|
|
- * the time delta accordingly to ensure the
|
|
|
- * clocksource does not wrap.
|
|
|
- */
|
|
|
- time_delta = min_t(u64, time_delta,
|
|
|
- tick_period.tv64 * delta_jiffies);
|
|
|
- }
|
|
|
-
|
|
|
- if (time_delta < KTIME_MAX)
|
|
|
- expires = ktime_add_ns(last_update, time_delta);
|
|
|
- else
|
|
|
- expires.tv64 = KTIME_MAX;
|
|
|
+ time_delta = min_t(u64, time_delta,
|
|
|
+ tick_period.tv64 * delta_jiffies);
|
|
|
+ }
|
|
|
|
|
|
- /* Skip reprogram of event if its not changed */
|
|
|
- if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
|
|
|
- goto out;
|
|
|
+ if (time_delta < KTIME_MAX)
|
|
|
+ expires = ktime_add_ns(last_update, time_delta);
|
|
|
+ else
|
|
|
+ expires.tv64 = KTIME_MAX;
|
|
|
|
|
|
- ret = expires;
|
|
|
+ /* Skip reprogram of event if its not changed */
|
|
|
+ if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
|
|
|
+ goto out;
|
|
|
|
|
|
- /*
|
|
|
- * nohz_stop_sched_tick can be called several times before
|
|
|
- * the nohz_restart_sched_tick is called. This happens when
|
|
|
- * interrupts arrive which do not cause a reschedule. In the
|
|
|
- * first call we save the current tick time, so we can restart
|
|
|
- * the scheduler tick in nohz_restart_sched_tick.
|
|
|
- */
|
|
|
- if (!ts->tick_stopped) {
|
|
|
- nohz_balance_enter_idle(cpu);
|
|
|
- calc_load_enter_idle();
|
|
|
+ ret = expires;
|
|
|
|
|
|
- ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
|
|
|
- ts->tick_stopped = 1;
|
|
|
- trace_tick_stop(1, " ");
|
|
|
- }
|
|
|
+ /*
|
|
|
+ * nohz_stop_sched_tick can be called several times before
|
|
|
+ * the nohz_restart_sched_tick is called. This happens when
|
|
|
+ * interrupts arrive which do not cause a reschedule. In the
|
|
|
+ * first call we save the current tick time, so we can restart
|
|
|
+ * the scheduler tick in nohz_restart_sched_tick.
|
|
|
+ */
|
|
|
+ if (!ts->tick_stopped) {
|
|
|
+ nohz_balance_enter_idle(cpu);
|
|
|
+ calc_load_enter_idle();
|
|
|
|
|
|
- /*
|
|
|
- * If the expiration time == KTIME_MAX, then
|
|
|
- * in this case we simply stop the tick timer.
|
|
|
- */
|
|
|
- if (unlikely(expires.tv64 == KTIME_MAX)) {
|
|
|
- if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
|
|
|
- hrtimer_cancel(&ts->sched_timer);
|
|
|
- goto out;
|
|
|
- }
|
|
|
+ ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
|
|
|
+ ts->tick_stopped = 1;
|
|
|
+ trace_tick_stop(1, " ");
|
|
|
+ }
|
|
|
|
|
|
- if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
|
|
|
- hrtimer_start(&ts->sched_timer, expires,
|
|
|
- HRTIMER_MODE_ABS_PINNED);
|
|
|
- else
|
|
|
- tick_program_event(expires, 1);
|
|
|
- } else {
|
|
|
- /* Tick is stopped, but required now. Enforce it */
|
|
|
- tick_nohz_restart(ts, now);
|
|
|
+ /*
|
|
|
+ * If the expiration time == KTIME_MAX, then
|
|
|
+ * in this case we simply stop the tick timer.
|
|
|
+ */
|
|
|
+ if (unlikely(expires.tv64 == KTIME_MAX)) {
|
|
|
+ if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
|
|
|
+ hrtimer_cancel(&ts->sched_timer);
|
|
|
+ goto out;
|
|
|
}
|
|
|
|
|
|
+ if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
|
|
|
+ hrtimer_start(&ts->sched_timer, expires,
|
|
|
+ HRTIMER_MODE_ABS_PINNED);
|
|
|
+ else
|
|
|
+ tick_program_event(expires, 1);
|
|
|
out:
|
|
|
ts->next_jiffies = next_jiffies;
|
|
|
ts->last_jiffies = last_jiffies;
|