|
@@ -364,18 +364,18 @@ static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
|
|
* slightly wrong timestamp (a few nanoseconds). See
|
|
* slightly wrong timestamp (a few nanoseconds). See
|
|
* @ktime_get_mono_fast_ns.
|
|
* @ktime_get_mono_fast_ns.
|
|
*/
|
|
*/
|
|
-static void update_fast_timekeeper(struct tk_read_base *tkr)
|
|
|
|
|
|
+static void update_fast_timekeeper(struct tk_read_base *tkr, struct tk_fast *tkf)
|
|
{
|
|
{
|
|
- struct tk_read_base *base = tk_fast_mono.base;
|
|
|
|
|
|
+ struct tk_read_base *base = tkf->base;
|
|
|
|
|
|
/* Force readers off to base[1] */
|
|
/* Force readers off to base[1] */
|
|
- raw_write_seqcount_latch(&tk_fast_mono.seq);
|
|
|
|
|
|
+ raw_write_seqcount_latch(&tkf->seq);
|
|
|
|
|
|
/* Update base[0] */
|
|
/* Update base[0] */
|
|
memcpy(base, tkr, sizeof(*base));
|
|
memcpy(base, tkr, sizeof(*base));
|
|
|
|
|
|
/* Force readers back to base[0] */
|
|
/* Force readers back to base[0] */
|
|
- raw_write_seqcount_latch(&tk_fast_mono.seq);
|
|
|
|
|
|
+ raw_write_seqcount_latch(&tkf->seq);
|
|
|
|
|
|
/* Update base[1] */
|
|
/* Update base[1] */
|
|
memcpy(base + 1, base, sizeof(*base));
|
|
memcpy(base + 1, base, sizeof(*base));
|
|
@@ -413,20 +413,25 @@ static void update_fast_timekeeper(struct tk_read_base *tkr)
|
|
* of the following timestamps. Callers need to be aware of that and
|
|
* of the following timestamps. Callers need to be aware of that and
|
|
* deal with it.
|
|
* deal with it.
|
|
*/
|
|
*/
|
|
-u64 notrace ktime_get_mono_fast_ns(void)
|
|
|
|
|
|
+static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
|
|
{
|
|
{
|
|
struct tk_read_base *tkr;
|
|
struct tk_read_base *tkr;
|
|
unsigned int seq;
|
|
unsigned int seq;
|
|
u64 now;
|
|
u64 now;
|
|
|
|
|
|
do {
|
|
do {
|
|
- seq = raw_read_seqcount(&tk_fast_mono.seq);
|
|
|
|
- tkr = tk_fast_mono.base + (seq & 0x01);
|
|
|
|
|
|
+ seq = raw_read_seqcount(&tkf->seq);
|
|
|
|
+ tkr = tkf->base + (seq & 0x01);
|
|
now = ktime_to_ns(tkr->base) + timekeeping_get_ns(tkr);
|
|
now = ktime_to_ns(tkr->base) + timekeeping_get_ns(tkr);
|
|
|
|
+ } while (read_seqcount_retry(&tkf->seq, seq));
|
|
|
|
|
|
- } while (read_seqcount_retry(&tk_fast_mono.seq, seq));
|
|
|
|
return now;
|
|
return now;
|
|
}
|
|
}
|
|
|
|
+
|
|
|
|
+u64 ktime_get_mono_fast_ns(void)
|
|
|
|
+{
|
|
|
|
+ return __ktime_get_fast_ns(&tk_fast_mono);
|
|
|
|
+}
|
|
EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
|
|
EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
|
|
|
|
|
|
/* Suspend-time cycles value for halted fast timekeeper. */
|
|
/* Suspend-time cycles value for halted fast timekeeper. */
|
|
@@ -455,7 +460,7 @@ static void halt_fast_timekeeper(struct timekeeper *tk)
|
|
memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
|
|
memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
|
|
cycles_at_suspend = tkr->read(tkr->clock);
|
|
cycles_at_suspend = tkr->read(tkr->clock);
|
|
tkr_dummy.read = dummy_clock_read;
|
|
tkr_dummy.read = dummy_clock_read;
|
|
- update_fast_timekeeper(&tkr_dummy);
|
|
|
|
|
|
+ update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
|
|
}
|
|
}
|
|
|
|
|
|
#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
|
|
#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
|
|
@@ -586,7 +591,7 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action)
|
|
memcpy(&shadow_timekeeper, &tk_core.timekeeper,
|
|
memcpy(&shadow_timekeeper, &tk_core.timekeeper,
|
|
sizeof(tk_core.timekeeper));
|
|
sizeof(tk_core.timekeeper));
|
|
|
|
|
|
- update_fast_timekeeper(&tk->tkr_mono);
|
|
|
|
|
|
+ update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
/**
|