|
|
@@ -756,10 +756,104 @@ static void hpet_reserve_msi_timers(struct hpet_data *hd)
|
|
|
/*
|
|
|
* Clock source related code
|
|
|
*/
|
|
|
+#if defined(CONFIG_SMP) && defined(CONFIG_64BIT)
|
|
|
+/*
|
|
|
+ * Reading the HPET counter is a very slow operation. If a large number of
|
|
|
+ * CPUs are trying to access the HPET counter simultaneously, it can cause
|
|
|
+ * massive delay and slow down system performance dramatically. This may
|
|
|
+ * happen when HPET is the default clock source instead of TSC. For a
|
|
|
+ * really large system with hundreds of CPUs, the slowdown may be so
|
|
|
+ * severe that it may actually crash the system because of a NMI watchdog
|
|
|
+ * soft lockup, for example.
|
|
|
+ *
|
|
|
+ * If multiple CPUs are trying to access the HPET counter at the same time,
|
|
|
+ * we don't actually need to read the counter multiple times. Instead, the
|
|
|
+ * other CPUs can use the counter value read by the first CPU in the group.
|
|
|
+ *
|
|
|
+ * This special feature is only enabled on x86-64 systems. It is unlikely
|
|
|
+ * that 32-bit x86 systems will have enough CPUs to require this feature
|
|
|
+ * with its associated locking overhead. And we also need 64-bit atomic
|
|
|
+ * read.
|
|
|
+ *
|
|
|
+ * The lock and the hpet value are stored together and can be read in a
|
|
|
+ * single atomic 64-bit read. It is explicitly assumed that arch_spinlock_t
|
|
|
+ * is 32 bits in size.
|
|
|
+ */
|
|
|
+union hpet_lock {
|
|
|
+ struct {
|
|
|
+ arch_spinlock_t lock;
|
|
|
+ u32 value;
|
|
|
+ };
|
|
|
+ u64 lockval;
|
|
|
+};
|
|
|
+
|
|
|
+static union hpet_lock hpet __cacheline_aligned = {
|
|
|
+ { .lock = __ARCH_SPIN_LOCK_UNLOCKED, },
|
|
|
+};
|
|
|
+
|
|
|
+static cycle_t read_hpet(struct clocksource *cs)
|
|
|
+{
|
|
|
+ unsigned long flags;
|
|
|
+ union hpet_lock old, new;
|
|
|
+
|
|
|
+ BUILD_BUG_ON(sizeof(union hpet_lock) != 8);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Read HPET directly if in NMI.
|
|
|
+ */
|
|
|
+ if (in_nmi())
|
|
|
+ return (cycle_t)hpet_readl(HPET_COUNTER);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Read the current state of the lock and HPET value atomically.
|
|
|
+ */
|
|
|
+ old.lockval = READ_ONCE(hpet.lockval);
|
|
|
+
|
|
|
+ if (arch_spin_is_locked(&old.lock))
|
|
|
+ goto contended;
|
|
|
+
|
|
|
+ local_irq_save(flags);
|
|
|
+ if (arch_spin_trylock(&hpet.lock)) {
|
|
|
+ new.value = hpet_readl(HPET_COUNTER);
|
|
|
+ /*
|
|
|
+ * Use WRITE_ONCE() to prevent store tearing.
|
|
|
+ */
|
|
|
+ WRITE_ONCE(hpet.value, new.value);
|
|
|
+ arch_spin_unlock(&hpet.lock);
|
|
|
+ local_irq_restore(flags);
|
|
|
+ return (cycle_t)new.value;
|
|
|
+ }
|
|
|
+ local_irq_restore(flags);
|
|
|
+
|
|
|
+contended:
|
|
|
+ /*
|
|
|
+ * Contended case
|
|
|
+ * --------------
|
|
|
+ * Wait until the HPET value change or the lock is free to indicate
|
|
|
+ * its value is up-to-date.
|
|
|
+ *
|
|
|
+ * It is possible that old.value has already contained the latest
|
|
|
+ * HPET value while the lock holder was in the process of releasing
|
|
|
+ * the lock. Checking for lock state change will enable us to return
|
|
|
+ * the value immediately instead of waiting for the next HPET reader
|
|
|
+ * to come along.
|
|
|
+ */
|
|
|
+ do {
|
|
|
+ cpu_relax();
|
|
|
+ new.lockval = READ_ONCE(hpet.lockval);
|
|
|
+ } while ((new.value == old.value) && arch_spin_is_locked(&new.lock));
|
|
|
+
|
|
|
+ return (cycle_t)new.value;
|
|
|
+}
|
|
|
+#else
|
|
|
+/*
|
|
|
+ * For UP or 32-bit.
|
|
|
+ */
|
|
|
static cycle_t read_hpet(struct clocksource *cs)
|
|
|
{
|
|
|
return (cycle_t)hpet_readl(HPET_COUNTER);
|
|
|
}
|
|
|
+#endif
|
|
|
|
|
|
static struct clocksource clocksource_hpet = {
|
|
|
.name = "hpet",
|
Waiman Long