drm/i915: Wake up all waiters before idling

When we idle, we wakeup the first waiter (checking to see if it missed
an earlier wakeup) and disarm the breadcrumbs. However, we now assert
that there are no waiter when the interrupt is disabled, triggering an
assert if there were multiple waiters when we idled.

[  420.842275] invalid opcode: 0000 [#1] PREEMPT SMP
[  420.842285] Modules linked in: vgem snd_hda_codec_realtek x86_pkg_temp_thermal snd_hda_codec_generic intel_powerclamp coretemp crct10dif_pclmul crc32_pclmul ghash_clmulni_intel snd_hda_intel snd_hda_codec snd_hwdep mei_me snd_hda_core mei snd_pcm lpc_ich i915 r8169 mii prime_numbers
[  420.842357] CPU: 4 PID: 8714 Comm: kms_pipe_crc_ba Tainted: G     U  W       4.10.0-CI-CI_DRM_2280+ #1
[  420.842377] Hardware name: Hewlett-Packard HP Pro 3500 Series/2ABF, BIOS 8.11 10/24/2012
[  420.842395] task: ffff880117ddce40 task.stack: ffffc90001114000
[  420.842439] RIP: 0010:__intel_engine_remove_wait+0x1f4/0x200 [i915]
[  420.842454] RSP: 0018:ffffc90001117b18 EFLAGS: 00010046
[  420.842467] RAX: 0000000000000000 RBX: ffff88010c25c2a8 RCX: 0000000000000001
[  420.842481] RDX: 0000000000000001 RSI: 00000000ffffffff RDI: ffffc90001117c50
[  420.842495] RBP: ffffc90001117b58 R08: 0000000011e52352 R09: c4d16acc00000000
[  420.842511] R10: ffffffff82789eb0 R11: ffff880117ddce40 R12: ffffc90001117c50
[  420.842525] R13: ffffc90001117c50 R14: 0000000000000078 R15: 0000000000000000
[  420.842540] FS:  00007fe47dda0a40(0000) GS:ffff88011fb00000(0000) knlGS:0000000000000000
[  420.842559] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  420.842571] CR2: 00007fd6c0a2cec4 CR3: 000000010a5e5000 CR4: 00000000001406e0
[  420.842586] Call Trace:
[  420.842595]  ? do_raw_spin_lock+0xad/0xb0
[  420.842635]  intel_engine_remove_wait.part.3+0x26/0x40 [i915]
[  420.842678]  intel_engine_remove_wait+0xe/0x20 [i915]
[  420.842721]  i915_wait_request+0x4f0/0x8c0 [i915]
[  420.842736]  ? wake_up_q+0x70/0x70
[  420.842747]  ? wake_up_q+0x70/0x70
[  420.842787]  i915_gem_object_wait_fence+0x7d/0x1a0 [i915]
[  420.842829]  i915_gem_object_wait+0x30d/0x520 [i915]
[  420.842842]  ? __this_cpu_preempt_check+0x13/0x20
[  420.842884]  i915_gem_wait_ioctl+0x12e/0x2e0 [i915]
[  420.842924]  ? i915_gem_wait_ioctl+0x22/0x2e0 [i915]
[  420.842939]  drm_ioctl+0x200/0x450
[  420.842976]  ? i915_gem_set_wedged+0x90/0x90 [i915]
[  420.842993]  do_vfs_ioctl+0x90/0x6e0
[  420.843003]  ? entry_SYSCALL_64_fastpath+0x5/0xb1
[  420.843017]  ? __this_cpu_preempt_check+0x13/0x20
[  420.843030]  ? trace_hardirqs_on_caller+0xe7/0x200
[  420.843042]  SyS_ioctl+0x3c/0x70
[  420.843054]  entry_SYSCALL_64_fastpath+0x1c/0xb1
[  420.843065] RIP: 0033:0x7fe47c4b9357
[  420.843075] RSP: 002b:00007ffc3c0633c8 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
[  420.843094] RAX: ffffffffffffffda RBX: ffffffff81482393 RCX: 00007fe47c4b9357
[  420.843109] RDX: 00007ffc3c063400 RSI: 00000000c010646c RDI: 0000000000000004
[  420.843123] RBP: ffffc90001117f88 R08: 0000000000000008 R09: 0000000000000000
[  420.843137] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000
[  420.843151] R13: 0000000000000004 R14: 00000000c010646c R15: 0000000000000000
[  420.843168]  ? __this_cpu_preempt_check+0x13/0x20
[  420.843180] Code: 81 48 c7 c1 40 6a 16 a0 48 c7 c2 47 29 15 a0 be 17 01 00 00 48 c7 c7 10 6a 16 a0 e8 c7 ea fe e0 e9 5d ff ff ff 0f 0b 0f 0b 0f 0b <0f> 0b 66 2e 0f 1f 84 00 00 00 00 00 55 48 89 e5 e8 67 41 7e e1
[  420.843325] RIP: __intel_engine_remove_wait+0x1f4/0x200 [i915] RSP: ffffc90001117b18

Fixes: b66255f0f779 ("drm/i915: Refactor wakeup of the next breadcrumb waiter")
Fixes: 67b807a89230 ("drm/i915: Delay disabling the user interrupt for breadcrumbs")
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Mika Kuoppala <mika.kuoppala@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170306092916.11623-2-chris@chris-wilson.co.uk
Reviewed-by: Mika Kuoppala <mika.kuoppala@intel.com>

drivers/gpu/drm/i915/intel_breadcrumbs.c

@@ -167,6 +167,7 @@ void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 
 	lockdep_assert_held(&b->irq_lock);
+	GEM_BUG_ON(b->irq_wait);
 
 	if (b->irq_enabled) {
 		irq_disable(engine);
@@ -179,23 +180,30 @@ void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 {
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
-	unsigned long flags;
+	struct intel_wait *wait, *n;
 
 	if (!b->irq_armed)
 		return;
 
-	spin_lock_irqsave(&b->irq_lock, flags);
-
 	/* We only disarm the irq when we are idle (all requests completed),
-	 * so if there remains a sleeping waiter, it missed the request
+	 * so if the bottom-half remains asleep, it missed the request
 	 * completion.
 	 */
-	if (__intel_breadcrumbs_wakeup(b) & ENGINE_WAKEUP_ASLEEP)
-		missed_breadcrumb(engine);
 
+	spin_lock_irq(&b->rb_lock);
+	rbtree_postorder_for_each_entry_safe(wait, n, &b->waiters, node) {
+		RB_CLEAR_NODE(&wait->node);
+		if (wake_up_process(wait->tsk) && wait == b->irq_wait)
+			missed_breadcrumb(engine);
+	}
+	b->waiters = RB_ROOT;
+
+	spin_lock(&b->irq_lock);
+	b->irq_wait = NULL;
 	__intel_engine_disarm_breadcrumbs(engine);
+	spin_unlock(&b->irq_lock);
 
-	spin_unlock_irqrestore(&b->irq_lock, flags);
+	spin_unlock_irq(&b->rb_lock);
 }
 
 static bool use_fake_irq(const struct intel_breadcrumbs *b)