|
@@ -53,12 +53,6 @@ void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
|
|
|
ringbuf->tail, ringbuf->size);
|
|
|
}
|
|
|
|
|
|
-int intel_ring_space(struct intel_ringbuffer *ringbuf)
|
|
|
-{
|
|
|
- intel_ring_update_space(ringbuf);
|
|
|
- return ringbuf->space;
|
|
|
-}
|
|
|
-
|
|
|
bool intel_engine_stopped(struct intel_engine_cs *engine)
|
|
|
{
|
|
|
struct drm_i915_private *dev_priv = engine->dev->dev_private;
|
|
@@ -1309,7 +1303,7 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
|
|
|
intel_ring_emit(signaller, seqno);
|
|
|
intel_ring_emit(signaller, 0);
|
|
|
intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
|
|
|
- MI_SEMAPHORE_TARGET(waiter->id));
|
|
|
+ MI_SEMAPHORE_TARGET(waiter->hw_id));
|
|
|
intel_ring_emit(signaller, 0);
|
|
|
}
|
|
|
|
|
@@ -1349,7 +1343,7 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
|
|
|
intel_ring_emit(signaller, upper_32_bits(gtt_offset));
|
|
|
intel_ring_emit(signaller, seqno);
|
|
|
intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
|
|
|
- MI_SEMAPHORE_TARGET(waiter->id));
|
|
|
+ MI_SEMAPHORE_TARGET(waiter->hw_id));
|
|
|
intel_ring_emit(signaller, 0);
|
|
|
}
|
|
|
|
|
@@ -1573,6 +1567,8 @@ pc_render_add_request(struct drm_i915_gem_request *req)
|
|
|
static void
|
|
|
gen6_seqno_barrier(struct intel_engine_cs *engine)
|
|
|
{
|
|
|
+ struct drm_i915_private *dev_priv = engine->dev->dev_private;
|
|
|
+
|
|
|
/* Workaround to force correct ordering between irq and seqno writes on
|
|
|
* ivb (and maybe also on snb) by reading from a CS register (like
|
|
|
* ACTHD) before reading the status page.
|
|
@@ -1584,9 +1580,13 @@ gen6_seqno_barrier(struct intel_engine_cs *engine)
|
|
|
* the write time to land, but that would incur a delay after every
|
|
|
* batch i.e. much more frequent than a delay when waiting for the
|
|
|
* interrupt (with the same net latency).
|
|
|
+ *
|
|
|
+ * Also note that to prevent whole machine hangs on gen7, we have to
|
|
|
+ * take the spinlock to guard against concurrent cacheline access.
|
|
|
*/
|
|
|
- struct drm_i915_private *dev_priv = engine->dev->dev_private;
|
|
|
+ spin_lock_irq(&dev_priv->uncore.lock);
|
|
|
POSTING_READ_FW(RING_ACTHD(engine->mmio_base));
|
|
|
+ spin_unlock_irq(&dev_priv->uncore.lock);
|
|
|
}
|
|
|
|
|
|
static u32
|
|
@@ -2312,51 +2312,6 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
|
|
|
engine->dev = NULL;
|
|
|
}
|
|
|
|
|
|
-static int ring_wait_for_space(struct intel_engine_cs *engine, int n)
|
|
|
-{
|
|
|
- struct intel_ringbuffer *ringbuf = engine->buffer;
|
|
|
- struct drm_i915_gem_request *request;
|
|
|
- unsigned space;
|
|
|
- int ret;
|
|
|
-
|
|
|
- if (intel_ring_space(ringbuf) >= n)
|
|
|
- return 0;
|
|
|
-
|
|
|
- /* The whole point of reserving space is to not wait! */
|
|
|
- WARN_ON(ringbuf->reserved_in_use);
|
|
|
-
|
|
|
- list_for_each_entry(request, &engine->request_list, list) {
|
|
|
- space = __intel_ring_space(request->postfix, ringbuf->tail,
|
|
|
- ringbuf->size);
|
|
|
- if (space >= n)
|
|
|
- break;
|
|
|
- }
|
|
|
-
|
|
|
- if (WARN_ON(&request->list == &engine->request_list))
|
|
|
- return -ENOSPC;
|
|
|
-
|
|
|
- ret = i915_wait_request(request);
|
|
|
- if (ret)
|
|
|
- return ret;
|
|
|
-
|
|
|
- ringbuf->space = space;
|
|
|
- return 0;
|
|
|
-}
|
|
|
-
|
|
|
-static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf)
|
|
|
-{
|
|
|
- uint32_t __iomem *virt;
|
|
|
- int rem = ringbuf->size - ringbuf->tail;
|
|
|
-
|
|
|
- virt = ringbuf->virtual_start + ringbuf->tail;
|
|
|
- rem /= 4;
|
|
|
- while (rem--)
|
|
|
- iowrite32(MI_NOOP, virt++);
|
|
|
-
|
|
|
- ringbuf->tail = 0;
|
|
|
- intel_ring_update_space(ringbuf);
|
|
|
-}
|
|
|
-
|
|
|
int intel_engine_idle(struct intel_engine_cs *engine)
|
|
|
{
|
|
|
struct drm_i915_gem_request *req;
|
|
@@ -2398,63 +2353,82 @@ int intel_ring_reserve_space(struct drm_i915_gem_request *request)
|
|
|
|
|
|
void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size)
|
|
|
{
|
|
|
- WARN_ON(ringbuf->reserved_size);
|
|
|
- WARN_ON(ringbuf->reserved_in_use);
|
|
|
-
|
|
|
+ GEM_BUG_ON(ringbuf->reserved_size);
|
|
|
ringbuf->reserved_size = size;
|
|
|
}
|
|
|
|
|
|
void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf)
|
|
|
{
|
|
|
- WARN_ON(ringbuf->reserved_in_use);
|
|
|
-
|
|
|
+ GEM_BUG_ON(!ringbuf->reserved_size);
|
|
|
ringbuf->reserved_size = 0;
|
|
|
- ringbuf->reserved_in_use = false;
|
|
|
}
|
|
|
|
|
|
void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf)
|
|
|
{
|
|
|
- WARN_ON(ringbuf->reserved_in_use);
|
|
|
-
|
|
|
- ringbuf->reserved_in_use = true;
|
|
|
- ringbuf->reserved_tail = ringbuf->tail;
|
|
|
+ GEM_BUG_ON(!ringbuf->reserved_size);
|
|
|
+ ringbuf->reserved_size = 0;
|
|
|
}
|
|
|
|
|
|
void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf)
|
|
|
{
|
|
|
- WARN_ON(!ringbuf->reserved_in_use);
|
|
|
- if (ringbuf->tail > ringbuf->reserved_tail) {
|
|
|
- WARN(ringbuf->tail > ringbuf->reserved_tail + ringbuf->reserved_size,
|
|
|
- "request reserved size too small: %d vs %d!\n",
|
|
|
- ringbuf->tail - ringbuf->reserved_tail, ringbuf->reserved_size);
|
|
|
- } else {
|
|
|
+ GEM_BUG_ON(ringbuf->reserved_size);
|
|
|
+}
|
|
|
+
|
|
|
+static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
|
|
|
+{
|
|
|
+ struct intel_ringbuffer *ringbuf = req->ringbuf;
|
|
|
+ struct intel_engine_cs *engine = req->engine;
|
|
|
+ struct drm_i915_gem_request *target;
|
|
|
+
|
|
|
+ intel_ring_update_space(ringbuf);
|
|
|
+ if (ringbuf->space >= bytes)
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Space is reserved in the ringbuffer for finalising the request,
|
|
|
+ * as that cannot be allowed to fail. During request finalisation,
|
|
|
+ * reserved_space is set to 0 to stop the overallocation and the
|
|
|
+ * assumption is that then we never need to wait (which has the
|
|
|
+ * risk of failing with EINTR).
|
|
|
+ *
|
|
|
+ * See also i915_gem_request_alloc() and i915_add_request().
|
|
|
+ */
|
|
|
+ GEM_BUG_ON(!ringbuf->reserved_size);
|
|
|
+
|
|
|
+ list_for_each_entry(target, &engine->request_list, list) {
|
|
|
+ unsigned space;
|
|
|
+
|
|
|
/*
|
|
|
- * The ring was wrapped while the reserved space was in use.
|
|
|
- * That means that some unknown amount of the ring tail was
|
|
|
- * no-op filled and skipped. Thus simply adding the ring size
|
|
|
- * to the tail and doing the above space check will not work.
|
|
|
- * Rather than attempt to track how much tail was skipped,
|
|
|
- * it is much simpler to say that also skipping the sanity
|
|
|
- * check every once in a while is not a big issue.
|
|
|
+ * The request queue is per-engine, so can contain requests
|
|
|
+ * from multiple ringbuffers. Here, we must ignore any that
|
|
|
+ * aren't from the ringbuffer we're considering.
|
|
|
*/
|
|
|
+ if (target->ringbuf != ringbuf)
|
|
|
+ continue;
|
|
|
+
|
|
|
+ /* Would completion of this request free enough space? */
|
|
|
+ space = __intel_ring_space(target->postfix, ringbuf->tail,
|
|
|
+ ringbuf->size);
|
|
|
+ if (space >= bytes)
|
|
|
+ break;
|
|
|
}
|
|
|
|
|
|
- ringbuf->reserved_size = 0;
|
|
|
- ringbuf->reserved_in_use = false;
|
|
|
+ if (WARN_ON(&target->list == &engine->request_list))
|
|
|
+ return -ENOSPC;
|
|
|
+
|
|
|
+ return i915_wait_request(target);
|
|
|
}
|
|
|
|
|
|
-static int __intel_ring_prepare(struct intel_engine_cs *engine, int bytes)
|
|
|
+int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
|
|
|
{
|
|
|
- struct intel_ringbuffer *ringbuf = engine->buffer;
|
|
|
- int remain_usable = ringbuf->effective_size - ringbuf->tail;
|
|
|
+ struct intel_ringbuffer *ringbuf = req->ringbuf;
|
|
|
int remain_actual = ringbuf->size - ringbuf->tail;
|
|
|
- int ret, total_bytes, wait_bytes = 0;
|
|
|
+ int remain_usable = ringbuf->effective_size - ringbuf->tail;
|
|
|
+ int bytes = num_dwords * sizeof(u32);
|
|
|
+ int total_bytes, wait_bytes;
|
|
|
bool need_wrap = false;
|
|
|
|
|
|
- if (ringbuf->reserved_in_use)
|
|
|
- total_bytes = bytes;
|
|
|
- else
|
|
|
- total_bytes = bytes + ringbuf->reserved_size;
|
|
|
+ total_bytes = bytes + ringbuf->reserved_size;
|
|
|
|
|
|
if (unlikely(bytes > remain_usable)) {
|
|
|
/*
|
|
@@ -2463,44 +2437,42 @@ static int __intel_ring_prepare(struct intel_engine_cs *engine, int bytes)
|
|
|
*/
|
|
|
wait_bytes = remain_actual + total_bytes;
|
|
|
need_wrap = true;
|
|
|
+ } else if (unlikely(total_bytes > remain_usable)) {
|
|
|
+ /*
|
|
|
+ * The base request will fit but the reserved space
|
|
|
+ * falls off the end. So we don't need an immediate wrap
|
|
|
+ * and only need to effectively wait for the reserved
|
|
|
+ * size space from the start of ringbuffer.
|
|
|
+ */
|
|
|
+ wait_bytes = remain_actual + ringbuf->reserved_size;
|
|
|
} else {
|
|
|
- if (unlikely(total_bytes > remain_usable)) {
|
|
|
- /*
|
|
|
- * The base request will fit but the reserved space
|
|
|
- * falls off the end. So don't need an immediate wrap
|
|
|
- * and only need to effectively wait for the reserved
|
|
|
- * size space from the start of ringbuffer.
|
|
|
- */
|
|
|
- wait_bytes = remain_actual + ringbuf->reserved_size;
|
|
|
- } else if (total_bytes > ringbuf->space) {
|
|
|
- /* No wrapping required, just waiting. */
|
|
|
- wait_bytes = total_bytes;
|
|
|
- }
|
|
|
+ /* No wrapping required, just waiting. */
|
|
|
+ wait_bytes = total_bytes;
|
|
|
}
|
|
|
|
|
|
- if (wait_bytes) {
|
|
|
- ret = ring_wait_for_space(engine, wait_bytes);
|
|
|
+ if (wait_bytes > ringbuf->space) {
|
|
|
+ int ret = wait_for_space(req, wait_bytes);
|
|
|
if (unlikely(ret))
|
|
|
return ret;
|
|
|
|
|
|
- if (need_wrap)
|
|
|
- __wrap_ring_buffer(ringbuf);
|
|
|
+ intel_ring_update_space(ringbuf);
|
|
|
+ if (unlikely(ringbuf->space < wait_bytes))
|
|
|
+ return -EAGAIN;
|
|
|
}
|
|
|
|
|
|
- return 0;
|
|
|
-}
|
|
|
+ if (unlikely(need_wrap)) {
|
|
|
+ GEM_BUG_ON(remain_actual > ringbuf->space);
|
|
|
+ GEM_BUG_ON(ringbuf->tail + remain_actual > ringbuf->size);
|
|
|
|
|
|
-int intel_ring_begin(struct drm_i915_gem_request *req,
|
|
|
- int num_dwords)
|
|
|
-{
|
|
|
- struct intel_engine_cs *engine = req->engine;
|
|
|
- int ret;
|
|
|
-
|
|
|
- ret = __intel_ring_prepare(engine, num_dwords * sizeof(uint32_t));
|
|
|
- if (ret)
|
|
|
- return ret;
|
|
|
+ /* Fill the tail with MI_NOOP */
|
|
|
+ memset(ringbuf->virtual_start + ringbuf->tail,
|
|
|
+ 0, remain_actual);
|
|
|
+ ringbuf->tail = 0;
|
|
|
+ ringbuf->space -= remain_actual;
|
|
|
+ }
|
|
|
|
|
|
- engine->buffer->space -= num_dwords * sizeof(uint32_t);
|
|
|
+ ringbuf->space -= bytes;
|
|
|
+ GEM_BUG_ON(ringbuf->space < 0);
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
@@ -2772,6 +2744,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
|
|
|
engine->name = "render ring";
|
|
|
engine->id = RCS;
|
|
|
engine->exec_id = I915_EXEC_RENDER;
|
|
|
+ engine->hw_id = 0;
|
|
|
engine->mmio_base = RENDER_RING_BASE;
|
|
|
|
|
|
if (INTEL_INFO(dev)->gen >= 8) {
|
|
@@ -2923,6 +2896,7 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
|
|
|
engine->name = "bsd ring";
|
|
|
engine->id = VCS;
|
|
|
engine->exec_id = I915_EXEC_BSD;
|
|
|
+ engine->hw_id = 1;
|
|
|
|
|
|
engine->write_tail = ring_write_tail;
|
|
|
if (INTEL_INFO(dev)->gen >= 6) {
|
|
@@ -3001,6 +2975,7 @@ int intel_init_bsd2_ring_buffer(struct drm_device *dev)
|
|
|
engine->name = "bsd2 ring";
|
|
|
engine->id = VCS2;
|
|
|
engine->exec_id = I915_EXEC_BSD;
|
|
|
+ engine->hw_id = 4;
|
|
|
|
|
|
engine->write_tail = ring_write_tail;
|
|
|
engine->mmio_base = GEN8_BSD2_RING_BASE;
|
|
@@ -3033,6 +3008,7 @@ int intel_init_blt_ring_buffer(struct drm_device *dev)
|
|
|
engine->name = "blitter ring";
|
|
|
engine->id = BCS;
|
|
|
engine->exec_id = I915_EXEC_BLT;
|
|
|
+ engine->hw_id = 2;
|
|
|
|
|
|
engine->mmio_base = BLT_RING_BASE;
|
|
|
engine->write_tail = ring_write_tail;
|
|
@@ -3092,6 +3068,7 @@ int intel_init_vebox_ring_buffer(struct drm_device *dev)
|
|
|
engine->name = "video enhancement ring";
|
|
|
engine->id = VECS;
|
|
|
engine->exec_id = I915_EXEC_VEBOX;
|
|
|
+ engine->hw_id = 3;
|
|
|
|
|
|
engine->mmio_base = VEBOX_RING_BASE;
|
|
|
engine->write_tail = ring_write_tail;
|