drm/nouveau/instmem/gk20a: add IOMMU support

Let GK20A's instmem take advantage of the IOMMU if it is present. Having
an IOMMU means that instmem is no longer allocated using the DMA API,
but instead obtained through page_alloc and made contiguous to the GPU
by IOMMU mappings.

Signed-off-by: Alexandre Courbot <acourbot@nvidia.com>
Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c

@@ -20,12 +20,32 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+/*
+ * GK20A does not have dedicated video memory, and to accurately represent this
+ * fact Nouveau will not create a RAM device for it. Therefore its instmem
+ * implementation must be done directly on top of system memory, while providing
+ * coherent read and write operations.
+ *
+ * Instmem can be allocated through two means:
+ * 1) If an IOMMU mapping has been probed, the IOMMU API is used to make memory
+ *    pages contiguous to the GPU. This is the preferred way.
+ * 2) If no IOMMU mapping is probed, the DMA API is used to allocate physically
+ *    contiguous memory.
+ *
+ * In both cases CPU read and writes are performed using PRAMIN (i.e. using the
+ * GPU path) to ensure these operations are coherent for the GPU. This allows us
+ * to use more "relaxed" allocation parameters when using the DMA API, since we
+ * never need a kernel mapping.
+ */
+
 #include <subdev/fb.h>
 #include <core/mm.h>
 #include <core/device.h>
 
 #ifdef __KERNEL__
 #include <linux/dma-attrs.h>
+#include <linux/iommu.h>
+#include <nouveau_platform.h>
 #endif
 
 #include "priv.h"
@@ -36,18 +56,53 @@ struct gk20a_instobj_priv {
 	struct nvkm_mem *mem;
 	/* Pointed by mem */
 	struct nvkm_mem _mem;
+};
+
+/*
+ * Used for objects allocated using the DMA API
+ */
+struct gk20a_instobj_dma {
+	struct gk20a_instobj_priv base;
+
 	void *cpuaddr;
 	dma_addr_t handle;
 	struct nvkm_mm_node r;
 };
 
+/*
+ * Used for objects flattened using the IOMMU API
+ */
+struct gk20a_instobj_iommu {
+	struct gk20a_instobj_priv base;
+
+	/* array of base.mem->size pages */
+	struct page *pages[];
+};
+
 struct gk20a_instmem_priv {
 	struct nvkm_instmem base;
 	spinlock_t lock;
 	u64 addr;
+
+	/* Only used if IOMMU if present */
+	struct mutex *mm_mutex;
+	struct nvkm_mm *mm;
+	struct iommu_domain *domain;
+	unsigned long iommu_pgshift;
+
+	/* Only used by DMA API */
 	struct dma_attrs attrs;
 };
 
+/*
+ * Use PRAMIN to read/write data and avoid coherency issues.
+ * PRAMIN uses the GPU path and ensures data will always be coherent.
+ *
+ * A dynamic mapping based solution would be desirable in the future, but
+ * the issue remains of how to maintain coherency efficiently. On ARM it is
+ * not easy (if possible at all?) to create uncached temporary mappings.
+ */
+
 static u32
 gk20a_instobj_rd32(struct nvkm_object *object, u64 offset)
 {
@@ -87,50 +142,79 @@ gk20a_instobj_wr32(struct nvkm_object *object, u64 offset, u32 data)
 }
 
 static void
-gk20a_instobj_dtor(struct nvkm_object *object)
+gk20a_instobj_dtor_dma(struct gk20a_instobj_priv *_node)
 {
-	struct gk20a_instobj_priv *node = (void *)object;
+	struct gk20a_instobj_dma *node = (void *)_node;
 	struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(node);
 	struct device *dev = nv_device_base(nv_device(priv));
 
 	if (unlikely(!node->handle))
 		return;
 
-	dma_free_attrs(dev, node->mem->size << PAGE_SHIFT, node->cpuaddr,
+	dma_free_attrs(dev, _node->mem->size << PAGE_SHIFT, node->cpuaddr,
 		       node->handle, &priv->attrs);
+}
+
+static void
+gk20a_instobj_dtor_iommu(struct gk20a_instobj_priv *_node)
+{
+	struct gk20a_instobj_iommu *node = (void *)_node;
+	struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(node);
+	struct nvkm_mm_node *r;
+	int i;
+
+	if (unlikely(list_empty(&_node->mem->regions)))
+		return;
+
+	r = list_first_entry(&_node->mem->regions, struct nvkm_mm_node,
+			     rl_entry);
+
+	/* clear bit 34 to unmap pages */
+	r->offset &= ~BIT(34 - priv->iommu_pgshift);
+
+	/* Unmap pages from GPU address space and free them */
+	for (i = 0; i < _node->mem->size; i++) {
+		iommu_unmap(priv->domain,
+			    (r->offset + i) << priv->iommu_pgshift, PAGE_SIZE);
+		__free_page(node->pages[i]);
+	}
+
+	/* Release area from GPU address space */
+	mutex_lock(priv->mm_mutex);
+	nvkm_mm_free(priv->mm, &r);
+	mutex_unlock(priv->mm_mutex);
+}
+
+static void
+gk20a_instobj_dtor(struct nvkm_object *object)
+{
+	struct gk20a_instobj_priv *node = (void *)object;
+	struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(node);
+
+	if (priv->domain)
+		gk20a_instobj_dtor_iommu(node);
+	else
+		gk20a_instobj_dtor_dma(node);
 
 	nvkm_instobj_destroy(&node->base);
 }
 
 static int
-gk20a_instobj_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
-		   struct nvkm_oclass *oclass, void *data, u32 _size,
-		   struct nvkm_object **pobject)
+gk20a_instobj_ctor_dma(struct nvkm_object *parent, struct nvkm_object *engine,
+		       struct nvkm_oclass *oclass, u32 npages, u32 align,
+		       struct gk20a_instobj_priv **_node)
 {
-	struct nvkm_instobj_args *args = data;
+	struct gk20a_instobj_dma *node;
 	struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(parent);
-	struct device *dev = nv_device_base(nv_device(priv));
-	struct gk20a_instobj_priv *node;
-	u32 size, align;
-	u32 npages;
+	struct device *dev = nv_device_base(nv_device(parent));
 	int ret;
 
-	nv_debug(parent, "%s: size: %x align: %x\n", __func__,
-		 args->size, args->align);
-
-	size  = max((args->size  + 4095) & ~4095, (u32)4096);
-	align = max((args->align + 4095) & ~4095, (u32)4096);
-
-	npages = size >> PAGE_SHIFT;
-
 	ret = nvkm_instobj_create_(parent, engine, oclass, sizeof(*node),
-				      (void **)&node);
-	*pobject = nv_object(node);
+				   (void **)&node);
+	*_node = &node->base;
 	if (ret)
 		return ret;
 
-	node->mem = &node->_mem;
-
 	node->cpuaddr = dma_alloc_attrs(dev, npages << PAGE_SHIFT,
 					&node->handle, GFP_KERNEL,
 					&priv->attrs);
@@ -144,16 +228,132 @@ gk20a_instobj_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
 		nv_warn(priv, "memory not aligned as requested: %pad (0x%x)\n",
 			&node->handle, align);
 
-	node->mem->offset = node->handle;
+	/* present memory for being mapped using small pages */
+	node->r.type = 12;
+	node->r.offset = node->handle >> 12;
+	node->r.length = (npages << PAGE_SHIFT) >> 12;
+
+	node->base._mem.offset = node->handle;
+
+	INIT_LIST_HEAD(&node->base._mem.regions);
+	list_add_tail(&node->r.rl_entry, &node->base._mem.regions);
+
+	return 0;
+}
+
+static int
+gk20a_instobj_ctor_iommu(struct nvkm_object *parent, struct nvkm_object *engine,
+			 struct nvkm_oclass *oclass, u32 npages, u32 align,
+			 struct gk20a_instobj_priv **_node)
+{
+	struct gk20a_instobj_iommu *node;
+	struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(parent);
+	struct nvkm_mm_node *r;
+	int ret;
+	int i;
+
+	ret = nvkm_instobj_create_(parent, engine, oclass,
+				sizeof(*node) + sizeof(node->pages[0]) * npages,
+				(void **)&node);
+	*_node = &node->base;
+	if (ret)
+		return ret;
+
+	/* Allocate backing memory */
+	for (i = 0; i < npages; i++) {
+		struct page *p = alloc_page(GFP_KERNEL);
+
+		if (p == NULL) {
+			ret = -ENOMEM;
+			goto free_pages;
+		}
+		node->pages[i] = p;
+	}
+
+	mutex_lock(priv->mm_mutex);
+	/* Reserve area from GPU address space */
+	ret = nvkm_mm_head(priv->mm, 0, 1, npages, npages,
+			   align >> priv->iommu_pgshift, &r);
+	mutex_unlock(priv->mm_mutex);
+	if (ret) {
+		nv_error(priv, "virtual space is full!\n");
+		goto free_pages;
+	}
+
+	/* Map into GPU address space */
+	for (i = 0; i < npages; i++) {
+		struct page *p = node->pages[i];
+		u32 offset = (r->offset + i) << priv->iommu_pgshift;
+
+		ret = iommu_map(priv->domain, offset, page_to_phys(p),
+				PAGE_SIZE, IOMMU_READ | IOMMU_WRITE);
+		if (ret < 0) {
+			nv_error(priv, "IOMMU mapping failure: %d\n", ret);
+
+			while (i-- > 0) {
+				offset -= PAGE_SIZE;
+				iommu_unmap(priv->domain, offset, PAGE_SIZE);
+			}
+			goto release_area;
+		}
+	}
+
+	/* Bit 34 tells that an address is to be resolved through the IOMMU */
+	r->offset |= BIT(34 - priv->iommu_pgshift);
+
+	node->base._mem.offset = ((u64)r->offset) << priv->iommu_pgshift;
+
+	INIT_LIST_HEAD(&node->base._mem.regions);
+	list_add_tail(&r->rl_entry, &node->base._mem.regions);
+
+	return 0;
+
+release_area:
+	mutex_lock(priv->mm_mutex);
+	nvkm_mm_free(priv->mm, &r);
+	mutex_unlock(priv->mm_mutex);
+
+free_pages:
+	for (i = 0; i < npages && node->pages[i] != NULL; i++)
+		__free_page(node->pages[i]);
+
+	return ret;
+}
+
+static int
+gk20a_instobj_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
+		   struct nvkm_oclass *oclass, void *data, u32 _size,
+		   struct nvkm_object **pobject)
+{
+	struct nvkm_instobj_args *args = data;
+	struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(parent);
+	struct gk20a_instobj_priv *node;
+	u32 size, align;
+	int ret;
+
+	nv_debug(parent, "%s (%s): size: %x align: %x\n", __func__,
+		 priv->domain ? "IOMMU" : "DMA", args->size, args->align);
+
+	/* Round size and align to page bounds */
+	size = max(roundup(args->size, PAGE_SIZE), PAGE_SIZE);
+	align = max(roundup(args->align, PAGE_SIZE), PAGE_SIZE);
+
+	if (priv->domain)
+		ret = gk20a_instobj_ctor_iommu(parent, engine, oclass,
+					      size >> PAGE_SHIFT, align, &node);
+	else
+		ret = gk20a_instobj_ctor_dma(parent, engine, oclass,
+					     size >> PAGE_SHIFT, align, &node);
+	*pobject = nv_object(node);
+	if (ret)
+		return ret;
+
+	node->mem = &node->_mem;
+
+	/* present memory for being mapped using small pages */
 	node->mem->size = size >> 12;
 	node->mem->memtype = 0;
 	node->mem->page_shift = 12;
-	INIT_LIST_HEAD(&node->mem->regions);
-
-	node->r.type = 12;
-	node->r.offset = node->handle >> 12;
-	node->r.length = npages;
-	list_add_tail(&node->r.rl_entry, &node->mem->regions);
 
 	node->base.addr = node->mem->offset;
 	node->base.size = size;
@@ -192,6 +392,7 @@ gk20a_instmem_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
 		   struct nvkm_object **pobject)
 {
 	struct gk20a_instmem_priv *priv;
+	struct nouveau_platform_device *plat;
 	int ret;
 
 	ret = nvkm_instmem_create(parent, engine, oclass, &priv);
@@ -201,15 +402,27 @@ gk20a_instmem_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
 
 	spin_lock_init(&priv->lock);
 
-	init_dma_attrs(&priv->attrs);
-	/*
-	 * We will access instmem through PRAMIN and thus do not need a
-	 * consistent CPU pointer or kernel mapping
-	 */
-	dma_set_attr(DMA_ATTR_NON_CONSISTENT, &priv->attrs);
-	dma_set_attr(DMA_ATTR_WEAK_ORDERING, &priv->attrs);
-	dma_set_attr(DMA_ATTR_WRITE_COMBINE, &priv->attrs);
-	dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &priv->attrs);
+	plat = nv_device_to_platform(nv_device(parent));
+	if (plat->gpu->iommu.domain) {
+		priv->domain = plat->gpu->iommu.domain;
+		priv->mm = plat->gpu->iommu.mm;
+		priv->iommu_pgshift = plat->gpu->iommu.pgshift;
+		priv->mm_mutex = &plat->gpu->iommu.mutex;
+
+		nv_info(priv, "using IOMMU\n");
+	} else {
+		init_dma_attrs(&priv->attrs);
+		/*
+		 * We will access instmem through PRAMIN and thus do not need a
+		 * consistent CPU pointer or kernel mapping
+		 */
+		dma_set_attr(DMA_ATTR_NON_CONSISTENT, &priv->attrs);
+		dma_set_attr(DMA_ATTR_WEAK_ORDERING, &priv->attrs);
+		dma_set_attr(DMA_ATTR_WRITE_COMBINE, &priv->attrs);
+		dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &priv->attrs);
+
+		nv_info(priv, "using DMA API\n");
+	}
 
 	return 0;
 }