iommu/amd: Make dma_ops_domain->next_index percpu

Make this pointer percpu so that we start searching for new
addresses in the range we last stopped and which is has a
higher probability of being still in the cache.

Signed-off-by: Joerg Roedel <jroedel@suse.de>

drivers/iommu/amd_iommu.c

@@ -35,6 +35,7 @@
 #include <linux/msi.h>
 #include <linux/dma-contiguous.h>
 #include <linux/irqdomain.h>
+#include <linux/percpu.h>
 #include <asm/irq_remapping.h>
 #include <asm/io_apic.h>
 #include <asm/apic.h>
@@ -147,7 +148,7 @@ struct dma_ops_domain {
 	unsigned long aperture_size;
 
 	/* aperture index we start searching for free addresses */
-	unsigned long next_index;
+	u32 __percpu *next_index;
 
 	/* address space relevant data */
 	struct aperture_range *aperture[APERTURE_MAX_RANGES];
@@ -1583,18 +1584,30 @@ static unsigned long dma_ops_area_alloc(struct device *dev,
 {
 	unsigned long boundary_size, mask;
 	unsigned long address = -1;
-	int start = dom->next_index;
-	int i;
+	u32 start, i;
+
+	preempt_disable();
 
 	mask = dma_get_seg_boundary(dev);
 
+	start = this_cpu_read(*dom->next_index);
+
+	/* Sanity check - is it really necessary? */
+	if (unlikely(start > APERTURE_MAX_RANGES)) {
+		start = 0;
+		this_cpu_write(*dom->next_index, 0);
+	}
+
 	boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
 				   1UL << (BITS_PER_LONG - PAGE_SHIFT);
 
 	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
 		struct aperture_range *range;
+		int index;
+
+		index = (start + i) % APERTURE_MAX_RANGES;
 
-		range = dom->aperture[(start + i) % APERTURE_MAX_RANGES];
+		range = dom->aperture[index];
 
 		if (!range || range->offset >= dma_mask)
 			continue;
@@ -1604,11 +1617,13 @@ static unsigned long dma_ops_area_alloc(struct device *dev,
 						 align_mask);
 		if (address != -1) {
 			address = range->offset + (address << PAGE_SHIFT);
-			dom->next_index = i;
+			this_cpu_write(*dom->next_index, index);
 			break;
 		}
 	}
 
+	preempt_enable();
+
 	return address;
 }
 
@@ -1620,10 +1635,6 @@ static unsigned long dma_ops_alloc_addresses(struct device *dev,
 {
 	unsigned long address = -1;
 
-#ifdef CONFIG_IOMMU_STRESS
-	dom->next_index = 0;
-#endif
-
 	while (address == -1) {
 		address = dma_ops_area_alloc(dev, dom, pages,
 					     align_mask, dma_mask);
@@ -1851,6 +1862,8 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 	if (!dom)
 		return;
 
+	free_percpu(dom->next_index);
+
 	del_domain_from_list(&dom->domain);
 
 	free_pagetable(&dom->domain);
@@ -1873,6 +1886,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 static struct dma_ops_domain *dma_ops_domain_alloc(void)
 {
 	struct dma_ops_domain *dma_dom;
+	int cpu;
 
 	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
 	if (!dma_dom)
@@ -1881,6 +1895,10 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
 	if (protection_domain_init(&dma_dom->domain))
 		goto free_dma_dom;
 
+	dma_dom->next_index = alloc_percpu(u32);
+	if (!dma_dom->next_index)
+		goto free_dma_dom;
+
 	dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
 	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
 	dma_dom->domain.flags = PD_DMA_OPS_MASK;
@@ -1898,8 +1916,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
 	 * a valid dma-address. So we can use 0 as error value
 	 */
 	dma_dom->aperture[0]->bitmap[0] = 1;
-	dma_dom->next_index = 0;
 
+	for_each_possible_cpu(cpu)
+		*per_cpu_ptr(dma_dom->next_index, cpu) = 0;
 
 	return dma_dom;