|
@@ -1,17 +1,6 @@
|
|
|
+// SPDX-License-Identifier: GPL-2.0
|
|
|
/*
|
|
|
- * Copyright (C) 2016 Etnaviv Project
|
|
|
- *
|
|
|
- * This program is free software; you can redistribute it and/or modify it
|
|
|
- * under the terms of the GNU General Public License version 2 as published by
|
|
|
- * the Free Software Foundation.
|
|
|
- *
|
|
|
- * This program is distributed in the hope that it will be useful, but WITHOUT
|
|
|
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
- * more details.
|
|
|
- *
|
|
|
- * You should have received a copy of the GNU General Public License along with
|
|
|
- * this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
+ * Copyright (C) 2016-2018 Etnaviv Project
|
|
|
*/
|
|
|
|
|
|
#include <linux/platform_device.h>
|
|
@@ -47,8 +36,8 @@ struct etnaviv_iommuv2_domain {
|
|
|
u32 *mtlb_cpu;
|
|
|
dma_addr_t mtlb_dma;
|
|
|
/* S(lave) TLB aka second level pagetable */
|
|
|
- u32 *stlb_cpu[1024];
|
|
|
- dma_addr_t stlb_dma[1024];
|
|
|
+ u32 *stlb_cpu[MMUv2_MAX_STLB_ENTRIES];
|
|
|
+ dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES];
|
|
|
};
|
|
|
|
|
|
static struct etnaviv_iommuv2_domain *
|
|
@@ -57,24 +46,54 @@ to_etnaviv_domain(struct etnaviv_iommu_domain *domain)
|
|
|
return container_of(domain, struct etnaviv_iommuv2_domain, base);
|
|
|
}
|
|
|
|
|
|
+static int
|
|
|
+etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_domain *etnaviv_domain,
|
|
|
+ int stlb)
|
|
|
+{
|
|
|
+ if (etnaviv_domain->stlb_cpu[stlb])
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ etnaviv_domain->stlb_cpu[stlb] =
|
|
|
+ dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K,
|
|
|
+ &etnaviv_domain->stlb_dma[stlb],
|
|
|
+ GFP_KERNEL);
|
|
|
+
|
|
|
+ if (!etnaviv_domain->stlb_cpu[stlb])
|
|
|
+ return -ENOMEM;
|
|
|
+
|
|
|
+ memset32(etnaviv_domain->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION,
|
|
|
+ SZ_4K / sizeof(u32));
|
|
|
+
|
|
|
+ etnaviv_domain->mtlb_cpu[stlb] = etnaviv_domain->stlb_dma[stlb] |
|
|
|
+ MMUv2_PTE_PRESENT;
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
static int etnaviv_iommuv2_map(struct etnaviv_iommu_domain *domain,
|
|
|
unsigned long iova, phys_addr_t paddr,
|
|
|
size_t size, int prot)
|
|
|
{
|
|
|
struct etnaviv_iommuv2_domain *etnaviv_domain =
|
|
|
to_etnaviv_domain(domain);
|
|
|
- int mtlb_entry, stlb_entry;
|
|
|
- u32 entry = (u32)paddr | MMUv2_PTE_PRESENT;
|
|
|
+ int mtlb_entry, stlb_entry, ret;
|
|
|
+ u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT;
|
|
|
|
|
|
if (size != SZ_4K)
|
|
|
return -EINVAL;
|
|
|
|
|
|
+ if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
|
|
|
+ entry |= (upper_32_bits(paddr) & 0xff) << 4;
|
|
|
+
|
|
|
if (prot & ETNAVIV_PROT_WRITE)
|
|
|
entry |= MMUv2_PTE_WRITEABLE;
|
|
|
|
|
|
mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
|
|
|
stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
|
|
|
|
|
|
+ ret = etnaviv_iommuv2_ensure_stlb(etnaviv_domain, mtlb_entry);
|
|
|
+ if (ret)
|
|
|
+ return ret;
|
|
|
+
|
|
|
etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry;
|
|
|
|
|
|
return 0;
|
|
@@ -101,14 +120,13 @@ static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_domain *domain,
|
|
|
static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
|
|
|
{
|
|
|
u32 *p;
|
|
|
- int ret, i, j;
|
|
|
+ int ret, i;
|
|
|
|
|
|
/* allocate scratch page */
|
|
|
- etnaviv_domain->base.bad_page_cpu = dma_alloc_coherent(
|
|
|
- etnaviv_domain->base.dev,
|
|
|
- SZ_4K,
|
|
|
- &etnaviv_domain->base.bad_page_dma,
|
|
|
- GFP_KERNEL);
|
|
|
+ etnaviv_domain->base.bad_page_cpu =
|
|
|
+ dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K,
|
|
|
+ &etnaviv_domain->base.bad_page_dma,
|
|
|
+ GFP_KERNEL);
|
|
|
if (!etnaviv_domain->base.bad_page_cpu) {
|
|
|
ret = -ENOMEM;
|
|
|
goto fail_mem;
|
|
@@ -117,67 +135,40 @@ static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
|
|
|
for (i = 0; i < SZ_4K / 4; i++)
|
|
|
*p++ = 0xdead55aa;
|
|
|
|
|
|
- etnaviv_domain->pta_cpu = dma_alloc_coherent(etnaviv_domain->base.dev,
|
|
|
- SZ_4K,
|
|
|
- &etnaviv_domain->pta_dma,
|
|
|
- GFP_KERNEL);
|
|
|
+ etnaviv_domain->pta_cpu = dma_alloc_wc(etnaviv_domain->base.dev,
|
|
|
+ SZ_4K, &etnaviv_domain->pta_dma,
|
|
|
+ GFP_KERNEL);
|
|
|
if (!etnaviv_domain->pta_cpu) {
|
|
|
ret = -ENOMEM;
|
|
|
goto fail_mem;
|
|
|
}
|
|
|
|
|
|
- etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->base.dev,
|
|
|
- SZ_4K,
|
|
|
- &etnaviv_domain->mtlb_dma,
|
|
|
- GFP_KERNEL);
|
|
|
+ etnaviv_domain->mtlb_cpu = dma_alloc_wc(etnaviv_domain->base.dev,
|
|
|
+ SZ_4K, &etnaviv_domain->mtlb_dma,
|
|
|
+ GFP_KERNEL);
|
|
|
if (!etnaviv_domain->mtlb_cpu) {
|
|
|
ret = -ENOMEM;
|
|
|
goto fail_mem;
|
|
|
}
|
|
|
|
|
|
- /* pre-populate STLB pages (may want to switch to on-demand later) */
|
|
|
- for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
|
|
|
- etnaviv_domain->stlb_cpu[i] =
|
|
|
- dma_alloc_coherent(etnaviv_domain->base.dev,
|
|
|
- SZ_4K,
|
|
|
- &etnaviv_domain->stlb_dma[i],
|
|
|
- GFP_KERNEL);
|
|
|
- if (!etnaviv_domain->stlb_cpu[i]) {
|
|
|
- ret = -ENOMEM;
|
|
|
- goto fail_mem;
|
|
|
- }
|
|
|
- p = etnaviv_domain->stlb_cpu[i];
|
|
|
- for (j = 0; j < SZ_4K / 4; j++)
|
|
|
- *p++ = MMUv2_PTE_EXCEPTION;
|
|
|
-
|
|
|
- etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] |
|
|
|
- MMUv2_PTE_PRESENT;
|
|
|
- }
|
|
|
+ memset32(etnaviv_domain->mtlb_cpu, MMUv2_PTE_EXCEPTION,
|
|
|
+ MMUv2_MAX_STLB_ENTRIES);
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
fail_mem:
|
|
|
if (etnaviv_domain->base.bad_page_cpu)
|
|
|
- dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
|
|
|
- etnaviv_domain->base.bad_page_cpu,
|
|
|
- etnaviv_domain->base.bad_page_dma);
|
|
|
+ dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
|
|
|
+ etnaviv_domain->base.bad_page_cpu,
|
|
|
+ etnaviv_domain->base.bad_page_dma);
|
|
|
|
|
|
if (etnaviv_domain->pta_cpu)
|
|
|
- dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
|
|
|
- etnaviv_domain->pta_cpu,
|
|
|
- etnaviv_domain->pta_dma);
|
|
|
+ dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
|
|
|
+ etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma);
|
|
|
|
|
|
if (etnaviv_domain->mtlb_cpu)
|
|
|
- dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
|
|
|
- etnaviv_domain->mtlb_cpu,
|
|
|
- etnaviv_domain->mtlb_dma);
|
|
|
-
|
|
|
- for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
|
|
|
- if (etnaviv_domain->stlb_cpu[i])
|
|
|
- dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
|
|
|
- etnaviv_domain->stlb_cpu[i],
|
|
|
- etnaviv_domain->stlb_dma[i]);
|
|
|
- }
|
|
|
+ dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
|
|
|
+ etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma);
|
|
|
|
|
|
return ret;
|
|
|
}
|
|
@@ -188,23 +179,21 @@ static void etnaviv_iommuv2_domain_free(struct etnaviv_iommu_domain *domain)
|
|
|
to_etnaviv_domain(domain);
|
|
|
int i;
|
|
|
|
|
|
- dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
|
|
|
- etnaviv_domain->base.bad_page_cpu,
|
|
|
- etnaviv_domain->base.bad_page_dma);
|
|
|
+ dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
|
|
|
+ etnaviv_domain->base.bad_page_cpu,
|
|
|
+ etnaviv_domain->base.bad_page_dma);
|
|
|
|
|
|
- dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
|
|
|
- etnaviv_domain->pta_cpu,
|
|
|
- etnaviv_domain->pta_dma);
|
|
|
+ dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
|
|
|
+ etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma);
|
|
|
|
|
|
- dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
|
|
|
- etnaviv_domain->mtlb_cpu,
|
|
|
- etnaviv_domain->mtlb_dma);
|
|
|
+ dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
|
|
|
+ etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma);
|
|
|
|
|
|
for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
|
|
|
if (etnaviv_domain->stlb_cpu[i])
|
|
|
- dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
|
|
|
- etnaviv_domain->stlb_cpu[i],
|
|
|
- etnaviv_domain->stlb_dma[i]);
|
|
|
+ dma_free_wc(etnaviv_domain->base.dev, SZ_4K,
|
|
|
+ etnaviv_domain->stlb_cpu[i],
|
|
|
+ etnaviv_domain->stlb_dma[i]);
|
|
|
}
|
|
|
|
|
|
vfree(etnaviv_domain);
|