linux_kernel/arch/csky/mm/dma-mapping.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.

#include <linux/cache.h>
#include <linux/dma-mapping.h>
#include <linux/dma-contiguous.h>
#include <linux/dma-noncoherent.h>
#include <linux/genalloc.h>
#include <linux/highmem.h>
#include <linux/io.h>
#include <linux/mm.h>
#include <linux/scatterlist.h>
#include <linux/types.h>
#include <linux/version.h>
#include <asm/cache.h>

static struct gen_pool *atomic_pool;
static size_t atomic_pool_size __initdata = SZ_256K;

static int __init early_coherent_pool(char *p)
{
	atomic_pool_size = memparse(p, &p);
	return 0;
}
early_param("coherent_pool", early_coherent_pool);

static int __init atomic_pool_init(void)
{
	struct page *page;
	size_t size = atomic_pool_size;
	void *ptr;
	int ret;

	atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
	if (!atomic_pool)
		BUG();

	page = alloc_pages(GFP_KERNEL | GFP_DMA, get_order(size));
	if (!page)
		BUG();

	ptr = dma_common_contiguous_remap(page, size, VM_ALLOC,
					  pgprot_noncached(PAGE_KERNEL),
					  __builtin_return_address(0));
	if (!ptr)
		BUG();

	ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
				page_to_phys(page), atomic_pool_size, -1);
	if (ret)
		BUG();

	gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);

	pr_info("DMA: preallocated %zu KiB pool for atomic coherent pool\n",
		atomic_pool_size / 1024);

	pr_info("DMA: vaddr: 0x%x phy: 0x%lx,\n", (unsigned int)ptr,
		page_to_phys(page));

	return 0;
}
postcore_initcall(atomic_pool_init);

static void *csky_dma_alloc_atomic(struct device *dev, size_t size,
				   dma_addr_t *dma_handle)
{
	unsigned long addr;

	addr = gen_pool_alloc(atomic_pool, size);
	if (addr)
		*dma_handle = gen_pool_virt_to_phys(atomic_pool, addr);

	return (void *)addr;
}

static void csky_dma_free_atomic(struct device *dev, size_t size, void *vaddr,
				 dma_addr_t dma_handle, unsigned long attrs)
{
	gen_pool_free(atomic_pool, (unsigned long)vaddr, size);
}

static void __dma_clear_buffer(struct page *page, size_t size)
{
	if (PageHighMem(page)) {
		unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;

		do {
			void *ptr = kmap_atomic(page);
			size_t _size = (size < PAGE_SIZE) ? size : PAGE_SIZE;

			memset(ptr, 0, _size);
			dma_wbinv_range((unsigned long)ptr,
					(unsigned long)ptr + _size);

			kunmap_atomic(ptr);

			page++;
			size -= PAGE_SIZE;
			count--;
		} while (count);
	} else {
		void *ptr = page_address(page);

		memset(ptr, 0, size);
		dma_wbinv_range((unsigned long)ptr, (unsigned long)ptr + size);
	}
}

static void *csky_dma_alloc_nonatomic(struct device *dev, size_t size,
				      dma_addr_t *dma_handle, gfp_t gfp,
				      unsigned long attrs)
{
	void  *vaddr;
	struct page *page;
	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;

	if (DMA_ATTR_NON_CONSISTENT & attrs) {
		pr_err("csky %s can't support DMA_ATTR_NON_CONSISTENT.\n", __func__);
		return NULL;
	}

	if (IS_ENABLED(CONFIG_DMA_CMA))
		page = dma_alloc_from_contiguous(dev, count, get_order(size),
						 gfp);
	else
		page = alloc_pages(gfp, get_order(size));

	if (!page) {
		pr_err("csky %s no more free pages.\n", __func__);
		return NULL;
	}

	*dma_handle = page_to_phys(page);

	__dma_clear_buffer(page, size);

	if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
		return page;

	vaddr = dma_common_contiguous_remap(page, PAGE_ALIGN(size), VM_USERMAP,
		pgprot_noncached(PAGE_KERNEL), __builtin_return_address(0));
	if (!vaddr)
		BUG();

	return vaddr;
}

static void csky_dma_free_nonatomic(
	struct device *dev,
	size_t size,
	void *vaddr,
	dma_addr_t dma_handle,
	unsigned long attrs
	)
{
	struct page *page = phys_to_page(dma_handle);
	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;

	if ((unsigned int)vaddr >= VMALLOC_START)
		dma_common_free_remap(vaddr, size, VM_USERMAP);

	if (IS_ENABLED(CONFIG_DMA_CMA))
		dma_release_from_contiguous(dev, page, count);
	else
		__free_pages(page, get_order(size));
}

void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
		     gfp_t gfp, unsigned long attrs)
{
	if (gfpflags_allow_blocking(gfp))
		return csky_dma_alloc_nonatomic(dev, size, dma_handle, gfp,
						attrs);
	else
		return csky_dma_alloc_atomic(dev, size, dma_handle);
}

void arch_dma_free(struct device *dev, size_t size, void *vaddr,
		   dma_addr_t dma_handle, unsigned long attrs)
{
	if (!addr_in_gen_pool(atomic_pool, (unsigned int) vaddr, size))
		csky_dma_free_nonatomic(dev, size, vaddr, dma_handle, attrs);
	else
		csky_dma_free_atomic(dev, size, vaddr, dma_handle, attrs);
}

static inline void cache_op(phys_addr_t paddr, size_t size,
			    void (*fn)(unsigned long start, unsigned long end))
{
	struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
	unsigned int offset = paddr & ~PAGE_MASK;
	size_t left = size;
	unsigned long start;

	do {
		size_t len = left;

		if (PageHighMem(page)) {
			void *addr;

			if (offset + len > PAGE_SIZE) {
				if (offset >= PAGE_SIZE) {
					page += offset >> PAGE_SHIFT;
					offset &= ~PAGE_MASK;
				}
				len = PAGE_SIZE - offset;
			}

			addr = kmap_atomic(page);
			start = (unsigned long)(addr + offset);
			fn(start, start + len);
			kunmap_atomic(addr);
		} else {
			start = (unsigned long)phys_to_virt(paddr);
			fn(start, start + size);
		}
		offset = 0;
		page++;
		left -= len;
	} while (left);
}

void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
			      size_t size, enum dma_data_direction dir)
{
	switch (dir) {
	case DMA_TO_DEVICE:
		cache_op(paddr, size, dma_wb_range);
		break;
	case DMA_FROM_DEVICE:
	case DMA_BIDIRECTIONAL:
		cache_op(paddr, size, dma_wbinv_range);
		break;
	default:
		BUG();
	}
}

void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
			   size_t size, enum dma_data_direction dir)
{
	switch (dir) {
	case DMA_TO_DEVICE:
		cache_op(paddr, size, dma_wb_range);
		break;
	case DMA_FROM_DEVICE:
	case DMA_BIDIRECTIONAL:
		cache_op(paddr, size, dma_wbinv_range);
		break;
	default:
		BUG();
	}
}