dma: MIC X100 DMA Driver

This patch implements DMA Engine API for DMA controller on MIC X100
Coprocessors. DMA h/w is shared between host and card s/w.
Channels 0 to 3 are used by host and 4 to 7 are used by card.
Since the DMA device doesn't show up as PCIe device, a virtual bus called mic
bus is created and virtual devices are added on that bus to follow device model.
Allowed dma transfer directions are host to card, card to host and card to card.

Reviewed-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
Reviewed-by: Nikhil Rao <nikhil.rao@intel.com>
Reviewed-by: Sudeep Dutt <sudeep.dutt@intel.com>
Signed-off-by: Siva Yerramreddy <yshivakrishna@gmail.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

drivers/dma/Kconfig

@@ -33,6 +33,25 @@ if DMADEVICES
 
 comment "DMA Devices"
 
+config INTEL_MIC_X100_DMA
+	tristate "Intel MIC X100 DMA Driver"
+	depends on 64BIT && X86 && INTEL_MIC_BUS
+	select DMAENGINE
+	default N
+	help
+	  This enables DMA support for the Intel Many Integrated Core
+	  (MIC) family of PCIe form factor coprocessor X100 devices that
+	  run a 64 bit Linux OS. This driver will be used by both MIC
+	  host and card drivers.
+
+	  If you are building host kernel with a MIC device or a card
+	  kernel for a MIC device, then say M (recommended) or Y, else
+	  say N. If unsure say N.
+
+	  More information about the Intel MIC family as well as the Linux
+	  OS and tools for MIC to use with this driver are available from
+	  <http://software.intel.com/en-us/mic-developer>.
+
 config INTEL_MID_DMAC
 	tristate "Intel MID DMA support for Peripheral DMA controllers"
 	depends on PCI && X86

drivers/dma/Makefile

@@ -47,3 +47,4 @@ obj-$(CONFIG_MOXART_DMA) += moxart-dma.o
 obj-$(CONFIG_FSL_EDMA) += fsl-edma.o
 obj-$(CONFIG_QCOM_BAM_DMA) += qcom_bam_dma.o
 obj-y += xilinx/
+obj-$(CONFIG_INTEL_MIC_X100_DMA) += mic_x100_dma.o

drivers/dma/mic_x100_dma.c

@@ -0,0 +1,774 @@
+/*
+ * Intel MIC Platform Software Stack (MPSS)
+ *
+ * Copyright(c) 2014 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * Intel MIC X100 DMA Driver.
+ *
+ * Adapted from IOAT dma driver.
+ */
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/seq_file.h>
+
+#include "mic_x100_dma.h"
+
+#define MIC_DMA_MAX_XFER_SIZE_CARD  (1 * 1024 * 1024 -\
+				       MIC_DMA_ALIGN_BYTES)
+#define MIC_DMA_MAX_XFER_SIZE_HOST  (1 * 1024 * 1024 >> 1)
+#define MIC_DMA_DESC_TYPE_SHIFT	60
+#define MIC_DMA_MEMCPY_LEN_SHIFT 46
+#define MIC_DMA_STAT_INTR_SHIFT 59
+
+/* high-water mark for pushing dma descriptors */
+static int mic_dma_pending_level = 4;
+
+/* Status descriptor is used to write a 64 bit value to a memory location */
+enum mic_dma_desc_format_type {
+	MIC_DMA_MEMCPY = 1,
+	MIC_DMA_STATUS,
+};
+
+static inline u32 mic_dma_hw_ring_inc(u32 val)
+{
+	return (val + 1) % MIC_DMA_DESC_RX_SIZE;
+}
+
+static inline u32 mic_dma_hw_ring_dec(u32 val)
+{
+	return val ? val - 1 : MIC_DMA_DESC_RX_SIZE - 1;
+}
+
+static inline void mic_dma_hw_ring_inc_head(struct mic_dma_chan *ch)
+{
+	ch->head = mic_dma_hw_ring_inc(ch->head);
+}
+
+/* Prepare a memcpy desc */
+static inline void mic_dma_memcpy_desc(struct mic_dma_desc *desc,
+	dma_addr_t src_phys, dma_addr_t dst_phys, u64 size)
+{
+	u64 qw0, qw1;
+
+	qw0 = src_phys;
+	qw0 |= (size >> MIC_DMA_ALIGN_SHIFT) << MIC_DMA_MEMCPY_LEN_SHIFT;
+	qw1 = MIC_DMA_MEMCPY;
+	qw1 <<= MIC_DMA_DESC_TYPE_SHIFT;
+	qw1 |= dst_phys;
+	desc->qw0 = qw0;
+	desc->qw1 = qw1;
+}
+
+/* Prepare a status desc. with @data to be written at @dst_phys */
+static inline void mic_dma_prep_status_desc(struct mic_dma_desc *desc, u64 data,
+	dma_addr_t dst_phys, bool generate_intr)
+{
+	u64 qw0, qw1;
+
+	qw0 = data;
+	qw1 = (u64) MIC_DMA_STATUS << MIC_DMA_DESC_TYPE_SHIFT | dst_phys;
+	if (generate_intr)
+		qw1 |= (1ULL << MIC_DMA_STAT_INTR_SHIFT);
+	desc->qw0 = qw0;
+	desc->qw1 = qw1;
+}
+
+static void mic_dma_cleanup(struct mic_dma_chan *ch)
+{
+	struct dma_async_tx_descriptor *tx;
+	u32 tail;
+	u32 last_tail;
+
+	spin_lock(&ch->cleanup_lock);
+	tail = mic_dma_read_cmp_cnt(ch);
+	/*
+	 * This is the barrier pair for smp_wmb() in fn.
+	 * mic_dma_tx_submit_unlock. It's required so that we read the
+	 * updated cookie value from tx->cookie.
+	 */
+	smp_rmb();
+	for (last_tail = ch->last_tail; tail != last_tail;) {
+		tx = &ch->tx_array[last_tail];
+		if (tx->cookie) {
+			dma_cookie_complete(tx);
+			if (tx->callback) {
+				tx->callback(tx->callback_param);
+				tx->callback = NULL;
+			}
+		}
+		last_tail = mic_dma_hw_ring_inc(last_tail);
+	}
+	/* finish all completion callbacks before incrementing tail */
+	smp_mb();
+	ch->last_tail = last_tail;
+	spin_unlock(&ch->cleanup_lock);
+}
+
+static u32 mic_dma_ring_count(u32 head, u32 tail)
+{
+	u32 count;
+
+	if (head >= tail)
+		count = (tail - 0) + (MIC_DMA_DESC_RX_SIZE - head);
+	else
+		count = tail - head;
+	return count - 1;
+}
+
+/* Returns the num. of free descriptors on success, -ENOMEM on failure */
+static int mic_dma_avail_desc_ring_space(struct mic_dma_chan *ch, int required)
+{
+	struct device *dev = mic_dma_ch_to_device(ch);
+	u32 count;
+
+	count = mic_dma_ring_count(ch->head, ch->last_tail);
+	if (count < required) {
+		mic_dma_cleanup(ch);
+		count = mic_dma_ring_count(ch->head, ch->last_tail);
+	}
+
+	if (count < required) {
+		dev_dbg(dev, "Not enough desc space");
+		dev_dbg(dev, "%s %d required=%u, avail=%u\n",
+			__func__, __LINE__, required, count);
+		return -ENOMEM;
+	} else {
+		return count;
+	}
+}
+
+/* Program memcpy descriptors into the descriptor ring and update s/w head ptr*/
+static int mic_dma_prog_memcpy_desc(struct mic_dma_chan *ch, dma_addr_t src,
+				    dma_addr_t dst, size_t len)
+{
+	size_t current_transfer_len;
+	size_t max_xfer_size = to_mic_dma_dev(ch)->max_xfer_size;
+	/* 3 is added to make sure we have enough space for status desc */
+	int num_desc = len / max_xfer_size + 3;
+	int ret;
+
+	if (len % max_xfer_size)
+		num_desc++;
+
+	ret = mic_dma_avail_desc_ring_space(ch, num_desc);
+	if (ret < 0)
+		return ret;
+	do {
+		current_transfer_len = min(len, max_xfer_size);
+		mic_dma_memcpy_desc(&ch->desc_ring[ch->head],
+				    src, dst, current_transfer_len);
+		mic_dma_hw_ring_inc_head(ch);
+		len -= current_transfer_len;
+		dst = dst + current_transfer_len;
+		src = src + current_transfer_len;
+	} while (len > 0);
+	return 0;
+}
+
+/* It's a h/w quirk and h/w needs 2 status descriptors for every status desc */
+static void mic_dma_prog_intr(struct mic_dma_chan *ch)
+{
+	mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
+				 ch->status_dest_micpa, false);
+	mic_dma_hw_ring_inc_head(ch);
+	mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
+				 ch->status_dest_micpa, true);
+	mic_dma_hw_ring_inc_head(ch);
+}
+
+/* Wrapper function to program memcpy descriptors/status descriptors */
+static int mic_dma_do_dma(struct mic_dma_chan *ch, int flags, dma_addr_t src,
+			  dma_addr_t dst, size_t len)
+{
+	if (-ENOMEM == mic_dma_prog_memcpy_desc(ch, src, dst, len))
+		return -ENOMEM;
+	/* Above mic_dma_prog_memcpy_desc() makes sure we have enough space */
+	if (flags & DMA_PREP_FENCE) {
+		mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0,
+					 ch->status_dest_micpa, false);
+		mic_dma_hw_ring_inc_head(ch);
+	}
+
+	if (flags & DMA_PREP_INTERRUPT)
+		mic_dma_prog_intr(ch);
+
+	return 0;
+}
+
+static inline void mic_dma_issue_pending(struct dma_chan *ch)
+{
+	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
+
+	spin_lock(&mic_ch->issue_lock);
+	/*
+	 * Write to head triggers h/w to act on the descriptors.
+	 * On MIC, writing the same head value twice causes
+	 * a h/w error. On second write, h/w assumes we filled
+	 * the entire ring & overwrote some of the descriptors.
+	 */
+	if (mic_ch->issued == mic_ch->submitted)
+		goto out;
+	mic_ch->issued = mic_ch->submitted;
+	/*
+	 * make descriptor updates visible before advancing head,
+	 * this is purposefully not smp_wmb() since we are also
+	 * publishing the descriptor updates to a dma device
+	 */
+	wmb();
+	mic_dma_write_reg(mic_ch, MIC_DMA_REG_DHPR, mic_ch->issued);
+out:
+	spin_unlock(&mic_ch->issue_lock);
+}
+
+static inline void mic_dma_update_pending(struct mic_dma_chan *ch)
+{
+	if (mic_dma_ring_count(ch->issued, ch->submitted)
+			> mic_dma_pending_level)
+		mic_dma_issue_pending(&ch->api_ch);
+}
+
+static dma_cookie_t mic_dma_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
+{
+	struct mic_dma_chan *mic_ch = to_mic_dma_chan(tx->chan);
+	dma_cookie_t cookie;
+
+	dma_cookie_assign(tx);
+	cookie = tx->cookie;
+	/*
+	 * We need an smp write barrier here because another CPU might see
+	 * an update to submitted and update h/w head even before we
+	 * assigned a cookie to this tx.
+	 */
+	smp_wmb();
+	mic_ch->submitted = mic_ch->head;
+	spin_unlock(&mic_ch->prep_lock);
+	mic_dma_update_pending(mic_ch);
+	return cookie;
+}
+
+static inline struct dma_async_tx_descriptor *
+allocate_tx(struct mic_dma_chan *ch)
+{
+	u32 idx = mic_dma_hw_ring_dec(ch->head);
+	struct dma_async_tx_descriptor *tx = &ch->tx_array[idx];
+
+	dma_async_tx_descriptor_init(tx, &ch->api_ch);
+	tx->tx_submit = mic_dma_tx_submit_unlock;
+	return tx;
+}
+
+/*
+ * Prepare a memcpy descriptor to be added to the ring.
+ * Note that the temporary descriptor adds an extra overhead of copying the
+ * descriptor to ring. So, we copy directly to the descriptor ring
+ */
+static struct dma_async_tx_descriptor *
+mic_dma_prep_memcpy_lock(struct dma_chan *ch, dma_addr_t dma_dest,
+			 dma_addr_t dma_src, size_t len, unsigned long flags)
+{
+	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
+	struct device *dev = mic_dma_ch_to_device(mic_ch);
+	int result;
+
+	if (!len && !flags)
+		return NULL;
+
+	spin_lock(&mic_ch->prep_lock);
+	result = mic_dma_do_dma(mic_ch, flags, dma_src, dma_dest, len);
+	if (result >= 0)
+		return allocate_tx(mic_ch);
+	dev_err(dev, "Error enqueueing dma, error=%d\n", result);
+	spin_unlock(&mic_ch->prep_lock);
+	return NULL;
+}
+
+static struct dma_async_tx_descriptor *
+mic_dma_prep_interrupt_lock(struct dma_chan *ch, unsigned long flags)
+{
+	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
+	int ret;
+
+	spin_lock(&mic_ch->prep_lock);
+	ret = mic_dma_do_dma(mic_ch, flags, 0, 0, 0);
+	if (!ret)
+		return allocate_tx(mic_ch);
+	spin_unlock(&mic_ch->prep_lock);
+	return NULL;
+}
+
+/* Return the status of the transaction */
+static enum dma_status
+mic_dma_tx_status(struct dma_chan *ch, dma_cookie_t cookie,
+		  struct dma_tx_state *txstate)
+{
+	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
+
+	if (DMA_COMPLETE != dma_cookie_status(ch, cookie, txstate))
+		mic_dma_cleanup(mic_ch);
+
+	return dma_cookie_status(ch, cookie, txstate);
+}
+
+static irqreturn_t mic_dma_thread_fn(int irq, void *data)
+{
+	mic_dma_cleanup((struct mic_dma_chan *)data);
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t mic_dma_intr_handler(int irq, void *data)
+{
+	struct mic_dma_chan *ch = ((struct mic_dma_chan *)data);
+
+	mic_dma_ack_interrupt(ch);
+	return IRQ_WAKE_THREAD;
+}
+
+static int mic_dma_alloc_desc_ring(struct mic_dma_chan *ch)
+{
+	u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
+	struct device *dev = &to_mbus_device(ch)->dev;
+
+	desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
+	ch->desc_ring = kzalloc(desc_ring_size, GFP_KERNEL);
+
+	if (!ch->desc_ring)
+		return -ENOMEM;
+
+	ch->desc_ring_micpa = dma_map_single(dev, ch->desc_ring,
+					     desc_ring_size, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(dev, ch->desc_ring_micpa))
+		goto map_error;
+
+	ch->tx_array = vzalloc(MIC_DMA_DESC_RX_SIZE * sizeof(*ch->tx_array));
+	if (!ch->tx_array)
+		goto tx_error;
+	return 0;
+tx_error:
+	dma_unmap_single(dev, ch->desc_ring_micpa, desc_ring_size,
+			 DMA_BIDIRECTIONAL);
+map_error:
+	kfree(ch->desc_ring);
+	return -ENOMEM;
+}
+
+static void mic_dma_free_desc_ring(struct mic_dma_chan *ch)
+{
+	u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring);
+
+	vfree(ch->tx_array);
+	desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES);
+	dma_unmap_single(&to_mbus_device(ch)->dev, ch->desc_ring_micpa,
+			 desc_ring_size, DMA_BIDIRECTIONAL);
+	kfree(ch->desc_ring);
+	ch->desc_ring = NULL;
+}
+
+static void mic_dma_free_status_dest(struct mic_dma_chan *ch)
+{
+	dma_unmap_single(&to_mbus_device(ch)->dev, ch->status_dest_micpa,
+			 L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
+	kfree(ch->status_dest);
+}
+
+static int mic_dma_alloc_status_dest(struct mic_dma_chan *ch)
+{
+	struct device *dev = &to_mbus_device(ch)->dev;
+
+	ch->status_dest = kzalloc(L1_CACHE_BYTES, GFP_KERNEL);
+	if (!ch->status_dest)
+		return -ENOMEM;
+	ch->status_dest_micpa = dma_map_single(dev, ch->status_dest,
+					L1_CACHE_BYTES, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(dev, ch->status_dest_micpa)) {
+		kfree(ch->status_dest);
+		ch->status_dest = NULL;
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int mic_dma_check_chan(struct mic_dma_chan *ch)
+{
+	if (mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR) ||
+	    mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) & MIC_DMA_CHAN_QUIESCE) {
+		mic_dma_disable_chan(ch);
+		mic_dma_chan_mask_intr(ch);
+		dev_err(mic_dma_ch_to_device(ch),
+			"%s %d error setting up mic dma chan %d\n",
+			__func__, __LINE__, ch->ch_num);
+		return -EBUSY;
+	}
+	return 0;
+}
+
+static int mic_dma_chan_setup(struct mic_dma_chan *ch)
+{
+	if (MIC_DMA_CHAN_MIC == ch->owner)
+		mic_dma_chan_set_owner(ch);
+	mic_dma_disable_chan(ch);
+	mic_dma_chan_mask_intr(ch);
+	mic_dma_write_reg(ch, MIC_DMA_REG_DCHERRMSK, 0);
+	mic_dma_chan_set_desc_ring(ch);
+	ch->last_tail = mic_dma_read_reg(ch, MIC_DMA_REG_DTPR);
+	ch->head = ch->last_tail;
+	ch->issued = 0;
+	mic_dma_chan_unmask_intr(ch);
+	mic_dma_enable_chan(ch);
+	return mic_dma_check_chan(ch);
+}
+
+static void mic_dma_chan_destroy(struct mic_dma_chan *ch)
+{
+	mic_dma_disable_chan(ch);
+	mic_dma_chan_mask_intr(ch);
+}
+
+static void mic_dma_unregister_dma_device(struct mic_dma_device *mic_dma_dev)
+{
+	dma_async_device_unregister(&mic_dma_dev->dma_dev);
+}
+
+static int mic_dma_setup_irq(struct mic_dma_chan *ch)
+{
+	ch->cookie =
+		to_mbus_hw_ops(ch)->request_threaded_irq(to_mbus_device(ch),
+			mic_dma_intr_handler, mic_dma_thread_fn,
+			"mic dma_channel", ch, ch->ch_num);
+	if (IS_ERR(ch->cookie))
+		return IS_ERR(ch->cookie);
+	return 0;
+}
+
+static inline void mic_dma_free_irq(struct mic_dma_chan *ch)
+{
+	to_mbus_hw_ops(ch)->free_irq(to_mbus_device(ch), ch->cookie, ch);
+}
+
+static int mic_dma_chan_init(struct mic_dma_chan *ch)
+{
+	int ret = mic_dma_alloc_desc_ring(ch);
+
+	if (ret)
+		goto ring_error;
+	ret = mic_dma_alloc_status_dest(ch);
+	if (ret)
+		goto status_error;
+	ret = mic_dma_chan_setup(ch);
+	if (ret)
+		goto chan_error;
+	return ret;
+chan_error:
+	mic_dma_free_status_dest(ch);
+status_error:
+	mic_dma_free_desc_ring(ch);
+ring_error:
+	return ret;
+}
+
+static int mic_dma_drain_chan(struct mic_dma_chan *ch)
+{
+	struct dma_async_tx_descriptor *tx;
+	int err = 0;
+	dma_cookie_t cookie;
+
+	tx = mic_dma_prep_memcpy_lock(&ch->api_ch, 0, 0, 0, DMA_PREP_FENCE);
+	if (!tx) {
+		err = -ENOMEM;
+		goto error;
+	}
+
+	cookie = tx->tx_submit(tx);
+	if (dma_submit_error(cookie))
+		err = -ENOMEM;
+	else
+		err = dma_sync_wait(&ch->api_ch, cookie);
+	if (err) {
+		dev_err(mic_dma_ch_to_device(ch), "%s %d TO chan 0x%x\n",
+			__func__, __LINE__, ch->ch_num);
+		err = -EIO;
+	}
+error:
+	mic_dma_cleanup(ch);
+	return err;
+}
+
+static inline void mic_dma_chan_uninit(struct mic_dma_chan *ch)
+{
+	mic_dma_chan_destroy(ch);
+	mic_dma_cleanup(ch);
+	mic_dma_free_status_dest(ch);
+	mic_dma_free_desc_ring(ch);
+}
+
+static int mic_dma_init(struct mic_dma_device *mic_dma_dev,
+			enum mic_dma_chan_owner owner)
+{
+	int i, first_chan = mic_dma_dev->start_ch;
+	struct mic_dma_chan *ch;
+	int ret;
+
+	for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
+		unsigned long data;
+		ch = &mic_dma_dev->mic_ch[i];
+		data = (unsigned long)ch;
+		ch->ch_num = i;
+		ch->owner = owner;
+		spin_lock_init(&ch->cleanup_lock);
+		spin_lock_init(&ch->prep_lock);
+		spin_lock_init(&ch->issue_lock);
+		ret = mic_dma_setup_irq(ch);
+		if (ret)
+			goto error;
+	}
+	return 0;
+error:
+	for (i = i - 1; i >= first_chan; i--)
+		mic_dma_free_irq(ch);
+	return ret;
+}
+
+static void mic_dma_uninit(struct mic_dma_device *mic_dma_dev)
+{
+	int i, first_chan = mic_dma_dev->start_ch;
+	struct mic_dma_chan *ch;
+
+	for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
+		ch = &mic_dma_dev->mic_ch[i];
+		mic_dma_free_irq(ch);
+	}
+}
+
+static int mic_dma_alloc_chan_resources(struct dma_chan *ch)
+{
+	int ret = mic_dma_chan_init(to_mic_dma_chan(ch));
+	if (ret)
+		return ret;
+	return MIC_DMA_DESC_RX_SIZE;
+}
+
+static void mic_dma_free_chan_resources(struct dma_chan *ch)
+{
+	struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch);
+	mic_dma_drain_chan(mic_ch);
+	mic_dma_chan_uninit(mic_ch);
+}
+
+/* Set the fn. handlers and register the dma device with dma api */
+static int mic_dma_register_dma_device(struct mic_dma_device *mic_dma_dev,
+				       enum mic_dma_chan_owner owner)
+{
+	int i, first_chan = mic_dma_dev->start_ch;
+
+	dma_cap_zero(mic_dma_dev->dma_dev.cap_mask);
+	/*
+	 * This dma engine is not capable of host memory to host memory
+	 * transfers
+	 */
+	dma_cap_set(DMA_MEMCPY, mic_dma_dev->dma_dev.cap_mask);
+
+	if (MIC_DMA_CHAN_HOST == owner)
+		dma_cap_set(DMA_PRIVATE, mic_dma_dev->dma_dev.cap_mask);
+	mic_dma_dev->dma_dev.device_alloc_chan_resources =
+		mic_dma_alloc_chan_resources;
+	mic_dma_dev->dma_dev.device_free_chan_resources =
+		mic_dma_free_chan_resources;
+	mic_dma_dev->dma_dev.device_tx_status = mic_dma_tx_status;
+	mic_dma_dev->dma_dev.device_prep_dma_memcpy = mic_dma_prep_memcpy_lock;
+	mic_dma_dev->dma_dev.device_prep_dma_interrupt =
+		mic_dma_prep_interrupt_lock;
+	mic_dma_dev->dma_dev.device_issue_pending = mic_dma_issue_pending;
+	mic_dma_dev->dma_dev.copy_align = MIC_DMA_ALIGN_SHIFT;
+	INIT_LIST_HEAD(&mic_dma_dev->dma_dev.channels);
+	for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
+		mic_dma_dev->mic_ch[i].api_ch.device = &mic_dma_dev->dma_dev;
+		dma_cookie_init(&mic_dma_dev->mic_ch[i].api_ch);
+		list_add_tail(&mic_dma_dev->mic_ch[i].api_ch.device_node,
+			      &mic_dma_dev->dma_dev.channels);
+	}
+	return dma_async_device_register(&mic_dma_dev->dma_dev);
+}
+
+/*
+ * Initializes dma channels and registers the dma device with the
+ * dma engine api.
+ */
+static struct mic_dma_device *mic_dma_dev_reg(struct mbus_device *mbdev,
+					      enum mic_dma_chan_owner owner)
+{
+	struct mic_dma_device *mic_dma_dev;
+	int ret;
+	struct device *dev = &mbdev->dev;
+
+	mic_dma_dev = kzalloc(sizeof(*mic_dma_dev), GFP_KERNEL);
+	if (!mic_dma_dev) {
+		ret = -ENOMEM;
+		goto alloc_error;
+	}
+	mic_dma_dev->mbdev = mbdev;
+	mic_dma_dev->dma_dev.dev = dev;
+	mic_dma_dev->mmio = mbdev->mmio_va;
+	if (MIC_DMA_CHAN_HOST == owner) {
+		mic_dma_dev->start_ch = 0;
+		mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_HOST;
+	} else {
+		mic_dma_dev->start_ch = 4;
+		mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_CARD;
+	}
+	ret = mic_dma_init(mic_dma_dev, owner);
+	if (ret)
+		goto init_error;
+	ret = mic_dma_register_dma_device(mic_dma_dev, owner);
+	if (ret)
+		goto reg_error;
+	return mic_dma_dev;
+reg_error:
+	mic_dma_uninit(mic_dma_dev);
+init_error:
+	kfree(mic_dma_dev);
+	mic_dma_dev = NULL;
+alloc_error:
+	dev_err(dev, "Error at %s %d ret=%d\n", __func__, __LINE__, ret);
+	return mic_dma_dev;
+}
+
+static void mic_dma_dev_unreg(struct mic_dma_device *mic_dma_dev)
+{
+	mic_dma_unregister_dma_device(mic_dma_dev);
+	mic_dma_uninit(mic_dma_dev);
+	kfree(mic_dma_dev);
+}
+
+/* DEBUGFS CODE */
+static int mic_dma_reg_seq_show(struct seq_file *s, void *pos)
+{
+	struct mic_dma_device *mic_dma_dev = s->private;
+	int i, chan_num, first_chan = mic_dma_dev->start_ch;
+	struct mic_dma_chan *ch;
+
+	seq_printf(s, "SBOX_DCR: %#x\n",
+		   mic_dma_mmio_read(&mic_dma_dev->mic_ch[first_chan],
+				     MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR));
+	seq_puts(s, "DMA Channel Registers\n");
+	seq_printf(s, "%-10s| %-10s %-10s %-10s %-10s %-10s",
+		   "Channel", "DCAR", "DTPR", "DHPR", "DRAR_HI", "DRAR_LO");
+	seq_printf(s, " %-11s %-14s %-10s\n", "DCHERR", "DCHERRMSK", "DSTAT");
+	for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
+		ch = &mic_dma_dev->mic_ch[i];
+		chan_num = ch->ch_num;
+		seq_printf(s, "%-10i| %-#10x %-#10x %-#10x %-#10x",
+			   chan_num,
+			   mic_dma_read_reg(ch, MIC_DMA_REG_DCAR),
+			   mic_dma_read_reg(ch, MIC_DMA_REG_DTPR),
+			   mic_dma_read_reg(ch, MIC_DMA_REG_DHPR),
+			   mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_HI));
+		seq_printf(s, " %-#10x %-#10x %-#14x %-#10x\n",
+			   mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_LO),
+			   mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR),
+			   mic_dma_read_reg(ch, MIC_DMA_REG_DCHERRMSK),
+			   mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT));
+	}
+	return 0;
+}
+
+static int mic_dma_reg_debug_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mic_dma_reg_seq_show, inode->i_private);
+}
+
+static int mic_dma_reg_debug_release(struct inode *inode, struct file *file)
+{
+	return single_release(inode, file);
+}
+
+static const struct file_operations mic_dma_reg_ops = {
+	.owner   = THIS_MODULE,
+	.open    = mic_dma_reg_debug_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = mic_dma_reg_debug_release
+};
+
+/* Debugfs parent dir */
+static struct dentry *mic_dma_dbg;
+
+static int mic_dma_driver_probe(struct mbus_device *mbdev)
+{
+	struct mic_dma_device *mic_dma_dev;
+	enum mic_dma_chan_owner owner;
+
+	if (MBUS_DEV_DMA_MIC == mbdev->id.device)
+		owner = MIC_DMA_CHAN_MIC;
+	else
+		owner = MIC_DMA_CHAN_HOST;
+
+	mic_dma_dev = mic_dma_dev_reg(mbdev, owner);
+	dev_set_drvdata(&mbdev->dev, mic_dma_dev);
+
+	if (mic_dma_dbg) {
+		mic_dma_dev->dbg_dir = debugfs_create_dir(dev_name(&mbdev->dev),
+							  mic_dma_dbg);
+		if (mic_dma_dev->dbg_dir)
+			debugfs_create_file("mic_dma_reg", 0444,
+					    mic_dma_dev->dbg_dir, mic_dma_dev,
+					    &mic_dma_reg_ops);
+	}
+	return 0;
+}
+
+static void mic_dma_driver_remove(struct mbus_device *mbdev)
+{
+	struct mic_dma_device *mic_dma_dev;
+
+	mic_dma_dev = dev_get_drvdata(&mbdev->dev);
+	debugfs_remove_recursive(mic_dma_dev->dbg_dir);
+	mic_dma_dev_unreg(mic_dma_dev);
+}
+
+static struct mbus_device_id id_table[] = {
+	{MBUS_DEV_DMA_MIC, MBUS_DEV_ANY_ID},
+	{MBUS_DEV_DMA_HOST, MBUS_DEV_ANY_ID},
+	{0},
+};
+
+static struct mbus_driver mic_dma_driver = {
+	.driver.name =	KBUILD_MODNAME,
+	.driver.owner =	THIS_MODULE,
+	.id_table = id_table,
+	.probe = mic_dma_driver_probe,
+	.remove = mic_dma_driver_remove,
+};
+
+static int __init mic_x100_dma_init(void)
+{
+	int rc = mbus_register_driver(&mic_dma_driver);
+	if (rc)
+		return rc;
+	mic_dma_dbg = debugfs_create_dir(KBUILD_MODNAME, NULL);
+	return 0;
+}
+
+static void __exit mic_x100_dma_exit(void)
+{
+	debugfs_remove_recursive(mic_dma_dbg);
+	mbus_unregister_driver(&mic_dma_driver);
+}
+
+module_init(mic_x100_dma_init);
+module_exit(mic_x100_dma_exit);
+
+MODULE_DEVICE_TABLE(mbus, id_table);
+MODULE_AUTHOR("Intel Corporation");
+MODULE_DESCRIPTION("Intel(R) MIC X100 DMA Driver");
+MODULE_LICENSE("GPL v2");

drivers/dma/mic_x100_dma.h

@@ -0,0 +1,286 @@
+/*
+ * Intel MIC Platform Software Stack (MPSS)
+ *
+ * Copyright(c) 2014 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * Intel MIC X100 DMA Driver.
+ *
+ * Adapted from IOAT dma driver.
+ */
+#ifndef _MIC_X100_DMA_H_
+#define _MIC_X100_DMA_H_
+
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+#include <linux/debugfs.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/mic_bus.h>
+
+#include "dmaengine.h"
+
+/*
+ * MIC has a total of 8 dma channels.
+ * Four channels are assigned for host SW use & the remaining for MIC SW.
+ * MIC DMA transfer size & addresses need to be 64 byte aligned.
+ */
+#define MIC_DMA_MAX_NUM_CHAN	8
+#define MIC_DMA_NUM_CHAN	4
+#define MIC_DMA_ALIGN_SHIFT	6
+#define MIC_DMA_ALIGN_BYTES	(1 << MIC_DMA_ALIGN_SHIFT)
+#define MIC_DMA_DESC_RX_SIZE	(128 * 1024 - 4)
+
+/*
+ * Register descriptions
+ * All the registers are 32 bit registers.
+ * DCR is a global register and all others are per-channel.
+ * DCR - bits 0, 2, 4, 6, 8, 10, 12, 14 - enable bits for channels 0 to 7
+ *	 bits 1, 3, 5, 7, 9, 11, 13, 15 - owner bits for channels 0 to 7
+ * DCAR - bit 24 & 25 interrupt masks for mic owned & host owned channels
+ * DHPR - head of the descriptor ring updated by s/w
+ * DTPR - tail of the descriptor ring updated by h/w
+ * DRAR_LO - lower 32 bits of descriptor ring's mic address
+ * DRAR_HI - 3:0 - remaining 4 bits of descriptor ring's mic address
+ *	     20:4 descriptor ring size
+ *	     25:21 mic smpt entry number
+ * DSTAT - 16:0 h/w completion count; 31:28 dma engine status
+ * DCHERR - this register is non-zero on error
+ * DCHERRMSK - interrupt mask register
+ */
+#define MIC_DMA_HW_CMP_CNT_MASK		0x1ffff
+#define MIC_DMA_CHAN_QUIESCE		0x20000000
+#define MIC_DMA_SBOX_BASE		0x00010000
+#define MIC_DMA_SBOX_DCR		0x0000A280
+#define MIC_DMA_SBOX_CH_BASE		0x0001A000
+#define MIC_DMA_SBOX_CHAN_OFF		0x40
+#define MIC_DMA_SBOX_DCAR_IM0		(0x1 << 24)
+#define MIC_DMA_SBOX_DCAR_IM1		(0x1 << 25)
+#define MIC_DMA_SBOX_DRARHI_SYS_MASK	(0x1 << 26)
+#define MIC_DMA_REG_DCAR		0
+#define MIC_DMA_REG_DHPR		4
+#define MIC_DMA_REG_DTPR		8
+#define MIC_DMA_REG_DRAR_LO		20
+#define MIC_DMA_REG_DRAR_HI		24
+#define MIC_DMA_REG_DSTAT		32
+#define MIC_DMA_REG_DCHERR		44
+#define MIC_DMA_REG_DCHERRMSK		48
+
+/* HW dma desc */
+struct mic_dma_desc {
+	u64 qw0;
+	u64 qw1;
+};
+
+enum mic_dma_chan_owner {
+	MIC_DMA_CHAN_MIC = 0,
+	MIC_DMA_CHAN_HOST
+};
+
+/*
+ * mic_dma_chan - channel specific information
+ * @ch_num: channel number
+ * @owner: owner of this channel
+ * @last_tail: cached value of descriptor ring tail
+ * @head: index of next descriptor in desc_ring
+ * @issued: hardware notification point
+ * @submitted: index that will be used to submit descriptors to h/w
+ * @api_ch: dma engine api channel
+ * @desc_ring: dma descriptor ring
+ * @desc_ring_micpa: mic physical address of desc_ring
+ * @status_dest: destination for status (fence) descriptor
+ * @status_dest_micpa: mic address for status_dest,
+ *		       DMA controller uses this address
+ * @tx_array: array of async_tx
+ * @cleanup_lock: lock held when processing completed tx
+ * @prep_lock: lock held in prep_memcpy & released in tx_submit
+ * @issue_lock: lock used to synchronize writes to head
+ * @cookie: mic_irq cookie used with mic irq request
+ */
+struct mic_dma_chan {
+	int ch_num;
+	enum mic_dma_chan_owner owner;
+	u32 last_tail;
+	u32 head;
+	u32 issued;
+	u32 submitted;
+	struct dma_chan api_ch;
+	struct mic_dma_desc *desc_ring;
+	dma_addr_t desc_ring_micpa;
+	u64 *status_dest;
+	dma_addr_t status_dest_micpa;
+	struct dma_async_tx_descriptor *tx_array;
+	spinlock_t cleanup_lock;
+	spinlock_t prep_lock;
+	spinlock_t issue_lock;
+	struct mic_irq *cookie;
+};
+
+/*
+ * struct mic_dma_device - per mic device
+ * @mic_ch: dma channels
+ * @dma_dev: underlying dma device
+ * @mbdev: mic bus dma device
+ * @mmio: virtual address of the mmio space
+ * @dbg_dir: debugfs directory
+ * @start_ch: first channel number that can be used
+ * @max_xfer_size: maximum transfer size per dma descriptor
+ */
+struct mic_dma_device {
+	struct mic_dma_chan mic_ch[MIC_DMA_MAX_NUM_CHAN];
+	struct dma_device dma_dev;
+	struct mbus_device *mbdev;
+	void __iomem *mmio;
+	struct dentry *dbg_dir;
+	int start_ch;
+	size_t max_xfer_size;
+};
+
+static inline struct mic_dma_chan *to_mic_dma_chan(struct dma_chan *ch)
+{
+	return container_of(ch, struct mic_dma_chan, api_ch);
+}
+
+static inline struct mic_dma_device *to_mic_dma_dev(struct mic_dma_chan *ch)
+{
+	return
+	container_of((const typeof(((struct mic_dma_device *)0)->mic_ch)*)
+		     (ch - ch->ch_num), struct mic_dma_device, mic_ch);
+}
+
+static inline struct mbus_device *to_mbus_device(struct mic_dma_chan *ch)
+{
+	return to_mic_dma_dev(ch)->mbdev;
+}
+
+static inline struct mbus_hw_ops *to_mbus_hw_ops(struct mic_dma_chan *ch)
+{
+	return to_mbus_device(ch)->hw_ops;
+}
+
+static inline struct device *mic_dma_ch_to_device(struct mic_dma_chan *ch)
+{
+	return to_mic_dma_dev(ch)->dma_dev.dev;
+}
+
+static inline void __iomem *mic_dma_chan_to_mmio(struct mic_dma_chan *ch)
+{
+	return to_mic_dma_dev(ch)->mmio;
+}
+
+static inline u32 mic_dma_read_reg(struct mic_dma_chan *ch, u32 reg)
+{
+	return ioread32(mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
+			ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
+}
+
+static inline void mic_dma_write_reg(struct mic_dma_chan *ch, u32 reg, u32 val)
+{
+	iowrite32(val, mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
+		  ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
+}
+
+static inline u32 mic_dma_mmio_read(struct mic_dma_chan *ch, u32 offset)
+{
+	return ioread32(mic_dma_chan_to_mmio(ch) + offset);
+}
+
+static inline void mic_dma_mmio_write(struct mic_dma_chan *ch, u32 val,
+				      u32 offset)
+{
+	iowrite32(val, mic_dma_chan_to_mmio(ch) + offset);
+}
+
+static inline u32 mic_dma_read_cmp_cnt(struct mic_dma_chan *ch)
+{
+	return mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) &
+	       MIC_DMA_HW_CMP_CNT_MASK;
+}
+
+static inline void mic_dma_chan_set_owner(struct mic_dma_chan *ch)
+{
+	u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
+	u32 chan_num = ch->ch_num;
+
+	dcr = (dcr & ~(0x1 << (chan_num * 2))) | (ch->owner << (chan_num * 2));
+	mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
+}
+
+static inline void mic_dma_enable_chan(struct mic_dma_chan *ch)
+{
+	u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
+
+	dcr |= 2 << (ch->ch_num << 1);
+	mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
+}
+
+static inline void mic_dma_disable_chan(struct mic_dma_chan *ch)
+{
+	u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
+
+	dcr &= ~(2 << (ch->ch_num << 1));
+	mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
+}
+
+static void mic_dma_chan_set_desc_ring(struct mic_dma_chan *ch)
+{
+	u32 drar_hi;
+	dma_addr_t desc_ring_micpa = ch->desc_ring_micpa;
+
+	drar_hi = (MIC_DMA_DESC_RX_SIZE & 0x1ffff) << 4;
+	if (MIC_DMA_CHAN_MIC == ch->owner) {
+		drar_hi |= (desc_ring_micpa >> 32) & 0xf;
+	} else {
+		drar_hi |= MIC_DMA_SBOX_DRARHI_SYS_MASK;
+		drar_hi |= ((desc_ring_micpa >> 34)
+			    & 0x1f) << 21;
+		drar_hi |= (desc_ring_micpa >> 32) & 0x3;
+	}
+	mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_LO, (u32) desc_ring_micpa);
+	mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_HI, drar_hi);
+}
+
+static inline void mic_dma_chan_mask_intr(struct mic_dma_chan *ch)
+{
+	u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
+
+	if (MIC_DMA_CHAN_MIC == ch->owner)
+		dcar |= MIC_DMA_SBOX_DCAR_IM0;
+	else
+		dcar |= MIC_DMA_SBOX_DCAR_IM1;
+	mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
+}
+
+static inline void mic_dma_chan_unmask_intr(struct mic_dma_chan *ch)
+{
+	u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
+
+	if (MIC_DMA_CHAN_MIC == ch->owner)
+		dcar &= ~MIC_DMA_SBOX_DCAR_IM0;
+	else
+		dcar &= ~MIC_DMA_SBOX_DCAR_IM1;
+	mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
+}
+
+static void mic_dma_ack_interrupt(struct mic_dma_chan *ch)
+{
+	if (MIC_DMA_CHAN_MIC == ch->owner) {
+		/* HW errata */
+		mic_dma_chan_mask_intr(ch);
+		mic_dma_chan_unmask_intr(ch);
+	}
+	to_mbus_hw_ops(ch)->ack_interrupt(to_mbus_device(ch), ch->ch_num);
+}
+#endif