|
@@ -0,0 +1,680 @@
|
|
|
+/*
|
|
|
+ * Copyright (C) 2012-2017 ARM Limited or its affiliates.
|
|
|
+ *
|
|
|
+ * This program is free software; you can redistribute it and/or modify
|
|
|
+ * it under the terms of the GNU General Public License version 2 as
|
|
|
+ * published by the Free Software Foundation.
|
|
|
+ *
|
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
+ * GNU General Public License for more details.
|
|
|
+ *
|
|
|
+ * You should have received a copy of the GNU General Public License
|
|
|
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
|
|
|
+ */
|
|
|
+
|
|
|
+#include "ssi_config.h"
|
|
|
+#include <linux/kernel.h>
|
|
|
+#include <linux/platform_device.h>
|
|
|
+#include <linux/interrupt.h>
|
|
|
+#include <linux/delay.h>
|
|
|
+#include <crypto/ctr.h>
|
|
|
+#ifdef FLUSH_CACHE_ALL
|
|
|
+#include <asm/cacheflush.h>
|
|
|
+#endif
|
|
|
+#include <linux/pm_runtime.h>
|
|
|
+#include "ssi_driver.h"
|
|
|
+#include "ssi_buffer_mgr.h"
|
|
|
+#include "ssi_request_mgr.h"
|
|
|
+#include "ssi_sysfs.h"
|
|
|
+#include "ssi_pm.h"
|
|
|
+
|
|
|
+#define SSI_MAX_POLL_ITER 10
|
|
|
+
|
|
|
+#define AXIM_MON_BASE_OFFSET CC_REG_OFFSET(CRY_KERNEL, AXIM_MON_COMP)
|
|
|
+
|
|
|
+#ifdef CC_CYCLE_COUNT
|
|
|
+
|
|
|
+#define MONITOR_CNTR_BIT 0
|
|
|
+
|
|
|
+/**
|
|
|
+ * Monitor descriptor.
|
|
|
+ * Used to measure CC performance.
|
|
|
+ */
|
|
|
+#define INIT_CC_MONITOR_DESC(desc_p) \
|
|
|
+do { \
|
|
|
+ HW_DESC_INIT(desc_p); \
|
|
|
+ HW_DESC_SET_DIN_MONITOR_CNTR(desc_p); \
|
|
|
+} while (0)
|
|
|
+
|
|
|
+/**
|
|
|
+ * Try adding monitor descriptor BEFORE enqueuing sequence.
|
|
|
+ */
|
|
|
+#define CC_CYCLE_DESC_HEAD(cc_base_addr, desc_p, lock_p, is_monitored_p) \
|
|
|
+do { \
|
|
|
+ if (!test_and_set_bit(MONITOR_CNTR_BIT, (lock_p))) { \
|
|
|
+ enqueue_seq((cc_base_addr), (desc_p), 1); \
|
|
|
+ *(is_monitored_p) = true; \
|
|
|
+ } else { \
|
|
|
+ *(is_monitored_p) = false; \
|
|
|
+ } \
|
|
|
+} while (0)
|
|
|
+
|
|
|
+/**
|
|
|
+ * If CC_CYCLE_DESC_HEAD was successfully added:
|
|
|
+ * 1. Add memory barrier descriptor to ensure last AXI transaction.
|
|
|
+ * 2. Add monitor descriptor to sequence tail AFTER enqueuing sequence.
|
|
|
+ */
|
|
|
+#define CC_CYCLE_DESC_TAIL(cc_base_addr, desc_p, is_monitored) \
|
|
|
+do { \
|
|
|
+ if ((is_monitored) == true) { \
|
|
|
+ HwDesc_s barrier_desc; \
|
|
|
+ HW_DESC_INIT(&barrier_desc); \
|
|
|
+ HW_DESC_SET_DIN_NO_DMA(&barrier_desc, 0, 0xfffff0); \
|
|
|
+ HW_DESC_SET_DOUT_NO_DMA(&barrier_desc, 0, 0, 1); \
|
|
|
+ enqueue_seq((cc_base_addr), &barrier_desc, 1); \
|
|
|
+ enqueue_seq((cc_base_addr), (desc_p), 1); \
|
|
|
+ } \
|
|
|
+} while (0)
|
|
|
+
|
|
|
+/**
|
|
|
+ * Try reading CC monitor counter value upon sequence complete.
|
|
|
+ * Can only succeed if the lock_p is taken by the owner of the given request.
|
|
|
+ */
|
|
|
+#define END_CC_MONITOR_COUNT(cc_base_addr, stat_op_type, stat_phase, monitor_null_cycles, lock_p, is_monitored) \
|
|
|
+do { \
|
|
|
+ uint32_t elapsed_cycles; \
|
|
|
+ if ((is_monitored) == true) { \
|
|
|
+ elapsed_cycles = READ_REGISTER((cc_base_addr) + CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_MEASURE_CNTR)); \
|
|
|
+ clear_bit(MONITOR_CNTR_BIT, (lock_p)); \
|
|
|
+ if (elapsed_cycles > 0) \
|
|
|
+ update_cc_stat(stat_op_type, stat_phase, (elapsed_cycles - monitor_null_cycles)); \
|
|
|
+ } \
|
|
|
+} while (0)
|
|
|
+
|
|
|
+#else /*CC_CYCLE_COUNT*/
|
|
|
+
|
|
|
+#define INIT_CC_MONITOR_DESC(desc_p) do { } while (0)
|
|
|
+#define CC_CYCLE_DESC_HEAD(cc_base_addr, desc_p, lock_p, is_monitored_p) do { } while (0)
|
|
|
+#define CC_CYCLE_DESC_TAIL(cc_base_addr, desc_p, is_monitored) do { } while (0)
|
|
|
+#define END_CC_MONITOR_COUNT(cc_base_addr, stat_op_type, stat_phase, monitor_null_cycles, lock_p, is_monitored) do { } while (0)
|
|
|
+#endif /*CC_CYCLE_COUNT*/
|
|
|
+
|
|
|
+
|
|
|
+struct ssi_request_mgr_handle {
|
|
|
+ /* Request manager resources */
|
|
|
+ unsigned int hw_queue_size; /* HW capability */
|
|
|
+ unsigned int min_free_hw_slots;
|
|
|
+ unsigned int max_used_sw_slots;
|
|
|
+ struct ssi_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];
|
|
|
+ uint32_t req_queue_head;
|
|
|
+ uint32_t req_queue_tail;
|
|
|
+ uint32_t axi_completed;
|
|
|
+ uint32_t q_free_slots;
|
|
|
+ spinlock_t hw_lock;
|
|
|
+ HwDesc_s compl_desc;
|
|
|
+ uint8_t *dummy_comp_buff;
|
|
|
+ dma_addr_t dummy_comp_buff_dma;
|
|
|
+ HwDesc_s monitor_desc;
|
|
|
+ volatile unsigned long monitor_lock;
|
|
|
+#ifdef COMP_IN_WQ
|
|
|
+ struct workqueue_struct *workq;
|
|
|
+ struct delayed_work compwork;
|
|
|
+#else
|
|
|
+ struct tasklet_struct comptask;
|
|
|
+#endif
|
|
|
+#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP)
|
|
|
+ bool is_runtime_suspended;
|
|
|
+#endif
|
|
|
+};
|
|
|
+
|
|
|
+static void comp_handler(unsigned long devarg);
|
|
|
+#ifdef COMP_IN_WQ
|
|
|
+static void comp_work_handler(struct work_struct *work);
|
|
|
+#endif
|
|
|
+
|
|
|
+void request_mgr_fini(struct ssi_drvdata *drvdata)
|
|
|
+{
|
|
|
+ struct ssi_request_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
|
|
|
+
|
|
|
+ if (req_mgr_h == NULL)
|
|
|
+ return; /* Not allocated */
|
|
|
+
|
|
|
+ if (req_mgr_h->dummy_comp_buff_dma != 0) {
|
|
|
+ SSI_RESTORE_DMA_ADDR_TO_48BIT(req_mgr_h->dummy_comp_buff_dma);
|
|
|
+ dma_free_coherent(&drvdata->plat_dev->dev,
|
|
|
+ sizeof(uint32_t), req_mgr_h->dummy_comp_buff,
|
|
|
+ req_mgr_h->dummy_comp_buff_dma);
|
|
|
+ }
|
|
|
+
|
|
|
+ SSI_LOG_DEBUG("max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -
|
|
|
+ req_mgr_h->min_free_hw_slots) );
|
|
|
+ SSI_LOG_DEBUG("max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
|
|
|
+
|
|
|
+#ifdef COMP_IN_WQ
|
|
|
+ flush_workqueue(req_mgr_h->workq);
|
|
|
+ destroy_workqueue(req_mgr_h->workq);
|
|
|
+#else
|
|
|
+ /* Kill tasklet */
|
|
|
+ tasklet_kill(&req_mgr_h->comptask);
|
|
|
+#endif
|
|
|
+ memset(req_mgr_h, 0, sizeof(struct ssi_request_mgr_handle));
|
|
|
+ kfree(req_mgr_h);
|
|
|
+ drvdata->request_mgr_handle = NULL;
|
|
|
+}
|
|
|
+
|
|
|
+int request_mgr_init(struct ssi_drvdata *drvdata)
|
|
|
+{
|
|
|
+#ifdef CC_CYCLE_COUNT
|
|
|
+ HwDesc_s monitor_desc[2];
|
|
|
+ struct ssi_crypto_req monitor_req = {0};
|
|
|
+#endif
|
|
|
+ struct ssi_request_mgr_handle *req_mgr_h;
|
|
|
+ int rc = 0;
|
|
|
+
|
|
|
+ req_mgr_h = kzalloc(sizeof(struct ssi_request_mgr_handle),GFP_KERNEL);
|
|
|
+ if (req_mgr_h == NULL) {
|
|
|
+ rc = -ENOMEM;
|
|
|
+ goto req_mgr_init_err;
|
|
|
+ }
|
|
|
+
|
|
|
+ drvdata->request_mgr_handle = req_mgr_h;
|
|
|
+
|
|
|
+ spin_lock_init(&req_mgr_h->hw_lock);
|
|
|
+#ifdef COMP_IN_WQ
|
|
|
+ SSI_LOG_DEBUG("Initializing completion workqueue\n");
|
|
|
+ req_mgr_h->workq = create_singlethread_workqueue("arm_cc7x_wq");
|
|
|
+ if (unlikely(req_mgr_h->workq == NULL)) {
|
|
|
+ SSI_LOG_ERR("Failed creating work queue\n");
|
|
|
+ rc = -ENOMEM;
|
|
|
+ goto req_mgr_init_err;
|
|
|
+ }
|
|
|
+ INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);
|
|
|
+#else
|
|
|
+ SSI_LOG_DEBUG("Initializing completion tasklet\n");
|
|
|
+ tasklet_init(&req_mgr_h->comptask, comp_handler, (unsigned long)drvdata);
|
|
|
+#endif
|
|
|
+ req_mgr_h->hw_queue_size = READ_REGISTER(drvdata->cc_base +
|
|
|
+ CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_SRAM_SIZE));
|
|
|
+ SSI_LOG_DEBUG("hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
|
|
|
+ if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
|
|
|
+ SSI_LOG_ERR("Invalid HW queue size = %u (Min. required is %u)\n",
|
|
|
+ req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
|
|
|
+ rc = -ENOMEM;
|
|
|
+ goto req_mgr_init_err;
|
|
|
+ }
|
|
|
+ req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;
|
|
|
+ req_mgr_h->max_used_sw_slots = 0;
|
|
|
+
|
|
|
+
|
|
|
+ /* Allocate DMA word for "dummy" completion descriptor use */
|
|
|
+ req_mgr_h->dummy_comp_buff = dma_alloc_coherent(&drvdata->plat_dev->dev,
|
|
|
+ sizeof(uint32_t), &req_mgr_h->dummy_comp_buff_dma, GFP_KERNEL);
|
|
|
+ if (!req_mgr_h->dummy_comp_buff) {
|
|
|
+ SSI_LOG_ERR("Not enough memory to allocate DMA (%zu) dropped "
|
|
|
+ "buffer\n", sizeof(uint32_t));
|
|
|
+ rc = -ENOMEM;
|
|
|
+ goto req_mgr_init_err;
|
|
|
+ }
|
|
|
+ SSI_UPDATE_DMA_ADDR_TO_48BIT(req_mgr_h->dummy_comp_buff_dma,
|
|
|
+ sizeof(uint32_t));
|
|
|
+
|
|
|
+ /* Init. "dummy" completion descriptor */
|
|
|
+ HW_DESC_INIT(&req_mgr_h->compl_desc);
|
|
|
+ HW_DESC_SET_DIN_CONST(&req_mgr_h->compl_desc, 0, sizeof(uint32_t));
|
|
|
+ HW_DESC_SET_DOUT_DLLI(&req_mgr_h->compl_desc,
|
|
|
+ req_mgr_h->dummy_comp_buff_dma,
|
|
|
+ sizeof(uint32_t), NS_BIT, 1);
|
|
|
+ HW_DESC_SET_FLOW_MODE(&req_mgr_h->compl_desc, BYPASS);
|
|
|
+ HW_DESC_SET_QUEUE_LAST_IND(&req_mgr_h->compl_desc);
|
|
|
+
|
|
|
+#ifdef CC_CYCLE_COUNT
|
|
|
+ /* For CC-HW cycle performance trace */
|
|
|
+ INIT_CC_MONITOR_DESC(&req_mgr_h->monitor_desc);
|
|
|
+ set_bit(MONITOR_CNTR_BIT, &req_mgr_h->monitor_lock);
|
|
|
+ monitor_desc[0] = req_mgr_h->monitor_desc;
|
|
|
+ monitor_desc[1] = req_mgr_h->monitor_desc;
|
|
|
+
|
|
|
+ rc = send_request(drvdata, &monitor_req, monitor_desc, 2, 0);
|
|
|
+ if (unlikely(rc != 0))
|
|
|
+ goto req_mgr_init_err;
|
|
|
+
|
|
|
+ drvdata->monitor_null_cycles = READ_REGISTER(drvdata->cc_base +
|
|
|
+ CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_MEASURE_CNTR));
|
|
|
+ SSI_LOG_ERR("Calibration time=0x%08x\n", drvdata->monitor_null_cycles);
|
|
|
+
|
|
|
+ clear_bit(MONITOR_CNTR_BIT, &req_mgr_h->monitor_lock);
|
|
|
+#endif
|
|
|
+
|
|
|
+ return 0;
|
|
|
+
|
|
|
+req_mgr_init_err:
|
|
|
+ request_mgr_fini(drvdata);
|
|
|
+ return rc;
|
|
|
+}
|
|
|
+
|
|
|
+static inline void enqueue_seq(
|
|
|
+ void __iomem *cc_base,
|
|
|
+ HwDesc_s seq[], unsigned int seq_len)
|
|
|
+{
|
|
|
+ int i;
|
|
|
+
|
|
|
+ for (i = 0; i < seq_len; i++) {
|
|
|
+ writel_relaxed(seq[i].word[0], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0)));
|
|
|
+ writel_relaxed(seq[i].word[1], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0)));
|
|
|
+ writel_relaxed(seq[i].word[2], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0)));
|
|
|
+ writel_relaxed(seq[i].word[3], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0)));
|
|
|
+ writel_relaxed(seq[i].word[4], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0)));
|
|
|
+ wmb();
|
|
|
+ writel_relaxed(seq[i].word[5], (volatile void __iomem *)(cc_base+CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0)));
|
|
|
+#ifdef DX_DUMP_DESCS
|
|
|
+ SSI_LOG_DEBUG("desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n", i,
|
|
|
+ seq[i].word[0], seq[i].word[1], seq[i].word[2], seq[i].word[3], seq[i].word[4], seq[i].word[5]);
|
|
|
+#endif
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/*!
|
|
|
+ * Completion will take place if and only if user requested completion
|
|
|
+ * by setting "is_dout = 0" in send_request().
|
|
|
+ *
|
|
|
+ * \param dev
|
|
|
+ * \param dx_compl_h The completion event to signal
|
|
|
+ */
|
|
|
+static void request_mgr_complete(struct device *dev, void *dx_compl_h, void __iomem *cc_base)
|
|
|
+{
|
|
|
+ struct completion *this_compl = dx_compl_h;
|
|
|
+ complete(this_compl);
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+static inline int request_mgr_queues_status_check(
|
|
|
+ struct ssi_request_mgr_handle *req_mgr_h,
|
|
|
+ void __iomem *cc_base,
|
|
|
+ unsigned int total_seq_len)
|
|
|
+{
|
|
|
+ unsigned long poll_queue;
|
|
|
+
|
|
|
+ /* SW queue is checked only once as it will not
|
|
|
+ be chaned during the poll becasue the spinlock_bh
|
|
|
+ is held by the thread */
|
|
|
+ if (unlikely(((req_mgr_h->req_queue_head + 1) &
|
|
|
+ (MAX_REQUEST_QUEUE_SIZE - 1)) ==
|
|
|
+ req_mgr_h->req_queue_tail)) {
|
|
|
+ SSI_LOG_ERR("SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
|
|
|
+ req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
|
|
|
+ return -EBUSY;
|
|
|
+ }
|
|
|
+
|
|
|
+ if ((likely(req_mgr_h->q_free_slots >= total_seq_len)) ) {
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ /* Wait for space in HW queue. Poll constant num of iterations. */
|
|
|
+ for (poll_queue =0; poll_queue < SSI_MAX_POLL_ITER ; poll_queue ++) {
|
|
|
+ req_mgr_h->q_free_slots =
|
|
|
+ CC_HAL_READ_REGISTER(
|
|
|
+ CC_REG_OFFSET(CRY_KERNEL,
|
|
|
+ DSCRPTR_QUEUE_CONTENT));
|
|
|
+ if (unlikely(req_mgr_h->q_free_slots <
|
|
|
+ req_mgr_h->min_free_hw_slots)) {
|
|
|
+ req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (likely (req_mgr_h->q_free_slots >= total_seq_len)) {
|
|
|
+ /* If there is enough place return */
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ SSI_LOG_DEBUG("HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
|
|
|
+ req_mgr_h->q_free_slots, total_seq_len);
|
|
|
+ }
|
|
|
+ /* No room in the HW queue try again later */
|
|
|
+ SSI_LOG_DEBUG("HW FIFO full, timeout. req_queue_head=%d "
|
|
|
+ "sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",
|
|
|
+ req_mgr_h->req_queue_head,
|
|
|
+ MAX_REQUEST_QUEUE_SIZE,
|
|
|
+ req_mgr_h->q_free_slots,
|
|
|
+ total_seq_len);
|
|
|
+ return -EAGAIN;
|
|
|
+}
|
|
|
+
|
|
|
+/*!
|
|
|
+ * Enqueue caller request to crypto hardware.
|
|
|
+ *
|
|
|
+ * \param drvdata
|
|
|
+ * \param ssi_req The request to enqueue
|
|
|
+ * \param desc The crypto sequence
|
|
|
+ * \param len The crypto sequence length
|
|
|
+ * \param is_dout If "true": completion is handled by the caller
|
|
|
+ * If "false": this function adds a dummy descriptor completion
|
|
|
+ * and waits upon completion signal.
|
|
|
+ *
|
|
|
+ * \return int Returns -EINPROGRESS if "is_dout=true"; "0" if "is_dout=false"
|
|
|
+ */
|
|
|
+int send_request(
|
|
|
+ struct ssi_drvdata *drvdata, struct ssi_crypto_req *ssi_req,
|
|
|
+ HwDesc_s *desc, unsigned int len, bool is_dout)
|
|
|
+{
|
|
|
+ void __iomem *cc_base = drvdata->cc_base;
|
|
|
+ struct ssi_request_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
|
|
|
+ unsigned int used_sw_slots;
|
|
|
+ unsigned int total_seq_len = len; /*initial sequence length*/
|
|
|
+ int rc;
|
|
|
+ unsigned int max_required_seq_len = total_seq_len + ((is_dout == 0) ? 1 : 0);
|
|
|
+ DECL_CYCLE_COUNT_RESOURCES;
|
|
|
+
|
|
|
+#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP)
|
|
|
+ rc = ssi_power_mgr_runtime_get(&drvdata->plat_dev->dev);
|
|
|
+ if (rc != 0) {
|
|
|
+ SSI_LOG_ERR("ssi_power_mgr_runtime_get returned %x\n",rc);
|
|
|
+ spin_unlock_bh(&req_mgr_h->hw_lock);
|
|
|
+ return rc;
|
|
|
+ }
|
|
|
+#endif
|
|
|
+
|
|
|
+ do {
|
|
|
+ spin_lock_bh(&req_mgr_h->hw_lock);
|
|
|
+
|
|
|
+ /* Check if there is enough place in the SW/HW queues
|
|
|
+ in case iv gen add the max size and in case of no dout add 1
|
|
|
+ for the internal completion descriptor */
|
|
|
+ rc = request_mgr_queues_status_check(req_mgr_h,
|
|
|
+ cc_base,
|
|
|
+ max_required_seq_len);
|
|
|
+ if (likely(rc == 0 ))
|
|
|
+ /* There is enough place in the queue */
|
|
|
+ break;
|
|
|
+ /* something wrong release the spinlock*/
|
|
|
+ spin_unlock_bh(&req_mgr_h->hw_lock);
|
|
|
+
|
|
|
+ if (rc != -EAGAIN) {
|
|
|
+ /* Any error other than HW queue full
|
|
|
+ (SW queue is full) */
|
|
|
+#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP)
|
|
|
+ ssi_power_mgr_runtime_put_suspend(&drvdata->plat_dev->dev);
|
|
|
+#endif
|
|
|
+ return rc;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* HW queue is full - short sleep */
|
|
|
+ msleep(1);
|
|
|
+ } while (1);
|
|
|
+
|
|
|
+ /* Additional completion descriptor is needed incase caller did not
|
|
|
+ enabled any DLLI/MLLI DOUT bit in the given sequence */
|
|
|
+ if (!is_dout) {
|
|
|
+ init_completion(&ssi_req->seq_compl);
|
|
|
+ ssi_req->user_cb = request_mgr_complete;
|
|
|
+ ssi_req->user_arg = &(ssi_req->seq_compl);
|
|
|
+ total_seq_len++;
|
|
|
+ }
|
|
|
+
|
|
|
+ used_sw_slots = ((req_mgr_h->req_queue_head - req_mgr_h->req_queue_tail) & (MAX_REQUEST_QUEUE_SIZE-1));
|
|
|
+ if (unlikely(used_sw_slots > req_mgr_h->max_used_sw_slots)) {
|
|
|
+ req_mgr_h->max_used_sw_slots = used_sw_slots;
|
|
|
+ }
|
|
|
+
|
|
|
+ CC_CYCLE_DESC_HEAD(cc_base, &req_mgr_h->monitor_desc,
|
|
|
+ &req_mgr_h->monitor_lock, &ssi_req->is_monitored_p);
|
|
|
+
|
|
|
+ /* Enqueue request - must be locked with HW lock*/
|
|
|
+ req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *ssi_req;
|
|
|
+ START_CYCLE_COUNT_AT(req_mgr_h->req_queue[req_mgr_h->req_queue_head].submit_cycle);
|
|
|
+ req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
|
|
|
+ /* TODO: Use circ_buf.h ? */
|
|
|
+
|
|
|
+ SSI_LOG_DEBUG("Enqueue request head=%u\n", req_mgr_h->req_queue_head);
|
|
|
+
|
|
|
+#ifdef FLUSH_CACHE_ALL
|
|
|
+ flush_cache_all();
|
|
|
+#endif
|
|
|
+
|
|
|
+ /* STAT_PHASE_4: Push sequence */
|
|
|
+ START_CYCLE_COUNT();
|
|
|
+ enqueue_seq(cc_base, desc, len);
|
|
|
+ enqueue_seq(cc_base, &req_mgr_h->compl_desc, (is_dout ? 0 : 1));
|
|
|
+ END_CYCLE_COUNT(ssi_req->op_type, STAT_PHASE_4);
|
|
|
+
|
|
|
+ CC_CYCLE_DESC_TAIL(cc_base, &req_mgr_h->monitor_desc, ssi_req->is_monitored_p);
|
|
|
+
|
|
|
+ if (unlikely(req_mgr_h->q_free_slots < total_seq_len)) {
|
|
|
+ /*This means that there was a problem with the resume*/
|
|
|
+ BUG();
|
|
|
+ }
|
|
|
+ /* Update the free slots in HW queue */
|
|
|
+ req_mgr_h->q_free_slots -= total_seq_len;
|
|
|
+
|
|
|
+ spin_unlock_bh(&req_mgr_h->hw_lock);
|
|
|
+
|
|
|
+ if (!is_dout) {
|
|
|
+ /* Wait upon sequence completion.
|
|
|
+ * Return "0" -Operation done successfully. */
|
|
|
+ return wait_for_completion_interruptible(&ssi_req->seq_compl);
|
|
|
+ } else {
|
|
|
+ /* Operation still in process */
|
|
|
+ return -EINPROGRESS;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+/*!
|
|
|
+ * Enqueue caller request to crypto hardware during init process.
|
|
|
+ * assume this function is not called in middle of a flow,
|
|
|
+ * since we set QUEUE_LAST_IND flag in the last descriptor.
|
|
|
+ *
|
|
|
+ * \param drvdata
|
|
|
+ * \param desc The crypto sequence
|
|
|
+ * \param len The crypto sequence length
|
|
|
+ *
|
|
|
+ * \return int Returns "0" upon success
|
|
|
+ */
|
|
|
+int send_request_init(
|
|
|
+ struct ssi_drvdata *drvdata, HwDesc_s *desc, unsigned int len)
|
|
|
+{
|
|
|
+ void __iomem *cc_base = drvdata->cc_base;
|
|
|
+ struct ssi_request_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
|
|
|
+ unsigned int total_seq_len = len; /*initial sequence length*/
|
|
|
+ int rc = 0;
|
|
|
+
|
|
|
+ /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT. */
|
|
|
+ rc = request_mgr_queues_status_check(req_mgr_h, cc_base, total_seq_len);
|
|
|
+ if (unlikely(rc != 0 )) {
|
|
|
+ return rc;
|
|
|
+ }
|
|
|
+ HW_DESC_SET_QUEUE_LAST_IND(&desc[len-1]);
|
|
|
+
|
|
|
+ enqueue_seq(cc_base, desc, len);
|
|
|
+
|
|
|
+ /* Update the free slots in HW queue */
|
|
|
+ req_mgr_h->q_free_slots = CC_HAL_READ_REGISTER(
|
|
|
+ CC_REG_OFFSET(CRY_KERNEL,
|
|
|
+ DSCRPTR_QUEUE_CONTENT));
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+void complete_request(struct ssi_drvdata *drvdata)
|
|
|
+{
|
|
|
+ struct ssi_request_mgr_handle *request_mgr_handle =
|
|
|
+ drvdata->request_mgr_handle;
|
|
|
+#ifdef COMP_IN_WQ
|
|
|
+ queue_delayed_work(request_mgr_handle->workq, &request_mgr_handle->compwork, 0);
|
|
|
+#else
|
|
|
+ tasklet_schedule(&request_mgr_handle->comptask);
|
|
|
+#endif
|
|
|
+}
|
|
|
+
|
|
|
+#ifdef COMP_IN_WQ
|
|
|
+static void comp_work_handler(struct work_struct *work)
|
|
|
+{
|
|
|
+ struct ssi_drvdata *drvdata =
|
|
|
+ container_of(work, struct ssi_drvdata, compwork.work);
|
|
|
+
|
|
|
+ comp_handler((unsigned long)drvdata);
|
|
|
+}
|
|
|
+#endif
|
|
|
+
|
|
|
+static void proc_completions(struct ssi_drvdata *drvdata)
|
|
|
+{
|
|
|
+ struct ssi_crypto_req *ssi_req;
|
|
|
+ struct platform_device *plat_dev = drvdata->plat_dev;
|
|
|
+ struct ssi_request_mgr_handle * request_mgr_handle =
|
|
|
+ drvdata->request_mgr_handle;
|
|
|
+#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP)
|
|
|
+ int rc = 0;
|
|
|
+#endif
|
|
|
+ DECL_CYCLE_COUNT_RESOURCES;
|
|
|
+
|
|
|
+ while(request_mgr_handle->axi_completed) {
|
|
|
+ request_mgr_handle->axi_completed--;
|
|
|
+
|
|
|
+ /* Dequeue request */
|
|
|
+ if (unlikely(request_mgr_handle->req_queue_head == request_mgr_handle->req_queue_tail)) {
|
|
|
+ SSI_LOG_ERR("Request queue is empty req_queue_head==req_queue_tail==%u\n", request_mgr_handle->req_queue_head);
|
|
|
+ BUG();
|
|
|
+ }
|
|
|
+
|
|
|
+ ssi_req = &request_mgr_handle->req_queue[request_mgr_handle->req_queue_tail];
|
|
|
+ END_CYCLE_COUNT_AT(ssi_req->submit_cycle, ssi_req->op_type, STAT_PHASE_5); /* Seq. Comp. */
|
|
|
+ END_CC_MONITOR_COUNT(drvdata->cc_base, ssi_req->op_type, STAT_PHASE_6,
|
|
|
+ drvdata->monitor_null_cycles, &request_mgr_handle->monitor_lock, ssi_req->is_monitored_p);
|
|
|
+
|
|
|
+#ifdef FLUSH_CACHE_ALL
|
|
|
+ flush_cache_all();
|
|
|
+#endif
|
|
|
+
|
|
|
+#ifdef COMPLETION_DELAY
|
|
|
+ /* Delay */
|
|
|
+ {
|
|
|
+ uint32_t axi_err;
|
|
|
+ int i;
|
|
|
+ SSI_LOG_INFO("Delay\n");
|
|
|
+ for (i=0;i<1000000;i++) {
|
|
|
+ axi_err = READ_REGISTER(drvdata->cc_base + CC_REG_OFFSET(CRY_KERNEL, AXIM_MON_ERR));
|
|
|
+ }
|
|
|
+ }
|
|
|
+#endif /* COMPLETION_DELAY */
|
|
|
+
|
|
|
+ if (likely(ssi_req->user_cb != NULL)) {
|
|
|
+ START_CYCLE_COUNT();
|
|
|
+ ssi_req->user_cb(&plat_dev->dev, ssi_req->user_arg, drvdata->cc_base);
|
|
|
+ END_CYCLE_COUNT(STAT_OP_TYPE_GENERIC, STAT_PHASE_3);
|
|
|
+ }
|
|
|
+ request_mgr_handle->req_queue_tail = (request_mgr_handle->req_queue_tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
|
|
|
+ SSI_LOG_DEBUG("Dequeue request tail=%u\n", request_mgr_handle->req_queue_tail);
|
|
|
+ SSI_LOG_DEBUG("Request completed. axi_completed=%d\n", request_mgr_handle->axi_completed);
|
|
|
+#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP)
|
|
|
+ rc = ssi_power_mgr_runtime_put_suspend(&plat_dev->dev);
|
|
|
+ if (rc != 0) {
|
|
|
+ SSI_LOG_ERR("Failed to set runtime suspension %d\n",rc);
|
|
|
+ }
|
|
|
+#endif
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/* Deferred service handler, run as interrupt-fired tasklet */
|
|
|
+static void comp_handler(unsigned long devarg)
|
|
|
+{
|
|
|
+ struct ssi_drvdata *drvdata = (struct ssi_drvdata *)devarg;
|
|
|
+ void __iomem *cc_base = drvdata->cc_base;
|
|
|
+ struct ssi_request_mgr_handle * request_mgr_handle =
|
|
|
+ drvdata->request_mgr_handle;
|
|
|
+
|
|
|
+ uint32_t irq;
|
|
|
+
|
|
|
+ DECL_CYCLE_COUNT_RESOURCES;
|
|
|
+
|
|
|
+ START_CYCLE_COUNT();
|
|
|
+
|
|
|
+ irq = (drvdata->irq & SSI_COMP_IRQ_MASK);
|
|
|
+
|
|
|
+ if (irq & SSI_COMP_IRQ_MASK) {
|
|
|
+ /* To avoid the interrupt from firing as we unmask it, we clear it now */
|
|
|
+ CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_ICR), SSI_COMP_IRQ_MASK);
|
|
|
+
|
|
|
+ /* Avoid race with above clear: Test completion counter once more */
|
|
|
+ request_mgr_handle->axi_completed += CC_REG_FLD_GET(CRY_KERNEL, AXIM_MON_COMP, VALUE,
|
|
|
+ CC_HAL_READ_REGISTER(AXIM_MON_BASE_OFFSET));
|
|
|
+
|
|
|
+ /* ISR-to-Tasklet latency */
|
|
|
+ if (request_mgr_handle->axi_completed) {
|
|
|
+ /* Only if actually reflects ISR-to-completion-handling latency, i.e.,
|
|
|
+ not duplicate as a result of interrupt after AXIM_MON_ERR clear, before end of loop */
|
|
|
+ END_CYCLE_COUNT_AT(drvdata->isr_exit_cycles, STAT_OP_TYPE_GENERIC, STAT_PHASE_1);
|
|
|
+ }
|
|
|
+
|
|
|
+ while (request_mgr_handle->axi_completed) {
|
|
|
+ do {
|
|
|
+ proc_completions(drvdata);
|
|
|
+ /* At this point (after proc_completions()), request_mgr_handle->axi_completed is always 0.
|
|
|
+ The following assignment was changed to = (previously was +=) to conform KW restrictions. */
|
|
|
+ request_mgr_handle->axi_completed = CC_REG_FLD_GET(CRY_KERNEL, AXIM_MON_COMP, VALUE,
|
|
|
+ CC_HAL_READ_REGISTER(AXIM_MON_BASE_OFFSET));
|
|
|
+ } while (request_mgr_handle->axi_completed > 0);
|
|
|
+
|
|
|
+ /* To avoid the interrupt from firing as we unmask it, we clear it now */
|
|
|
+ CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_ICR), SSI_COMP_IRQ_MASK);
|
|
|
+
|
|
|
+ /* Avoid race with above clear: Test completion counter once more */
|
|
|
+ request_mgr_handle->axi_completed += CC_REG_FLD_GET(CRY_KERNEL, AXIM_MON_COMP, VALUE,
|
|
|
+ CC_HAL_READ_REGISTER(AXIM_MON_BASE_OFFSET));
|
|
|
+ };
|
|
|
+
|
|
|
+ }
|
|
|
+ /* after verifing that there is nothing to do, Unmask AXI completion interrupt */
|
|
|
+ CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IMR),
|
|
|
+ CC_HAL_READ_REGISTER(
|
|
|
+ CC_REG_OFFSET(HOST_RGF, HOST_IMR)) & ~irq);
|
|
|
+ END_CYCLE_COUNT(STAT_OP_TYPE_GENERIC, STAT_PHASE_2);
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+resume the queue configuration - no need to take the lock as this happens inside
|
|
|
+the spin lock protection
|
|
|
+*/
|
|
|
+#if defined (CONFIG_PM_RUNTIME) || defined (CONFIG_PM_SLEEP)
|
|
|
+int ssi_request_mgr_runtime_resume_queue(struct ssi_drvdata *drvdata)
|
|
|
+{
|
|
|
+ struct ssi_request_mgr_handle * request_mgr_handle = drvdata->request_mgr_handle;
|
|
|
+
|
|
|
+ spin_lock_bh(&request_mgr_handle->hw_lock);
|
|
|
+ request_mgr_handle->is_runtime_suspended = false;
|
|
|
+ spin_unlock_bh(&request_mgr_handle->hw_lock);
|
|
|
+
|
|
|
+ return 0 ;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+suspend the queue configuration. Since it is used for the runtime suspend
|
|
|
+only verify that the queue can be suspended.
|
|
|
+*/
|
|
|
+int ssi_request_mgr_runtime_suspend_queue(struct ssi_drvdata *drvdata)
|
|
|
+{
|
|
|
+ struct ssi_request_mgr_handle * request_mgr_handle =
|
|
|
+ drvdata->request_mgr_handle;
|
|
|
+
|
|
|
+ /* lock the send_request */
|
|
|
+ spin_lock_bh(&request_mgr_handle->hw_lock);
|
|
|
+ if (request_mgr_handle->req_queue_head !=
|
|
|
+ request_mgr_handle->req_queue_tail) {
|
|
|
+ spin_unlock_bh(&request_mgr_handle->hw_lock);
|
|
|
+ return -EBUSY;
|
|
|
+ }
|
|
|
+ request_mgr_handle->is_runtime_suspended = true;
|
|
|
+ spin_unlock_bh(&request_mgr_handle->hw_lock);
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+bool ssi_request_mgr_is_queue_runtime_suspend(struct ssi_drvdata *drvdata)
|
|
|
+{
|
|
|
+ struct ssi_request_mgr_handle * request_mgr_handle =
|
|
|
+ drvdata->request_mgr_handle;
|
|
|
+
|
|
|
+ return request_mgr_handle->is_runtime_suspended;
|
|
|
+}
|
|
|
+
|
|
|
+#endif
|
|
|
+
|