|
|
@@ -406,7 +406,7 @@ static void free_tx_desc(struct adapter *adap, struct sge_txq *q,
|
|
|
*/
|
|
|
static inline int reclaimable(const struct sge_txq *q)
|
|
|
{
|
|
|
- int hw_cidx = ntohs(q->stat->cidx);
|
|
|
+ int hw_cidx = ntohs(ACCESS_ONCE(q->stat->cidx));
|
|
|
hw_cidx -= q->cidx;
|
|
|
return hw_cidx < 0 ? hw_cidx + q->size : hw_cidx;
|
|
|
}
|
|
|
@@ -613,6 +613,7 @@ static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n,
|
|
|
PCI_DMA_FROMDEVICE);
|
|
|
if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
|
|
|
__free_pages(pg, s->fl_pg_order);
|
|
|
+ q->mapping_err++;
|
|
|
goto out; /* do not try small pages for this error */
|
|
|
}
|
|
|
mapping |= RX_LARGE_PG_BUF;
|
|
|
@@ -642,6 +643,7 @@ alloc_small_pages:
|
|
|
PCI_DMA_FROMDEVICE);
|
|
|
if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
|
|
|
put_page(pg);
|
|
|
+ q->mapping_err++;
|
|
|
goto out;
|
|
|
}
|
|
|
*d++ = cpu_to_be64(mapping);
|
|
|
@@ -663,6 +665,7 @@ out: cred = q->avail - cred;
|
|
|
|
|
|
if (unlikely(fl_starving(adap, q))) {
|
|
|
smp_wmb();
|
|
|
+ q->low++;
|
|
|
set_bit(q->cntxt_id - adap->sge.egr_start,
|
|
|
adap->sge.starving_fl);
|
|
|
}
|
|
|
@@ -1029,6 +1032,30 @@ static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q,
|
|
|
*p = 0;
|
|
|
}
|
|
|
|
|
|
+static void *inline_tx_skb_header(const struct sk_buff *skb,
|
|
|
+ const struct sge_txq *q, void *pos,
|
|
|
+ int length)
|
|
|
+{
|
|
|
+ u64 *p;
|
|
|
+ int left = (void *)q->stat - pos;
|
|
|
+
|
|
|
+ if (likely(length <= left)) {
|
|
|
+ memcpy(pos, skb->data, length);
|
|
|
+ pos += length;
|
|
|
+ } else {
|
|
|
+ memcpy(pos, skb->data, left);
|
|
|
+ memcpy(q->desc, skb->data + left, length - left);
|
|
|
+ pos = (void *)q->desc + (length - left);
|
|
|
+ }
|
|
|
+ /* 0-pad to multiple of 16 */
|
|
|
+ p = PTR_ALIGN(pos, 8);
|
|
|
+ if ((uintptr_t)p & 8) {
|
|
|
+ *p = 0;
|
|
|
+ return p + 1;
|
|
|
+ }
|
|
|
+ return p;
|
|
|
+}
|
|
|
+
|
|
|
/*
|
|
|
* Figure out what HW csum a packet wants and return the appropriate control
|
|
|
* bits.
|
|
|
@@ -1320,7 +1347,7 @@ out_free: dev_kfree_skb_any(skb);
|
|
|
*/
|
|
|
static inline void reclaim_completed_tx_imm(struct sge_txq *q)
|
|
|
{
|
|
|
- int hw_cidx = ntohs(q->stat->cidx);
|
|
|
+ int hw_cidx = ntohs(ACCESS_ONCE(q->stat->cidx));
|
|
|
int reclaim = hw_cidx - q->cidx;
|
|
|
|
|
|
if (reclaim < 0)
|
|
|
@@ -1542,24 +1569,50 @@ static void ofldtxq_stop(struct sge_ofld_txq *q, struct sk_buff *skb)
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * service_ofldq - restart a suspended offload queue
|
|
|
+ * service_ofldq - service/restart a suspended offload queue
|
|
|
* @q: the offload queue
|
|
|
*
|
|
|
- * Services an offload Tx queue by moving packets from its packet queue
|
|
|
- * to the HW Tx ring. The function starts and ends with the queue locked.
|
|
|
+ * Services an offload Tx queue by moving packets from its Pending Send
|
|
|
+ * Queue to the Hardware TX ring. The function starts and ends with the
|
|
|
+ * Send Queue locked, but drops the lock while putting the skb at the
|
|
|
+ * head of the Send Queue onto the Hardware TX Ring. Dropping the lock
|
|
|
+ * allows more skbs to be added to the Send Queue by other threads.
|
|
|
+ * The packet being processed at the head of the Pending Send Queue is
|
|
|
+ * left on the queue in case we experience DMA Mapping errors, etc.
|
|
|
+ * and need to give up and restart later.
|
|
|
+ *
|
|
|
+ * service_ofldq() can be thought of as a task which opportunistically
|
|
|
+ * uses other threads execution contexts. We use the Offload Queue
|
|
|
+ * boolean "service_ofldq_running" to make sure that only one instance
|
|
|
+ * is ever running at a time ...
|
|
|
*/
|
|
|
static void service_ofldq(struct sge_ofld_txq *q)
|
|
|
{
|
|
|
- u64 *pos;
|
|
|
+ u64 *pos, *before, *end;
|
|
|
int credits;
|
|
|
struct sk_buff *skb;
|
|
|
+ struct sge_txq *txq;
|
|
|
+ unsigned int left;
|
|
|
unsigned int written = 0;
|
|
|
unsigned int flits, ndesc;
|
|
|
|
|
|
+ /* If another thread is currently in service_ofldq() processing the
|
|
|
+ * Pending Send Queue then there's nothing to do. Otherwise, flag
|
|
|
+ * that we're doing the work and continue. Examining/modifying
|
|
|
+ * the Offload Queue boolean "service_ofldq_running" must be done
|
|
|
+ * while holding the Pending Send Queue Lock.
|
|
|
+ */
|
|
|
+ if (q->service_ofldq_running)
|
|
|
+ return;
|
|
|
+ q->service_ofldq_running = true;
|
|
|
+
|
|
|
while ((skb = skb_peek(&q->sendq)) != NULL && !q->full) {
|
|
|
- /*
|
|
|
- * We drop the lock but leave skb on sendq, thus retaining
|
|
|
- * exclusive access to the state of the queue.
|
|
|
+ /* We drop the lock while we're working with the skb at the
|
|
|
+ * head of the Pending Send Queue. This allows more skbs to
|
|
|
+ * be added to the Pending Send Queue while we're working on
|
|
|
+ * this one. We don't need to lock to guard the TX Ring
|
|
|
+ * updates because only one thread of execution is ever
|
|
|
+ * allowed into service_ofldq() at a time.
|
|
|
*/
|
|
|
spin_unlock(&q->sendq.lock);
|
|
|
|
|
|
@@ -1583,9 +1636,32 @@ static void service_ofldq(struct sge_ofld_txq *q)
|
|
|
} else {
|
|
|
int last_desc, hdr_len = skb_transport_offset(skb);
|
|
|
|
|
|
- memcpy(pos, skb->data, hdr_len);
|
|
|
- write_sgl(skb, &q->q, (void *)pos + hdr_len,
|
|
|
- pos + flits, hdr_len,
|
|
|
+ /* The WR headers may not fit within one descriptor.
|
|
|
+ * So we need to deal with wrap-around here.
|
|
|
+ */
|
|
|
+ before = (u64 *)pos;
|
|
|
+ end = (u64 *)pos + flits;
|
|
|
+ txq = &q->q;
|
|
|
+ pos = (void *)inline_tx_skb_header(skb, &q->q,
|
|
|
+ (void *)pos,
|
|
|
+ hdr_len);
|
|
|
+ if (before > (u64 *)pos) {
|
|
|
+ left = (u8 *)end - (u8 *)txq->stat;
|
|
|
+ end = (void *)txq->desc + left;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* If current position is already at the end of the
|
|
|
+ * ofld queue, reset the current to point to
|
|
|
+ * start of the queue and update the end ptr as well.
|
|
|
+ */
|
|
|
+ if (pos == (u64 *)txq->stat) {
|
|
|
+ left = (u8 *)end - (u8 *)txq->stat;
|
|
|
+ end = (void *)txq->desc + left;
|
|
|
+ pos = (void *)txq->desc;
|
|
|
+ }
|
|
|
+
|
|
|
+ write_sgl(skb, &q->q, (void *)pos,
|
|
|
+ end, hdr_len,
|
|
|
(dma_addr_t *)skb->head);
|
|
|
#ifdef CONFIG_NEED_DMA_MAP_STATE
|
|
|
skb->dev = q->adap->port[0];
|
|
|
@@ -1604,6 +1680,11 @@ static void service_ofldq(struct sge_ofld_txq *q)
|
|
|
written = 0;
|
|
|
}
|
|
|
|
|
|
+ /* Reacquire the Pending Send Queue Lock so we can unlink the
|
|
|
+ * skb we've just successfully transferred to the TX Ring and
|
|
|
+ * loop for the next skb which may be at the head of the
|
|
|
+ * Pending Send Queue.
|
|
|
+ */
|
|
|
spin_lock(&q->sendq.lock);
|
|
|
__skb_unlink(skb, &q->sendq);
|
|
|
if (is_ofld_imm(skb))
|
|
|
@@ -1611,6 +1692,11 @@ static void service_ofldq(struct sge_ofld_txq *q)
|
|
|
}
|
|
|
if (likely(written))
|
|
|
ring_tx_db(q->adap, &q->q, written);
|
|
|
+
|
|
|
+ /*Indicate that no thread is processing the Pending Send Queue
|
|
|
+ * currently.
|
|
|
+ */
|
|
|
+ q->service_ofldq_running = false;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -1624,9 +1710,19 @@ static int ofld_xmit(struct sge_ofld_txq *q, struct sk_buff *skb)
|
|
|
{
|
|
|
skb->priority = calc_tx_flits_ofld(skb); /* save for restart */
|
|
|
spin_lock(&q->sendq.lock);
|
|
|
+
|
|
|
+ /* Queue the new skb onto the Offload Queue's Pending Send Queue. If
|
|
|
+ * that results in this new skb being the only one on the queue, start
|
|
|
+ * servicing it. If there are other skbs already on the list, then
|
|
|
+ * either the queue is currently being processed or it's been stopped
|
|
|
+ * for some reason and it'll be restarted at a later time. Restart
|
|
|
+ * paths are triggered by events like experiencing a DMA Mapping Error
|
|
|
+ * or filling the Hardware TX Ring.
|
|
|
+ */
|
|
|
__skb_queue_tail(&q->sendq, skb);
|
|
|
if (q->sendq.qlen == 1)
|
|
|
service_ofldq(q);
|
|
|
+
|
|
|
spin_unlock(&q->sendq.lock);
|
|
|
return NET_XMIT_SUCCESS;
|
|
|
}
|
David S. Miller