|
|
@@ -1032,22 +1032,6 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
|
|
|
if (!rx_ring->rx_bi)
|
|
|
return;
|
|
|
|
|
|
- if (ring_is_ps_enabled(rx_ring)) {
|
|
|
- int bufsz = ALIGN(rx_ring->rx_hdr_len, 256) * rx_ring->count;
|
|
|
-
|
|
|
- rx_bi = &rx_ring->rx_bi[0];
|
|
|
- if (rx_bi->hdr_buf) {
|
|
|
- dma_free_coherent(dev,
|
|
|
- bufsz,
|
|
|
- rx_bi->hdr_buf,
|
|
|
- rx_bi->dma);
|
|
|
- for (i = 0; i < rx_ring->count; i++) {
|
|
|
- rx_bi = &rx_ring->rx_bi[i];
|
|
|
- rx_bi->dma = 0;
|
|
|
- rx_bi->hdr_buf = NULL;
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
/* Free all the Rx ring sk_buffs */
|
|
|
for (i = 0; i < rx_ring->count; i++) {
|
|
|
rx_bi = &rx_ring->rx_bi[i];
|
|
|
@@ -1502,230 +1486,6 @@ static inline void i40e_rx_hash(struct i40e_ring *ring,
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-/**
|
|
|
- * i40e_clean_rx_irq_ps - Reclaim resources after receive; packet split
|
|
|
- * @rx_ring: rx ring to clean
|
|
|
- * @budget: how many cleans we're allowed
|
|
|
- *
|
|
|
- * Returns true if there's any budget left (e.g. the clean is finished)
|
|
|
- **/
|
|
|
-static int i40e_clean_rx_irq_ps(struct i40e_ring *rx_ring, const int budget)
|
|
|
-{
|
|
|
- unsigned int total_rx_bytes = 0, total_rx_packets = 0;
|
|
|
- u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
|
|
|
- u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
|
|
|
- struct i40e_vsi *vsi = rx_ring->vsi;
|
|
|
- u16 i = rx_ring->next_to_clean;
|
|
|
- union i40e_rx_desc *rx_desc;
|
|
|
- u32 rx_error, rx_status;
|
|
|
- bool failure = false;
|
|
|
- u8 rx_ptype;
|
|
|
- u64 qword;
|
|
|
- u32 copysize;
|
|
|
-
|
|
|
- if (budget <= 0)
|
|
|
- return 0;
|
|
|
-
|
|
|
- do {
|
|
|
- struct i40e_rx_buffer *rx_bi;
|
|
|
- struct sk_buff *skb;
|
|
|
- u16 vlan_tag;
|
|
|
- /* return some buffers to hardware, one at a time is too slow */
|
|
|
- if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
|
|
|
- failure = failure ||
|
|
|
- i40e_alloc_rx_buffers_ps(rx_ring,
|
|
|
- cleaned_count);
|
|
|
- cleaned_count = 0;
|
|
|
- }
|
|
|
-
|
|
|
- i = rx_ring->next_to_clean;
|
|
|
- rx_desc = I40E_RX_DESC(rx_ring, i);
|
|
|
- qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
|
|
|
- rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
|
|
|
- I40E_RXD_QW1_STATUS_SHIFT;
|
|
|
-
|
|
|
- if (!(rx_status & BIT(I40E_RX_DESC_STATUS_DD_SHIFT)))
|
|
|
- break;
|
|
|
-
|
|
|
- /* This memory barrier is needed to keep us from reading
|
|
|
- * any other fields out of the rx_desc until we know the
|
|
|
- * DD bit is set.
|
|
|
- */
|
|
|
- dma_rmb();
|
|
|
- /* sync header buffer for reading */
|
|
|
- dma_sync_single_range_for_cpu(rx_ring->dev,
|
|
|
- rx_ring->rx_bi[0].dma,
|
|
|
- i * rx_ring->rx_hdr_len,
|
|
|
- rx_ring->rx_hdr_len,
|
|
|
- DMA_FROM_DEVICE);
|
|
|
- if (i40e_rx_is_programming_status(qword)) {
|
|
|
- i40e_clean_programming_status(rx_ring, rx_desc);
|
|
|
- I40E_RX_INCREMENT(rx_ring, i);
|
|
|
- continue;
|
|
|
- }
|
|
|
- rx_bi = &rx_ring->rx_bi[i];
|
|
|
- skb = rx_bi->skb;
|
|
|
- if (likely(!skb)) {
|
|
|
- skb = __netdev_alloc_skb_ip_align(rx_ring->netdev,
|
|
|
- rx_ring->rx_hdr_len,
|
|
|
- GFP_ATOMIC |
|
|
|
- __GFP_NOWARN);
|
|
|
- if (!skb) {
|
|
|
- rx_ring->rx_stats.alloc_buff_failed++;
|
|
|
- failure = true;
|
|
|
- break;
|
|
|
- }
|
|
|
-
|
|
|
- /* initialize queue mapping */
|
|
|
- skb_record_rx_queue(skb, rx_ring->queue_index);
|
|
|
- /* we are reusing so sync this buffer for CPU use */
|
|
|
- dma_sync_single_range_for_cpu(rx_ring->dev,
|
|
|
- rx_ring->rx_bi[0].dma,
|
|
|
- i * rx_ring->rx_hdr_len,
|
|
|
- rx_ring->rx_hdr_len,
|
|
|
- DMA_FROM_DEVICE);
|
|
|
- }
|
|
|
- rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
|
|
|
- I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
|
|
|
- rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
|
|
|
- I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
|
|
|
- rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
|
|
|
- I40E_RXD_QW1_LENGTH_SPH_SHIFT;
|
|
|
-
|
|
|
- rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
|
|
|
- I40E_RXD_QW1_ERROR_SHIFT;
|
|
|
- rx_hbo = rx_error & BIT(I40E_RX_DESC_ERROR_HBO_SHIFT);
|
|
|
- rx_error &= ~BIT(I40E_RX_DESC_ERROR_HBO_SHIFT);
|
|
|
-
|
|
|
- rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
|
|
|
- I40E_RXD_QW1_PTYPE_SHIFT;
|
|
|
- /* sync half-page for reading */
|
|
|
- dma_sync_single_range_for_cpu(rx_ring->dev,
|
|
|
- rx_bi->page_dma,
|
|
|
- rx_bi->page_offset,
|
|
|
- PAGE_SIZE / 2,
|
|
|
- DMA_FROM_DEVICE);
|
|
|
- prefetch(page_address(rx_bi->page) + rx_bi->page_offset);
|
|
|
- rx_bi->skb = NULL;
|
|
|
- cleaned_count++;
|
|
|
- copysize = 0;
|
|
|
- if (rx_hbo || rx_sph) {
|
|
|
- int len;
|
|
|
-
|
|
|
- if (rx_hbo)
|
|
|
- len = I40E_RX_HDR_SIZE;
|
|
|
- else
|
|
|
- len = rx_header_len;
|
|
|
- memcpy(__skb_put(skb, len), rx_bi->hdr_buf, len);
|
|
|
- } else if (skb->len == 0) {
|
|
|
- int len;
|
|
|
- unsigned char *va = page_address(rx_bi->page) +
|
|
|
- rx_bi->page_offset;
|
|
|
-
|
|
|
- len = min(rx_packet_len, rx_ring->rx_hdr_len);
|
|
|
- memcpy(__skb_put(skb, len), va, len);
|
|
|
- copysize = len;
|
|
|
- rx_packet_len -= len;
|
|
|
- }
|
|
|
- /* Get the rest of the data if this was a header split */
|
|
|
- if (rx_packet_len) {
|
|
|
- skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
|
|
|
- rx_bi->page,
|
|
|
- rx_bi->page_offset + copysize,
|
|
|
- rx_packet_len, I40E_RXBUFFER_2048);
|
|
|
-
|
|
|
- /* If the page count is more than 2, then both halves
|
|
|
- * of the page are used and we need to free it. Do it
|
|
|
- * here instead of in the alloc code. Otherwise one
|
|
|
- * of the half-pages might be released between now and
|
|
|
- * then, and we wouldn't know which one to use.
|
|
|
- * Don't call get_page and free_page since those are
|
|
|
- * both expensive atomic operations that just change
|
|
|
- * the refcount in opposite directions. Just give the
|
|
|
- * page to the stack; he can have our refcount.
|
|
|
- */
|
|
|
- if (page_count(rx_bi->page) > 2) {
|
|
|
- dma_unmap_page(rx_ring->dev,
|
|
|
- rx_bi->page_dma,
|
|
|
- PAGE_SIZE,
|
|
|
- DMA_FROM_DEVICE);
|
|
|
- rx_bi->page = NULL;
|
|
|
- rx_bi->page_dma = 0;
|
|
|
- rx_ring->rx_stats.realloc_count++;
|
|
|
- } else {
|
|
|
- get_page(rx_bi->page);
|
|
|
- /* switch to the other half-page here; the
|
|
|
- * allocation code programs the right addr
|
|
|
- * into HW. If we haven't used this half-page,
|
|
|
- * the address won't be changed, and HW can
|
|
|
- * just use it next time through.
|
|
|
- */
|
|
|
- rx_bi->page_offset ^= PAGE_SIZE / 2;
|
|
|
- }
|
|
|
-
|
|
|
- }
|
|
|
- I40E_RX_INCREMENT(rx_ring, i);
|
|
|
-
|
|
|
- if (unlikely(
|
|
|
- !(rx_status & BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
|
|
|
- struct i40e_rx_buffer *next_buffer;
|
|
|
-
|
|
|
- next_buffer = &rx_ring->rx_bi[i];
|
|
|
- next_buffer->skb = skb;
|
|
|
- rx_ring->rx_stats.non_eop_descs++;
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- /* ERR_MASK will only have valid bits if EOP set */
|
|
|
- if (unlikely(rx_error & BIT(I40E_RX_DESC_ERROR_RXE_SHIFT))) {
|
|
|
- dev_kfree_skb_any(skb);
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
|
|
|
-
|
|
|
- if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
|
|
|
- i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
|
|
|
- I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
|
|
|
- I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
|
|
|
- rx_ring->last_rx_timestamp = jiffies;
|
|
|
- }
|
|
|
-
|
|
|
- /* probably a little skewed due to removing CRC */
|
|
|
- total_rx_bytes += skb->len;
|
|
|
- total_rx_packets++;
|
|
|
-
|
|
|
- skb->protocol = eth_type_trans(skb, rx_ring->netdev);
|
|
|
-
|
|
|
- i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
|
|
|
-
|
|
|
- vlan_tag = rx_status & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
|
|
|
- ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
|
|
|
- : 0;
|
|
|
-#ifdef I40E_FCOE
|
|
|
- if (unlikely(
|
|
|
- i40e_rx_is_fcoe(rx_ptype) &&
|
|
|
- !i40e_fcoe_handle_offload(rx_ring, rx_desc, skb))) {
|
|
|
- dev_kfree_skb_any(skb);
|
|
|
- continue;
|
|
|
- }
|
|
|
-#endif
|
|
|
- i40e_receive_skb(rx_ring, skb, vlan_tag);
|
|
|
-
|
|
|
- rx_desc->wb.qword1.status_error_len = 0;
|
|
|
-
|
|
|
- } while (likely(total_rx_packets < budget));
|
|
|
-
|
|
|
- u64_stats_update_begin(&rx_ring->syncp);
|
|
|
- rx_ring->stats.packets += total_rx_packets;
|
|
|
- rx_ring->stats.bytes += total_rx_bytes;
|
|
|
- u64_stats_update_end(&rx_ring->syncp);
|
|
|
- rx_ring->q_vector->rx.total_packets += total_rx_packets;
|
|
|
- rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
|
|
|
-
|
|
|
- return failure ? budget : total_rx_packets;
|
|
|
-}
|
|
|
-
|
|
|
/**
|
|
|
* i40e_clean_rx_irq_1buf - Reclaim resources after receive; single buffer
|
|
|
* @rx_ring: rx ring to clean
|
|
|
@@ -2001,10 +1761,7 @@ int i40e_napi_poll(struct napi_struct *napi, int budget)
|
|
|
i40e_for_each_ring(ring, q_vector->rx) {
|
|
|
int cleaned;
|
|
|
|
|
|
- if (ring_is_ps_enabled(ring))
|
|
|
- cleaned = i40e_clean_rx_irq_ps(ring, budget_per_ring);
|
|
|
- else
|
|
|
- cleaned = i40e_clean_rx_irq_1buf(ring, budget_per_ring);
|
|
|
+ cleaned = i40e_clean_rx_irq_1buf(ring, budget_per_ring);
|
|
|
|
|
|
work_done += cleaned;
|
|
|
/* if we clean as many as budgeted, we must not be done */
|
Jesse Brandeburg