|
@@ -42,6 +42,9 @@
|
|
|
#ifdef CONFIG_RFS_ACCEL
|
|
|
#include <linux/cpu_rmap.h>
|
|
|
#endif
|
|
|
+#ifdef CONFIG_NET_RX_BUSY_POLL
|
|
|
+#include <net/busy_poll.h>
|
|
|
+#endif
|
|
|
|
|
|
#include "cq_enet_desc.h"
|
|
|
#include "vnic_dev.h"
|
|
@@ -1053,10 +1056,12 @@ static void enic_rq_indicate_buf(struct vnic_rq *rq,
|
|
|
if (vlan_stripped)
|
|
|
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci);
|
|
|
|
|
|
- if (netdev->features & NETIF_F_GRO)
|
|
|
- napi_gro_receive(&enic->napi[q_number], skb);
|
|
|
- else
|
|
|
+ skb_mark_napi_id(skb, &enic->napi[rq->index]);
|
|
|
+ if (enic_poll_busy_polling(rq) ||
|
|
|
+ !(netdev->features & NETIF_F_GRO))
|
|
|
netif_receive_skb(skb);
|
|
|
+ else
|
|
|
+ napi_gro_receive(&enic->napi[q_number], skb);
|
|
|
if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
|
|
|
enic_intr_update_pkt_size(&cq->pkt_size_counter,
|
|
|
bytes_written);
|
|
@@ -1093,16 +1098,22 @@ static int enic_poll(struct napi_struct *napi, int budget)
|
|
|
unsigned int work_done, rq_work_done = 0, wq_work_done;
|
|
|
int err;
|
|
|
|
|
|
- /* Service RQ (first) and WQ
|
|
|
- */
|
|
|
+ wq_work_done = vnic_cq_service(&enic->cq[cq_wq], wq_work_to_do,
|
|
|
+ enic_wq_service, NULL);
|
|
|
+
|
|
|
+ if (!enic_poll_lock_napi(&enic->rq[cq_rq])) {
|
|
|
+ if (wq_work_done > 0)
|
|
|
+ vnic_intr_return_credits(&enic->intr[intr],
|
|
|
+ wq_work_done,
|
|
|
+ 0 /* dont unmask intr */,
|
|
|
+ 0 /* dont reset intr timer */);
|
|
|
+ return rq_work_done;
|
|
|
+ }
|
|
|
|
|
|
if (budget > 0)
|
|
|
rq_work_done = vnic_cq_service(&enic->cq[cq_rq],
|
|
|
rq_work_to_do, enic_rq_service, NULL);
|
|
|
|
|
|
- wq_work_done = vnic_cq_service(&enic->cq[cq_wq],
|
|
|
- wq_work_to_do, enic_wq_service, NULL);
|
|
|
-
|
|
|
/* Accumulate intr event credits for this polling
|
|
|
* cycle. An intr event is the completion of a
|
|
|
* a WQ or RQ packet.
|
|
@@ -1134,6 +1145,7 @@ static int enic_poll(struct napi_struct *napi, int budget)
|
|
|
napi_complete(napi);
|
|
|
vnic_intr_unmask(&enic->intr[intr]);
|
|
|
}
|
|
|
+ enic_poll_unlock_napi(&enic->rq[cq_rq]);
|
|
|
|
|
|
return rq_work_done;
|
|
|
}
|
|
@@ -1234,6 +1246,34 @@ static void enic_set_rx_cpu_rmap(struct enic *enic)
|
|
|
|
|
|
#endif /* CONFIG_RFS_ACCEL */
|
|
|
|
|
|
+#ifdef CONFIG_NET_RX_BUSY_POLL
|
|
|
+int enic_busy_poll(struct napi_struct *napi)
|
|
|
+{
|
|
|
+ struct net_device *netdev = napi->dev;
|
|
|
+ struct enic *enic = netdev_priv(netdev);
|
|
|
+ unsigned int rq = (napi - &enic->napi[0]);
|
|
|
+ unsigned int cq = enic_cq_rq(enic, rq);
|
|
|
+ unsigned int intr = enic_msix_rq_intr(enic, rq);
|
|
|
+ unsigned int work_to_do = -1; /* clean all pkts possible */
|
|
|
+ unsigned int work_done;
|
|
|
+
|
|
|
+ if (!enic_poll_lock_poll(&enic->rq[rq]))
|
|
|
+ return LL_FLUSH_BUSY;
|
|
|
+ work_done = vnic_cq_service(&enic->cq[cq], work_to_do,
|
|
|
+ enic_rq_service, NULL);
|
|
|
+
|
|
|
+ if (work_done > 0)
|
|
|
+ vnic_intr_return_credits(&enic->intr[intr],
|
|
|
+ work_done, 0, 0);
|
|
|
+ vnic_rq_fill(&enic->rq[rq], enic_rq_alloc_buf);
|
|
|
+ if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
|
|
|
+ enic_calc_int_moderation(enic, &enic->rq[rq]);
|
|
|
+ enic_poll_unlock_poll(&enic->rq[rq]);
|
|
|
+
|
|
|
+ return work_done;
|
|
|
+}
|
|
|
+#endif /* CONFIG_NET_RX_BUSY_POLL */
|
|
|
+
|
|
|
static int enic_poll_msix(struct napi_struct *napi, int budget)
|
|
|
{
|
|
|
struct net_device *netdev = napi->dev;
|
|
@@ -1245,6 +1285,8 @@ static int enic_poll_msix(struct napi_struct *napi, int budget)
|
|
|
unsigned int work_done = 0;
|
|
|
int err;
|
|
|
|
|
|
+ if (!enic_poll_lock_napi(&enic->rq[rq]))
|
|
|
+ return work_done;
|
|
|
/* Service RQ
|
|
|
*/
|
|
|
|
|
@@ -1290,6 +1332,7 @@ static int enic_poll_msix(struct napi_struct *napi, int budget)
|
|
|
enic_set_int_moderation(enic, &enic->rq[rq]);
|
|
|
vnic_intr_unmask(&enic->intr[intr]);
|
|
|
}
|
|
|
+ enic_poll_unlock_napi(&enic->rq[rq]);
|
|
|
|
|
|
return work_done;
|
|
|
}
|
|
@@ -1538,8 +1581,10 @@ static int enic_open(struct net_device *netdev)
|
|
|
|
|
|
netif_tx_wake_all_queues(netdev);
|
|
|
|
|
|
- for (i = 0; i < enic->rq_count; i++)
|
|
|
+ for (i = 0; i < enic->rq_count; i++) {
|
|
|
+ enic_busy_poll_init_lock(&enic->rq[i]);
|
|
|
napi_enable(&enic->napi[i]);
|
|
|
+ }
|
|
|
|
|
|
enic_dev_enable(enic);
|
|
|
|
|
@@ -1578,8 +1623,13 @@ static int enic_stop(struct net_device *netdev)
|
|
|
|
|
|
enic_dev_disable(enic);
|
|
|
|
|
|
- for (i = 0; i < enic->rq_count; i++)
|
|
|
+ local_bh_disable();
|
|
|
+ for (i = 0; i < enic->rq_count; i++) {
|
|
|
napi_disable(&enic->napi[i]);
|
|
|
+ while (!enic_poll_lock_napi(&enic->rq[i]))
|
|
|
+ mdelay(1);
|
|
|
+ }
|
|
|
+ local_bh_enable();
|
|
|
|
|
|
netif_carrier_off(netdev);
|
|
|
netif_tx_disable(netdev);
|
|
@@ -2070,6 +2120,9 @@ static const struct net_device_ops enic_netdev_dynamic_ops = {
|
|
|
#ifdef CONFIG_RFS_ACCEL
|
|
|
.ndo_rx_flow_steer = enic_rx_flow_steer,
|
|
|
#endif
|
|
|
+#ifdef CONFIG_NET_RX_BUSY_POLL
|
|
|
+ .ndo_busy_poll = enic_busy_poll,
|
|
|
+#endif
|
|
|
};
|
|
|
|
|
|
static const struct net_device_ops enic_netdev_ops = {
|
|
@@ -2093,14 +2146,19 @@ static const struct net_device_ops enic_netdev_ops = {
|
|
|
#ifdef CONFIG_RFS_ACCEL
|
|
|
.ndo_rx_flow_steer = enic_rx_flow_steer,
|
|
|
#endif
|
|
|
+#ifdef CONFIG_NET_RX_BUSY_POLL
|
|
|
+ .ndo_busy_poll = enic_busy_poll,
|
|
|
+#endif
|
|
|
};
|
|
|
|
|
|
static void enic_dev_deinit(struct enic *enic)
|
|
|
{
|
|
|
unsigned int i;
|
|
|
|
|
|
- for (i = 0; i < enic->rq_count; i++)
|
|
|
+ for (i = 0; i < enic->rq_count; i++) {
|
|
|
+ napi_hash_del(&enic->napi[i]);
|
|
|
netif_napi_del(&enic->napi[i]);
|
|
|
+ }
|
|
|
|
|
|
enic_free_vnic_resources(enic);
|
|
|
enic_clear_intr_mode(enic);
|
|
@@ -2166,11 +2224,14 @@ static int enic_dev_init(struct enic *enic)
|
|
|
switch (vnic_dev_get_intr_mode(enic->vdev)) {
|
|
|
default:
|
|
|
netif_napi_add(netdev, &enic->napi[0], enic_poll, 64);
|
|
|
+ napi_hash_add(&enic->napi[0]);
|
|
|
break;
|
|
|
case VNIC_DEV_INTR_MODE_MSIX:
|
|
|
- for (i = 0; i < enic->rq_count; i++)
|
|
|
+ for (i = 0; i < enic->rq_count; i++) {
|
|
|
netif_napi_add(netdev, &enic->napi[i],
|
|
|
enic_poll_msix, 64);
|
|
|
+ napi_hash_add(&enic->napi[i]);
|
|
|
+ }
|
|
|
break;
|
|
|
}
|
|
|
|