|
|
@@ -41,22 +41,27 @@
|
|
|
#include <linux/delay.h>
|
|
|
#include <linux/interrupt.h>
|
|
|
#include <linux/if_ether.h>
|
|
|
-
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+#include <linux/dca.h>
|
|
|
+#endif
|
|
|
#include "igb.h"
|
|
|
|
|
|
-#define DRV_VERSION "1.0.8-k2"
|
|
|
+#define DRV_VERSION "1.2.45-k2"
|
|
|
char igb_driver_name[] = "igb";
|
|
|
char igb_driver_version[] = DRV_VERSION;
|
|
|
static const char igb_driver_string[] =
|
|
|
"Intel(R) Gigabit Ethernet Network Driver";
|
|
|
-static const char igb_copyright[] = "Copyright (c) 2007 Intel Corporation.";
|
|
|
-
|
|
|
+static const char igb_copyright[] = "Copyright (c) 2008 Intel Corporation.";
|
|
|
|
|
|
static const struct e1000_info *igb_info_tbl[] = {
|
|
|
[board_82575] = &e1000_82575_info,
|
|
|
};
|
|
|
|
|
|
static struct pci_device_id igb_pci_tbl[] = {
|
|
|
+ { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
|
|
|
+ { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
|
|
|
+ { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
|
|
|
+ { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
|
|
|
{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
|
|
|
{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
|
|
|
{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
|
|
|
@@ -102,10 +107,18 @@ static irqreturn_t igb_msix_other(int irq, void *);
|
|
|
static irqreturn_t igb_msix_rx(int irq, void *);
|
|
|
static irqreturn_t igb_msix_tx(int irq, void *);
|
|
|
static int igb_clean_rx_ring_msix(struct napi_struct *, int);
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+static void igb_update_rx_dca(struct igb_ring *);
|
|
|
+static void igb_update_tx_dca(struct igb_ring *);
|
|
|
+static void igb_setup_dca(struct igb_adapter *);
|
|
|
+#endif /* CONFIG_DCA */
|
|
|
static bool igb_clean_tx_irq(struct igb_ring *);
|
|
|
-static int igb_clean(struct napi_struct *, int);
|
|
|
+static int igb_poll(struct napi_struct *, int);
|
|
|
static bool igb_clean_rx_irq_adv(struct igb_ring *, int *, int);
|
|
|
static void igb_alloc_rx_buffers_adv(struct igb_ring *, int);
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+static int igb_get_skb_hdr(struct sk_buff *skb, void **, void **, u64 *, void *);
|
|
|
+#endif
|
|
|
static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
|
|
|
static void igb_tx_timeout(struct net_device *);
|
|
|
static void igb_reset_task(struct work_struct *);
|
|
|
@@ -119,6 +132,14 @@ static int igb_suspend(struct pci_dev *, pm_message_t);
|
|
|
static int igb_resume(struct pci_dev *);
|
|
|
#endif
|
|
|
static void igb_shutdown(struct pci_dev *);
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
|
|
|
+static struct notifier_block dca_notifier = {
|
|
|
+ .notifier_call = igb_notify_dca,
|
|
|
+ .next = NULL,
|
|
|
+ .priority = 0
|
|
|
+};
|
|
|
+#endif
|
|
|
|
|
|
#ifdef CONFIG_NET_POLL_CONTROLLER
|
|
|
/* for netdump / net console */
|
|
|
@@ -151,6 +172,8 @@ static struct pci_driver igb_driver = {
|
|
|
.err_handler = &igb_err_handler
|
|
|
};
|
|
|
|
|
|
+static int global_quad_port_a; /* global quad port a indication */
|
|
|
+
|
|
|
MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
|
|
|
MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
|
|
|
MODULE_LICENSE("GPL");
|
|
|
@@ -182,7 +205,12 @@ static int __init igb_init_module(void)
|
|
|
|
|
|
printk(KERN_INFO "%s\n", igb_copyright);
|
|
|
|
|
|
+ global_quad_port_a = 0;
|
|
|
+
|
|
|
ret = pci_register_driver(&igb_driver);
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ dca_register_notify(&dca_notifier);
|
|
|
+#endif
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
@@ -196,6 +224,9 @@ module_init(igb_init_module);
|
|
|
**/
|
|
|
static void __exit igb_exit_module(void)
|
|
|
{
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ dca_unregister_notify(&dca_notifier);
|
|
|
+#endif
|
|
|
pci_unregister_driver(&igb_driver);
|
|
|
}
|
|
|
|
|
|
@@ -224,6 +255,13 @@ static int igb_alloc_queues(struct igb_adapter *adapter)
|
|
|
return -ENOMEM;
|
|
|
}
|
|
|
|
|
|
+ adapter->rx_ring->buddy = adapter->tx_ring;
|
|
|
+
|
|
|
+ for (i = 0; i < adapter->num_tx_queues; i++) {
|
|
|
+ struct igb_ring *ring = &(adapter->tx_ring[i]);
|
|
|
+ ring->adapter = adapter;
|
|
|
+ ring->queue_index = i;
|
|
|
+ }
|
|
|
for (i = 0; i < adapter->num_rx_queues; i++) {
|
|
|
struct igb_ring *ring = &(adapter->rx_ring[i]);
|
|
|
ring->adapter = adapter;
|
|
|
@@ -231,17 +269,32 @@ static int igb_alloc_queues(struct igb_adapter *adapter)
|
|
|
ring->itr_register = E1000_ITR;
|
|
|
|
|
|
/* set a default napi handler for each rx_ring */
|
|
|
- netif_napi_add(adapter->netdev, &ring->napi, igb_clean, 64);
|
|
|
+ netif_napi_add(adapter->netdev, &ring->napi, igb_poll, 64);
|
|
|
}
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
+static void igb_free_queues(struct igb_adapter *adapter)
|
|
|
+{
|
|
|
+ int i;
|
|
|
+
|
|
|
+ for (i = 0; i < adapter->num_rx_queues; i++)
|
|
|
+ netif_napi_del(&adapter->rx_ring[i].napi);
|
|
|
+
|
|
|
+ kfree(adapter->tx_ring);
|
|
|
+ kfree(adapter->rx_ring);
|
|
|
+}
|
|
|
+
|
|
|
#define IGB_N0_QUEUE -1
|
|
|
static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue,
|
|
|
int tx_queue, int msix_vector)
|
|
|
{
|
|
|
u32 msixbm = 0;
|
|
|
struct e1000_hw *hw = &adapter->hw;
|
|
|
+ u32 ivar, index;
|
|
|
+
|
|
|
+ switch (hw->mac.type) {
|
|
|
+ case e1000_82575:
|
|
|
/* The 82575 assigns vectors using a bitmask, which matches the
|
|
|
bitmask for the EICR/EIMS/EIMC registers. To assign one
|
|
|
or more queues to a vector, we write the appropriate bits
|
|
|
@@ -256,6 +309,47 @@ static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue,
|
|
|
E1000_EICR_TX_QUEUE0 << tx_queue;
|
|
|
}
|
|
|
array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
|
|
|
+ break;
|
|
|
+ case e1000_82576:
|
|
|
+ /* Kawela uses a table-based method for assigning vectors.
|
|
|
+ Each queue has a single entry in the table to which we write
|
|
|
+ a vector number along with a "valid" bit. Sadly, the layout
|
|
|
+ of the table is somewhat counterintuitive. */
|
|
|
+ if (rx_queue > IGB_N0_QUEUE) {
|
|
|
+ index = (rx_queue & 0x7);
|
|
|
+ ivar = array_rd32(E1000_IVAR0, index);
|
|
|
+ if (rx_queue < 8) {
|
|
|
+ /* vector goes into low byte of register */
|
|
|
+ ivar = ivar & 0xFFFFFF00;
|
|
|
+ ivar |= msix_vector | E1000_IVAR_VALID;
|
|
|
+ } else {
|
|
|
+ /* vector goes into third byte of register */
|
|
|
+ ivar = ivar & 0xFF00FFFF;
|
|
|
+ ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
|
|
|
+ }
|
|
|
+ adapter->rx_ring[rx_queue].eims_value= 1 << msix_vector;
|
|
|
+ array_wr32(E1000_IVAR0, index, ivar);
|
|
|
+ }
|
|
|
+ if (tx_queue > IGB_N0_QUEUE) {
|
|
|
+ index = (tx_queue & 0x7);
|
|
|
+ ivar = array_rd32(E1000_IVAR0, index);
|
|
|
+ if (tx_queue < 8) {
|
|
|
+ /* vector goes into second byte of register */
|
|
|
+ ivar = ivar & 0xFFFF00FF;
|
|
|
+ ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
|
|
|
+ } else {
|
|
|
+ /* vector goes into high byte of register */
|
|
|
+ ivar = ivar & 0x00FFFFFF;
|
|
|
+ ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
|
|
|
+ }
|
|
|
+ adapter->tx_ring[tx_queue].eims_value= 1 << msix_vector;
|
|
|
+ array_wr32(E1000_IVAR0, index, ivar);
|
|
|
+ }
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ BUG();
|
|
|
+ break;
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -271,13 +365,19 @@ static void igb_configure_msix(struct igb_adapter *adapter)
|
|
|
struct e1000_hw *hw = &adapter->hw;
|
|
|
|
|
|
adapter->eims_enable_mask = 0;
|
|
|
+ if (hw->mac.type == e1000_82576)
|
|
|
+ /* Turn on MSI-X capability first, or our settings
|
|
|
+ * won't stick. And it will take days to debug. */
|
|
|
+ wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
|
|
|
+ E1000_GPIE_PBA | E1000_GPIE_EIAME |
|
|
|
+ E1000_GPIE_NSICR);
|
|
|
|
|
|
for (i = 0; i < adapter->num_tx_queues; i++) {
|
|
|
struct igb_ring *tx_ring = &adapter->tx_ring[i];
|
|
|
igb_assign_vector(adapter, IGB_N0_QUEUE, i, vector++);
|
|
|
adapter->eims_enable_mask |= tx_ring->eims_value;
|
|
|
if (tx_ring->itr_val)
|
|
|
- writel(1000000000 / (tx_ring->itr_val * 256),
|
|
|
+ writel(tx_ring->itr_val,
|
|
|
hw->hw_addr + tx_ring->itr_register);
|
|
|
else
|
|
|
writel(1, hw->hw_addr + tx_ring->itr_register);
|
|
|
@@ -285,10 +385,11 @@ static void igb_configure_msix(struct igb_adapter *adapter)
|
|
|
|
|
|
for (i = 0; i < adapter->num_rx_queues; i++) {
|
|
|
struct igb_ring *rx_ring = &adapter->rx_ring[i];
|
|
|
+ rx_ring->buddy = 0;
|
|
|
igb_assign_vector(adapter, i, IGB_N0_QUEUE, vector++);
|
|
|
adapter->eims_enable_mask |= rx_ring->eims_value;
|
|
|
if (rx_ring->itr_val)
|
|
|
- writel(1000000000 / (rx_ring->itr_val * 256),
|
|
|
+ writel(rx_ring->itr_val,
|
|
|
hw->hw_addr + rx_ring->itr_register);
|
|
|
else
|
|
|
writel(1, hw->hw_addr + rx_ring->itr_register);
|
|
|
@@ -296,6 +397,8 @@ static void igb_configure_msix(struct igb_adapter *adapter)
|
|
|
|
|
|
|
|
|
/* set vector for other causes, i.e. link changes */
|
|
|
+ switch (hw->mac.type) {
|
|
|
+ case e1000_82575:
|
|
|
array_wr32(E1000_MSIXBM(0), vector++,
|
|
|
E1000_EIMS_OTHER);
|
|
|
|
|
|
@@ -311,6 +414,19 @@ static void igb_configure_msix(struct igb_adapter *adapter)
|
|
|
adapter->eims_enable_mask |= E1000_EIMS_OTHER;
|
|
|
adapter->eims_other = E1000_EIMS_OTHER;
|
|
|
|
|
|
+ break;
|
|
|
+
|
|
|
+ case e1000_82576:
|
|
|
+ tmp = (vector++ | E1000_IVAR_VALID) << 8;
|
|
|
+ wr32(E1000_IVAR_MISC, tmp);
|
|
|
+
|
|
|
+ adapter->eims_enable_mask = (1 << (vector)) - 1;
|
|
|
+ adapter->eims_other = 1 << (vector - 1);
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ /* do nothing, since nothing else supports MSI-X */
|
|
|
+ break;
|
|
|
+ } /* switch (hw->mac.type) */
|
|
|
wrfl();
|
|
|
}
|
|
|
|
|
|
@@ -336,7 +452,7 @@ static int igb_request_msix(struct igb_adapter *adapter)
|
|
|
if (err)
|
|
|
goto out;
|
|
|
ring->itr_register = E1000_EITR(0) + (vector << 2);
|
|
|
- ring->itr_val = adapter->itr;
|
|
|
+ ring->itr_val = 976; /* ~4000 ints/sec */
|
|
|
vector++;
|
|
|
}
|
|
|
for (i = 0; i < adapter->num_rx_queues; i++) {
|
|
|
@@ -375,7 +491,7 @@ static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
|
|
|
pci_disable_msix(adapter->pdev);
|
|
|
kfree(adapter->msix_entries);
|
|
|
adapter->msix_entries = NULL;
|
|
|
- } else if (adapter->msi_enabled)
|
|
|
+ } else if (adapter->flags & IGB_FLAG_HAS_MSI)
|
|
|
pci_disable_msi(adapter->pdev);
|
|
|
return;
|
|
|
}
|
|
|
@@ -412,8 +528,14 @@ static void igb_set_interrupt_capability(struct igb_adapter *adapter)
|
|
|
/* If we can't do MSI-X, try MSI */
|
|
|
msi_only:
|
|
|
adapter->num_rx_queues = 1;
|
|
|
+ adapter->num_tx_queues = 1;
|
|
|
if (!pci_enable_msi(adapter->pdev))
|
|
|
- adapter->msi_enabled = 1;
|
|
|
+ adapter->flags |= IGB_FLAG_HAS_MSI;
|
|
|
+
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ /* Notify the stack of the (possibly) reduced Tx Queue count. */
|
|
|
+ adapter->netdev->egress_subqueue_count = adapter->num_tx_queues;
|
|
|
+#endif
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
@@ -436,24 +558,33 @@ static int igb_request_irq(struct igb_adapter *adapter)
|
|
|
/* fall back to MSI */
|
|
|
igb_reset_interrupt_capability(adapter);
|
|
|
if (!pci_enable_msi(adapter->pdev))
|
|
|
- adapter->msi_enabled = 1;
|
|
|
+ adapter->flags |= IGB_FLAG_HAS_MSI;
|
|
|
igb_free_all_tx_resources(adapter);
|
|
|
igb_free_all_rx_resources(adapter);
|
|
|
adapter->num_rx_queues = 1;
|
|
|
igb_alloc_queues(adapter);
|
|
|
} else {
|
|
|
- wr32(E1000_MSIXBM(0), (E1000_EICR_RX_QUEUE0 |
|
|
|
- E1000_EIMS_OTHER));
|
|
|
+ switch (hw->mac.type) {
|
|
|
+ case e1000_82575:
|
|
|
+ wr32(E1000_MSIXBM(0),
|
|
|
+ (E1000_EICR_RX_QUEUE0 | E1000_EIMS_OTHER));
|
|
|
+ break;
|
|
|
+ case e1000_82576:
|
|
|
+ wr32(E1000_IVAR0, E1000_IVAR_VALID);
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ break;
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
- if (adapter->msi_enabled) {
|
|
|
+ if (adapter->flags & IGB_FLAG_HAS_MSI) {
|
|
|
err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
|
|
|
netdev->name, netdev);
|
|
|
if (!err)
|
|
|
goto request_done;
|
|
|
/* fall back to legacy interrupts */
|
|
|
igb_reset_interrupt_capability(adapter);
|
|
|
- adapter->msi_enabled = 0;
|
|
|
+ adapter->flags &= ~IGB_FLAG_HAS_MSI;
|
|
|
}
|
|
|
|
|
|
err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
|
|
|
@@ -693,6 +824,10 @@ void igb_down(struct igb_adapter *adapter)
|
|
|
/* flush and sleep below */
|
|
|
|
|
|
netif_stop_queue(netdev);
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ for (i = 0; i < adapter->num_tx_queues; i++)
|
|
|
+ netif_stop_subqueue(netdev, i);
|
|
|
+#endif
|
|
|
|
|
|
/* disable transmits in the hardware */
|
|
|
tctl = rd32(E1000_TCTL);
|
|
|
@@ -734,16 +869,23 @@ void igb_reinit_locked(struct igb_adapter *adapter)
|
|
|
void igb_reset(struct igb_adapter *adapter)
|
|
|
{
|
|
|
struct e1000_hw *hw = &adapter->hw;
|
|
|
- struct e1000_fc_info *fc = &adapter->hw.fc;
|
|
|
+ struct e1000_mac_info *mac = &hw->mac;
|
|
|
+ struct e1000_fc_info *fc = &hw->fc;
|
|
|
u32 pba = 0, tx_space, min_tx_space, min_rx_space;
|
|
|
u16 hwm;
|
|
|
|
|
|
/* Repartition Pba for greater than 9k mtu
|
|
|
* To take effect CTRL.RST is required.
|
|
|
*/
|
|
|
+ if (mac->type != e1000_82576) {
|
|
|
pba = E1000_PBA_34K;
|
|
|
+ }
|
|
|
+ else {
|
|
|
+ pba = E1000_PBA_64K;
|
|
|
+ }
|
|
|
|
|
|
- if (adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) {
|
|
|
+ if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
|
|
|
+ (mac->type < e1000_82576)) {
|
|
|
/* adjust PBA for jumbo frames */
|
|
|
wr32(E1000_PBA, pba);
|
|
|
|
|
|
@@ -782,8 +924,8 @@ void igb_reset(struct igb_adapter *adapter)
|
|
|
if (pba < min_rx_space)
|
|
|
pba = min_rx_space;
|
|
|
}
|
|
|
+ wr32(E1000_PBA, pba);
|
|
|
}
|
|
|
- wr32(E1000_PBA, pba);
|
|
|
|
|
|
/* flow control settings */
|
|
|
/* The high water mark must be low enough to fit one full frame
|
|
|
@@ -792,10 +934,15 @@ void igb_reset(struct igb_adapter *adapter)
|
|
|
* - 90% of the Rx FIFO size, or
|
|
|
* - the full Rx FIFO size minus one full frame */
|
|
|
hwm = min(((pba << 10) * 9 / 10),
|
|
|
- ((pba << 10) - adapter->max_frame_size));
|
|
|
+ ((pba << 10) - 2 * adapter->max_frame_size));
|
|
|
|
|
|
- fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
|
|
|
- fc->low_water = fc->high_water - 8;
|
|
|
+ if (mac->type < e1000_82576) {
|
|
|
+ fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
|
|
|
+ fc->low_water = fc->high_water - 8;
|
|
|
+ } else {
|
|
|
+ fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
|
|
|
+ fc->low_water = fc->high_water - 16;
|
|
|
+ }
|
|
|
fc->pause_time = 0xFFFF;
|
|
|
fc->send_xon = 1;
|
|
|
fc->type = fc->original_type;
|
|
|
@@ -895,7 +1042,11 @@ static int __devinit igb_probe(struct pci_dev *pdev,
|
|
|
pci_save_state(pdev);
|
|
|
|
|
|
err = -ENOMEM;
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ netdev = alloc_etherdev_mq(sizeof(struct igb_adapter), IGB_MAX_TX_QUEUES);
|
|
|
+#else
|
|
|
netdev = alloc_etherdev(sizeof(struct igb_adapter));
|
|
|
+#endif /* CONFIG_NETDEVICES_MULTIQUEUE */
|
|
|
if (!netdev)
|
|
|
goto err_alloc_etherdev;
|
|
|
|
|
|
@@ -966,6 +1117,17 @@ static int __devinit igb_probe(struct pci_dev *pdev,
|
|
|
|
|
|
igb_get_bus_info_pcie(hw);
|
|
|
|
|
|
+ /* set flags */
|
|
|
+ switch (hw->mac.type) {
|
|
|
+ case e1000_82576:
|
|
|
+ case e1000_82575:
|
|
|
+ adapter->flags |= IGB_FLAG_HAS_DCA;
|
|
|
+ adapter->flags |= IGB_FLAG_NEED_CTX_IDX;
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ break;
|
|
|
+ }
|
|
|
+
|
|
|
hw->phy.autoneg_wait_to_complete = false;
|
|
|
hw->mac.adaptive_ifs = true;
|
|
|
|
|
|
@@ -989,6 +1151,10 @@ static int __devinit igb_probe(struct pci_dev *pdev,
|
|
|
netdev->features |= NETIF_F_TSO;
|
|
|
netdev->features |= NETIF_F_TSO6;
|
|
|
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+ netdev->features |= NETIF_F_LRO;
|
|
|
+#endif
|
|
|
+
|
|
|
netdev->vlan_features |= NETIF_F_TSO;
|
|
|
netdev->vlan_features |= NETIF_F_TSO6;
|
|
|
netdev->vlan_features |= NETIF_F_HW_CSUM;
|
|
|
@@ -997,6 +1163,10 @@ static int __devinit igb_probe(struct pci_dev *pdev,
|
|
|
if (pci_using_dac)
|
|
|
netdev->features |= NETIF_F_HIGHDMA;
|
|
|
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ netdev->features |= NETIF_F_MULTI_QUEUE;
|
|
|
+#endif
|
|
|
+
|
|
|
netdev->features |= NETIF_F_LLTX;
|
|
|
adapter->en_mng_pt = igb_enable_mng_pass_thru(&adapter->hw);
|
|
|
|
|
|
@@ -1077,11 +1247,23 @@ static int __devinit igb_probe(struct pci_dev *pdev,
|
|
|
adapter->eeprom_wol = 0;
|
|
|
break;
|
|
|
case E1000_DEV_ID_82575EB_FIBER_SERDES:
|
|
|
+ case E1000_DEV_ID_82576_FIBER:
|
|
|
+ case E1000_DEV_ID_82576_SERDES:
|
|
|
/* Wake events only supported on port A for dual fiber
|
|
|
* regardless of eeprom setting */
|
|
|
if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
|
|
|
adapter->eeprom_wol = 0;
|
|
|
break;
|
|
|
+ case E1000_DEV_ID_82576_QUAD_COPPER:
|
|
|
+ /* if quad port adapter, disable WoL on all but port A */
|
|
|
+ if (global_quad_port_a != 0)
|
|
|
+ adapter->eeprom_wol = 0;
|
|
|
+ else
|
|
|
+ adapter->flags |= IGB_FLAG_QUAD_PORT_A;
|
|
|
+ /* Reset for multiple quad port adapters */
|
|
|
+ if (++global_quad_port_a == 4)
|
|
|
+ global_quad_port_a = 0;
|
|
|
+ break;
|
|
|
}
|
|
|
|
|
|
/* initialize the wol settings based on the eeprom settings */
|
|
|
@@ -1097,12 +1279,28 @@ static int __devinit igb_probe(struct pci_dev *pdev,
|
|
|
/* tell the stack to leave us alone until igb_open() is called */
|
|
|
netif_carrier_off(netdev);
|
|
|
netif_stop_queue(netdev);
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ for (i = 0; i < adapter->num_tx_queues; i++)
|
|
|
+ netif_stop_subqueue(netdev, i);
|
|
|
+#endif
|
|
|
|
|
|
strcpy(netdev->name, "eth%d");
|
|
|
err = register_netdev(netdev);
|
|
|
if (err)
|
|
|
goto err_register;
|
|
|
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ if ((adapter->flags & IGB_FLAG_HAS_DCA) &&
|
|
|
+ (dca_add_requester(&pdev->dev) == 0)) {
|
|
|
+ adapter->flags |= IGB_FLAG_DCA_ENABLED;
|
|
|
+ dev_info(&pdev->dev, "DCA enabled\n");
|
|
|
+ /* Always use CB2 mode, difference is masked
|
|
|
+ * in the CB driver. */
|
|
|
+ wr32(E1000_DCA_CTRL, 2);
|
|
|
+ igb_setup_dca(adapter);
|
|
|
+ }
|
|
|
+#endif
|
|
|
+
|
|
|
dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
|
|
|
/* print bus type/speed/width info */
|
|
|
dev_info(&pdev->dev,
|
|
|
@@ -1123,7 +1321,7 @@ static int __devinit igb_probe(struct pci_dev *pdev,
|
|
|
dev_info(&pdev->dev,
|
|
|
"Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
|
|
|
adapter->msix_entries ? "MSI-X" :
|
|
|
- adapter->msi_enabled ? "MSI" : "legacy",
|
|
|
+ (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
|
|
|
adapter->num_rx_queues, adapter->num_tx_queues);
|
|
|
|
|
|
return 0;
|
|
|
@@ -1138,8 +1336,7 @@ static int __devinit igb_probe(struct pci_dev *pdev,
|
|
|
iounmap(hw->flash_address);
|
|
|
|
|
|
igb_remove_device(hw);
|
|
|
- kfree(adapter->tx_ring);
|
|
|
- kfree(adapter->rx_ring);
|
|
|
+ igb_free_queues(adapter);
|
|
|
err_sw_init:
|
|
|
err_hw_init:
|
|
|
iounmap(hw->hw_addr);
|
|
|
@@ -1166,6 +1363,9 @@ static void __devexit igb_remove(struct pci_dev *pdev)
|
|
|
{
|
|
|
struct net_device *netdev = pci_get_drvdata(pdev);
|
|
|
struct igb_adapter *adapter = netdev_priv(netdev);
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ struct e1000_hw *hw = &adapter->hw;
|
|
|
+#endif
|
|
|
|
|
|
/* flush_scheduled work may reschedule our watchdog task, so
|
|
|
* explicitly disable watchdog tasks from being rescheduled */
|
|
|
@@ -1175,6 +1375,15 @@ static void __devexit igb_remove(struct pci_dev *pdev)
|
|
|
|
|
|
flush_scheduled_work();
|
|
|
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
|
|
|
+ dev_info(&pdev->dev, "DCA disabled\n");
|
|
|
+ dca_remove_requester(&pdev->dev);
|
|
|
+ adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
|
|
|
+ wr32(E1000_DCA_CTRL, 1);
|
|
|
+ }
|
|
|
+#endif
|
|
|
+
|
|
|
/* Release control of h/w to f/w. If f/w is AMT enabled, this
|
|
|
* would have already happened in close and is redundant. */
|
|
|
igb_release_hw_control(adapter);
|
|
|
@@ -1187,8 +1396,7 @@ static void __devexit igb_remove(struct pci_dev *pdev)
|
|
|
igb_remove_device(&adapter->hw);
|
|
|
igb_reset_interrupt_capability(adapter);
|
|
|
|
|
|
- kfree(adapter->tx_ring);
|
|
|
- kfree(adapter->rx_ring);
|
|
|
+ igb_free_queues(adapter);
|
|
|
|
|
|
iounmap(adapter->hw.hw_addr);
|
|
|
if (adapter->hw.flash_address)
|
|
|
@@ -1223,9 +1431,15 @@ static int __devinit igb_sw_init(struct igb_adapter *adapter)
|
|
|
|
|
|
/* Number of supported queues. */
|
|
|
/* Having more queues than CPUs doesn't make sense. */
|
|
|
+ adapter->num_rx_queues = min((u32)IGB_MAX_RX_QUEUES, (u32)num_online_cpus());
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ adapter->num_tx_queues = min(IGB_MAX_TX_QUEUES, num_online_cpus());
|
|
|
+#else
|
|
|
adapter->num_tx_queues = 1;
|
|
|
- adapter->num_rx_queues = min(IGB_MAX_RX_QUEUES, num_online_cpus());
|
|
|
+#endif /* CONFIG_NET_MULTI_QUEUE_DEVICE */
|
|
|
|
|
|
+ /* This call may decrease the number of queues depending on
|
|
|
+ * interrupt mode. */
|
|
|
igb_set_interrupt_capability(adapter);
|
|
|
|
|
|
if (igb_alloc_queues(adapter)) {
|
|
|
@@ -1386,8 +1600,6 @@ int igb_setup_tx_resources(struct igb_adapter *adapter,
|
|
|
tx_ring->adapter = adapter;
|
|
|
tx_ring->next_to_use = 0;
|
|
|
tx_ring->next_to_clean = 0;
|
|
|
- spin_lock_init(&tx_ring->tx_clean_lock);
|
|
|
- spin_lock_init(&tx_ring->tx_lock);
|
|
|
return 0;
|
|
|
|
|
|
err:
|
|
|
@@ -1407,6 +1619,9 @@ int igb_setup_tx_resources(struct igb_adapter *adapter,
|
|
|
static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
|
|
|
{
|
|
|
int i, err = 0;
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ int r_idx;
|
|
|
+#endif
|
|
|
|
|
|
for (i = 0; i < adapter->num_tx_queues; i++) {
|
|
|
err = igb_setup_tx_resources(adapter, &adapter->tx_ring[i]);
|
|
|
@@ -1419,6 +1634,12 @@ static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
|
|
|
+ r_idx = i % adapter->num_tx_queues;
|
|
|
+ adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
|
|
|
+ }
|
|
|
+#endif
|
|
|
return err;
|
|
|
}
|
|
|
|
|
|
@@ -1505,6 +1726,14 @@ int igb_setup_rx_resources(struct igb_adapter *adapter,
|
|
|
struct pci_dev *pdev = adapter->pdev;
|
|
|
int size, desc_len;
|
|
|
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+ size = sizeof(struct net_lro_desc) * MAX_LRO_DESCRIPTORS;
|
|
|
+ rx_ring->lro_mgr.lro_arr = vmalloc(size);
|
|
|
+ if (!rx_ring->lro_mgr.lro_arr)
|
|
|
+ goto err;
|
|
|
+ memset(rx_ring->lro_mgr.lro_arr, 0, size);
|
|
|
+#endif
|
|
|
+
|
|
|
size = sizeof(struct igb_buffer) * rx_ring->count;
|
|
|
rx_ring->buffer_info = vmalloc(size);
|
|
|
if (!rx_ring->buffer_info)
|
|
|
@@ -1525,13 +1754,16 @@ int igb_setup_rx_resources(struct igb_adapter *adapter,
|
|
|
|
|
|
rx_ring->next_to_clean = 0;
|
|
|
rx_ring->next_to_use = 0;
|
|
|
- rx_ring->pending_skb = NULL;
|
|
|
|
|
|
rx_ring->adapter = adapter;
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
err:
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+ vfree(rx_ring->lro_mgr.lro_arr);
|
|
|
+ rx_ring->lro_mgr.lro_arr = NULL;
|
|
|
+#endif
|
|
|
vfree(rx_ring->buffer_info);
|
|
|
dev_err(&adapter->pdev->dev, "Unable to allocate memory for "
|
|
|
"the receive descriptor ring\n");
|
|
|
@@ -1582,10 +1814,12 @@ static void igb_setup_rctl(struct igb_adapter *adapter)
|
|
|
E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
|
|
|
(adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
|
|
|
|
|
|
- /* disable the stripping of CRC because it breaks
|
|
|
- * BMC firmware connected over SMBUS
|
|
|
- rctl |= E1000_RCTL_SECRC;
|
|
|
+ /*
|
|
|
+ * enable stripping of CRC. It's unlikely this will break BMC
|
|
|
+ * redirection as it did with e1000. Newer features require
|
|
|
+ * that the HW strips the CRC.
|
|
|
*/
|
|
|
+ rctl |= E1000_RCTL_SECRC;
|
|
|
|
|
|
rctl &= ~E1000_RCTL_SBP;
|
|
|
|
|
|
@@ -1615,15 +1849,6 @@ static void igb_setup_rctl(struct igb_adapter *adapter)
|
|
|
rctl |= E1000_RCTL_SZ_2048;
|
|
|
rctl &= ~E1000_RCTL_BSEX;
|
|
|
break;
|
|
|
- case IGB_RXBUFFER_4096:
|
|
|
- rctl |= E1000_RCTL_SZ_4096;
|
|
|
- break;
|
|
|
- case IGB_RXBUFFER_8192:
|
|
|
- rctl |= E1000_RCTL_SZ_8192;
|
|
|
- break;
|
|
|
- case IGB_RXBUFFER_16384:
|
|
|
- rctl |= E1000_RCTL_SZ_16384;
|
|
|
- break;
|
|
|
}
|
|
|
} else {
|
|
|
rctl &= ~E1000_RCTL_BSEX;
|
|
|
@@ -1641,10 +1866,8 @@ static void igb_setup_rctl(struct igb_adapter *adapter)
|
|
|
* so only enable packet split for jumbo frames */
|
|
|
if (rctl & E1000_RCTL_LPE) {
|
|
|
adapter->rx_ps_hdr_size = IGB_RXBUFFER_128;
|
|
|
- srrctl = adapter->rx_ps_hdr_size <<
|
|
|
+ srrctl |= adapter->rx_ps_hdr_size <<
|
|
|
E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
|
|
|
- /* buffer size is ALWAYS one page */
|
|
|
- srrctl |= PAGE_SIZE >> E1000_SRRCTL_BSIZEPKT_SHIFT;
|
|
|
srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
|
|
|
} else {
|
|
|
adapter->rx_ps_hdr_size = 0;
|
|
|
@@ -1678,8 +1901,7 @@ static void igb_configure_rx(struct igb_adapter *adapter)
|
|
|
mdelay(10);
|
|
|
|
|
|
if (adapter->itr_setting > 3)
|
|
|
- wr32(E1000_ITR,
|
|
|
- 1000000000 / (adapter->itr * 256));
|
|
|
+ wr32(E1000_ITR, adapter->itr);
|
|
|
|
|
|
/* Setup the HW Rx Head and Tail Descriptor Pointers and
|
|
|
* the Base and Length of the Rx Descriptor Ring */
|
|
|
@@ -1704,6 +1926,16 @@ static void igb_configure_rx(struct igb_adapter *adapter)
|
|
|
rxdctl |= IGB_RX_HTHRESH << 8;
|
|
|
rxdctl |= IGB_RX_WTHRESH << 16;
|
|
|
wr32(E1000_RXDCTL(i), rxdctl);
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+ /* Intitial LRO Settings */
|
|
|
+ ring->lro_mgr.max_aggr = MAX_LRO_AGGR;
|
|
|
+ ring->lro_mgr.max_desc = MAX_LRO_DESCRIPTORS;
|
|
|
+ ring->lro_mgr.get_skb_header = igb_get_skb_hdr;
|
|
|
+ ring->lro_mgr.features = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
|
|
|
+ ring->lro_mgr.dev = adapter->netdev;
|
|
|
+ ring->lro_mgr.ip_summed = CHECKSUM_UNNECESSARY;
|
|
|
+ ring->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
|
|
|
+#endif
|
|
|
}
|
|
|
|
|
|
if (adapter->num_rx_queues > 1) {
|
|
|
@@ -1717,7 +1949,10 @@ static void igb_configure_rx(struct igb_adapter *adapter)
|
|
|
|
|
|
get_random_bytes(&random[0], 40);
|
|
|
|
|
|
- shift = 6;
|
|
|
+ if (hw->mac.type >= e1000_82576)
|
|
|
+ shift = 0;
|
|
|
+ else
|
|
|
+ shift = 6;
|
|
|
for (j = 0; j < (32 * 4); j++) {
|
|
|
reta.bytes[j & 3] =
|
|
|
(j % adapter->num_rx_queues) << shift;
|
|
|
@@ -1892,6 +2127,11 @@ static void igb_free_rx_resources(struct igb_ring *rx_ring)
|
|
|
vfree(rx_ring->buffer_info);
|
|
|
rx_ring->buffer_info = NULL;
|
|
|
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+ vfree(rx_ring->lro_mgr.lro_arr);
|
|
|
+ rx_ring->lro_mgr.lro_arr = NULL;
|
|
|
+#endif
|
|
|
+
|
|
|
pci_free_consistent(pdev, rx_ring->size, rx_ring->desc, rx_ring->dma);
|
|
|
|
|
|
rx_ring->desc = NULL;
|
|
|
@@ -1946,20 +2186,17 @@ static void igb_clean_rx_ring(struct igb_ring *rx_ring)
|
|
|
buffer_info->skb = NULL;
|
|
|
}
|
|
|
if (buffer_info->page) {
|
|
|
- pci_unmap_page(pdev, buffer_info->page_dma,
|
|
|
- PAGE_SIZE, PCI_DMA_FROMDEVICE);
|
|
|
+ if (buffer_info->page_dma)
|
|
|
+ pci_unmap_page(pdev, buffer_info->page_dma,
|
|
|
+ PAGE_SIZE / 2,
|
|
|
+ PCI_DMA_FROMDEVICE);
|
|
|
put_page(buffer_info->page);
|
|
|
buffer_info->page = NULL;
|
|
|
buffer_info->page_dma = 0;
|
|
|
+ buffer_info->page_offset = 0;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- /* there also may be some cached data from a chained receive */
|
|
|
- if (rx_ring->pending_skb) {
|
|
|
- dev_kfree_skb(rx_ring->pending_skb);
|
|
|
- rx_ring->pending_skb = NULL;
|
|
|
- }
|
|
|
-
|
|
|
size = sizeof(struct igb_buffer) * rx_ring->count;
|
|
|
memset(rx_ring->buffer_info, 0, size);
|
|
|
|
|
|
@@ -2043,7 +2280,7 @@ static void igb_set_multi(struct net_device *netdev)
|
|
|
|
|
|
if (!netdev->mc_count) {
|
|
|
/* nothing to program, so clear mc list */
|
|
|
- igb_update_mc_addr_list(hw, NULL, 0, 1,
|
|
|
+ igb_update_mc_addr_list_82575(hw, NULL, 0, 1,
|
|
|
mac->rar_entry_count);
|
|
|
return;
|
|
|
}
|
|
|
@@ -2061,7 +2298,8 @@ static void igb_set_multi(struct net_device *netdev)
|
|
|
memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
|
|
|
mc_ptr = mc_ptr->next;
|
|
|
}
|
|
|
- igb_update_mc_addr_list(hw, mta_list, i, 1, mac->rar_entry_count);
|
|
|
+ igb_update_mc_addr_list_82575(hw, mta_list, i, 1,
|
|
|
+ mac->rar_entry_count);
|
|
|
kfree(mta_list);
|
|
|
}
|
|
|
|
|
|
@@ -2096,6 +2334,9 @@ static void igb_watchdog_task(struct work_struct *work)
|
|
|
struct e1000_mac_info *mac = &adapter->hw.mac;
|
|
|
u32 link;
|
|
|
s32 ret_val;
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ int i;
|
|
|
+#endif
|
|
|
|
|
|
if ((netif_carrier_ok(netdev)) &&
|
|
|
(rd32(E1000_STATUS) & E1000_STATUS_LU))
|
|
|
@@ -2152,6 +2393,10 @@ static void igb_watchdog_task(struct work_struct *work)
|
|
|
|
|
|
netif_carrier_on(netdev);
|
|
|
netif_wake_queue(netdev);
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ for (i = 0; i < adapter->num_tx_queues; i++)
|
|
|
+ netif_wake_subqueue(netdev, i);
|
|
|
+#endif
|
|
|
|
|
|
if (!test_bit(__IGB_DOWN, &adapter->state))
|
|
|
mod_timer(&adapter->phy_info_timer,
|
|
|
@@ -2164,6 +2409,10 @@ static void igb_watchdog_task(struct work_struct *work)
|
|
|
dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
|
|
|
netif_carrier_off(netdev);
|
|
|
netif_stop_queue(netdev);
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ for (i = 0; i < adapter->num_tx_queues; i++)
|
|
|
+ netif_stop_subqueue(netdev, i);
|
|
|
+#endif
|
|
|
if (!test_bit(__IGB_DOWN, &adapter->state))
|
|
|
mod_timer(&adapter->phy_info_timer,
|
|
|
round_jiffies(jiffies + 2 * HZ));
|
|
|
@@ -2216,38 +2465,60 @@ enum latency_range {
|
|
|
};
|
|
|
|
|
|
|
|
|
-static void igb_lower_rx_eitr(struct igb_adapter *adapter,
|
|
|
- struct igb_ring *rx_ring)
|
|
|
+/**
|
|
|
+ * igb_update_ring_itr - update the dynamic ITR value based on packet size
|
|
|
+ *
|
|
|
+ * Stores a new ITR value based on strictly on packet size. This
|
|
|
+ * algorithm is less sophisticated than that used in igb_update_itr,
|
|
|
+ * due to the difficulty of synchronizing statistics across multiple
|
|
|
+ * receive rings. The divisors and thresholds used by this fuction
|
|
|
+ * were determined based on theoretical maximum wire speed and testing
|
|
|
+ * data, in order to minimize response time while increasing bulk
|
|
|
+ * throughput.
|
|
|
+ * This functionality is controlled by the InterruptThrottleRate module
|
|
|
+ * parameter (see igb_param.c)
|
|
|
+ * NOTE: This function is called only when operating in a multiqueue
|
|
|
+ * receive environment.
|
|
|
+ * @rx_ring: pointer to ring
|
|
|
+ **/
|
|
|
+static void igb_update_ring_itr(struct igb_ring *rx_ring)
|
|
|
{
|
|
|
- struct e1000_hw *hw = &adapter->hw;
|
|
|
- int new_val;
|
|
|
+ int new_val = rx_ring->itr_val;
|
|
|
+ int avg_wire_size = 0;
|
|
|
+ struct igb_adapter *adapter = rx_ring->adapter;
|
|
|
|
|
|
- new_val = rx_ring->itr_val / 2;
|
|
|
- if (new_val < IGB_MIN_DYN_ITR)
|
|
|
- new_val = IGB_MIN_DYN_ITR;
|
|
|
+ if (!rx_ring->total_packets)
|
|
|
+ goto clear_counts; /* no packets, so don't do anything */
|
|
|
|
|
|
- if (new_val != rx_ring->itr_val) {
|
|
|
- rx_ring->itr_val = new_val;
|
|
|
- wr32(rx_ring->itr_register,
|
|
|
- 1000000000 / (new_val * 256));
|
|
|
+ /* For non-gigabit speeds, just fix the interrupt rate at 4000
|
|
|
+ * ints/sec - ITR timer value of 120 ticks.
|
|
|
+ */
|
|
|
+ if (adapter->link_speed != SPEED_1000) {
|
|
|
+ new_val = 120;
|
|
|
+ goto set_itr_val;
|
|
|
}
|
|
|
-}
|
|
|
+ avg_wire_size = rx_ring->total_bytes / rx_ring->total_packets;
|
|
|
|
|
|
-static void igb_raise_rx_eitr(struct igb_adapter *adapter,
|
|
|
- struct igb_ring *rx_ring)
|
|
|
-{
|
|
|
- struct e1000_hw *hw = &adapter->hw;
|
|
|
- int new_val;
|
|
|
+ /* Add 24 bytes to size to account for CRC, preamble, and gap */
|
|
|
+ avg_wire_size += 24;
|
|
|
|
|
|
- new_val = rx_ring->itr_val * 2;
|
|
|
- if (new_val > IGB_MAX_DYN_ITR)
|
|
|
- new_val = IGB_MAX_DYN_ITR;
|
|
|
+ /* Don't starve jumbo frames */
|
|
|
+ avg_wire_size = min(avg_wire_size, 3000);
|
|
|
|
|
|
+ /* Give a little boost to mid-size frames */
|
|
|
+ if ((avg_wire_size > 300) && (avg_wire_size < 1200))
|
|
|
+ new_val = avg_wire_size / 3;
|
|
|
+ else
|
|
|
+ new_val = avg_wire_size / 2;
|
|
|
+
|
|
|
+set_itr_val:
|
|
|
if (new_val != rx_ring->itr_val) {
|
|
|
rx_ring->itr_val = new_val;
|
|
|
- wr32(rx_ring->itr_register,
|
|
|
- 1000000000 / (new_val * 256));
|
|
|
+ rx_ring->set_itr = 1;
|
|
|
}
|
|
|
+clear_counts:
|
|
|
+ rx_ring->total_bytes = 0;
|
|
|
+ rx_ring->total_packets = 0;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -2314,8 +2585,7 @@ static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
|
|
|
return retval;
|
|
|
}
|
|
|
|
|
|
-static void igb_set_itr(struct igb_adapter *adapter, u16 itr_register,
|
|
|
- int rx_only)
|
|
|
+static void igb_set_itr(struct igb_adapter *adapter)
|
|
|
{
|
|
|
u16 current_itr;
|
|
|
u32 new_itr = adapter->itr;
|
|
|
@@ -2331,26 +2601,23 @@ static void igb_set_itr(struct igb_adapter *adapter, u16 itr_register,
|
|
|
adapter->rx_itr,
|
|
|
adapter->rx_ring->total_packets,
|
|
|
adapter->rx_ring->total_bytes);
|
|
|
- /* conservative mode (itr 3) eliminates the lowest_latency setting */
|
|
|
- if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
|
|
|
- adapter->rx_itr = low_latency;
|
|
|
|
|
|
- if (!rx_only) {
|
|
|
+ if (adapter->rx_ring->buddy) {
|
|
|
adapter->tx_itr = igb_update_itr(adapter,
|
|
|
adapter->tx_itr,
|
|
|
adapter->tx_ring->total_packets,
|
|
|
adapter->tx_ring->total_bytes);
|
|
|
- /* conservative mode (itr 3) eliminates the
|
|
|
- * lowest_latency setting */
|
|
|
- if (adapter->itr_setting == 3 &&
|
|
|
- adapter->tx_itr == lowest_latency)
|
|
|
- adapter->tx_itr = low_latency;
|
|
|
|
|
|
current_itr = max(adapter->rx_itr, adapter->tx_itr);
|
|
|
} else {
|
|
|
current_itr = adapter->rx_itr;
|
|
|
}
|
|
|
|
|
|
+ /* conservative mode (itr 3) eliminates the lowest_latency setting */
|
|
|
+ if (adapter->itr_setting == 3 &&
|
|
|
+ current_itr == lowest_latency)
|
|
|
+ current_itr = low_latency;
|
|
|
+
|
|
|
switch (current_itr) {
|
|
|
/* counts and packets in update_itr are dependent on these numbers */
|
|
|
case lowest_latency:
|
|
|
@@ -2367,6 +2634,13 @@ static void igb_set_itr(struct igb_adapter *adapter, u16 itr_register,
|
|
|
}
|
|
|
|
|
|
set_itr_now:
|
|
|
+ adapter->rx_ring->total_bytes = 0;
|
|
|
+ adapter->rx_ring->total_packets = 0;
|
|
|
+ if (adapter->rx_ring->buddy) {
|
|
|
+ adapter->rx_ring->buddy->total_bytes = 0;
|
|
|
+ adapter->rx_ring->buddy->total_packets = 0;
|
|
|
+ }
|
|
|
+
|
|
|
if (new_itr != adapter->itr) {
|
|
|
/* this attempts to bias the interrupt rate towards Bulk
|
|
|
* by adding intermediate steps when interrupt rate is
|
|
|
@@ -2381,7 +2655,8 @@ static void igb_set_itr(struct igb_adapter *adapter, u16 itr_register,
|
|
|
* ends up being correct.
|
|
|
*/
|
|
|
adapter->itr = new_itr;
|
|
|
- adapter->set_itr = 1;
|
|
|
+ adapter->rx_ring->itr_val = 1000000000 / (new_itr * 256);
|
|
|
+ adapter->rx_ring->set_itr = 1;
|
|
|
}
|
|
|
|
|
|
return;
|
|
|
@@ -2457,9 +2732,9 @@ static inline int igb_tso_adv(struct igb_adapter *adapter,
|
|
|
mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
|
|
|
mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
|
|
|
|
|
|
- /* Context index must be unique per ring. Luckily, so is the interrupt
|
|
|
- * mask value. */
|
|
|
- mss_l4len_idx |= tx_ring->eims_value >> 4;
|
|
|
+ /* Context index must be unique per ring. */
|
|
|
+ if (adapter->flags & IGB_FLAG_NEED_CTX_IDX)
|
|
|
+ mss_l4len_idx |= tx_ring->queue_index << 4;
|
|
|
|
|
|
context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
|
|
|
context_desc->seqnum_seed = 0;
|
|
|
@@ -2523,8 +2798,9 @@ static inline bool igb_tx_csum_adv(struct igb_adapter *adapter,
|
|
|
|
|
|
context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
|
|
|
context_desc->seqnum_seed = 0;
|
|
|
- context_desc->mss_l4len_idx =
|
|
|
- cpu_to_le32(tx_ring->eims_value >> 4);
|
|
|
+ if (adapter->flags & IGB_FLAG_NEED_CTX_IDX)
|
|
|
+ context_desc->mss_l4len_idx =
|
|
|
+ cpu_to_le32(tx_ring->queue_index << 4);
|
|
|
|
|
|
buffer_info->time_stamp = jiffies;
|
|
|
buffer_info->dma = 0;
|
|
|
@@ -2625,9 +2901,10 @@ static inline void igb_tx_queue_adv(struct igb_adapter *adapter,
|
|
|
olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
|
|
|
}
|
|
|
|
|
|
- if (tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_TSO |
|
|
|
- IGB_TX_FLAGS_VLAN))
|
|
|
- olinfo_status |= tx_ring->eims_value >> 4;
|
|
|
+ if ((adapter->flags & IGB_FLAG_NEED_CTX_IDX) &&
|
|
|
+ (tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_TSO |
|
|
|
+ IGB_TX_FLAGS_VLAN)))
|
|
|
+ olinfo_status |= tx_ring->queue_index << 4;
|
|
|
|
|
|
olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
|
|
|
|
|
|
@@ -2663,7 +2940,12 @@ static int __igb_maybe_stop_tx(struct net_device *netdev,
|
|
|
{
|
|
|
struct igb_adapter *adapter = netdev_priv(netdev);
|
|
|
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ netif_stop_subqueue(netdev, tx_ring->queue_index);
|
|
|
+#else
|
|
|
netif_stop_queue(netdev);
|
|
|
+#endif
|
|
|
+
|
|
|
/* Herbert's original patch had:
|
|
|
* smp_mb__after_netif_stop_queue();
|
|
|
* but since that doesn't exist yet, just open code it. */
|
|
|
@@ -2675,7 +2957,11 @@ static int __igb_maybe_stop_tx(struct net_device *netdev,
|
|
|
return -EBUSY;
|
|
|
|
|
|
/* A reprieve! */
|
|
|
- netif_start_queue(netdev);
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ netif_wake_subqueue(netdev, tx_ring->queue_index);
|
|
|
+#else
|
|
|
+ netif_wake_queue(netdev);
|
|
|
+#endif
|
|
|
++adapter->restart_queue;
|
|
|
return 0;
|
|
|
}
|
|
|
@@ -2697,7 +2983,6 @@ static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
|
|
|
struct igb_adapter *adapter = netdev_priv(netdev);
|
|
|
unsigned int tx_flags = 0;
|
|
|
unsigned int len;
|
|
|
- unsigned long irq_flags;
|
|
|
u8 hdr_len = 0;
|
|
|
int tso = 0;
|
|
|
|
|
|
@@ -2713,10 +2998,6 @@ static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
|
|
|
return NETDEV_TX_OK;
|
|
|
}
|
|
|
|
|
|
- if (!spin_trylock_irqsave(&tx_ring->tx_lock, irq_flags))
|
|
|
- /* Collision - tell upper layer to requeue */
|
|
|
- return NETDEV_TX_LOCKED;
|
|
|
-
|
|
|
/* need: 1 descriptor per page,
|
|
|
* + 2 desc gap to keep tail from touching head,
|
|
|
* + 1 desc for skb->data,
|
|
|
@@ -2724,21 +3005,23 @@ static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
|
|
|
* otherwise try next time */
|
|
|
if (igb_maybe_stop_tx(netdev, tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
|
|
|
/* this is a hard error */
|
|
|
- spin_unlock_irqrestore(&tx_ring->tx_lock, irq_flags);
|
|
|
return NETDEV_TX_BUSY;
|
|
|
}
|
|
|
+ skb_orphan(skb);
|
|
|
|
|
|
if (adapter->vlgrp && vlan_tx_tag_present(skb)) {
|
|
|
tx_flags |= IGB_TX_FLAGS_VLAN;
|
|
|
tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
|
|
|
}
|
|
|
|
|
|
+ if (skb->protocol == htons(ETH_P_IP))
|
|
|
+ tx_flags |= IGB_TX_FLAGS_IPV4;
|
|
|
+
|
|
|
tso = skb_is_gso(skb) ? igb_tso_adv(adapter, tx_ring, skb, tx_flags,
|
|
|
&hdr_len) : 0;
|
|
|
|
|
|
if (tso < 0) {
|
|
|
dev_kfree_skb_any(skb);
|
|
|
- spin_unlock_irqrestore(&tx_ring->tx_lock, irq_flags);
|
|
|
return NETDEV_TX_OK;
|
|
|
}
|
|
|
|
|
|
@@ -2748,9 +3031,6 @@ static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
|
|
|
if (skb->ip_summed == CHECKSUM_PARTIAL)
|
|
|
tx_flags |= IGB_TX_FLAGS_CSUM;
|
|
|
|
|
|
- if (skb->protocol == htons(ETH_P_IP))
|
|
|
- tx_flags |= IGB_TX_FLAGS_IPV4;
|
|
|
-
|
|
|
igb_tx_queue_adv(adapter, tx_ring, tx_flags,
|
|
|
igb_tx_map_adv(adapter, tx_ring, skb),
|
|
|
skb->len, hdr_len);
|
|
|
@@ -2760,14 +3040,22 @@ static int igb_xmit_frame_ring_adv(struct sk_buff *skb,
|
|
|
/* Make sure there is space in the ring for the next send. */
|
|
|
igb_maybe_stop_tx(netdev, tx_ring, MAX_SKB_FRAGS + 4);
|
|
|
|
|
|
- spin_unlock_irqrestore(&tx_ring->tx_lock, irq_flags);
|
|
|
return NETDEV_TX_OK;
|
|
|
}
|
|
|
|
|
|
static int igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *netdev)
|
|
|
{
|
|
|
struct igb_adapter *adapter = netdev_priv(netdev);
|
|
|
- struct igb_ring *tx_ring = &adapter->tx_ring[0];
|
|
|
+ struct igb_ring *tx_ring;
|
|
|
+
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ int r_idx = 0;
|
|
|
+ r_idx = skb->queue_mapping & (IGB_MAX_TX_QUEUES - 1);
|
|
|
+ tx_ring = adapter->multi_tx_table[r_idx];
|
|
|
+#else
|
|
|
+ tx_ring = &adapter->tx_ring[0];
|
|
|
+#endif
|
|
|
+
|
|
|
|
|
|
/* This goes back to the question of how to logically map a tx queue
|
|
|
* to a flow. Right now, performance is impacted slightly negatively
|
|
|
@@ -2862,7 +3150,11 @@ static int igb_change_mtu(struct net_device *netdev, int new_mtu)
|
|
|
else if (max_frame <= IGB_RXBUFFER_2048)
|
|
|
adapter->rx_buffer_len = IGB_RXBUFFER_2048;
|
|
|
else
|
|
|
- adapter->rx_buffer_len = IGB_RXBUFFER_4096;
|
|
|
+#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
|
|
|
+ adapter->rx_buffer_len = IGB_RXBUFFER_16384;
|
|
|
+#else
|
|
|
+ adapter->rx_buffer_len = PAGE_SIZE / 2;
|
|
|
+#endif
|
|
|
/* adjust allocation if LPE protects us, and we aren't using SBP */
|
|
|
if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
|
|
|
(max_frame == MAXIMUM_ETHERNET_VLAN_SIZE))
|
|
|
@@ -3035,7 +3327,7 @@ static irqreturn_t igb_msix_other(int irq, void *data)
|
|
|
/* guard against interrupt when we're going down */
|
|
|
if (!test_bit(__IGB_DOWN, &adapter->state))
|
|
|
mod_timer(&adapter->watchdog_timer, jiffies + 1);
|
|
|
-
|
|
|
+
|
|
|
no_link_interrupt:
|
|
|
wr32(E1000_IMS, E1000_IMS_LSC);
|
|
|
wr32(E1000_EIMS, adapter->eims_other);
|
|
|
@@ -3049,42 +3341,186 @@ static irqreturn_t igb_msix_tx(int irq, void *data)
|
|
|
struct igb_adapter *adapter = tx_ring->adapter;
|
|
|
struct e1000_hw *hw = &adapter->hw;
|
|
|
|
|
|
- if (!tx_ring->itr_val)
|
|
|
- wr32(E1000_EIMC, tx_ring->eims_value);
|
|
|
-
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ if (adapter->flags & IGB_FLAG_DCA_ENABLED)
|
|
|
+ igb_update_tx_dca(tx_ring);
|
|
|
+#endif
|
|
|
tx_ring->total_bytes = 0;
|
|
|
tx_ring->total_packets = 0;
|
|
|
+
|
|
|
+ /* auto mask will automatically reenable the interrupt when we write
|
|
|
+ * EICS */
|
|
|
if (!igb_clean_tx_irq(tx_ring))
|
|
|
/* Ring was not completely cleaned, so fire another interrupt */
|
|
|
wr32(E1000_EICS, tx_ring->eims_value);
|
|
|
-
|
|
|
- if (!tx_ring->itr_val)
|
|
|
+ else
|
|
|
wr32(E1000_EIMS, tx_ring->eims_value);
|
|
|
+
|
|
|
return IRQ_HANDLED;
|
|
|
}
|
|
|
|
|
|
+static void igb_write_itr(struct igb_ring *ring)
|
|
|
+{
|
|
|
+ struct e1000_hw *hw = &ring->adapter->hw;
|
|
|
+ if ((ring->adapter->itr_setting & 3) && ring->set_itr) {
|
|
|
+ switch (hw->mac.type) {
|
|
|
+ case e1000_82576:
|
|
|
+ wr32(ring->itr_register,
|
|
|
+ ring->itr_val |
|
|
|
+ 0x80000000);
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ wr32(ring->itr_register,
|
|
|
+ ring->itr_val |
|
|
|
+ (ring->itr_val << 16));
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ ring->set_itr = 0;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
static irqreturn_t igb_msix_rx(int irq, void *data)
|
|
|
{
|
|
|
struct igb_ring *rx_ring = data;
|
|
|
struct igb_adapter *adapter = rx_ring->adapter;
|
|
|
- struct e1000_hw *hw = &adapter->hw;
|
|
|
|
|
|
/* Write the ITR value calculated at the end of the
|
|
|
* previous interrupt.
|
|
|
*/
|
|
|
|
|
|
- if (adapter->set_itr) {
|
|
|
- wr32(rx_ring->itr_register,
|
|
|
- 1000000000 / (rx_ring->itr_val * 256));
|
|
|
- adapter->set_itr = 0;
|
|
|
- }
|
|
|
+ igb_write_itr(rx_ring);
|
|
|
|
|
|
if (netif_rx_schedule_prep(adapter->netdev, &rx_ring->napi))
|
|
|
__netif_rx_schedule(adapter->netdev, &rx_ring->napi);
|
|
|
|
|
|
- return IRQ_HANDLED;
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ if (adapter->flags & IGB_FLAG_DCA_ENABLED)
|
|
|
+ igb_update_rx_dca(rx_ring);
|
|
|
+#endif
|
|
|
+ return IRQ_HANDLED;
|
|
|
}
|
|
|
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+static void igb_update_rx_dca(struct igb_ring *rx_ring)
|
|
|
+{
|
|
|
+ u32 dca_rxctrl;
|
|
|
+ struct igb_adapter *adapter = rx_ring->adapter;
|
|
|
+ struct e1000_hw *hw = &adapter->hw;
|
|
|
+ int cpu = get_cpu();
|
|
|
+ int q = rx_ring - adapter->rx_ring;
|
|
|
+
|
|
|
+ if (rx_ring->cpu != cpu) {
|
|
|
+ dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
|
|
|
+ if (hw->mac.type == e1000_82576) {
|
|
|
+ dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
|
|
|
+ dca_rxctrl |= dca_get_tag(cpu) <<
|
|
|
+ E1000_DCA_RXCTRL_CPUID_SHIFT;
|
|
|
+ } else {
|
|
|
+ dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
|
|
|
+ dca_rxctrl |= dca_get_tag(cpu);
|
|
|
+ }
|
|
|
+ dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
|
|
|
+ dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
|
|
|
+ dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
|
|
|
+ wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
|
|
|
+ rx_ring->cpu = cpu;
|
|
|
+ }
|
|
|
+ put_cpu();
|
|
|
+}
|
|
|
+
|
|
|
+static void igb_update_tx_dca(struct igb_ring *tx_ring)
|
|
|
+{
|
|
|
+ u32 dca_txctrl;
|
|
|
+ struct igb_adapter *adapter = tx_ring->adapter;
|
|
|
+ struct e1000_hw *hw = &adapter->hw;
|
|
|
+ int cpu = get_cpu();
|
|
|
+ int q = tx_ring - adapter->tx_ring;
|
|
|
+
|
|
|
+ if (tx_ring->cpu != cpu) {
|
|
|
+ dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
|
|
|
+ if (hw->mac.type == e1000_82576) {
|
|
|
+ dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
|
|
|
+ dca_txctrl |= dca_get_tag(cpu) <<
|
|
|
+ E1000_DCA_TXCTRL_CPUID_SHIFT;
|
|
|
+ } else {
|
|
|
+ dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
|
|
|
+ dca_txctrl |= dca_get_tag(cpu);
|
|
|
+ }
|
|
|
+ dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
|
|
|
+ wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
|
|
|
+ tx_ring->cpu = cpu;
|
|
|
+ }
|
|
|
+ put_cpu();
|
|
|
+}
|
|
|
+
|
|
|
+static void igb_setup_dca(struct igb_adapter *adapter)
|
|
|
+{
|
|
|
+ int i;
|
|
|
+
|
|
|
+ if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
|
|
|
+ return;
|
|
|
+
|
|
|
+ for (i = 0; i < adapter->num_tx_queues; i++) {
|
|
|
+ adapter->tx_ring[i].cpu = -1;
|
|
|
+ igb_update_tx_dca(&adapter->tx_ring[i]);
|
|
|
+ }
|
|
|
+ for (i = 0; i < adapter->num_rx_queues; i++) {
|
|
|
+ adapter->rx_ring[i].cpu = -1;
|
|
|
+ igb_update_rx_dca(&adapter->rx_ring[i]);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static int __igb_notify_dca(struct device *dev, void *data)
|
|
|
+{
|
|
|
+ struct net_device *netdev = dev_get_drvdata(dev);
|
|
|
+ struct igb_adapter *adapter = netdev_priv(netdev);
|
|
|
+ struct e1000_hw *hw = &adapter->hw;
|
|
|
+ unsigned long event = *(unsigned long *)data;
|
|
|
+
|
|
|
+ if (!(adapter->flags & IGB_FLAG_HAS_DCA))
|
|
|
+ goto out;
|
|
|
+
|
|
|
+ switch (event) {
|
|
|
+ case DCA_PROVIDER_ADD:
|
|
|
+ /* if already enabled, don't do it again */
|
|
|
+ if (adapter->flags & IGB_FLAG_DCA_ENABLED)
|
|
|
+ break;
|
|
|
+ adapter->flags |= IGB_FLAG_DCA_ENABLED;
|
|
|
+ /* Always use CB2 mode, difference is masked
|
|
|
+ * in the CB driver. */
|
|
|
+ wr32(E1000_DCA_CTRL, 2);
|
|
|
+ if (dca_add_requester(dev) == 0) {
|
|
|
+ dev_info(&adapter->pdev->dev, "DCA enabled\n");
|
|
|
+ igb_setup_dca(adapter);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ /* Fall Through since DCA is disabled. */
|
|
|
+ case DCA_PROVIDER_REMOVE:
|
|
|
+ if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
|
|
|
+ /* without this a class_device is left
|
|
|
+ * hanging around in the sysfs model */
|
|
|
+ dca_remove_requester(dev);
|
|
|
+ dev_info(&adapter->pdev->dev, "DCA disabled\n");
|
|
|
+ adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
|
|
|
+ wr32(E1000_DCA_CTRL, 1);
|
|
|
+ }
|
|
|
+ break;
|
|
|
+ }
|
|
|
+out:
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
|
|
|
+ void *p)
|
|
|
+{
|
|
|
+ int ret_val;
|
|
|
+
|
|
|
+ ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
|
|
|
+ __igb_notify_dca);
|
|
|
+
|
|
|
+ return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
|
|
|
+}
|
|
|
+#endif /* CONFIG_DCA */
|
|
|
|
|
|
/**
|
|
|
* igb_intr_msi - Interrupt Handler
|
|
|
@@ -3099,13 +3535,7 @@ static irqreturn_t igb_intr_msi(int irq, void *data)
|
|
|
/* read ICR disables interrupts using IAM */
|
|
|
u32 icr = rd32(E1000_ICR);
|
|
|
|
|
|
- /* Write the ITR value calculated at the end of the
|
|
|
- * previous interrupt.
|
|
|
- */
|
|
|
- if (adapter->set_itr) {
|
|
|
- wr32(E1000_ITR, 1000000000 / (adapter->itr * 256));
|
|
|
- adapter->set_itr = 0;
|
|
|
- }
|
|
|
+ igb_write_itr(adapter->rx_ring);
|
|
|
|
|
|
if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
|
|
|
hw->mac.get_link_status = 1;
|
|
|
@@ -3135,13 +3565,7 @@ static irqreturn_t igb_intr(int irq, void *data)
|
|
|
if (!icr)
|
|
|
return IRQ_NONE; /* Not our interrupt */
|
|
|
|
|
|
- /* Write the ITR value calculated at the end of the
|
|
|
- * previous interrupt.
|
|
|
- */
|
|
|
- if (adapter->set_itr) {
|
|
|
- wr32(E1000_ITR, 1000000000 / (adapter->itr * 256));
|
|
|
- adapter->set_itr = 0;
|
|
|
- }
|
|
|
+ igb_write_itr(adapter->rx_ring);
|
|
|
|
|
|
/* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
|
|
|
* not set, then the adapter didn't send an interrupt */
|
|
|
@@ -3163,44 +3587,35 @@ static irqreturn_t igb_intr(int irq, void *data)
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * igb_clean - NAPI Rx polling callback
|
|
|
- * @adapter: board private structure
|
|
|
+ * igb_poll - NAPI Rx polling callback
|
|
|
+ * @napi: napi polling structure
|
|
|
+ * @budget: count of how many packets we should handle
|
|
|
**/
|
|
|
-static int igb_clean(struct napi_struct *napi, int budget)
|
|
|
+static int igb_poll(struct napi_struct *napi, int budget)
|
|
|
{
|
|
|
- struct igb_adapter *adapter = container_of(napi, struct igb_adapter,
|
|
|
- napi);
|
|
|
+ struct igb_ring *rx_ring = container_of(napi, struct igb_ring, napi);
|
|
|
+ struct igb_adapter *adapter = rx_ring->adapter;
|
|
|
struct net_device *netdev = adapter->netdev;
|
|
|
- int tx_clean_complete = 1, work_done = 0;
|
|
|
- int i;
|
|
|
-
|
|
|
- /* Must NOT use netdev_priv macro here. */
|
|
|
- adapter = netdev->priv;
|
|
|
-
|
|
|
- /* Keep link state information with original netdev */
|
|
|
- if (!netif_carrier_ok(netdev))
|
|
|
- goto quit_polling;
|
|
|
+ int tx_clean_complete, work_done = 0;
|
|
|
|
|
|
- /* igb_clean is called per-cpu. This lock protects tx_ring[i] from
|
|
|
- * being cleaned by multiple cpus simultaneously. A failure obtaining
|
|
|
- * the lock means tx_ring[i] is currently being cleaned anyway. */
|
|
|
- for (i = 0; i < adapter->num_tx_queues; i++) {
|
|
|
- if (spin_trylock(&adapter->tx_ring[i].tx_clean_lock)) {
|
|
|
- tx_clean_complete &= igb_clean_tx_irq(&adapter->tx_ring[i]);
|
|
|
- spin_unlock(&adapter->tx_ring[i].tx_clean_lock);
|
|
|
- }
|
|
|
- }
|
|
|
+ /* this poll routine only supports one tx and one rx queue */
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ if (adapter->flags & IGB_FLAG_DCA_ENABLED)
|
|
|
+ igb_update_tx_dca(&adapter->tx_ring[0]);
|
|
|
+#endif
|
|
|
+ tx_clean_complete = igb_clean_tx_irq(&adapter->tx_ring[0]);
|
|
|
|
|
|
- for (i = 0; i < adapter->num_rx_queues; i++)
|
|
|
- igb_clean_rx_irq_adv(&adapter->rx_ring[i], &work_done,
|
|
|
- adapter->rx_ring[i].napi.weight);
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ if (adapter->flags & IGB_FLAG_DCA_ENABLED)
|
|
|
+ igb_update_rx_dca(&adapter->rx_ring[0]);
|
|
|
+#endif
|
|
|
+ igb_clean_rx_irq_adv(&adapter->rx_ring[0], &work_done, budget);
|
|
|
|
|
|
/* If no Tx and not enough Rx work done, exit the polling mode */
|
|
|
if ((tx_clean_complete && (work_done < budget)) ||
|
|
|
!netif_running(netdev)) {
|
|
|
-quit_polling:
|
|
|
if (adapter->itr_setting & 3)
|
|
|
- igb_set_itr(adapter, E1000_ITR, false);
|
|
|
+ igb_set_itr(adapter);
|
|
|
netif_rx_complete(netdev, napi);
|
|
|
if (!test_bit(__IGB_DOWN, &adapter->state))
|
|
|
igb_irq_enable(adapter);
|
|
|
@@ -3222,6 +3637,10 @@ static int igb_clean_rx_ring_msix(struct napi_struct *napi, int budget)
|
|
|
if (!netif_carrier_ok(netdev))
|
|
|
goto quit_polling;
|
|
|
|
|
|
+#ifdef CONFIG_DCA
|
|
|
+ if (adapter->flags & IGB_FLAG_DCA_ENABLED)
|
|
|
+ igb_update_rx_dca(rx_ring);
|
|
|
+#endif
|
|
|
igb_clean_rx_irq_adv(rx_ring, &work_done, budget);
|
|
|
|
|
|
|
|
|
@@ -3230,15 +3649,11 @@ static int igb_clean_rx_ring_msix(struct napi_struct *napi, int budget)
|
|
|
quit_polling:
|
|
|
netif_rx_complete(netdev, napi);
|
|
|
|
|
|
- wr32(E1000_EIMS, rx_ring->eims_value);
|
|
|
- if ((adapter->itr_setting & 3) && !rx_ring->no_itr_adjust &&
|
|
|
- (rx_ring->total_packets > IGB_DYN_ITR_PACKET_THRESHOLD)) {
|
|
|
- int mean_size = rx_ring->total_bytes /
|
|
|
- rx_ring->total_packets;
|
|
|
- if (mean_size < IGB_DYN_ITR_LENGTH_LOW)
|
|
|
- igb_raise_rx_eitr(adapter, rx_ring);
|
|
|
- else if (mean_size > IGB_DYN_ITR_LENGTH_HIGH)
|
|
|
- igb_lower_rx_eitr(adapter, rx_ring);
|
|
|
+ if (adapter->itr_setting & 3) {
|
|
|
+ if (adapter->num_rx_queues == 1)
|
|
|
+ igb_set_itr(adapter);
|
|
|
+ else
|
|
|
+ igb_update_ring_itr(rx_ring);
|
|
|
}
|
|
|
|
|
|
if (!test_bit(__IGB_DOWN, &adapter->state))
|
|
|
@@ -3327,11 +3742,19 @@ static bool igb_clean_tx_irq(struct igb_ring *tx_ring)
|
|
|
* sees the new next_to_clean.
|
|
|
*/
|
|
|
smp_mb();
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
|
|
|
+ !(test_bit(__IGB_DOWN, &adapter->state))) {
|
|
|
+ netif_wake_subqueue(netdev, tx_ring->queue_index);
|
|
|
+ ++adapter->restart_queue;
|
|
|
+ }
|
|
|
+#else
|
|
|
if (netif_queue_stopped(netdev) &&
|
|
|
!(test_bit(__IGB_DOWN, &adapter->state))) {
|
|
|
netif_wake_queue(netdev);
|
|
|
++adapter->restart_queue;
|
|
|
}
|
|
|
+#endif
|
|
|
}
|
|
|
|
|
|
if (tx_ring->detect_tx_hung) {
|
|
|
@@ -3348,7 +3771,7 @@ static bool igb_clean_tx_irq(struct igb_ring *tx_ring)
|
|
|
/* detected Tx unit hang */
|
|
|
dev_err(&adapter->pdev->dev,
|
|
|
"Detected Tx Unit Hang\n"
|
|
|
- " Tx Queue <%lu>\n"
|
|
|
+ " Tx Queue <%d>\n"
|
|
|
" TDH <%x>\n"
|
|
|
" TDT <%x>\n"
|
|
|
" next_to_use <%x>\n"
|
|
|
@@ -3358,8 +3781,7 @@ static bool igb_clean_tx_irq(struct igb_ring *tx_ring)
|
|
|
" time_stamp <%lx>\n"
|
|
|
" jiffies <%lx>\n"
|
|
|
" desc.status <%x>\n",
|
|
|
- (unsigned long)((tx_ring - adapter->tx_ring) /
|
|
|
- sizeof(struct igb_ring)),
|
|
|
+ tx_ring->queue_index,
|
|
|
readl(adapter->hw.hw_addr + tx_ring->head),
|
|
|
readl(adapter->hw.hw_addr + tx_ring->tail),
|
|
|
tx_ring->next_to_use,
|
|
|
@@ -3368,32 +3790,91 @@ static bool igb_clean_tx_irq(struct igb_ring *tx_ring)
|
|
|
tx_ring->buffer_info[i].time_stamp,
|
|
|
jiffies,
|
|
|
tx_desc->upper.fields.status);
|
|
|
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
|
|
|
+ netif_stop_subqueue(netdev, tx_ring->queue_index);
|
|
|
+#else
|
|
|
netif_stop_queue(netdev);
|
|
|
+#endif
|
|
|
}
|
|
|
}
|
|
|
tx_ring->total_bytes += total_bytes;
|
|
|
tx_ring->total_packets += total_packets;
|
|
|
+ tx_ring->tx_stats.bytes += total_bytes;
|
|
|
+ tx_ring->tx_stats.packets += total_packets;
|
|
|
adapter->net_stats.tx_bytes += total_bytes;
|
|
|
adapter->net_stats.tx_packets += total_packets;
|
|
|
return retval;
|
|
|
}
|
|
|
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+ /**
|
|
|
+ * igb_get_skb_hdr - helper function for LRO header processing
|
|
|
+ * @skb: pointer to sk_buff to be added to LRO packet
|
|
|
+ * @iphdr: pointer to ip header structure
|
|
|
+ * @tcph: pointer to tcp header structure
|
|
|
+ * @hdr_flags: pointer to header flags
|
|
|
+ * @priv: pointer to the receive descriptor for the current sk_buff
|
|
|
+ **/
|
|
|
+static int igb_get_skb_hdr(struct sk_buff *skb, void **iphdr, void **tcph,
|
|
|
+ u64 *hdr_flags, void *priv)
|
|
|
+{
|
|
|
+ union e1000_adv_rx_desc *rx_desc = priv;
|
|
|
+ u16 pkt_type = rx_desc->wb.lower.lo_dword.pkt_info &
|
|
|
+ (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP);
|
|
|
+
|
|
|
+ /* Verify that this is a valid IPv4 TCP packet */
|
|
|
+ if (pkt_type != (E1000_RXDADV_PKTTYPE_IPV4 |
|
|
|
+ E1000_RXDADV_PKTTYPE_TCP))
|
|
|
+ return -1;
|
|
|
+
|
|
|
+ /* Set network headers */
|
|
|
+ skb_reset_network_header(skb);
|
|
|
+ skb_set_transport_header(skb, ip_hdrlen(skb));
|
|
|
+ *iphdr = ip_hdr(skb);
|
|
|
+ *tcph = tcp_hdr(skb);
|
|
|
+ *hdr_flags = LRO_IPV4 | LRO_TCP;
|
|
|
+
|
|
|
+ return 0;
|
|
|
+
|
|
|
+}
|
|
|
+#endif /* CONFIG_IGB_LRO */
|
|
|
|
|
|
/**
|
|
|
* igb_receive_skb - helper function to handle rx indications
|
|
|
- * @adapter: board private structure
|
|
|
+ * @ring: pointer to receive ring receving this packet
|
|
|
* @status: descriptor status field as written by hardware
|
|
|
* @vlan: descriptor vlan field as written by hardware (no le/be conversion)
|
|
|
* @skb: pointer to sk_buff to be indicated to stack
|
|
|
**/
|
|
|
-static void igb_receive_skb(struct igb_adapter *adapter, u8 status, __le16 vlan,
|
|
|
- struct sk_buff *skb)
|
|
|
+static void igb_receive_skb(struct igb_ring *ring, u8 status,
|
|
|
+ union e1000_adv_rx_desc * rx_desc,
|
|
|
+ struct sk_buff *skb)
|
|
|
{
|
|
|
- if (adapter->vlgrp && (status & E1000_RXD_STAT_VP))
|
|
|
- vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
|
|
|
- le16_to_cpu(vlan));
|
|
|
- else
|
|
|
- netif_receive_skb(skb);
|
|
|
+ struct igb_adapter * adapter = ring->adapter;
|
|
|
+ bool vlan_extracted = (adapter->vlgrp && (status & E1000_RXD_STAT_VP));
|
|
|
+
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+ if (adapter->netdev->features & NETIF_F_LRO &&
|
|
|
+ skb->ip_summed == CHECKSUM_UNNECESSARY) {
|
|
|
+ if (vlan_extracted)
|
|
|
+ lro_vlan_hwaccel_receive_skb(&ring->lro_mgr, skb,
|
|
|
+ adapter->vlgrp,
|
|
|
+ le16_to_cpu(rx_desc->wb.upper.vlan),
|
|
|
+ rx_desc);
|
|
|
+ else
|
|
|
+ lro_receive_skb(&ring->lro_mgr,skb, rx_desc);
|
|
|
+ ring->lro_used = 1;
|
|
|
+ } else {
|
|
|
+#endif
|
|
|
+ if (vlan_extracted)
|
|
|
+ vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
|
|
|
+ le16_to_cpu(rx_desc->wb.upper.vlan));
|
|
|
+ else
|
|
|
+
|
|
|
+ netif_receive_skb(skb);
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+ }
|
|
|
+#endif
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -3428,7 +3909,7 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring,
|
|
|
union e1000_adv_rx_desc *rx_desc , *next_rxd;
|
|
|
struct igb_buffer *buffer_info , *next_buffer;
|
|
|
struct sk_buff *skb;
|
|
|
- unsigned int i, j;
|
|
|
+ unsigned int i;
|
|
|
u32 length, hlen, staterr;
|
|
|
bool cleaned = false;
|
|
|
int cleaned_count = 0;
|
|
|
@@ -3458,64 +3939,48 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring,
|
|
|
cleaned = true;
|
|
|
cleaned_count++;
|
|
|
|
|
|
- if (rx_ring->pending_skb != NULL) {
|
|
|
- skb = rx_ring->pending_skb;
|
|
|
- rx_ring->pending_skb = NULL;
|
|
|
- j = rx_ring->pending_skb_page;
|
|
|
- } else {
|
|
|
- skb = buffer_info->skb;
|
|
|
- prefetch(skb->data - NET_IP_ALIGN);
|
|
|
- buffer_info->skb = NULL;
|
|
|
- if (hlen) {
|
|
|
- pci_unmap_single(pdev, buffer_info->dma,
|
|
|
- adapter->rx_ps_hdr_size +
|
|
|
- NET_IP_ALIGN,
|
|
|
- PCI_DMA_FROMDEVICE);
|
|
|
- skb_put(skb, hlen);
|
|
|
- } else {
|
|
|
- pci_unmap_single(pdev, buffer_info->dma,
|
|
|
- adapter->rx_buffer_len +
|
|
|
- NET_IP_ALIGN,
|
|
|
- PCI_DMA_FROMDEVICE);
|
|
|
- skb_put(skb, length);
|
|
|
- goto send_up;
|
|
|
- }
|
|
|
- j = 0;
|
|
|
+ skb = buffer_info->skb;
|
|
|
+ prefetch(skb->data - NET_IP_ALIGN);
|
|
|
+ buffer_info->skb = NULL;
|
|
|
+ if (!adapter->rx_ps_hdr_size) {
|
|
|
+ pci_unmap_single(pdev, buffer_info->dma,
|
|
|
+ adapter->rx_buffer_len +
|
|
|
+ NET_IP_ALIGN,
|
|
|
+ PCI_DMA_FROMDEVICE);
|
|
|
+ skb_put(skb, length);
|
|
|
+ goto send_up;
|
|
|
}
|
|
|
|
|
|
- while (length) {
|
|
|
+ if (!skb_shinfo(skb)->nr_frags) {
|
|
|
+ pci_unmap_single(pdev, buffer_info->dma,
|
|
|
+ adapter->rx_ps_hdr_size +
|
|
|
+ NET_IP_ALIGN,
|
|
|
+ PCI_DMA_FROMDEVICE);
|
|
|
+ skb_put(skb, hlen);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (length) {
|
|
|
pci_unmap_page(pdev, buffer_info->page_dma,
|
|
|
- PAGE_SIZE, PCI_DMA_FROMDEVICE);
|
|
|
+ PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
|
|
|
buffer_info->page_dma = 0;
|
|
|
- skb_fill_page_desc(skb, j, buffer_info->page,
|
|
|
- 0, length);
|
|
|
- buffer_info->page = NULL;
|
|
|
+
|
|
|
+ skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
|
|
|
+ buffer_info->page,
|
|
|
+ buffer_info->page_offset,
|
|
|
+ length);
|
|
|
+
|
|
|
+ if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||
|
|
|
+ (page_count(buffer_info->page) != 1))
|
|
|
+ buffer_info->page = NULL;
|
|
|
+ else
|
|
|
+ get_page(buffer_info->page);
|
|
|
|
|
|
skb->len += length;
|
|
|
skb->data_len += length;
|
|
|
- skb->truesize += length;
|
|
|
- rx_desc->wb.upper.status_error = 0;
|
|
|
- if (staterr & E1000_RXD_STAT_EOP)
|
|
|
- break;
|
|
|
|
|
|
- j++;
|
|
|
- cleaned_count++;
|
|
|
- i++;
|
|
|
- if (i == rx_ring->count)
|
|
|
- i = 0;
|
|
|
-
|
|
|
- buffer_info = &rx_ring->buffer_info[i];
|
|
|
- rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
|
|
|
- staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
|
|
|
- length = le16_to_cpu(rx_desc->wb.upper.length);
|
|
|
- if (!(staterr & E1000_RXD_STAT_DD)) {
|
|
|
- rx_ring->pending_skb = skb;
|
|
|
- rx_ring->pending_skb_page = j;
|
|
|
- goto out;
|
|
|
- }
|
|
|
+ skb->truesize += length;
|
|
|
}
|
|
|
send_up:
|
|
|
- pskb_trim(skb, skb->len - 4);
|
|
|
i++;
|
|
|
if (i == rx_ring->count)
|
|
|
i = 0;
|
|
|
@@ -3523,11 +3988,16 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring,
|
|
|
prefetch(next_rxd);
|
|
|
next_buffer = &rx_ring->buffer_info[i];
|
|
|
|
|
|
+ if (!(staterr & E1000_RXD_STAT_EOP)) {
|
|
|
+ buffer_info->skb = xchg(&next_buffer->skb, skb);
|
|
|
+ buffer_info->dma = xchg(&next_buffer->dma, 0);
|
|
|
+ goto next_desc;
|
|
|
+ }
|
|
|
+
|
|
|
if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
|
|
|
dev_kfree_skb_irq(skb);
|
|
|
goto next_desc;
|
|
|
}
|
|
|
- rx_ring->no_itr_adjust |= (staterr & E1000_RXD_STAT_DYNINT);
|
|
|
|
|
|
total_bytes += skb->len;
|
|
|
total_packets++;
|
|
|
@@ -3536,7 +4006,7 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring,
|
|
|
|
|
|
skb->protocol = eth_type_trans(skb, netdev);
|
|
|
|
|
|
- igb_receive_skb(adapter, staterr, rx_desc->wb.upper.vlan, skb);
|
|
|
+ igb_receive_skb(rx_ring, staterr, rx_desc, skb);
|
|
|
|
|
|
netdev->last_rx = jiffies;
|
|
|
|
|
|
@@ -3555,10 +4025,17 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring,
|
|
|
|
|
|
staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
|
|
|
}
|
|
|
-out:
|
|
|
+
|
|
|
rx_ring->next_to_clean = i;
|
|
|
cleaned_count = IGB_DESC_UNUSED(rx_ring);
|
|
|
|
|
|
+#ifdef CONFIG_IGB_LRO
|
|
|
+ if (rx_ring->lro_used) {
|
|
|
+ lro_flush_all(&rx_ring->lro_mgr);
|
|
|
+ rx_ring->lro_used = 0;
|
|
|
+ }
|
|
|
+#endif
|
|
|
+
|
|
|
if (cleaned_count)
|
|
|
igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
|
|
|
|
|
|
@@ -3593,16 +4070,22 @@ static void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring,
|
|
|
while (cleaned_count--) {
|
|
|
rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
|
|
|
|
|
|
- if (adapter->rx_ps_hdr_size && !buffer_info->page) {
|
|
|
- buffer_info->page = alloc_page(GFP_ATOMIC);
|
|
|
+ if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {
|
|
|
if (!buffer_info->page) {
|
|
|
- adapter->alloc_rx_buff_failed++;
|
|
|
- goto no_buffers;
|
|
|
+ buffer_info->page = alloc_page(GFP_ATOMIC);
|
|
|
+ if (!buffer_info->page) {
|
|
|
+ adapter->alloc_rx_buff_failed++;
|
|
|
+ goto no_buffers;
|
|
|
+ }
|
|
|
+ buffer_info->page_offset = 0;
|
|
|
+ } else {
|
|
|
+ buffer_info->page_offset ^= PAGE_SIZE / 2;
|
|
|
}
|
|
|
buffer_info->page_dma =
|
|
|
pci_map_page(pdev,
|
|
|
buffer_info->page,
|
|
|
- 0, PAGE_SIZE,
|
|
|
+ buffer_info->page_offset,
|
|
|
+ PAGE_SIZE / 2,
|
|
|
PCI_DMA_FROMDEVICE);
|
|
|
}
|
|
|
|
|
|
@@ -3869,7 +4352,7 @@ static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
|
|
|
struct net_device *netdev = pci_get_drvdata(pdev);
|
|
|
struct igb_adapter *adapter = netdev_priv(netdev);
|
|
|
struct e1000_hw *hw = &adapter->hw;
|
|
|
- u32 ctrl, ctrl_ext, rctl, status;
|
|
|
+ u32 ctrl, rctl, status;
|
|
|
u32 wufc = adapter->wol;
|
|
|
#ifdef CONFIG_PM
|
|
|
int retval = 0;
|
|
|
@@ -3877,11 +4360,12 @@ static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
|
|
|
|
|
|
netif_device_detach(netdev);
|
|
|
|
|
|
- if (netif_running(netdev)) {
|
|
|
- WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
|
|
|
- igb_down(adapter);
|
|
|
- igb_free_irq(adapter);
|
|
|
- }
|
|
|
+ if (netif_running(netdev))
|
|
|
+ igb_close(netdev);
|
|
|
+
|
|
|
+ igb_reset_interrupt_capability(adapter);
|
|
|
+
|
|
|
+ igb_free_queues(adapter);
|
|
|
|
|
|
#ifdef CONFIG_PM
|
|
|
retval = pci_save_state(pdev);
|
|
|
@@ -3912,33 +4396,24 @@ static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
|
|
|
ctrl |= E1000_CTRL_ADVD3WUC;
|
|
|
wr32(E1000_CTRL, ctrl);
|
|
|
|
|
|
- if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
|
|
|
- adapter->hw.phy.media_type ==
|
|
|
- e1000_media_type_internal_serdes) {
|
|
|
- /* keep the laser running in D3 */
|
|
|
- ctrl_ext = rd32(E1000_CTRL_EXT);
|
|
|
- ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
|
|
|
- wr32(E1000_CTRL_EXT, ctrl_ext);
|
|
|
- }
|
|
|
-
|
|
|
/* Allow time for pending master requests to run */
|
|
|
igb_disable_pcie_master(&adapter->hw);
|
|
|
|
|
|
wr32(E1000_WUC, E1000_WUC_PME_EN);
|
|
|
wr32(E1000_WUFC, wufc);
|
|
|
- pci_enable_wake(pdev, PCI_D3hot, 1);
|
|
|
- pci_enable_wake(pdev, PCI_D3cold, 1);
|
|
|
} else {
|
|
|
wr32(E1000_WUC, 0);
|
|
|
wr32(E1000_WUFC, 0);
|
|
|
- pci_enable_wake(pdev, PCI_D3hot, 0);
|
|
|
- pci_enable_wake(pdev, PCI_D3cold, 0);
|
|
|
}
|
|
|
|
|
|
- /* make sure adapter isn't asleep if manageability is enabled */
|
|
|
- if (adapter->en_mng_pt) {
|
|
|
+ /* make sure adapter isn't asleep if manageability/wol is enabled */
|
|
|
+ if (wufc || adapter->en_mng_pt) {
|
|
|
pci_enable_wake(pdev, PCI_D3hot, 1);
|
|
|
pci_enable_wake(pdev, PCI_D3cold, 1);
|
|
|
+ } else {
|
|
|
+ igb_shutdown_fiber_serdes_link_82575(hw);
|
|
|
+ pci_enable_wake(pdev, PCI_D3hot, 0);
|
|
|
+ pci_enable_wake(pdev, PCI_D3cold, 0);
|
|
|
}
|
|
|
|
|
|
/* Release control of h/w to f/w. If f/w is AMT enabled, this
|
|
|
@@ -3977,10 +4452,11 @@ static int igb_resume(struct pci_dev *pdev)
|
|
|
pci_enable_wake(pdev, PCI_D3hot, 0);
|
|
|
pci_enable_wake(pdev, PCI_D3cold, 0);
|
|
|
|
|
|
- if (netif_running(netdev)) {
|
|
|
- err = igb_request_irq(adapter);
|
|
|
- if (err)
|
|
|
- return err;
|
|
|
+ igb_set_interrupt_capability(adapter);
|
|
|
+
|
|
|
+ if (igb_alloc_queues(adapter)) {
|
|
|
+ dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
|
|
|
+ return -ENOMEM;
|
|
|
}
|
|
|
|
|
|
/* e1000_power_up_phy(adapter); */
|
|
|
@@ -3988,10 +4464,11 @@ static int igb_resume(struct pci_dev *pdev)
|
|
|
igb_reset(adapter);
|
|
|
wr32(E1000_WUS, ~0);
|
|
|
|
|
|
- igb_init_manageability(adapter);
|
|
|
-
|
|
|
- if (netif_running(netdev))
|
|
|
- igb_up(adapter);
|
|
|
+ if (netif_running(netdev)) {
|
|
|
+ err = igb_open(netdev);
|
|
|
+ if (err)
|
|
|
+ return err;
|
|
|
+ }
|
|
|
|
|
|
netif_device_attach(netdev);
|
|
|
|
|
|
@@ -4021,6 +4498,8 @@ static void igb_netpoll(struct net_device *netdev)
|
|
|
int work_done = 0;
|
|
|
|
|
|
igb_irq_disable(adapter);
|
|
|
+ adapter->flags |= IGB_FLAG_IN_NETPOLL;
|
|
|
+
|
|
|
for (i = 0; i < adapter->num_tx_queues; i++)
|
|
|
igb_clean_tx_irq(&adapter->tx_ring[i]);
|
|
|
|
|
|
@@ -4029,6 +4508,7 @@ static void igb_netpoll(struct net_device *netdev)
|
|
|
&work_done,
|
|
|
adapter->rx_ring[i].napi.weight);
|
|
|
|
|
|
+ adapter->flags &= ~IGB_FLAG_IN_NETPOLL;
|
|
|
igb_irq_enable(adapter);
|
|
|
}
|
|
|
#endif /* CONFIG_NET_POLL_CONTROLLER */
|