|
@@ -28,6 +28,7 @@
|
|
|
#include <linux/pm_runtime.h>
|
|
|
#include <linux/pci_hotplug.h>
|
|
|
#include <linux/vmalloc.h>
|
|
|
+#include <linux/pci-ats.h>
|
|
|
#include <asm/setup.h>
|
|
|
#include <asm/dma.h>
|
|
|
#include <linux/aer.h>
|
|
@@ -1173,6 +1174,8 @@ void pci_restore_state(struct pci_dev *dev)
|
|
|
|
|
|
/* PCI Express register must be restored first */
|
|
|
pci_restore_pcie_state(dev);
|
|
|
+ pci_restore_pasid_state(dev);
|
|
|
+ pci_restore_pri_state(dev);
|
|
|
pci_restore_ats_state(dev);
|
|
|
pci_restore_vc_state(dev);
|
|
|
|
|
@@ -4033,40 +4036,6 @@ static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe)
|
|
|
return pci_reset_hotplug_slot(dev->slot->hotplug, probe);
|
|
|
}
|
|
|
|
|
|
-static int __pci_dev_reset(struct pci_dev *dev, int probe)
|
|
|
-{
|
|
|
- int rc;
|
|
|
-
|
|
|
- might_sleep();
|
|
|
-
|
|
|
- rc = pci_dev_specific_reset(dev, probe);
|
|
|
- if (rc != -ENOTTY)
|
|
|
- goto done;
|
|
|
-
|
|
|
- if (pcie_has_flr(dev)) {
|
|
|
- if (!probe)
|
|
|
- pcie_flr(dev);
|
|
|
- rc = 0;
|
|
|
- goto done;
|
|
|
- }
|
|
|
-
|
|
|
- rc = pci_af_flr(dev, probe);
|
|
|
- if (rc != -ENOTTY)
|
|
|
- goto done;
|
|
|
-
|
|
|
- rc = pci_pm_reset(dev, probe);
|
|
|
- if (rc != -ENOTTY)
|
|
|
- goto done;
|
|
|
-
|
|
|
- rc = pci_dev_reset_slot_function(dev, probe);
|
|
|
- if (rc != -ENOTTY)
|
|
|
- goto done;
|
|
|
-
|
|
|
- rc = pci_parent_bus_reset(dev, probe);
|
|
|
-done:
|
|
|
- return rc;
|
|
|
-}
|
|
|
-
|
|
|
static void pci_dev_lock(struct pci_dev *dev)
|
|
|
{
|
|
|
pci_cfg_access_lock(dev);
|
|
@@ -4092,26 +4061,18 @@ static void pci_dev_unlock(struct pci_dev *dev)
|
|
|
pci_cfg_access_unlock(dev);
|
|
|
}
|
|
|
|
|
|
-/**
|
|
|
- * pci_reset_notify - notify device driver of reset
|
|
|
- * @dev: device to be notified of reset
|
|
|
- * @prepare: 'true' if device is about to be reset; 'false' if reset attempt
|
|
|
- * completed
|
|
|
- *
|
|
|
- * Must be called prior to device access being disabled and after device
|
|
|
- * access is restored.
|
|
|
- */
|
|
|
-static void pci_reset_notify(struct pci_dev *dev, bool prepare)
|
|
|
+static void pci_dev_save_and_disable(struct pci_dev *dev)
|
|
|
{
|
|
|
const struct pci_error_handlers *err_handler =
|
|
|
dev->driver ? dev->driver->err_handler : NULL;
|
|
|
- if (err_handler && err_handler->reset_notify)
|
|
|
- err_handler->reset_notify(dev, prepare);
|
|
|
-}
|
|
|
|
|
|
-static void pci_dev_save_and_disable(struct pci_dev *dev)
|
|
|
-{
|
|
|
- pci_reset_notify(dev, true);
|
|
|
+ /*
|
|
|
+ * dev->driver->err_handler->reset_prepare() is protected against
|
|
|
+ * races with ->remove() by the device lock, which must be held by
|
|
|
+ * the caller.
|
|
|
+ */
|
|
|
+ if (err_handler && err_handler->reset_prepare)
|
|
|
+ err_handler->reset_prepare(dev);
|
|
|
|
|
|
/*
|
|
|
* Wake-up device prior to save. PM registers default to D0 after
|
|
@@ -4133,23 +4094,18 @@ static void pci_dev_save_and_disable(struct pci_dev *dev)
|
|
|
|
|
|
static void pci_dev_restore(struct pci_dev *dev)
|
|
|
{
|
|
|
- pci_restore_state(dev);
|
|
|
- pci_reset_notify(dev, false);
|
|
|
-}
|
|
|
-
|
|
|
-static int pci_dev_reset(struct pci_dev *dev, int probe)
|
|
|
-{
|
|
|
- int rc;
|
|
|
-
|
|
|
- if (!probe)
|
|
|
- pci_dev_lock(dev);
|
|
|
-
|
|
|
- rc = __pci_dev_reset(dev, probe);
|
|
|
+ const struct pci_error_handlers *err_handler =
|
|
|
+ dev->driver ? dev->driver->err_handler : NULL;
|
|
|
|
|
|
- if (!probe)
|
|
|
- pci_dev_unlock(dev);
|
|
|
+ pci_restore_state(dev);
|
|
|
|
|
|
- return rc;
|
|
|
+ /*
|
|
|
+ * dev->driver->err_handler->reset_done() is protected against
|
|
|
+ * races with ->remove() by the device lock, which must be held by
|
|
|
+ * the caller.
|
|
|
+ */
|
|
|
+ if (err_handler && err_handler->reset_done)
|
|
|
+ err_handler->reset_done(dev);
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -4171,7 +4127,13 @@ static int pci_dev_reset(struct pci_dev *dev, int probe)
|
|
|
*/
|
|
|
int __pci_reset_function(struct pci_dev *dev)
|
|
|
{
|
|
|
- return pci_dev_reset(dev, 0);
|
|
|
+ int ret;
|
|
|
+
|
|
|
+ pci_dev_lock(dev);
|
|
|
+ ret = __pci_reset_function_locked(dev);
|
|
|
+ pci_dev_unlock(dev);
|
|
|
+
|
|
|
+ return ret;
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(__pci_reset_function);
|
|
|
|
|
@@ -4196,7 +4158,27 @@ EXPORT_SYMBOL_GPL(__pci_reset_function);
|
|
|
*/
|
|
|
int __pci_reset_function_locked(struct pci_dev *dev)
|
|
|
{
|
|
|
- return __pci_dev_reset(dev, 0);
|
|
|
+ int rc;
|
|
|
+
|
|
|
+ might_sleep();
|
|
|
+
|
|
|
+ rc = pci_dev_specific_reset(dev, 0);
|
|
|
+ if (rc != -ENOTTY)
|
|
|
+ return rc;
|
|
|
+ if (pcie_has_flr(dev)) {
|
|
|
+ pcie_flr(dev);
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ rc = pci_af_flr(dev, 0);
|
|
|
+ if (rc != -ENOTTY)
|
|
|
+ return rc;
|
|
|
+ rc = pci_pm_reset(dev, 0);
|
|
|
+ if (rc != -ENOTTY)
|
|
|
+ return rc;
|
|
|
+ rc = pci_dev_reset_slot_function(dev, 0);
|
|
|
+ if (rc != -ENOTTY)
|
|
|
+ return rc;
|
|
|
+ return pci_parent_bus_reset(dev, 0);
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
|
|
|
|
|
@@ -4213,7 +4195,26 @@ EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
|
|
|
*/
|
|
|
int pci_probe_reset_function(struct pci_dev *dev)
|
|
|
{
|
|
|
- return pci_dev_reset(dev, 1);
|
|
|
+ int rc;
|
|
|
+
|
|
|
+ might_sleep();
|
|
|
+
|
|
|
+ rc = pci_dev_specific_reset(dev, 1);
|
|
|
+ if (rc != -ENOTTY)
|
|
|
+ return rc;
|
|
|
+ if (pcie_has_flr(dev))
|
|
|
+ return 0;
|
|
|
+ rc = pci_af_flr(dev, 1);
|
|
|
+ if (rc != -ENOTTY)
|
|
|
+ return rc;
|
|
|
+ rc = pci_pm_reset(dev, 1);
|
|
|
+ if (rc != -ENOTTY)
|
|
|
+ return rc;
|
|
|
+ rc = pci_dev_reset_slot_function(dev, 1);
|
|
|
+ if (rc != -ENOTTY)
|
|
|
+ return rc;
|
|
|
+
|
|
|
+ return pci_parent_bus_reset(dev, 1);
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -4236,15 +4237,17 @@ int pci_reset_function(struct pci_dev *dev)
|
|
|
{
|
|
|
int rc;
|
|
|
|
|
|
- rc = pci_dev_reset(dev, 1);
|
|
|
+ rc = pci_probe_reset_function(dev);
|
|
|
if (rc)
|
|
|
return rc;
|
|
|
|
|
|
+ pci_dev_lock(dev);
|
|
|
pci_dev_save_and_disable(dev);
|
|
|
|
|
|
- rc = pci_dev_reset(dev, 0);
|
|
|
+ rc = __pci_reset_function_locked(dev);
|
|
|
|
|
|
pci_dev_restore(dev);
|
|
|
+ pci_dev_unlock(dev);
|
|
|
|
|
|
return rc;
|
|
|
}
|
|
@@ -4260,20 +4263,18 @@ int pci_try_reset_function(struct pci_dev *dev)
|
|
|
{
|
|
|
int rc;
|
|
|
|
|
|
- rc = pci_dev_reset(dev, 1);
|
|
|
+ rc = pci_probe_reset_function(dev);
|
|
|
if (rc)
|
|
|
return rc;
|
|
|
|
|
|
- pci_dev_save_and_disable(dev);
|
|
|
+ if (!pci_dev_trylock(dev))
|
|
|
+ return -EAGAIN;
|
|
|
|
|
|
- if (pci_dev_trylock(dev)) {
|
|
|
- rc = __pci_dev_reset(dev, 0);
|
|
|
- pci_dev_unlock(dev);
|
|
|
- } else
|
|
|
- rc = -EAGAIN;
|
|
|
+ pci_dev_save_and_disable(dev);
|
|
|
+ rc = __pci_reset_function_locked(dev);
|
|
|
+ pci_dev_unlock(dev);
|
|
|
|
|
|
pci_dev_restore(dev);
|
|
|
-
|
|
|
return rc;
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(pci_try_reset_function);
|
|
@@ -4423,7 +4424,9 @@ static void pci_bus_save_and_disable(struct pci_bus *bus)
|
|
|
struct pci_dev *dev;
|
|
|
|
|
|
list_for_each_entry(dev, &bus->devices, bus_list) {
|
|
|
+ pci_dev_lock(dev);
|
|
|
pci_dev_save_and_disable(dev);
|
|
|
+ pci_dev_unlock(dev);
|
|
|
if (dev->subordinate)
|
|
|
pci_bus_save_and_disable(dev->subordinate);
|
|
|
}
|
|
@@ -4438,7 +4441,9 @@ static void pci_bus_restore(struct pci_bus *bus)
|
|
|
struct pci_dev *dev;
|
|
|
|
|
|
list_for_each_entry(dev, &bus->devices, bus_list) {
|
|
|
+ pci_dev_lock(dev);
|
|
|
pci_dev_restore(dev);
|
|
|
+ pci_dev_unlock(dev);
|
|
|
if (dev->subordinate)
|
|
|
pci_bus_restore(dev->subordinate);
|
|
|
}
|