vfio_pci.c 25 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054
  1. /*
  2. * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
  3. * Author: Alex Williamson <alex.williamson@redhat.com>
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License version 2 as
  7. * published by the Free Software Foundation.
  8. *
  9. * Derived from original vfio:
  10. * Copyright 2010 Cisco Systems, Inc. All rights reserved.
  11. * Author: Tom Lyon, pugs@cisco.com
  12. */
  13. #include <linux/device.h>
  14. #include <linux/eventfd.h>
  15. #include <linux/file.h>
  16. #include <linux/interrupt.h>
  17. #include <linux/iommu.h>
  18. #include <linux/module.h>
  19. #include <linux/mutex.h>
  20. #include <linux/notifier.h>
  21. #include <linux/pci.h>
  22. #include <linux/pm_runtime.h>
  23. #include <linux/slab.h>
  24. #include <linux/types.h>
  25. #include <linux/uaccess.h>
  26. #include <linux/vfio.h>
  27. #include "vfio_pci_private.h"
  28. #define DRIVER_VERSION "0.2"
  29. #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
  30. #define DRIVER_DESC "VFIO PCI - User Level meta-driver"
  31. static bool nointxmask;
  32. module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR);
  33. MODULE_PARM_DESC(nointxmask,
  34. "Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");
  35. static DEFINE_MUTEX(driver_lock);
  36. static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev);
  37. static int vfio_pci_enable(struct vfio_pci_device *vdev)
  38. {
  39. struct pci_dev *pdev = vdev->pdev;
  40. int ret;
  41. u16 cmd;
  42. u8 msix_pos;
  43. /* Don't allow our initial saved state to include busmaster */
  44. pci_clear_master(pdev);
  45. ret = pci_enable_device(pdev);
  46. if (ret)
  47. return ret;
  48. vdev->reset_works = (pci_reset_function(pdev) == 0);
  49. pci_save_state(pdev);
  50. vdev->pci_saved_state = pci_store_saved_state(pdev);
  51. if (!vdev->pci_saved_state)
  52. pr_debug("%s: Couldn't store %s saved state\n",
  53. __func__, dev_name(&pdev->dev));
  54. ret = vfio_config_init(vdev);
  55. if (ret) {
  56. kfree(vdev->pci_saved_state);
  57. vdev->pci_saved_state = NULL;
  58. pci_disable_device(pdev);
  59. return ret;
  60. }
  61. if (likely(!nointxmask))
  62. vdev->pci_2_3 = pci_intx_mask_supported(pdev);
  63. pci_read_config_word(pdev, PCI_COMMAND, &cmd);
  64. if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
  65. cmd &= ~PCI_COMMAND_INTX_DISABLE;
  66. pci_write_config_word(pdev, PCI_COMMAND, cmd);
  67. }
  68. msix_pos = pdev->msix_cap;
  69. if (msix_pos) {
  70. u16 flags;
  71. u32 table;
  72. pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
  73. pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
  74. vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
  75. vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
  76. vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
  77. } else
  78. vdev->msix_bar = 0xFF;
  79. #ifdef CONFIG_VFIO_PCI_VGA
  80. if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
  81. vdev->has_vga = true;
  82. #endif
  83. return 0;
  84. }
  85. static void vfio_pci_disable(struct vfio_pci_device *vdev)
  86. {
  87. struct pci_dev *pdev = vdev->pdev;
  88. int bar;
  89. /* Stop the device from further DMA */
  90. pci_clear_master(pdev);
  91. vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |
  92. VFIO_IRQ_SET_ACTION_TRIGGER,
  93. vdev->irq_type, 0, 0, NULL);
  94. vdev->virq_disabled = false;
  95. vfio_config_free(vdev);
  96. for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
  97. if (!vdev->barmap[bar])
  98. continue;
  99. pci_iounmap(pdev, vdev->barmap[bar]);
  100. pci_release_selected_regions(pdev, 1 << bar);
  101. vdev->barmap[bar] = NULL;
  102. }
  103. vdev->needs_reset = true;
  104. /*
  105. * If we have saved state, restore it. If we can reset the device,
  106. * even better. Resetting with current state seems better than
  107. * nothing, but saving and restoring current state without reset
  108. * is just busy work.
  109. */
  110. if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
  111. pr_info("%s: Couldn't reload %s saved state\n",
  112. __func__, dev_name(&pdev->dev));
  113. if (!vdev->reset_works)
  114. goto out;
  115. pci_save_state(pdev);
  116. }
  117. /*
  118. * Disable INTx and MSI, presumably to avoid spurious interrupts
  119. * during reset. Stolen from pci_reset_function()
  120. */
  121. pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
  122. /*
  123. * Try to reset the device. The success of this is dependent on
  124. * being able to lock the device, which is not always possible.
  125. */
  126. if (vdev->reset_works) {
  127. int ret = pci_try_reset_function(pdev);
  128. if (ret)
  129. pr_warn("%s: Failed to reset device %s (%d)\n",
  130. __func__, dev_name(&pdev->dev), ret);
  131. else
  132. vdev->needs_reset = false;
  133. }
  134. pci_restore_state(pdev);
  135. out:
  136. pci_disable_device(pdev);
  137. vfio_pci_try_bus_reset(vdev);
  138. }
  139. static void vfio_pci_release(void *device_data)
  140. {
  141. struct vfio_pci_device *vdev = device_data;
  142. mutex_lock(&driver_lock);
  143. if (!(--vdev->refcnt)) {
  144. vfio_spapr_pci_eeh_release(vdev->pdev);
  145. vfio_pci_disable(vdev);
  146. }
  147. mutex_unlock(&driver_lock);
  148. module_put(THIS_MODULE);
  149. }
  150. static int vfio_pci_open(void *device_data)
  151. {
  152. struct vfio_pci_device *vdev = device_data;
  153. int ret = 0;
  154. if (!try_module_get(THIS_MODULE))
  155. return -ENODEV;
  156. mutex_lock(&driver_lock);
  157. if (!vdev->refcnt) {
  158. ret = vfio_pci_enable(vdev);
  159. if (ret)
  160. goto error;
  161. vfio_spapr_pci_eeh_open(vdev->pdev);
  162. }
  163. vdev->refcnt++;
  164. error:
  165. mutex_unlock(&driver_lock);
  166. if (ret)
  167. module_put(THIS_MODULE);
  168. return ret;
  169. }
  170. static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
  171. {
  172. if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
  173. u8 pin;
  174. pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
  175. if (IS_ENABLED(CONFIG_VFIO_PCI_INTX) && pin)
  176. return 1;
  177. } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
  178. u8 pos;
  179. u16 flags;
  180. pos = vdev->pdev->msi_cap;
  181. if (pos) {
  182. pci_read_config_word(vdev->pdev,
  183. pos + PCI_MSI_FLAGS, &flags);
  184. return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
  185. }
  186. } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
  187. u8 pos;
  188. u16 flags;
  189. pos = vdev->pdev->msix_cap;
  190. if (pos) {
  191. pci_read_config_word(vdev->pdev,
  192. pos + PCI_MSIX_FLAGS, &flags);
  193. return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
  194. }
  195. } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX)
  196. if (pci_is_pcie(vdev->pdev))
  197. return 1;
  198. return 0;
  199. }
  200. static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)
  201. {
  202. (*(int *)data)++;
  203. return 0;
  204. }
  205. struct vfio_pci_fill_info {
  206. int max;
  207. int cur;
  208. struct vfio_pci_dependent_device *devices;
  209. };
  210. static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)
  211. {
  212. struct vfio_pci_fill_info *fill = data;
  213. struct iommu_group *iommu_group;
  214. if (fill->cur == fill->max)
  215. return -EAGAIN; /* Something changed, try again */
  216. iommu_group = iommu_group_get(&pdev->dev);
  217. if (!iommu_group)
  218. return -EPERM; /* Cannot reset non-isolated devices */
  219. fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);
  220. fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);
  221. fill->devices[fill->cur].bus = pdev->bus->number;
  222. fill->devices[fill->cur].devfn = pdev->devfn;
  223. fill->cur++;
  224. iommu_group_put(iommu_group);
  225. return 0;
  226. }
  227. struct vfio_pci_group_entry {
  228. struct vfio_group *group;
  229. int id;
  230. };
  231. struct vfio_pci_group_info {
  232. int count;
  233. struct vfio_pci_group_entry *groups;
  234. };
  235. static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data)
  236. {
  237. struct vfio_pci_group_info *info = data;
  238. struct iommu_group *group;
  239. int id, i;
  240. group = iommu_group_get(&pdev->dev);
  241. if (!group)
  242. return -EPERM;
  243. id = iommu_group_id(group);
  244. for (i = 0; i < info->count; i++)
  245. if (info->groups[i].id == id)
  246. break;
  247. iommu_group_put(group);
  248. return (i == info->count) ? -EINVAL : 0;
  249. }
  250. static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)
  251. {
  252. for (; pdev; pdev = pdev->bus->self)
  253. if (pdev->bus == slot->bus)
  254. return (pdev->slot == slot);
  255. return false;
  256. }
  257. struct vfio_pci_walk_info {
  258. int (*fn)(struct pci_dev *, void *data);
  259. void *data;
  260. struct pci_dev *pdev;
  261. bool slot;
  262. int ret;
  263. };
  264. static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)
  265. {
  266. struct vfio_pci_walk_info *walk = data;
  267. if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
  268. walk->ret = walk->fn(pdev, walk->data);
  269. return walk->ret;
  270. }
  271. static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
  272. int (*fn)(struct pci_dev *,
  273. void *data), void *data,
  274. bool slot)
  275. {
  276. struct vfio_pci_walk_info walk = {
  277. .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
  278. };
  279. pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
  280. return walk.ret;
  281. }
  282. static long vfio_pci_ioctl(void *device_data,
  283. unsigned int cmd, unsigned long arg)
  284. {
  285. struct vfio_pci_device *vdev = device_data;
  286. unsigned long minsz;
  287. if (cmd == VFIO_DEVICE_GET_INFO) {
  288. struct vfio_device_info info;
  289. minsz = offsetofend(struct vfio_device_info, num_irqs);
  290. if (copy_from_user(&info, (void __user *)arg, minsz))
  291. return -EFAULT;
  292. if (info.argsz < minsz)
  293. return -EINVAL;
  294. info.flags = VFIO_DEVICE_FLAGS_PCI;
  295. if (vdev->reset_works)
  296. info.flags |= VFIO_DEVICE_FLAGS_RESET;
  297. info.num_regions = VFIO_PCI_NUM_REGIONS;
  298. info.num_irqs = VFIO_PCI_NUM_IRQS;
  299. return copy_to_user((void __user *)arg, &info, minsz);
  300. } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
  301. struct pci_dev *pdev = vdev->pdev;
  302. struct vfio_region_info info;
  303. minsz = offsetofend(struct vfio_region_info, offset);
  304. if (copy_from_user(&info, (void __user *)arg, minsz))
  305. return -EFAULT;
  306. if (info.argsz < minsz)
  307. return -EINVAL;
  308. switch (info.index) {
  309. case VFIO_PCI_CONFIG_REGION_INDEX:
  310. info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
  311. info.size = pdev->cfg_size;
  312. info.flags = VFIO_REGION_INFO_FLAG_READ |
  313. VFIO_REGION_INFO_FLAG_WRITE;
  314. break;
  315. case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
  316. info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
  317. info.size = pci_resource_len(pdev, info.index);
  318. if (!info.size) {
  319. info.flags = 0;
  320. break;
  321. }
  322. info.flags = VFIO_REGION_INFO_FLAG_READ |
  323. VFIO_REGION_INFO_FLAG_WRITE;
  324. if (IS_ENABLED(CONFIG_VFIO_PCI_MMAP) &&
  325. pci_resource_flags(pdev, info.index) &
  326. IORESOURCE_MEM && info.size >= PAGE_SIZE)
  327. info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
  328. break;
  329. case VFIO_PCI_ROM_REGION_INDEX:
  330. {
  331. void __iomem *io;
  332. size_t size;
  333. info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
  334. info.flags = 0;
  335. /* Report the BAR size, not the ROM size */
  336. info.size = pci_resource_len(pdev, info.index);
  337. if (!info.size)
  338. break;
  339. /* Is it really there? */
  340. io = pci_map_rom(pdev, &size);
  341. if (!io || !size) {
  342. info.size = 0;
  343. break;
  344. }
  345. pci_unmap_rom(pdev, io);
  346. info.flags = VFIO_REGION_INFO_FLAG_READ;
  347. break;
  348. }
  349. case VFIO_PCI_VGA_REGION_INDEX:
  350. if (!vdev->has_vga)
  351. return -EINVAL;
  352. info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
  353. info.size = 0xc0000;
  354. info.flags = VFIO_REGION_INFO_FLAG_READ |
  355. VFIO_REGION_INFO_FLAG_WRITE;
  356. break;
  357. default:
  358. return -EINVAL;
  359. }
  360. return copy_to_user((void __user *)arg, &info, minsz);
  361. } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
  362. struct vfio_irq_info info;
  363. minsz = offsetofend(struct vfio_irq_info, count);
  364. if (copy_from_user(&info, (void __user *)arg, minsz))
  365. return -EFAULT;
  366. if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
  367. return -EINVAL;
  368. switch (info.index) {
  369. case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
  370. break;
  371. case VFIO_PCI_ERR_IRQ_INDEX:
  372. if (pci_is_pcie(vdev->pdev))
  373. break;
  374. /* pass thru to return error */
  375. default:
  376. return -EINVAL;
  377. }
  378. info.flags = VFIO_IRQ_INFO_EVENTFD;
  379. info.count = vfio_pci_get_irq_count(vdev, info.index);
  380. if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
  381. info.flags |= (VFIO_IRQ_INFO_MASKABLE |
  382. VFIO_IRQ_INFO_AUTOMASKED);
  383. else
  384. info.flags |= VFIO_IRQ_INFO_NORESIZE;
  385. return copy_to_user((void __user *)arg, &info, minsz);
  386. } else if (cmd == VFIO_DEVICE_SET_IRQS) {
  387. struct vfio_irq_set hdr;
  388. u8 *data = NULL;
  389. int ret = 0;
  390. minsz = offsetofend(struct vfio_irq_set, count);
  391. if (copy_from_user(&hdr, (void __user *)arg, minsz))
  392. return -EFAULT;
  393. if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS ||
  394. hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
  395. VFIO_IRQ_SET_ACTION_TYPE_MASK))
  396. return -EINVAL;
  397. if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
  398. size_t size;
  399. int max = vfio_pci_get_irq_count(vdev, hdr.index);
  400. if (hdr.flags & VFIO_IRQ_SET_DATA_BOOL)
  401. size = sizeof(uint8_t);
  402. else if (hdr.flags & VFIO_IRQ_SET_DATA_EVENTFD)
  403. size = sizeof(int32_t);
  404. else
  405. return -EINVAL;
  406. if (hdr.argsz - minsz < hdr.count * size ||
  407. hdr.start >= max || hdr.start + hdr.count > max)
  408. return -EINVAL;
  409. data = memdup_user((void __user *)(arg + minsz),
  410. hdr.count * size);
  411. if (IS_ERR(data))
  412. return PTR_ERR(data);
  413. }
  414. mutex_lock(&vdev->igate);
  415. ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
  416. hdr.start, hdr.count, data);
  417. mutex_unlock(&vdev->igate);
  418. kfree(data);
  419. return ret;
  420. } else if (cmd == VFIO_DEVICE_RESET) {
  421. return vdev->reset_works ?
  422. pci_try_reset_function(vdev->pdev) : -EINVAL;
  423. } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
  424. struct vfio_pci_hot_reset_info hdr;
  425. struct vfio_pci_fill_info fill = { 0 };
  426. struct vfio_pci_dependent_device *devices = NULL;
  427. bool slot = false;
  428. int ret = 0;
  429. minsz = offsetofend(struct vfio_pci_hot_reset_info, count);
  430. if (copy_from_user(&hdr, (void __user *)arg, minsz))
  431. return -EFAULT;
  432. if (hdr.argsz < minsz)
  433. return -EINVAL;
  434. hdr.flags = 0;
  435. /* Can we do a slot or bus reset or neither? */
  436. if (!pci_probe_reset_slot(vdev->pdev->slot))
  437. slot = true;
  438. else if (pci_probe_reset_bus(vdev->pdev->bus))
  439. return -ENODEV;
  440. /* How many devices are affected? */
  441. ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
  442. vfio_pci_count_devs,
  443. &fill.max, slot);
  444. if (ret)
  445. return ret;
  446. WARN_ON(!fill.max); /* Should always be at least one */
  447. /*
  448. * If there's enough space, fill it now, otherwise return
  449. * -ENOSPC and the number of devices affected.
  450. */
  451. if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
  452. ret = -ENOSPC;
  453. hdr.count = fill.max;
  454. goto reset_info_exit;
  455. }
  456. devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
  457. if (!devices)
  458. return -ENOMEM;
  459. fill.devices = devices;
  460. ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
  461. vfio_pci_fill_devs,
  462. &fill, slot);
  463. /*
  464. * If a device was removed between counting and filling,
  465. * we may come up short of fill.max. If a device was
  466. * added, we'll have a return of -EAGAIN above.
  467. */
  468. if (!ret)
  469. hdr.count = fill.cur;
  470. reset_info_exit:
  471. if (copy_to_user((void __user *)arg, &hdr, minsz))
  472. ret = -EFAULT;
  473. if (!ret) {
  474. if (copy_to_user((void __user *)(arg + minsz), devices,
  475. hdr.count * sizeof(*devices)))
  476. ret = -EFAULT;
  477. }
  478. kfree(devices);
  479. return ret;
  480. } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {
  481. struct vfio_pci_hot_reset hdr;
  482. int32_t *group_fds;
  483. struct vfio_pci_group_entry *groups;
  484. struct vfio_pci_group_info info;
  485. bool slot = false;
  486. int i, count = 0, ret = 0;
  487. minsz = offsetofend(struct vfio_pci_hot_reset, count);
  488. if (copy_from_user(&hdr, (void __user *)arg, minsz))
  489. return -EFAULT;
  490. if (hdr.argsz < minsz || hdr.flags)
  491. return -EINVAL;
  492. /* Can we do a slot or bus reset or neither? */
  493. if (!pci_probe_reset_slot(vdev->pdev->slot))
  494. slot = true;
  495. else if (pci_probe_reset_bus(vdev->pdev->bus))
  496. return -ENODEV;
  497. /*
  498. * We can't let userspace give us an arbitrarily large
  499. * buffer to copy, so verify how many we think there
  500. * could be. Note groups can have multiple devices so
  501. * one group per device is the max.
  502. */
  503. ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
  504. vfio_pci_count_devs,
  505. &count, slot);
  506. if (ret)
  507. return ret;
  508. /* Somewhere between 1 and count is OK */
  509. if (!hdr.count || hdr.count > count)
  510. return -EINVAL;
  511. group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
  512. groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);
  513. if (!group_fds || !groups) {
  514. kfree(group_fds);
  515. kfree(groups);
  516. return -ENOMEM;
  517. }
  518. if (copy_from_user(group_fds, (void __user *)(arg + minsz),
  519. hdr.count * sizeof(*group_fds))) {
  520. kfree(group_fds);
  521. kfree(groups);
  522. return -EFAULT;
  523. }
  524. /*
  525. * For each group_fd, get the group through the vfio external
  526. * user interface and store the group and iommu ID. This
  527. * ensures the group is held across the reset.
  528. */
  529. for (i = 0; i < hdr.count; i++) {
  530. struct vfio_group *group;
  531. struct fd f = fdget(group_fds[i]);
  532. if (!f.file) {
  533. ret = -EBADF;
  534. break;
  535. }
  536. group = vfio_group_get_external_user(f.file);
  537. fdput(f);
  538. if (IS_ERR(group)) {
  539. ret = PTR_ERR(group);
  540. break;
  541. }
  542. groups[i].group = group;
  543. groups[i].id = vfio_external_user_iommu_id(group);
  544. }
  545. kfree(group_fds);
  546. /* release reference to groups on error */
  547. if (ret)
  548. goto hot_reset_release;
  549. info.count = hdr.count;
  550. info.groups = groups;
  551. /*
  552. * Test whether all the affected devices are contained
  553. * by the set of groups provided by the user.
  554. */
  555. ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
  556. vfio_pci_validate_devs,
  557. &info, slot);
  558. if (!ret)
  559. /* User has access, do the reset */
  560. ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :
  561. pci_try_reset_bus(vdev->pdev->bus);
  562. hot_reset_release:
  563. for (i--; i >= 0; i--)
  564. vfio_group_put_external_user(groups[i].group);
  565. kfree(groups);
  566. return ret;
  567. }
  568. return -ENOTTY;
  569. }
  570. static ssize_t vfio_pci_rw(void *device_data, char __user *buf,
  571. size_t count, loff_t *ppos, bool iswrite)
  572. {
  573. unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
  574. struct vfio_pci_device *vdev = device_data;
  575. if (index >= VFIO_PCI_NUM_REGIONS)
  576. return -EINVAL;
  577. switch (index) {
  578. case VFIO_PCI_CONFIG_REGION_INDEX:
  579. return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
  580. case VFIO_PCI_ROM_REGION_INDEX:
  581. if (iswrite)
  582. return -EINVAL;
  583. return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
  584. case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
  585. return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
  586. case VFIO_PCI_VGA_REGION_INDEX:
  587. return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
  588. }
  589. return -EINVAL;
  590. }
  591. static ssize_t vfio_pci_read(void *device_data, char __user *buf,
  592. size_t count, loff_t *ppos)
  593. {
  594. if (!count)
  595. return 0;
  596. return vfio_pci_rw(device_data, buf, count, ppos, false);
  597. }
  598. static ssize_t vfio_pci_write(void *device_data, const char __user *buf,
  599. size_t count, loff_t *ppos)
  600. {
  601. if (!count)
  602. return 0;
  603. return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true);
  604. }
  605. static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
  606. {
  607. struct vfio_pci_device *vdev = device_data;
  608. struct pci_dev *pdev = vdev->pdev;
  609. unsigned int index;
  610. u64 phys_len, req_len, pgoff, req_start;
  611. int ret;
  612. index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
  613. if (vma->vm_end < vma->vm_start)
  614. return -EINVAL;
  615. if ((vma->vm_flags & VM_SHARED) == 0)
  616. return -EINVAL;
  617. if (index >= VFIO_PCI_ROM_REGION_INDEX)
  618. return -EINVAL;
  619. if (!(pci_resource_flags(pdev, index) & IORESOURCE_MEM))
  620. return -EINVAL;
  621. phys_len = pci_resource_len(pdev, index);
  622. req_len = vma->vm_end - vma->vm_start;
  623. pgoff = vma->vm_pgoff &
  624. ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
  625. req_start = pgoff << PAGE_SHIFT;
  626. if (phys_len < PAGE_SIZE || req_start + req_len > phys_len)
  627. return -EINVAL;
  628. if (index == vdev->msix_bar) {
  629. /*
  630. * Disallow mmaps overlapping the MSI-X table; users don't
  631. * get to touch this directly. We could find somewhere
  632. * else to map the overlap, but page granularity is only
  633. * a recommendation, not a requirement, so the user needs
  634. * to know which bits are real. Requiring them to mmap
  635. * around the table makes that clear.
  636. */
  637. /* If neither entirely above nor below, then it overlaps */
  638. if (!(req_start >= vdev->msix_offset + vdev->msix_size ||
  639. req_start + req_len <= vdev->msix_offset))
  640. return -EINVAL;
  641. }
  642. /*
  643. * Even though we don't make use of the barmap for the mmap,
  644. * we need to request the region and the barmap tracks that.
  645. */
  646. if (!vdev->barmap[index]) {
  647. ret = pci_request_selected_regions(pdev,
  648. 1 << index, "vfio-pci");
  649. if (ret)
  650. return ret;
  651. vdev->barmap[index] = pci_iomap(pdev, index, 0);
  652. }
  653. vma->vm_private_data = vdev;
  654. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  655. vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
  656. return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
  657. req_len, vma->vm_page_prot);
  658. }
  659. static const struct vfio_device_ops vfio_pci_ops = {
  660. .name = "vfio-pci",
  661. .open = vfio_pci_open,
  662. .release = vfio_pci_release,
  663. .ioctl = vfio_pci_ioctl,
  664. .read = vfio_pci_read,
  665. .write = vfio_pci_write,
  666. .mmap = vfio_pci_mmap,
  667. };
  668. static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
  669. {
  670. u8 type;
  671. struct vfio_pci_device *vdev;
  672. struct iommu_group *group;
  673. int ret;
  674. pci_read_config_byte(pdev, PCI_HEADER_TYPE, &type);
  675. if ((type & PCI_HEADER_TYPE) != PCI_HEADER_TYPE_NORMAL)
  676. return -EINVAL;
  677. group = iommu_group_get(&pdev->dev);
  678. if (!group)
  679. return -EINVAL;
  680. vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
  681. if (!vdev) {
  682. iommu_group_put(group);
  683. return -ENOMEM;
  684. }
  685. vdev->pdev = pdev;
  686. vdev->irq_type = VFIO_PCI_NUM_IRQS;
  687. mutex_init(&vdev->igate);
  688. spin_lock_init(&vdev->irqlock);
  689. ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
  690. if (ret) {
  691. iommu_group_put(group);
  692. kfree(vdev);
  693. }
  694. return ret;
  695. }
  696. static void vfio_pci_remove(struct pci_dev *pdev)
  697. {
  698. struct vfio_pci_device *vdev;
  699. vdev = vfio_del_group_dev(&pdev->dev);
  700. if (vdev) {
  701. iommu_group_put(pdev->dev.iommu_group);
  702. kfree(vdev);
  703. }
  704. }
  705. static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
  706. pci_channel_state_t state)
  707. {
  708. struct vfio_pci_device *vdev;
  709. struct vfio_device *device;
  710. device = vfio_device_get_from_dev(&pdev->dev);
  711. if (device == NULL)
  712. return PCI_ERS_RESULT_DISCONNECT;
  713. vdev = vfio_device_data(device);
  714. if (vdev == NULL) {
  715. vfio_device_put(device);
  716. return PCI_ERS_RESULT_DISCONNECT;
  717. }
  718. mutex_lock(&vdev->igate);
  719. if (vdev->err_trigger)
  720. eventfd_signal(vdev->err_trigger, 1);
  721. mutex_unlock(&vdev->igate);
  722. vfio_device_put(device);
  723. return PCI_ERS_RESULT_CAN_RECOVER;
  724. }
  725. static struct pci_error_handlers vfio_err_handlers = {
  726. .error_detected = vfio_pci_aer_err_detected,
  727. };
  728. static struct pci_driver vfio_pci_driver = {
  729. .name = "vfio-pci",
  730. .id_table = NULL, /* only dynamic ids */
  731. .probe = vfio_pci_probe,
  732. .remove = vfio_pci_remove,
  733. .err_handler = &vfio_err_handlers,
  734. };
  735. struct vfio_devices {
  736. struct vfio_device **devices;
  737. int cur_index;
  738. int max_index;
  739. };
  740. static int vfio_pci_get_devs(struct pci_dev *pdev, void *data)
  741. {
  742. struct vfio_devices *devs = data;
  743. struct pci_driver *pci_drv = ACCESS_ONCE(pdev->driver);
  744. if (pci_drv != &vfio_pci_driver)
  745. return -EBUSY;
  746. if (devs->cur_index == devs->max_index)
  747. return -ENOSPC;
  748. devs->devices[devs->cur_index] = vfio_device_get_from_dev(&pdev->dev);
  749. if (!devs->devices[devs->cur_index])
  750. return -EINVAL;
  751. devs->cur_index++;
  752. return 0;
  753. }
  754. /*
  755. * Attempt to do a bus/slot reset if there are devices affected by a reset for
  756. * this device that are needs_reset and all of the affected devices are unused
  757. * (!refcnt). Callers are required to hold driver_lock when calling this to
  758. * prevent device opens and concurrent bus reset attempts. We prevent device
  759. * unbinds by acquiring and holding a reference to the vfio_device.
  760. *
  761. * NB: vfio-core considers a group to be viable even if some devices are
  762. * bound to drivers like pci-stub or pcieport. Here we require all devices
  763. * to be bound to vfio_pci since that's the only way we can be sure they
  764. * stay put.
  765. */
  766. static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)
  767. {
  768. struct vfio_devices devs = { .cur_index = 0 };
  769. int i = 0, ret = -EINVAL;
  770. bool needs_reset = false, slot = false;
  771. struct vfio_pci_device *tmp;
  772. if (!pci_probe_reset_slot(vdev->pdev->slot))
  773. slot = true;
  774. else if (pci_probe_reset_bus(vdev->pdev->bus))
  775. return;
  776. if (vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
  777. &i, slot) || !i)
  778. return;
  779. devs.max_index = i;
  780. devs.devices = kcalloc(i, sizeof(struct vfio_device *), GFP_KERNEL);
  781. if (!devs.devices)
  782. return;
  783. if (vfio_pci_for_each_slot_or_bus(vdev->pdev,
  784. vfio_pci_get_devs, &devs, slot))
  785. goto put_devs;
  786. for (i = 0; i < devs.cur_index; i++) {
  787. tmp = vfio_device_data(devs.devices[i]);
  788. if (tmp->needs_reset)
  789. needs_reset = true;
  790. if (tmp->refcnt)
  791. goto put_devs;
  792. }
  793. if (needs_reset)
  794. ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :
  795. pci_try_reset_bus(vdev->pdev->bus);
  796. put_devs:
  797. for (i = 0; i < devs.cur_index; i++) {
  798. if (!ret) {
  799. tmp = vfio_device_data(devs.devices[i]);
  800. tmp->needs_reset = false;
  801. }
  802. vfio_device_put(devs.devices[i]);
  803. }
  804. kfree(devs.devices);
  805. }
  806. static void __exit vfio_pci_cleanup(void)
  807. {
  808. pci_unregister_driver(&vfio_pci_driver);
  809. vfio_pci_virqfd_exit();
  810. vfio_pci_uninit_perm_bits();
  811. }
  812. static int __init vfio_pci_init(void)
  813. {
  814. int ret;
  815. /* Allocate shared config space permision data used by all devices */
  816. ret = vfio_pci_init_perm_bits();
  817. if (ret)
  818. return ret;
  819. /* Start the virqfd cleanup handler */
  820. ret = vfio_pci_virqfd_init();
  821. if (ret)
  822. goto out_virqfd;
  823. /* Register and scan for devices */
  824. ret = pci_register_driver(&vfio_pci_driver);
  825. if (ret)
  826. goto out_driver;
  827. return 0;
  828. out_driver:
  829. vfio_pci_virqfd_exit();
  830. out_virqfd:
  831. vfio_pci_uninit_perm_bits();
  832. return ret;
  833. }
  834. module_init(vfio_pci_init);
  835. module_exit(vfio_pci_cleanup);
  836. MODULE_VERSION(DRIVER_VERSION);
  837. MODULE_LICENSE("GPL v2");
  838. MODULE_AUTHOR(DRIVER_AUTHOR);
  839. MODULE_DESCRIPTION(DRIVER_DESC);