|
|
@@ -1,14 +1,17 @@
|
|
|
-/*
|
|
|
- * VFIO Mediated devices
|
|
|
- *
|
|
|
- * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
|
|
|
- * Author: Neo Jia <cjia@nvidia.com>
|
|
|
- * Kirti Wankhede <kwankhede@nvidia.com>
|
|
|
- *
|
|
|
- * This program is free software; you can redistribute it and/or modify
|
|
|
- * it under the terms of the GNU General Public License version 2 as
|
|
|
- * published by the Free Software Foundation.
|
|
|
- */
|
|
|
+.. include:: <isonum.txt>
|
|
|
+
|
|
|
+=====================
|
|
|
+VFIO Mediated devices
|
|
|
+=====================
|
|
|
+
|
|
|
+:Copyright: |copy| 2016, NVIDIA CORPORATION. All rights reserved.
|
|
|
+:Author: Neo Jia <cjia@nvidia.com>
|
|
|
+:Author: Kirti Wankhede <kwankhede@nvidia.com>
|
|
|
+
|
|
|
+This program is free software; you can redistribute it and/or modify
|
|
|
+it under the terms of the GNU General Public License version 2 as
|
|
|
+published by the Free Software Foundation.
|
|
|
+
|
|
|
|
|
|
Virtual Function I/O (VFIO) Mediated devices[1]
|
|
|
===============================================
|
|
|
@@ -42,7 +45,7 @@ removes it from a VFIO group.
|
|
|
|
|
|
The following high-level block diagram shows the main components and interfaces
|
|
|
in the VFIO mediated driver framework. The diagram shows NVIDIA, Intel, and IBM
|
|
|
-devices as examples, as these devices are the first devices to use this module.
|
|
|
+devices as examples, as these devices are the first devices to use this module::
|
|
|
|
|
|
+---------------+
|
|
|
| |
|
|
|
@@ -91,7 +94,7 @@ Registration Interface for a Mediated Bus Driver
|
|
|
------------------------------------------------
|
|
|
|
|
|
The registration interface for a mediated bus driver provides the following
|
|
|
-structure to represent a mediated device's driver:
|
|
|
+structure to represent a mediated device's driver::
|
|
|
|
|
|
/*
|
|
|
* struct mdev_driver [2] - Mediated device's driver
|
|
|
@@ -110,14 +113,14 @@ structure to represent a mediated device's driver:
|
|
|
A mediated bus driver for mdev should use this structure in the function calls
|
|
|
to register and unregister itself with the core driver:
|
|
|
|
|
|
-* Register:
|
|
|
+* Register::
|
|
|
|
|
|
- extern int mdev_register_driver(struct mdev_driver *drv,
|
|
|
+ extern int mdev_register_driver(struct mdev_driver *drv,
|
|
|
struct module *owner);
|
|
|
|
|
|
-* Unregister:
|
|
|
+* Unregister::
|
|
|
|
|
|
- extern void mdev_unregister_driver(struct mdev_driver *drv);
|
|
|
+ extern void mdev_unregister_driver(struct mdev_driver *drv);
|
|
|
|
|
|
The mediated bus driver is responsible for adding mediated devices to the VFIO
|
|
|
group when devices are bound to the driver and removing mediated devices from
|
|
|
@@ -152,15 +155,15 @@ The callbacks in the mdev_parent_ops structure are as follows:
|
|
|
* mmap: mmap emulation callback
|
|
|
|
|
|
A driver should use the mdev_parent_ops structure in the function call to
|
|
|
-register itself with the mdev core driver:
|
|
|
+register itself with the mdev core driver::
|
|
|
|
|
|
-extern int mdev_register_device(struct device *dev,
|
|
|
- const struct mdev_parent_ops *ops);
|
|
|
+ extern int mdev_register_device(struct device *dev,
|
|
|
+ const struct mdev_parent_ops *ops);
|
|
|
|
|
|
However, the mdev_parent_ops structure is not required in the function call
|
|
|
-that a driver should use to unregister itself with the mdev core driver:
|
|
|
+that a driver should use to unregister itself with the mdev core driver::
|
|
|
|
|
|
-extern void mdev_unregister_device(struct device *dev);
|
|
|
+ extern void mdev_unregister_device(struct device *dev);
|
|
|
|
|
|
|
|
|
Mediated Device Management Interface Through sysfs
|
|
|
@@ -183,30 +186,32 @@ with the mdev core driver.
|
|
|
Directories and files under the sysfs for Each Physical Device
|
|
|
--------------------------------------------------------------
|
|
|
|
|
|
-|- [parent physical device]
|
|
|
-|--- Vendor-specific-attributes [optional]
|
|
|
-|--- [mdev_supported_types]
|
|
|
-| |--- [<type-id>]
|
|
|
-| | |--- create
|
|
|
-| | |--- name
|
|
|
-| | |--- available_instances
|
|
|
-| | |--- device_api
|
|
|
-| | |--- description
|
|
|
-| | |--- [devices]
|
|
|
-| |--- [<type-id>]
|
|
|
-| | |--- create
|
|
|
-| | |--- name
|
|
|
-| | |--- available_instances
|
|
|
-| | |--- device_api
|
|
|
-| | |--- description
|
|
|
-| | |--- [devices]
|
|
|
-| |--- [<type-id>]
|
|
|
-| |--- create
|
|
|
-| |--- name
|
|
|
-| |--- available_instances
|
|
|
-| |--- device_api
|
|
|
-| |--- description
|
|
|
-| |--- [devices]
|
|
|
+::
|
|
|
+
|
|
|
+ |- [parent physical device]
|
|
|
+ |--- Vendor-specific-attributes [optional]
|
|
|
+ |--- [mdev_supported_types]
|
|
|
+ | |--- [<type-id>]
|
|
|
+ | | |--- create
|
|
|
+ | | |--- name
|
|
|
+ | | |--- available_instances
|
|
|
+ | | |--- device_api
|
|
|
+ | | |--- description
|
|
|
+ | | |--- [devices]
|
|
|
+ | |--- [<type-id>]
|
|
|
+ | | |--- create
|
|
|
+ | | |--- name
|
|
|
+ | | |--- available_instances
|
|
|
+ | | |--- device_api
|
|
|
+ | | |--- description
|
|
|
+ | | |--- [devices]
|
|
|
+ | |--- [<type-id>]
|
|
|
+ | |--- create
|
|
|
+ | |--- name
|
|
|
+ | |--- available_instances
|
|
|
+ | |--- device_api
|
|
|
+ | |--- description
|
|
|
+ | |--- [devices]
|
|
|
|
|
|
* [mdev_supported_types]
|
|
|
|
|
|
@@ -219,12 +224,12 @@ Directories and files under the sysfs for Each Physical Device
|
|
|
|
|
|
The [<type-id>] name is created by adding the device driver string as a prefix
|
|
|
to the string provided by the vendor driver. This format of this name is as
|
|
|
- follows:
|
|
|
+ follows::
|
|
|
|
|
|
sprintf(buf, "%s-%s", dev_driver_string(parent->dev), group->name);
|
|
|
|
|
|
(or using mdev_parent_dev(mdev) to arrive at the parent device outside
|
|
|
- of the core mdev code)
|
|
|
+ of the core mdev code)
|
|
|
|
|
|
* device_api
|
|
|
|
|
|
@@ -239,7 +244,7 @@ Directories and files under the sysfs for Each Physical Device
|
|
|
* [device]
|
|
|
|
|
|
This directory contains links to the devices of type <type-id> that have been
|
|
|
-created.
|
|
|
+ created.
|
|
|
|
|
|
* name
|
|
|
|
|
|
@@ -253,21 +258,25 @@ created.
|
|
|
Directories and Files Under the sysfs for Each mdev Device
|
|
|
----------------------------------------------------------
|
|
|
|
|
|
-|- [parent phy device]
|
|
|
-|--- [$MDEV_UUID]
|
|
|
+::
|
|
|
+
|
|
|
+ |- [parent phy device]
|
|
|
+ |--- [$MDEV_UUID]
|
|
|
|--- remove
|
|
|
|--- mdev_type {link to its type}
|
|
|
|--- vendor-specific-attributes [optional]
|
|
|
|
|
|
* remove (write only)
|
|
|
+
|
|
|
Writing '1' to the 'remove' file destroys the mdev device. The vendor driver can
|
|
|
fail the remove() callback if that device is active and the vendor driver
|
|
|
doesn't support hot unplug.
|
|
|
|
|
|
-Example:
|
|
|
+Example::
|
|
|
+
|
|
|
# echo 1 > /sys/bus/mdev/devices/$mdev_UUID/remove
|
|
|
|
|
|
-Mediated device Hot plug:
|
|
|
+Mediated device Hot plug
|
|
|
------------------------
|
|
|
|
|
|
Mediated devices can be created and assigned at runtime. The procedure to hot
|
|
|
@@ -277,13 +286,13 @@ Translation APIs for Mediated Devices
|
|
|
=====================================
|
|
|
|
|
|
The following APIs are provided for translating user pfn to host pfn in a VFIO
|
|
|
-driver:
|
|
|
+driver::
|
|
|
|
|
|
-extern int vfio_pin_pages(struct device *dev, unsigned long *user_pfn,
|
|
|
- int npage, int prot, unsigned long *phys_pfn);
|
|
|
+ extern int vfio_pin_pages(struct device *dev, unsigned long *user_pfn,
|
|
|
+ int npage, int prot, unsigned long *phys_pfn);
|
|
|
|
|
|
-extern int vfio_unpin_pages(struct device *dev, unsigned long *user_pfn,
|
|
|
- int npage);
|
|
|
+ extern int vfio_unpin_pages(struct device *dev, unsigned long *user_pfn,
|
|
|
+ int npage);
|
|
|
|
|
|
These functions call back into the back-end IOMMU module by using the pin_pages
|
|
|
and unpin_pages callbacks of the struct vfio_iommu_driver_ops[4]. Currently
|
|
|
@@ -304,81 +313,80 @@ card.
|
|
|
|
|
|
This step creates a dummy device, /sys/devices/virtual/mtty/mtty/
|
|
|
|
|
|
- Files in this device directory in sysfs are similar to the following:
|
|
|
-
|
|
|
- # tree /sys/devices/virtual/mtty/mtty/
|
|
|
- /sys/devices/virtual/mtty/mtty/
|
|
|
- |-- mdev_supported_types
|
|
|
- | |-- mtty-1
|
|
|
- | | |-- available_instances
|
|
|
- | | |-- create
|
|
|
- | | |-- device_api
|
|
|
- | | |-- devices
|
|
|
- | | `-- name
|
|
|
- | `-- mtty-2
|
|
|
- | |-- available_instances
|
|
|
- | |-- create
|
|
|
- | |-- device_api
|
|
|
- | |-- devices
|
|
|
- | `-- name
|
|
|
- |-- mtty_dev
|
|
|
- | `-- sample_mtty_dev
|
|
|
- |-- power
|
|
|
- | |-- autosuspend_delay_ms
|
|
|
- | |-- control
|
|
|
- | |-- runtime_active_time
|
|
|
- | |-- runtime_status
|
|
|
- | `-- runtime_suspended_time
|
|
|
- |-- subsystem -> ../../../../class/mtty
|
|
|
- `-- uevent
|
|
|
+ Files in this device directory in sysfs are similar to the following::
|
|
|
+
|
|
|
+ # tree /sys/devices/virtual/mtty/mtty/
|
|
|
+ /sys/devices/virtual/mtty/mtty/
|
|
|
+ |-- mdev_supported_types
|
|
|
+ | |-- mtty-1
|
|
|
+ | | |-- available_instances
|
|
|
+ | | |-- create
|
|
|
+ | | |-- device_api
|
|
|
+ | | |-- devices
|
|
|
+ | | `-- name
|
|
|
+ | `-- mtty-2
|
|
|
+ | |-- available_instances
|
|
|
+ | |-- create
|
|
|
+ | |-- device_api
|
|
|
+ | |-- devices
|
|
|
+ | `-- name
|
|
|
+ |-- mtty_dev
|
|
|
+ | `-- sample_mtty_dev
|
|
|
+ |-- power
|
|
|
+ | |-- autosuspend_delay_ms
|
|
|
+ | |-- control
|
|
|
+ | |-- runtime_active_time
|
|
|
+ | |-- runtime_status
|
|
|
+ | `-- runtime_suspended_time
|
|
|
+ |-- subsystem -> ../../../../class/mtty
|
|
|
+ `-- uevent
|
|
|
|
|
|
2. Create a mediated device by using the dummy device that you created in the
|
|
|
- previous step.
|
|
|
+ previous step::
|
|
|
|
|
|
- # echo "83b8f4f2-509f-382f-3c1e-e6bfe0fa1001" > \
|
|
|
+ # echo "83b8f4f2-509f-382f-3c1e-e6bfe0fa1001" > \
|
|
|
/sys/devices/virtual/mtty/mtty/mdev_supported_types/mtty-2/create
|
|
|
|
|
|
-3. Add parameters to qemu-kvm.
|
|
|
+3. Add parameters to qemu-kvm::
|
|
|
|
|
|
- -device vfio-pci,\
|
|
|
- sysfsdev=/sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1001
|
|
|
+ -device vfio-pci,\
|
|
|
+ sysfsdev=/sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1001
|
|
|
|
|
|
4. Boot the VM.
|
|
|
|
|
|
In the Linux guest VM, with no hardware on the host, the device appears
|
|
|
- as follows:
|
|
|
-
|
|
|
- # lspci -s 00:05.0 -xxvv
|
|
|
- 00:05.0 Serial controller: Device 4348:3253 (rev 10) (prog-if 02 [16550])
|
|
|
- Subsystem: Device 4348:3253
|
|
|
- Physical Slot: 5
|
|
|
- Control: I/O+ Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr-
|
|
|
- Stepping- SERR- FastB2B- DisINTx-
|
|
|
- Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort-
|
|
|
- <TAbort- <MAbort- >SERR- <PERR- INTx-
|
|
|
- Interrupt: pin A routed to IRQ 10
|
|
|
- Region 0: I/O ports at c150 [size=8]
|
|
|
- Region 1: I/O ports at c158 [size=8]
|
|
|
- Kernel driver in use: serial
|
|
|
- 00: 48 43 53 32 01 00 00 02 10 02 00 07 00 00 00 00
|
|
|
- 10: 51 c1 00 00 59 c1 00 00 00 00 00 00 00 00 00 00
|
|
|
- 20: 00 00 00 00 00 00 00 00 00 00 00 00 48 43 53 32
|
|
|
- 30: 00 00 00 00 00 00 00 00 00 00 00 00 0a 01 00 00
|
|
|
-
|
|
|
- In the Linux guest VM, dmesg output for the device is as follows:
|
|
|
-
|
|
|
- serial 0000:00:05.0: PCI INT A -> Link[LNKA] -> GSI 10 (level, high) -> IRQ
|
|
|
-10
|
|
|
- 0000:00:05.0: ttyS1 at I/O 0xc150 (irq = 10) is a 16550A
|
|
|
- 0000:00:05.0: ttyS2 at I/O 0xc158 (irq = 10) is a 16550A
|
|
|
-
|
|
|
-
|
|
|
-5. In the Linux guest VM, check the serial ports.
|
|
|
-
|
|
|
- # setserial -g /dev/ttyS*
|
|
|
- /dev/ttyS0, UART: 16550A, Port: 0x03f8, IRQ: 4
|
|
|
- /dev/ttyS1, UART: 16550A, Port: 0xc150, IRQ: 10
|
|
|
- /dev/ttyS2, UART: 16550A, Port: 0xc158, IRQ: 10
|
|
|
+ as follows::
|
|
|
+
|
|
|
+ # lspci -s 00:05.0 -xxvv
|
|
|
+ 00:05.0 Serial controller: Device 4348:3253 (rev 10) (prog-if 02 [16550])
|
|
|
+ Subsystem: Device 4348:3253
|
|
|
+ Physical Slot: 5
|
|
|
+ Control: I/O+ Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr-
|
|
|
+ Stepping- SERR- FastB2B- DisINTx-
|
|
|
+ Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort-
|
|
|
+ <TAbort- <MAbort- >SERR- <PERR- INTx-
|
|
|
+ Interrupt: pin A routed to IRQ 10
|
|
|
+ Region 0: I/O ports at c150 [size=8]
|
|
|
+ Region 1: I/O ports at c158 [size=8]
|
|
|
+ Kernel driver in use: serial
|
|
|
+ 00: 48 43 53 32 01 00 00 02 10 02 00 07 00 00 00 00
|
|
|
+ 10: 51 c1 00 00 59 c1 00 00 00 00 00 00 00 00 00 00
|
|
|
+ 20: 00 00 00 00 00 00 00 00 00 00 00 00 48 43 53 32
|
|
|
+ 30: 00 00 00 00 00 00 00 00 00 00 00 00 0a 01 00 00
|
|
|
+
|
|
|
+ In the Linux guest VM, dmesg output for the device is as follows:
|
|
|
+
|
|
|
+ serial 0000:00:05.0: PCI INT A -> Link[LNKA] -> GSI 10 (level, high) -> IRQ 10
|
|
|
+ 0000:00:05.0: ttyS1 at I/O 0xc150 (irq = 10) is a 16550A
|
|
|
+ 0000:00:05.0: ttyS2 at I/O 0xc158 (irq = 10) is a 16550A
|
|
|
+
|
|
|
+
|
|
|
+5. In the Linux guest VM, check the serial ports::
|
|
|
+
|
|
|
+ # setserial -g /dev/ttyS*
|
|
|
+ /dev/ttyS0, UART: 16550A, Port: 0x03f8, IRQ: 4
|
|
|
+ /dev/ttyS1, UART: 16550A, Port: 0xc150, IRQ: 10
|
|
|
+ /dev/ttyS2, UART: 16550A, Port: 0xc158, IRQ: 10
|
|
|
|
|
|
6. Using minicom or any terminal emulation program, open port /dev/ttyS1 or
|
|
|
/dev/ttyS2 with hardware flow control disabled.
|
|
|
@@ -388,14 +396,14 @@ card.
|
|
|
|
|
|
Data is loop backed from hosts mtty driver.
|
|
|
|
|
|
-8. Destroy the mediated device that you created.
|
|
|
+8. Destroy the mediated device that you created::
|
|
|
|
|
|
- # echo 1 > /sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1001/remove
|
|
|
+ # echo 1 > /sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1001/remove
|
|
|
|
|
|
References
|
|
|
==========
|
|
|
|
|
|
-[1] See Documentation/vfio.txt for more information on VFIO.
|
|
|
-[2] struct mdev_driver in include/linux/mdev.h
|
|
|
-[3] struct mdev_parent_ops in include/linux/mdev.h
|
|
|
-[4] struct vfio_iommu_driver_ops in include/linux/vfio.h
|
|
|
+1. See Documentation/vfio.txt for more information on VFIO.
|
|
|
+2. struct mdev_driver in include/linux/mdev.h
|
|
|
+3. struct mdev_parent_ops in include/linux/mdev.h
|
|
|
+4. struct vfio_iommu_driver_ops in include/linux/vfio.h
|