drivers/staging: Gasket driver framework + Apex driver

The Gasket (Google ASIC Software, Kernel Extensions, and Tools) kernel
framework is a generic, flexible system that supports thin kernel
drivers. Gasket kernel drivers are expected to handle opening and
closing devices, mmap'ing BAR space as requested, a small selection of
ioctls, and handling page table translation (covered below). Any other
functions should be handled by userspace code.

The Gasket common module is not enough to run a device. In order to
customize the Gasket code for a given piece of hardware, a device
specific module must be created. At a minimum, this module must define a
struct gasket_driver_desc containing the device-specific data for use by
the framework; in addition, the module must declare an __init function
that calls gasket_register_device with the module's gasket_driver_desc
struct. Finally, the driver must define an exit function that calls
gasket_unregister_device with the module's gasket_driver_desc struct.

One of the core assumptions of the Gasket framework is that precisely
one process is allowed to have an open write handle to the device node
at any given time. (That process may, once it has one write handle, open
any number of additional write handles.) This is accomplished by
tracking open and close data for each driver instance.

Signed-off-by: Rob Springer <rspringer@google.com>
Signed-off-by: John Joseph <jnjoseph@google.com>
Signed-off-by: Simon Que <sque@chromium.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

MAINTAINERS

@@ -5928,6 +5928,13 @@ F:	scripts/gcc-plugin.sh
 F:	scripts/Makefile.gcc-plugins
 F:	Documentation/gcc-plugins.txt
 
+GASKET DRIVER FRAMEWORK
+M:	Rob Springer <rspringer@google.com>
+M:	John Joseph <jnjoseph@google.com>
+M:	Ben Chan <benchan@chromium.org>
+S:	Maintained
+F:	drivers/staging/gasket/
+
 GCOV BASED KERNEL PROFILING
 M:	Peter Oberparleiter <oberpar@linux.ibm.com>
 S:	Maintained

drivers/staging/Kconfig

@@ -124,4 +124,6 @@ source "drivers/staging/mt7621-eth/Kconfig"
 
 source "drivers/staging/mt7621-dts/Kconfig"
 
+source "drivers/staging/gasket/Kconfig"
+
 endif # STAGING

drivers/staging/Makefile

@@ -53,3 +53,4 @@ obj-$(CONFIG_SOC_MT7621)	+= mt7621-dma/
 obj-$(CONFIG_SOC_MT7621)	+= mt7621-mmc/
 obj-$(CONFIG_SOC_MT7621)	+= mt7621-eth/
 obj-$(CONFIG_SOC_MT7621)	+= mt7621-dts/
+obj-$(CONFIG_STAGING_GASKET_FRAMEWORK)	+= gasket/

drivers/staging/gasket/Kconfig

@@ -0,0 +1,23 @@
+menu "Gasket devices"
+
+config STAGING_GASKET_FRAMEWORK
+	tristate "Gasket framework"
+	depends on PCI && X86_64
+	help
+	  This framework supports Gasket-compatible devices, such as Apex.
+	  It is required for any of the following module(s).
+
+	  To compile this driver as a module, choose M here.  The module
+	  will be called "gasket".
+
+config STAGING_APEX_DRIVER
+	tristate "Apex Driver"
+	depends on STAGING_GASKET_FRAMEWORK
+	help
+	  This driver supports the Apex device.  Say Y if you want to
+	  include this driver in the kernel.
+
+	  To compile this driver as a module, choose M here.  The module
+	  will be called "apex".
+
+endmenu

drivers/staging/gasket/Makefile

@@ -0,0 +1,9 @@
+#
+#  Makefile for Gasket framework and dependent drivers.
+#
+
+obj-$(CONFIG_STAGING_GASKET_FRAMEWORK)	+= gasket.o
+obj-$(CONFIG_STAGING_APEX_DRIVER)	+= apex.o
+
+gasket-objs	:= gasket_core.o gasket_ioctl.o gasket_interrupt.o gasket_page_table.o gasket_sysfs.o
+apex-objs	:= apex_driver.o

drivers/staging/gasket/TODO

@@ -0,0 +1,17 @@
+This is a list of things that need to be done to get this driver out of the
+staging directory.
+- Use SPDX tags to show the license of the file, and no more "boiler-plate"
+  license text is needed.
+- Remove static function declarations.
+- Document sysfs files with Documentation/ABI/ entries.
+- Use misc interface instead of major number for driver version description.
+- Add descriptions of module_param's
+- Remove gasket-specific logging functions.
+- apex_get_status() should actually check status.
+- Static functions don't need kernel doc formatting, can be simplified.
+- Fix multi-line alignment formatting to look like:
+      int ret = long_function_name(device, VARIABLE1, VARIABLE2,
+                                   VARIABLE3, VARIABLE4);
+- "drivers" should never be dealing with "raw" sysfs calls or mess around with
+  kobjects at all. The driver core should handle all of this for you
+  automaically. There should not be a need for raw attribute macros.

drivers/staging/gasket/apex.h

@@ -0,0 +1,95 @@
+/*
+ * Apex kernel-userspace interface definition(s).
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef __APEX_H__
+#define __APEX_H__
+
+#include <linux/ioctl.h>
+#include <linux/bitops.h>
+
+#include "gasket.h"
+
+/* Structural definitions/macros. */
+/* The number of PCI BARs. */
+#define APEX_NUM_BARS 3
+
+/* Size of a memory page in bytes, and the related number of bits to shift. */
+#define APEX_PAGE_SHIFT 12
+#define APEX_PAGE_SIZE BIT(APEX_PAGE_SHIFT)
+
+#define APEX_EXTENDED_SHIFT 63 /* Extended address bit position. */
+
+/* Addresses are 2^3=8 bytes each. */
+/* page in second level page table */
+/* holds APEX_PAGE_SIZE/8 addresses  */
+#define APEX_ADDR_SHIFT 3
+#define APEX_LEVEL_SHIFT (APEX_PAGE_SHIFT - APEX_ADDR_SHIFT)
+#define APEX_LEVEL_SIZE BIT(APEX_LEVEL_SHIFT)
+
+#define APEX_PAGE_TABLE_MAX 65536
+#define APEX_SIMPLE_PAGE_MAX APEX_PAGE_TABLE_MAX
+#define APEX_EXTENDED_PAGE_MAX (APEX_PAGE_TABLE_MAX << APEX_LEVEL_SHIFT)
+
+/* Check reset 120 times */
+#define APEX_RESET_RETRY 120
+/* Wait 100 ms between checks. Total 12 sec wait maximum. */
+#define APEX_RESET_DELAY 100
+
+#define APEX_CHIP_INIT_DONE 2
+#define APEX_RESET_ACCEPTED 0
+
+enum apex_reset_types {
+	APEX_CHIP_REINIT_RESET = 3,
+};
+
+/* Interrupt defines */
+/* Gasket device interrupts enums must be dense (i.e., no empty slots). */
+enum apex_interrupt {
+	APEX_INTERRUPT_INSTR_QUEUE = 0,
+	APEX_INTERRUPT_INPUT_ACTV_QUEUE = 1,
+	APEX_INTERRUPT_PARAM_QUEUE = 2,
+	APEX_INTERRUPT_OUTPUT_ACTV_QUEUE = 3,
+	APEX_INTERRUPT_SC_HOST_0 = 4,
+	APEX_INTERRUPT_SC_HOST_1 = 5,
+	APEX_INTERRUPT_SC_HOST_2 = 6,
+	APEX_INTERRUPT_SC_HOST_3 = 7,
+	APEX_INTERRUPT_TOP_LEVEL_0 = 8,
+	APEX_INTERRUPT_TOP_LEVEL_1 = 9,
+	APEX_INTERRUPT_TOP_LEVEL_2 = 10,
+	APEX_INTERRUPT_TOP_LEVEL_3 = 11,
+	APEX_INTERRUPT_FATAL_ERR = 12,
+	APEX_INTERRUPT_COUNT = 13,
+};
+
+/*
+ * Clock Gating ioctl.
+ */
+struct apex_gate_clock_ioctl {
+	/* Enter or leave clock gated state. */
+	u64 enable;
+
+	/* If set, enter clock gating state, regardless of custom block's
+	 * internal idle state
+	 */
+	u64 force_idle;
+};
+
+/* Base number for all Apex-common IOCTLs */
+#define APEX_IOCTL_BASE 0x7F
+
+/* Enable/Disable clock gating. */
+#define APEX_IOCTL_GATE_CLOCK                                                  \
+	_IOW(APEX_IOCTL_BASE, 0, struct apex_gate_clock_ioctl)
+
+#endif /* __APEX_H__ */

drivers/staging/gasket/apex_driver.c

@@ -0,0 +1,771 @@
+/* Driver for the Apex chip.
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/delay.h>
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/sched.h>
+#include <linux/uaccess.h>
+
+#include "apex.h"
+
+#include "gasket_core.h"
+#include "gasket_interrupt.h"
+#include "gasket_logging.h"
+#include "gasket_page_table.h"
+#include "gasket_sysfs.h"
+
+/* Constants */
+#define APEX_DEVICE_NAME "Apex"
+#define APEX_DRIVER_VERSION "1.0"
+
+/* CSRs are in BAR 2. */
+#define APEX_BAR_INDEX 2
+
+#define APEX_PCI_VENDOR_ID 0x1ac1
+#define APEX_PCI_DEVICE_ID 0x089a
+
+/* Bar Offsets. */
+#define APEX_BAR_OFFSET 0
+#define APEX_CM_OFFSET 0x1000000
+
+/* The sizes of each Apex BAR 2. */
+#define APEX_BAR_BYTES 0x100000
+#define APEX_CH_MEM_BYTES (PAGE_SIZE * MAX_NUM_COHERENT_PAGES)
+
+/* The number of user-mappable memory ranges in BAR2 of a Apex chip. */
+#define NUM_REGIONS 3
+
+/* The number of nodes in a Apex chip. */
+#define NUM_NODES 1
+
+/*
+ * The total number of entries in the page table. Should match the value read
+ * from the register APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE.
+ */
+#define APEX_PAGE_TABLE_TOTAL_ENTRIES 8192
+
+/* Enumeration of the supported sysfs entries. */
+enum sysfs_attribute_type {
+	ATTR_KERNEL_HIB_PAGE_TABLE_SIZE,
+	ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE,
+	ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES,
+};
+
+/*
+ * Register offsets into BAR2 memory.
+ * Only values necessary for driver implementation are defined.
+ */
+enum apex_bar2_regs {
+	APEX_BAR2_REG_SCU_BASE = 0x1A300,
+	APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE = 0x46000,
+	APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE = 0x46008,
+	APEX_BAR2_REG_KERNEL_HIB_TRANSLATION_ENABLE = 0x46010,
+	APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL = 0x46018,
+	APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL = 0x46020,
+	APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL = 0x46028,
+	APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL = 0x46030,
+	APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL = 0x46038,
+	APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL = 0x46040,
+	APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL = 0x46048,
+	APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE = 0x46050,
+	APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE_MASK = 0x46058,
+	APEX_BAR2_REG_KERNEL_HIB_STATUS_BLOCK_DELAY = 0x46060,
+	APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY0 = 0x46068,
+	APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY1 = 0x46070,
+	APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT = 0x46078,
+	APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT = 0x46080,
+	APEX_BAR2_REG_KERNEL_WIRE_INT_PENDING_BIT_ARRAY = 0x48778,
+	APEX_BAR2_REG_KERNEL_WIRE_INT_MASK_ARRAY = 0x48780,
+	APEX_BAR2_REG_USER_HIB_DMA_PAUSE = 0x486D8,
+	APEX_BAR2_REG_USER_HIB_DMA_PAUSED = 0x486E0,
+	APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER = 0x4A000,
+	APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE = 0x50000,
+
+	/* Error registers - Used mostly for debug */
+	APEX_BAR2_REG_USER_HIB_ERROR_STATUS = 0x86f0,
+	APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS = 0x41a0,
+};
+
+/* Addresses for packed registers. */
+#define APEX_BAR2_REG_AXI_QUIESCE (APEX_BAR2_REG_SCU_BASE + 0x2C)
+#define APEX_BAR2_REG_GCB_CLOCK_GATE (APEX_BAR2_REG_SCU_BASE + 0x14)
+#define APEX_BAR2_REG_SCU_0 (APEX_BAR2_REG_SCU_BASE + 0xc)
+#define APEX_BAR2_REG_SCU_1 (APEX_BAR2_REG_SCU_BASE + 0x10)
+#define APEX_BAR2_REG_SCU_2 (APEX_BAR2_REG_SCU_BASE + 0x14)
+#define APEX_BAR2_REG_SCU_3 (APEX_BAR2_REG_SCU_BASE + 0x18)
+#define APEX_BAR2_REG_SCU_4 (APEX_BAR2_REG_SCU_BASE + 0x1c)
+#define APEX_BAR2_REG_SCU_5 (APEX_BAR2_REG_SCU_BASE + 0x20)
+
+#define SCU3_RG_PWR_STATE_OVR_BIT_OFFSET 26
+#define SCU3_RG_PWR_STATE_OVR_MASK_WIDTH 2
+#define SCU3_CUR_RST_GCB_BIT_MASK 0x10
+#define SCU2_RG_RST_GCB_BIT_MASK 0xc
+
+/* Configuration for page table. */
+static struct gasket_page_table_config apex_page_table_configs[NUM_NODES] = {
+	{
+		.id = 0,
+		.mode = GASKET_PAGE_TABLE_MODE_NORMAL,
+		.total_entries = APEX_PAGE_TABLE_TOTAL_ENTRIES,
+		.base_reg = APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE,
+		.extended_reg = APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE,
+		.extended_bit = APEX_EXTENDED_SHIFT,
+	},
+};
+
+/* Function declarations */
+static int __init apex_init(void);
+static void apex_exit(void);
+
+static int apex_add_dev_cb(struct gasket_dev *gasket_dev);
+
+static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev);
+
+static int apex_device_cleanup(struct gasket_dev *gasket_dev);
+
+static int apex_device_open_cb(struct gasket_dev *gasket_dev);
+
+static ssize_t sysfs_show(
+	struct device *device, struct device_attribute *attr, char *buf);
+
+static int apex_reset(struct gasket_dev *gasket_dev, uint type);
+
+static int apex_get_status(struct gasket_dev *gasket_dev);
+
+static uint apex_ioctl_check_permissions(struct file *file, uint cmd);
+
+static long apex_ioctl(struct file *file, uint cmd, ulong arg);
+
+static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg);
+
+static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type);
+
+static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type);
+
+static bool is_gcb_in_reset(struct gasket_dev *gasket_dev);
+
+/* Data definitions */
+
+/* The data necessary to display this file's sysfs entries. */
+static struct gasket_sysfs_attribute apex_sysfs_attrs[] = {
+	GASKET_SYSFS_RO(node_0_page_table_entries, sysfs_show,
+			ATTR_KERNEL_HIB_PAGE_TABLE_SIZE),
+	GASKET_SYSFS_RO(node_0_simple_page_table_entries, sysfs_show,
+			ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE),
+	GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show,
+			ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES),
+	GASKET_END_OF_ATTR_ARRAY
+};
+
+static const struct pci_device_id apex_pci_ids[] = {
+	{ PCI_DEVICE(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID) }, { 0 }
+};
+
+/* The regions in the BAR2 space that can be mapped into user space. */
+static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = {
+	{ 0x40000, 0x1000 },
+	{ 0x44000, 0x1000 },
+	{ 0x48000, 0x1000 },
+};
+
+static const struct gasket_mappable_region cm_mappable_regions[1] = { { 0x0,
+	APEX_CH_MEM_BYTES } };
+
+/* Interrupt descriptors for Apex */
+static struct gasket_interrupt_desc apex_interrupts[] = {
+	{
+		APEX_INTERRUPT_INSTR_QUEUE,
+		APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL,
+		UNPACKED,
+	},
+	{
+		APEX_INTERRUPT_INPUT_ACTV_QUEUE,
+		APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL,
+		UNPACKED
+	},
+	{
+		APEX_INTERRUPT_PARAM_QUEUE,
+		APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL,
+		UNPACKED
+	},
+	{
+		APEX_INTERRUPT_OUTPUT_ACTV_QUEUE,
+		APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL,
+		UNPACKED
+	},
+	{
+		APEX_INTERRUPT_SC_HOST_0,
+		APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
+		PACK_0
+	},
+	{
+		APEX_INTERRUPT_SC_HOST_1,
+		APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
+		PACK_1
+	},
+	{
+		APEX_INTERRUPT_SC_HOST_2,
+		APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
+		PACK_2
+	},
+	{
+		APEX_INTERRUPT_SC_HOST_3,
+		APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL,
+		PACK_3
+	},
+	{
+		APEX_INTERRUPT_TOP_LEVEL_0,
+		APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
+		PACK_0
+	},
+	{
+		APEX_INTERRUPT_TOP_LEVEL_1,
+		APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
+		PACK_1
+	},
+	{
+		APEX_INTERRUPT_TOP_LEVEL_2,
+		APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
+		PACK_2
+	},
+	{
+		APEX_INTERRUPT_TOP_LEVEL_3,
+		APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL,
+		PACK_3
+	},
+	{
+		APEX_INTERRUPT_FATAL_ERR,
+		APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL,
+		UNPACKED
+	},
+};
+
+static struct gasket_driver_desc apex_desc = {
+	.name = "apex",
+	.driver_version = APEX_DRIVER_VERSION,
+	.major = 120,
+	.minor = 0,
+	.module = THIS_MODULE,
+	.pci_id_table = apex_pci_ids,
+
+	.num_page_tables = NUM_NODES,
+	.page_table_bar_index = APEX_BAR_INDEX,
+	.page_table_configs = apex_page_table_configs,
+	.page_table_extended_bit = APEX_EXTENDED_SHIFT,
+
+	.bar_descriptions = {
+		GASKET_UNUSED_BAR,
+		GASKET_UNUSED_BAR,
+		{ APEX_BAR_BYTES, (VM_WRITE | VM_READ), APEX_BAR_OFFSET,
+			NUM_REGIONS, mappable_regions, PCI_BAR },
+		GASKET_UNUSED_BAR,
+		GASKET_UNUSED_BAR,
+		GASKET_UNUSED_BAR,
+	},
+	.coherent_buffer_description = {
+		APEX_CH_MEM_BYTES,
+		(VM_WRITE | VM_READ),
+		APEX_CM_OFFSET,
+	},
+	.interrupt_type = PCI_MSIX,
+	.interrupt_bar_index = APEX_BAR_INDEX,
+	.num_interrupts = APEX_INTERRUPT_COUNT,
+	.interrupts = apex_interrupts,
+	.interrupt_pack_width = 7,
+
+	.add_dev_cb = apex_add_dev_cb,
+	.remove_dev_cb = NULL,
+
+	.enable_dev_cb = NULL,
+	.disable_dev_cb = NULL,
+
+	.sysfs_setup_cb = apex_sysfs_setup_cb,
+	.sysfs_cleanup_cb = NULL,
+
+	.device_open_cb = apex_device_open_cb,
+	.device_close_cb = apex_device_cleanup,
+
+	.ioctl_handler_cb = apex_ioctl,
+	.device_status_cb = apex_get_status,
+	.hardware_revision_cb = NULL,
+	.device_reset_cb = apex_reset,
+};
+
+/* Module registration boilerplate */
+MODULE_DESCRIPTION("Google Apex driver");
+MODULE_VERSION(APEX_DRIVER_VERSION);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("John Joseph <jnjoseph@google.com>");
+MODULE_DEVICE_TABLE(pci, apex_pci_ids);
+module_init(apex_init);
+module_exit(apex_exit);
+
+/* Allows device to enter power save upon driver close(). */
+static int allow_power_save;
+
+/* Allows SW based clock gating. */
+static int allow_sw_clock_gating;
+
+/* Allows HW based clock gating. */
+/* Note: this is not mutual exclusive with SW clock gating. */
+static int allow_hw_clock_gating = 1;
+
+/* Act as if only GCB is instantiated. */
+static int bypass_top_level;
+
+module_param(allow_power_save, int, 0644);
+module_param(allow_sw_clock_gating, int, 0644);
+module_param(allow_hw_clock_gating, int, 0644);
+module_param(bypass_top_level, int, 0644);
+
+static int __init apex_init(void)
+{
+	return gasket_register_device(&apex_desc);
+}
+
+static void apex_exit(void)
+{
+	gasket_unregister_device(&apex_desc);
+}
+
+static int apex_add_dev_cb(struct gasket_dev *gasket_dev)
+{
+	ulong page_table_ready, msix_table_ready;
+	int retries = 0;
+
+	gasket_log_error(gasket_dev, "apex_add_dev_cb.");
+
+	apex_reset(gasket_dev, 0);
+
+	while (retries < APEX_RESET_RETRY) {
+		page_table_ready =
+			gasket_dev_read_64(
+				gasket_dev, APEX_BAR_INDEX,
+				APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT);
+		msix_table_ready =
+			gasket_dev_read_64(
+				gasket_dev, APEX_BAR_INDEX,
+				APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT);
+		if (page_table_ready && msix_table_ready)
+			break;
+		schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY));
+		retries++;
+	}
+
+	if (retries == APEX_RESET_RETRY) {
+		if (!page_table_ready)
+			gasket_log_error(
+				gasket_dev, "Page table init timed out.");
+		if (!msix_table_ready)
+			gasket_log_error(
+				gasket_dev, "MSI-X table init timed out.");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int apex_sysfs_setup_cb(struct gasket_dev *gasket_dev)
+{
+	return gasket_sysfs_create_entries(
+		gasket_dev->dev_info.device, apex_sysfs_attrs);
+}
+
+/* On device open, we want to perform a core reinit reset. */
+static int apex_device_open_cb(struct gasket_dev *gasket_dev)
+{
+	return gasket_reset_nolock(gasket_dev, APEX_CHIP_REINIT_RESET);
+}
+
+/**
+ * apex_get_status - Set device status.
+ * @dev: Apex device struct.
+ *
+ * Description: Check the device status registers and set the driver status
+ *		to ALIVE or DEAD.
+ *
+ *		Returns 0 if status is ALIVE, a negative error number otherwise.
+ */
+static int apex_get_status(struct gasket_dev *gasket_dev)
+{
+	/* TODO: Check device status. */
+	return GASKET_STATUS_ALIVE;
+}
+
+/**
+ * apex_device_cleanup - Clean up Apex HW after close.
+ * @gasket_dev: Gasket device pointer.
+ *
+ * Description: Resets the Apex hardware. Called on final close via
+ * device_close_cb.
+ */
+static int apex_device_cleanup(struct gasket_dev *gasket_dev)
+{
+	u64 scalar_error;
+	u64 hib_error;
+	int ret = 0;
+
+	hib_error = gasket_dev_read_64(
+		gasket_dev, APEX_BAR_INDEX,
+		APEX_BAR2_REG_USER_HIB_ERROR_STATUS);
+	scalar_error = gasket_dev_read_64(
+		gasket_dev, APEX_BAR_INDEX,
+		APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS);
+
+	gasket_log_info(
+		gasket_dev,
+		"apex_device_cleanup 0x%p hib_error 0x%llx scalar_error "
+		"0x%llx.",
+		gasket_dev, hib_error, scalar_error);
+
+	if (allow_power_save)
+		ret = apex_enter_reset(gasket_dev, APEX_CHIP_REINIT_RESET);
+
+	return ret;
+}
+
+/**
+ * apex_reset - Quits reset.
+ * @gasket_dev: Gasket device pointer.
+ *
+ * Description: Resets the hardware, then quits reset.
+ * Called on device open.
+ *
+ */
+static int apex_reset(struct gasket_dev *gasket_dev, uint type)
+{
+	int ret;
+
+	if (bypass_top_level)
+		return 0;
+
+	gasket_log_debug(gasket_dev, "apex_reset.");
+
+	if (!is_gcb_in_reset(gasket_dev)) {
+		/* We are not in reset - toggle the reset bit so as to force
+		 * re-init of custom block
+		 */
+		gasket_log_debug(gasket_dev, "apex_reset: toggle reset.");
+
+		ret = apex_enter_reset(gasket_dev, type);
+		if (ret)
+			return ret;
+	}
+	ret = apex_quit_reset(gasket_dev, type);
+
+	return ret;
+}
+
+/*
+ * Enters GCB reset state.
+ */
+static int apex_enter_reset(struct gasket_dev *gasket_dev, uint type)
+{
+	if (bypass_top_level)
+		return 0;
+
+	gasket_log_debug(gasket_dev, "apex_enter_reset.");
+
+	/*
+	 * Software reset:
+	 * Enable sleep mode
+	 *  - Software force GCB idle
+	 *    - Enable GCB idle
+	 */
+	gasket_read_modify_write_64(
+		gasket_dev, APEX_BAR_INDEX,
+		APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER, 0x0, 1, 32);
+
+	/*    - Initiate DMA pause */
+	gasket_dev_write_64(gasket_dev, 1, APEX_BAR_INDEX,
+			    APEX_BAR2_REG_USER_HIB_DMA_PAUSE);
+
+	/*    - Wait for DMA pause complete. */
+	if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
+					APEX_BAR2_REG_USER_HIB_DMA_PAUSED, 1, 1,
+					APEX_RESET_DELAY, APEX_RESET_RETRY)) {
+		gasket_log_error(gasket_dev,
+				 "DMAs did not quiece within timeout (%d ms)",
+				 APEX_RESET_RETRY * APEX_RESET_DELAY);
+		return -EINVAL;
+	}
+
+	/*  - Enable GCB reset (0x1 to rg_rst_gcb) */
+	gasket_read_modify_write_32(
+		gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x1, 2, 2);
+
+	/*  - Enable GCB clock Gate (0x1 to rg_gated_gcb) */
+	gasket_read_modify_write_32(
+		gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x1, 2, 18);
+
+	/*  - Enable GCB memory shut down (0x3 to rg_force_ram_sd) */
+	gasket_read_modify_write_32(
+		gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 0x3, 2, 14);
+
+	/*    - Wait for RAM shutdown. */
+	if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
+					APEX_BAR2_REG_SCU_3, 1 << 6, 1 << 6,
+					APEX_RESET_DELAY, APEX_RESET_RETRY)) {
+		gasket_log_error(
+			gasket_dev,
+			"RAM did not shut down within timeout (%d ms)",
+			APEX_RESET_RETRY * APEX_RESET_DELAY);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/*
+ * Quits GCB reset state.
+ */
+static int apex_quit_reset(struct gasket_dev *gasket_dev, uint type)
+{
+	u32 val0, val1;
+
+	if (bypass_top_level)
+		return 0;
+
+	gasket_log_debug(gasket_dev, "apex_quit_reset.");
+
+	/*
+	 * Disable sleep mode:
+	 *  - Disable GCB memory shut down:
+	 *    - b00: Not forced (HW controlled)
+	 *    - b1x: Force disable
+	 */
+	gasket_read_modify_write_32(
+		gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 0x0, 2, 14);
+
+	/*
+	 *  - Disable software clock gate:
+	 *    - b00: Not forced (HW controlled)
+	 *    - b1x: Force disable
+	 */
+	gasket_read_modify_write_32(
+		gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x0, 2, 18);
+
+	/*
+	 *  - Disable GCB reset (rg_rst_gcb):
+	 *    - b00: Not forced (HW controlled)
+	 *    - b1x: Force disable = Force not Reset
+	 */
+	gasket_read_modify_write_32(
+		gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_2, 0x2, 2, 2);
+
+	/*    - Wait for RAM enable. */
+	if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
+					APEX_BAR2_REG_SCU_3, 1 << 6, 0,
+					APEX_RESET_DELAY, APEX_RESET_RETRY)) {
+		gasket_log_error(
+			gasket_dev,
+			"RAM did not enable within timeout (%d ms)",
+			APEX_RESET_RETRY * APEX_RESET_DELAY);
+		return -EINVAL;
+	}
+
+	/*    - Wait for Reset complete. */
+	if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX,
+					APEX_BAR2_REG_SCU_3,
+					SCU3_CUR_RST_GCB_BIT_MASK, 0,
+					APEX_RESET_DELAY, APEX_RESET_RETRY)) {
+		gasket_log_error(
+			gasket_dev,
+			"GCB did not leave reset within timeout (%d ms)",
+			APEX_RESET_RETRY * APEX_RESET_DELAY);
+		return -EINVAL;
+	}
+
+	if (!allow_hw_clock_gating) {
+		val0 = gasket_dev_read_32(
+			gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3);
+		/* Inactive and Sleep mode are disabled. */
+		gasket_read_modify_write_32(
+			gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 0x3,
+			SCU3_RG_PWR_STATE_OVR_MASK_WIDTH,
+			SCU3_RG_PWR_STATE_OVR_BIT_OFFSET);
+		val1 = gasket_dev_read_32(
+			gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3);
+		gasket_log_error(
+			gasket_dev, "Disallow HW clock gating 0x%x -> 0x%x",
+			val0, val1);
+	} else {
+		val0 = gasket_dev_read_32(
+			gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3);
+		/* Inactive mode enabled - Sleep mode disabled. */
+		gasket_read_modify_write_32(
+			gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, 2,
+			SCU3_RG_PWR_STATE_OVR_MASK_WIDTH,
+			SCU3_RG_PWR_STATE_OVR_BIT_OFFSET);
+		val1 = gasket_dev_read_32(
+			gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3);
+		gasket_log_error(
+			gasket_dev, "Allow HW clock gating 0x%x -> 0x%x", val0,
+			val1);
+	}
+
+	return 0;
+}
+
+/*
+ * Determines if GCB is in reset state.
+ */
+static bool is_gcb_in_reset(struct gasket_dev *gasket_dev)
+{
+	u32 val = gasket_dev_read_32(
+		gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3);
+
+	/* Masks rg_rst_gcb bit of SCU_CTRL_2 */
+	return (val & SCU3_CUR_RST_GCB_BIT_MASK);
+}
+
+/*
+ * Check permissions for Apex ioctls.
+ * @file: File pointer from ioctl.
+ * @cmd: ioctl command.
+ *
+ * Returns 1 if the current user may execute this ioctl, and 0 otherwise.
+ */
+static uint apex_ioctl_check_permissions(struct file *filp, uint cmd)
+{
+	struct gasket_dev *gasket_dev = filp->private_data;
+	int root = capable(CAP_SYS_ADMIN);
+	int is_owner = gasket_dev->dev_info.ownership.is_owned &&
+		       current->tgid == gasket_dev->dev_info.ownership.owner;
+
+	if (root || is_owner)
+		return 1;
+	return 0;
+}
+
+/*
+ * Apex-specific ioctl handler.
+ */
+static long apex_ioctl(struct file *filp, uint cmd, ulong arg)
+{
+	struct gasket_dev *gasket_dev = filp->private_data;
+
+	if (!apex_ioctl_check_permissions(filp, cmd))
+		return -EPERM;
+
+	switch (cmd) {
+	case APEX_IOCTL_GATE_CLOCK:
+		return apex_clock_gating(gasket_dev, arg);
+	default:
+		return -ENOTTY; /* unknown command */
+	}
+}
+
+/*
+ * Gates or un-gates Apex clock.
+ * @gasket_dev: Gasket device pointer.
+ * @arg: User ioctl arg, in this case to a apex_gate_clock_ioctl struct.
+ */
+static long apex_clock_gating(struct gasket_dev *gasket_dev, ulong arg)
+{
+	struct apex_gate_clock_ioctl ibuf;
+
+	if (bypass_top_level)
+		return 0;
+
+	if (allow_sw_clock_gating) {
+		if (copy_from_user(&ibuf, (void __user *)arg, sizeof(ibuf)))
+			return -EFAULT;
+
+		gasket_log_error(
+			gasket_dev, "apex_clock_gating %llu", ibuf.enable);
+
+		if (ibuf.enable) {
+			/* Quiesce AXI, gate GCB clock. */
+			gasket_read_modify_write_32(
+				gasket_dev, APEX_BAR_INDEX,
+				APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1, 16);
+			gasket_read_modify_write_32(
+				gasket_dev, APEX_BAR_INDEX,
+				APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1, 2, 18);
+		} else {
+			/* Un-gate GCB clock, un-quiesce AXI. */
+			gasket_read_modify_write_32(
+				gasket_dev, APEX_BAR_INDEX,
+				APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0, 2, 18);
+			gasket_read_modify_write_32(
+				gasket_dev, APEX_BAR_INDEX,
+				APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1, 16);
+		}
+	}
+	return 0;
+}
+
+/*
+ * Display driver sysfs entries.
+ * @device: Kernel device structure.
+ * @attr: Attribute to display.
+ * @buf: Buffer to which to write output.
+ *
+ * Description: Looks up the driver data and file-specific attribute data (the
+ * type of the attribute), then fills "buf" accordingly.
+ */
+static ssize_t sysfs_show(
+	struct device *device, struct device_attribute *attr, char *buf)
+{
+	int ret;
+	struct gasket_dev *gasket_dev;
+	struct gasket_sysfs_attribute *gasket_attr;
+	enum sysfs_attribute_type type;
+
+	gasket_dev = gasket_sysfs_get_device_data(device);
+	if (!gasket_dev) {
+		gasket_nodev_error("No Apex device sysfs mapping found");
+		return 0;
+	}
+
+	gasket_attr = gasket_sysfs_get_attr(device, attr);
+	if (!gasket_attr) {
+		gasket_nodev_error("No Apex device sysfs attr data found");
+		gasket_sysfs_put_device_data(device, gasket_dev);
+		return 0;
+	}
+
+	type = (enum sysfs_attribute_type)gasket_sysfs_get_attr(device, attr);
+	switch (type) {
+	case ATTR_KERNEL_HIB_PAGE_TABLE_SIZE:
+		ret = scnprintf(buf, PAGE_SIZE, "%u\n",
+				gasket_page_table_num_entries(
+					gasket_dev->page_table[0]));
+		break;
+	case ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE:
+		ret = scnprintf(buf, PAGE_SIZE, "%u\n",
+				gasket_page_table_num_entries(
+					gasket_dev->page_table[0]));
+		break;
+	case ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES:
+		ret = scnprintf(buf, PAGE_SIZE, "%u\n",
+				gasket_page_table_num_active_pages(
+					gasket_dev->page_table[0]));
+		break;
+	default:
+		gasket_log_error(
+			gasket_dev, "Unknown attribute: %s", attr->attr.name);
+		ret = 0;
+		break;
+	}
+
+	gasket_sysfs_put_attr(device, gasket_attr);
+	gasket_sysfs_put_device_data(device, gasket_dev);
+	return ret;
+}

drivers/staging/gasket/gasket.h

@@ -0,0 +1,129 @@
+/* Common Gasket device kernel and user space declarations.
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef __GASKET_H__
+#define __GASKET_H__
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+/* ioctl structure declarations */
+
+/* Ioctl structures are padded to a multiple of 64 bits */
+/* and padded to put 64 bit values on 64 bit boundaries. */
+/* Unsigned 64 bit integers are used to hold pointers. */
+/* This helps compatibility between 32 and 64 bits. */
+
+/*
+ * Common structure for ioctls associating an eventfd with a device interrupt,
+ * when using the Gasket interrupt module.
+ */
+struct gasket_interrupt_eventfd {
+	u64 interrupt;
+	u64 event_fd;
+};
+
+/*
+ * Common structure for ioctls mapping and unmapping buffers when using the
+ * Gasket page_table module.
+ */
+struct gasket_page_table_ioctl {
+	u64 page_table_index;
+	u64 size;
+	u64 host_address;
+	u64 device_address;
+};
+
+/*
+ * Common structure for ioctls mapping and unmapping buffers when using the
+ * Gasket page_table module.
+ * dma_address: phys addr start of coherent memory, allocated by kernel
+ */
+struct gasket_coherent_alloc_config_ioctl {
+	u64 page_table_index;
+	u64 enable;
+	u64 size;
+	u64 dma_address;
+};
+
+/* Base number for all Gasket-common IOCTLs */
+#define GASKET_IOCTL_BASE 0xDC
+
+/* Reset the device using the specified reset type. */
+#define GASKET_IOCTL_RESET _IOW(GASKET_IOCTL_BASE, 0, unsigned long)
+
+/* Associate the specified [event]fd with the specified interrupt. */
+#define GASKET_IOCTL_SET_EVENTFD                                               \
+	_IOW(GASKET_IOCTL_BASE, 1, struct gasket_interrupt_eventfd)
+
+/*
+ * Clears any eventfd associated with the specified interrupt. The (ulong)
+ * argument is the interrupt number to clear.
+ */
+#define GASKET_IOCTL_CLEAR_EVENTFD _IOW(GASKET_IOCTL_BASE, 2, unsigned long)
+
+/*
+ * [Loopbacks only] Requests that the loopback device send the specified
+ * interrupt to the host. The (ulong) argument is the number of the interrupt to
+ * send.
+ */
+#define GASKET_IOCTL_LOOPBACK_INTERRUPT                                        \
+	_IOW(GASKET_IOCTL_BASE, 3, unsigned long)
+
+/* Queries the kernel for the number of page tables supported by the device. */
+#define GASKET_IOCTL_NUMBER_PAGE_TABLES _IOR(GASKET_IOCTL_BASE, 4, u64)
+
+/*
+ * Queries the kernel for the maximum size of the page table.  Only the size and
+ * page_table_index fields are used from the struct gasket_page_table_ioctl.
+ */
+#define GASKET_IOCTL_PAGE_TABLE_SIZE                                           \
+	_IOWR(GASKET_IOCTL_BASE, 5, struct gasket_page_table_ioctl)
+
+/*
+ * Queries the kernel for the current simple page table size.  Only the size and
+ * page_table_index fields are used from the struct gasket_page_table_ioctl.
+ */
+#define GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE                                    \
+	_IOWR(GASKET_IOCTL_BASE, 6, struct gasket_page_table_ioctl)
+
+/*
+ * Tells the kernel to change the split between the number of simple and
+ * extended entries in the given page table. Only the size and page_table_index
+ * fields are used from the struct gasket_page_table_ioctl.
+ */
+#define GASKET_IOCTL_PARTITION_PAGE_TABLE                                      \
+	_IOW(GASKET_IOCTL_BASE, 7, struct gasket_page_table_ioctl)
+
+/*
+ * Tells the kernel to map size bytes at host_address to device_address in
+ * page_table_index page table.
+ */
+#define GASKET_IOCTL_MAP_BUFFER                                                \
+	_IOW(GASKET_IOCTL_BASE, 8, struct gasket_page_table_ioctl)
+
+/*
+ * Tells the kernel to unmap size bytes at host_address from device_address in
+ * page_table_index page table.
+ */
+#define GASKET_IOCTL_UNMAP_BUFFER                                              \
+	_IOW(GASKET_IOCTL_BASE, 9, struct gasket_page_table_ioctl)
+
+/* Clear the interrupt counts stored for this device. */
+#define GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS _IO(GASKET_IOCTL_BASE, 10)
+
+/* Enable/Disable and configure the coherent allocator. */
+#define GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR                                 \
+	_IOWR(GASKET_IOCTL_BASE, 11, struct gasket_coherent_alloc_config_ioctl)
+
+#endif /* __GASKET_H__ */

drivers/staging/gasket/gasket_constants.h

@@ -0,0 +1,56 @@
+/* Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef __GASKET_CONSTANTS_H__
+#define __GASKET_CONSTANTS_H__
+
+#define GASKET_FRAMEWORK_VERSION "1.1.1"
+
+/*
+ * The maximum number of simultaneous device types supported by the framework.
+ */
+#define GASKET_FRAMEWORK_DESC_MAX 2
+
+/* The maximum devices per each type. */
+#define GASKET_DEV_MAX 256
+
+/* The number of supported (and possible) PCI BARs. */
+#define GASKET_NUM_BARS 6
+
+/* The number of supported Gasket page tables per device. */
+#define GASKET_MAX_NUM_PAGE_TABLES 1
+
+/* Maximum length of device names (driver name + minor number suffix + NULL). */
+#define GASKET_NAME_MAX 32
+
+/* Device status enumeration. */
+enum gasket_status {
+	/*
+	 * A device is DEAD if it has not been initialized or has had an error.
+	 */
+	GASKET_STATUS_DEAD = 0,
+	/*
+	 * A device is LAMED if the hardware is healthy but the kernel was
+	 * unable to enable some functionality (e.g. interrupts).
+	 */
+	GASKET_STATUS_LAMED,
+
+	/* A device is ALIVE if it is ready for operation. */
+	GASKET_STATUS_ALIVE,
+
+	/*
+	 * This status is set when the driver is exiting and waiting for all
+	 * handles to be closed.
+	 */
+	GASKET_STATUS_DRIVER_EXIT,
+};
+
+#endif

drivers/staging/gasket/gasket_core.c

@@ -0,0 +1,2157 @@
+/* Gasket generic driver framework. This file contains the implementation
+ * for the Gasket generic driver framework - the functionality that is common
+ * across Gasket devices.
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include "gasket_core.h"
+
+#include "gasket_interrupt.h"
+#include "gasket_ioctl.h"
+#include "gasket_logging.h"
+#include "gasket_page_table.h"
+#include "gasket_sysfs.h"
+
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/of.h>
+
+#ifdef GASKET_KERNEL_TRACE_SUPPORT
+#define CREATE_TRACE_POINTS
+#include <trace/events/gasket_mmap.h>
+#else
+#define trace_gasket_mmap_exit(x)
+#define trace_gasket_mmap_entry(x, ...)
+#endif
+
+/*
+ * "Private" members of gasket_driver_desc.
+ *
+ * Contains internal per-device type tracking data, i.e., data not appropriate
+ * as part of the public interface for the generic framework.
+ */
+struct gasket_internal_desc {
+	/* Device-specific-driver-provided configuration information. */
+	const struct gasket_driver_desc *driver_desc;
+
+	/* Protects access to per-driver data (i.e. this structure). */
+	struct mutex mutex;
+
+	/* Kernel-internal device class. */
+	struct class *class;
+
+	/* PCI subsystem metadata associated with this driver. */
+	struct pci_driver pci;
+
+	/* Instantiated / present devices of this type. */
+	struct gasket_dev *devs[GASKET_DEV_MAX];
+};
+
+/* do_map_region() needs be able to return more than just true/false. */
+enum do_map_region_status {
+	/* The region was successfully mapped. */
+	DO_MAP_REGION_SUCCESS,
+
+	/* Attempted to map region and failed. */
+	DO_MAP_REGION_FAILURE,
+
+	/* The requested region to map was not part of a mappable region. */
+	DO_MAP_REGION_INVALID,
+};
+
+/* Function declarations; comments are with definitions. */
+static int __init gasket_init(void);
+static void __exit gasket_exit(void);
+
+static int gasket_pci_probe(
+	struct pci_dev *pci_dev, const struct pci_device_id *id);
+static void gasket_pci_remove(struct pci_dev *pci_dev);
+
+static int gasket_setup_pci(struct pci_dev *pci_dev, struct gasket_dev *dev);
+static void gasket_cleanup_pci(struct gasket_dev *dev);
+
+static int gasket_map_pci_bar(struct gasket_dev *dev, int bar_num);
+static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num);
+
+static int gasket_alloc_dev(
+	struct gasket_internal_desc *internal_desc, struct device *dev,
+	struct gasket_dev **pdev, const char *kobj_name);
+static void gasket_free_dev(struct gasket_dev *dev);
+
+static int gasket_find_dev_slot(
+	struct gasket_internal_desc *internal_desc, const char *kobj_name);
+
+static int gasket_add_cdev(
+	struct gasket_cdev_info *dev_info,
+	const struct file_operations *file_ops, struct module *owner);
+
+static int gasket_enable_dev(
+	struct gasket_internal_desc *internal_desc,
+	struct gasket_dev *gasket_dev);
+static void gasket_disable_dev(struct gasket_dev *gasket_dev);
+
+static struct gasket_internal_desc *lookup_internal_desc(
+	struct pci_dev *pci_dev);
+
+static ssize_t gasket_sysfs_data_show(
+	struct device *device, struct device_attribute *attr, char *buf);
+
+static int gasket_mmap(struct file *filp, struct vm_area_struct *vma);
+static int gasket_open(struct inode *inode, struct file *file);
+static int gasket_release(struct inode *inode, struct file *file);
+static long gasket_ioctl(struct file *filp, uint cmd, ulong arg);
+
+static int gasket_mm_vma_bar_offset(
+	const struct gasket_dev *gasket_dev, const struct vm_area_struct *vma,
+	ulong *bar_offset);
+static bool gasket_mm_get_mapping_addrs(
+	const struct gasket_mappable_region *region, ulong bar_offset,
+	ulong requested_length, struct gasket_mappable_region *mappable_region,
+	ulong *virt_offset);
+static enum do_map_region_status do_map_region(
+	const struct gasket_dev *gasket_dev, struct vm_area_struct *vma,
+	struct gasket_mappable_region *map_region);
+
+static int gasket_get_hw_status(struct gasket_dev *gasket_dev);
+
+/* Global data definitions. */
+/* Mutex - only for framework-wide data. Other data should be protected by
+ * finer-grained locks.
+ */
+static DEFINE_MUTEX(g_mutex);
+
+/* List of all registered device descriptions & their supporting data. */
+static struct gasket_internal_desc g_descs[GASKET_FRAMEWORK_DESC_MAX];
+
+/* Mapping of statuses to human-readable strings. Must end with {0,NULL}. */
+static const struct gasket_num_name gasket_status_name_table[] = {
+	{ GASKET_STATUS_DEAD, "DEAD" },
+	{ GASKET_STATUS_ALIVE, "ALIVE" },
+	{ GASKET_STATUS_LAMED, "LAMED" },
+	{ GASKET_STATUS_DRIVER_EXIT, "DRIVER_EXITING" },
+	{ 0, NULL },
+};
+
+/* Enumeration of the automatic Gasket framework sysfs nodes. */
+enum gasket_sysfs_attribute_type {
+	ATTR_BAR_OFFSETS,
+	ATTR_BAR_SIZES,
+	ATTR_DRIVER_VERSION,
+	ATTR_FRAMEWORK_VERSION,
+	ATTR_DEVICE_TYPE,
+	ATTR_HARDWARE_REVISION,
+	ATTR_PCI_ADDRESS,
+	ATTR_STATUS,
+	ATTR_IS_DEVICE_OWNED,
+	ATTR_DEVICE_OWNER,
+	ATTR_WRITE_OPEN_COUNT,
+	ATTR_RESET_COUNT,
+	ATTR_USER_MEM_RANGES
+};
+
+/* File operations for all Gasket devices. */
+static const struct file_operations gasket_file_ops = {
+	.owner = THIS_MODULE,
+	.llseek = no_llseek,
+	.mmap = gasket_mmap,
+	.open = gasket_open,
+	.release = gasket_release,
+	.unlocked_ioctl = gasket_ioctl,
+};
+
+/* These attributes apply to all Gasket driver instances. */
+static const struct gasket_sysfs_attribute gasket_sysfs_generic_attrs[] = {
+	GASKET_SYSFS_RO(bar_offsets, gasket_sysfs_data_show, ATTR_BAR_OFFSETS),
+	GASKET_SYSFS_RO(bar_sizes, gasket_sysfs_data_show, ATTR_BAR_SIZES),
+	GASKET_SYSFS_RO(driver_version, gasket_sysfs_data_show,
+			ATTR_DRIVER_VERSION),
+	GASKET_SYSFS_RO(framework_version, gasket_sysfs_data_show,
+			ATTR_FRAMEWORK_VERSION),
+	GASKET_SYSFS_RO(device_type, gasket_sysfs_data_show, ATTR_DEVICE_TYPE),
+	GASKET_SYSFS_RO(revision, gasket_sysfs_data_show,
+			ATTR_HARDWARE_REVISION),
+	GASKET_SYSFS_RO(pci_address, gasket_sysfs_data_show, ATTR_PCI_ADDRESS),
+	GASKET_SYSFS_RO(status, gasket_sysfs_data_show, ATTR_STATUS),
+	GASKET_SYSFS_RO(is_device_owned, gasket_sysfs_data_show,
+			ATTR_IS_DEVICE_OWNED),
+	GASKET_SYSFS_RO(device_owner, gasket_sysfs_data_show,
+			ATTR_DEVICE_OWNER),
+	GASKET_SYSFS_RO(write_open_count, gasket_sysfs_data_show,
+			ATTR_WRITE_OPEN_COUNT),
+	GASKET_SYSFS_RO(reset_count, gasket_sysfs_data_show, ATTR_RESET_COUNT),
+	GASKET_SYSFS_RO(user_mem_ranges, gasket_sysfs_data_show,
+			ATTR_USER_MEM_RANGES),
+	GASKET_END_OF_ATTR_ARRAY
+};
+
+MODULE_DESCRIPTION("Google Gasket driver framework");
+MODULE_VERSION(GASKET_FRAMEWORK_VERSION);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Rob Springer <rspringer@google.com>");
+module_init(gasket_init);
+module_exit(gasket_exit);
+
+/*
+ * Perform a standard Gasket callback.
+ * @gasket_dev: Device specific pointer to forward.
+ * @cb_function: Standard callback to perform.
+ */
+static inline int check_and_invoke_callback(
+	struct gasket_dev *gasket_dev, int (*cb_function)(struct gasket_dev *))
+{
+	int ret = 0;
+
+	gasket_nodev_error("check_and_invoke_callback %p", gasket_dev);
+	if (cb_function) {
+		mutex_lock(&gasket_dev->mutex);
+		ret = cb_function(gasket_dev);
+		mutex_unlock(&gasket_dev->mutex);
+	}
+	return ret;
+}
+
+/*
+ * Perform a standard Gasket callback
+ * without grabbing gasket_dev->mutex.
+ * @gasket_dev: Device specific pointer to forward.
+ * @cb_function: Standard callback to perform.
+ *
+ */
+static inline int gasket_check_and_invoke_callback_nolock(
+	struct gasket_dev *gasket_dev, int (*cb_function)(struct gasket_dev *))
+{
+	int ret = 0;
+
+	if (cb_function) {
+		gasket_log_info(
+			gasket_dev, "Invoking device-specific callback.");
+		ret = cb_function(gasket_dev);
+	}
+	return ret;
+}
+
+/*
+ * Retrieve device-specific data via cdev pointer.
+ * @cdev_ptr: Character device pointer associated with the device.
+ *
+ * This function returns the pointer to the device-specific data allocated in
+ * add_dev_cb for the device associated with cdev_ptr.
+ */
+static struct gasket_cdev_info *gasket_cdev_get_info(struct cdev *cdev_ptr)
+{
+	return container_of(cdev_ptr, struct gasket_cdev_info, cdev);
+}
+
+/*
+ * Returns nonzero if the gasket_cdev_info is owned by the current thread group
+ * ID.
+ * @info: Device node info.
+ */
+static int gasket_owned_by_current_tgid(struct gasket_cdev_info *info)
+{
+	return (info->ownership.is_owned &&
+		(info->ownership.owner == current->tgid));
+}
+
+static int __init gasket_init(void)
+{
+	int i;
+
+	gasket_nodev_info("Performing one-time init of the Gasket framework.");
+	/* Check for duplicates and find a free slot. */
+	mutex_lock(&g_mutex);
+	for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
+		g_descs[i].driver_desc = NULL;
+		mutex_init(&g_descs[i].mutex);
+	}
+
+	gasket_sysfs_init();
+
+	mutex_unlock(&g_mutex);
+	return 0;
+}
+
+static void __exit gasket_exit(void)
+{
+	/* No deinit/dealloc needed at present. */
+	gasket_nodev_info("Removing Gasket framework module.");
+}
+
+/* See gasket_core.h for description. */
+int gasket_register_device(const struct gasket_driver_desc *driver_desc)
+{
+	int i, ret;
+	int desc_idx = -1;
+	struct gasket_internal_desc *internal;
+
+	gasket_nodev_info("Initializing Gasket framework device");
+	/* Check for duplicates and find a free slot. */
+	mutex_lock(&g_mutex);
+
+	for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
+		if (g_descs[i].driver_desc == driver_desc) {
+			gasket_nodev_error(
+				"%s driver already loaded/registered",
+				driver_desc->name);
+			mutex_unlock(&g_mutex);
+			return -EBUSY;
+		}
+	}
+
+	/* This and the above loop could be combined, but this reads easier. */
+	for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
+		if (!g_descs[i].driver_desc) {
+			g_descs[i].driver_desc = driver_desc;
+			desc_idx = i;
+			break;
+		}
+	}
+	mutex_unlock(&g_mutex);
+
+	gasket_nodev_info("Loaded %s driver, framework version %s",
+			  driver_desc->name, GASKET_FRAMEWORK_VERSION);
+
+	if (desc_idx == -1) {
+		gasket_nodev_error("Too many Gasket drivers loaded: %d\n",
+				   GASKET_FRAMEWORK_DESC_MAX);
+		return -EBUSY;
+	}
+
+	/* Internal structure setup. */
+	gasket_nodev_info("Performing initial internal structure setup.");
+	internal = &g_descs[desc_idx];
+	mutex_init(&internal->mutex);
+	memset(internal->devs, 0, sizeof(struct gasket_dev *) * GASKET_DEV_MAX);
+	memset(&internal->pci, 0, sizeof(internal->pci));
+	internal->pci.name = driver_desc->name;
+	internal->pci.id_table = driver_desc->pci_id_table;
+	internal->pci.probe = gasket_pci_probe;
+	internal->pci.remove = gasket_pci_remove;
+	internal->class =
+		class_create(driver_desc->module, driver_desc->name);
+
+	if (IS_ERR_OR_NULL(internal->class)) {
+		gasket_nodev_error("Cannot register %s class [ret=%ld]",
+				   driver_desc->name, PTR_ERR(internal->class));
+		return PTR_ERR(internal->class);
+	}
+
+	/*
+	 * Not using pci_register_driver() (without underscores), as it
+	 * depends on KBUILD_MODNAME, and this is a shared file.
+	 */
+	gasket_nodev_info("Registering PCI driver.");
+	ret = __pci_register_driver(
+		&internal->pci, driver_desc->module, driver_desc->name);
+	if (ret) {
+		gasket_nodev_error(
+			"cannot register pci driver [ret=%d]", ret);
+		goto fail1;
+	}
+
+	gasket_nodev_info("Registering char driver.");
+	ret = register_chrdev_region(
+		MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX,
+		driver_desc->name);
+	if (ret) {
+		gasket_nodev_error("cannot register char driver [ret=%d]", ret);
+		goto fail2;
+	}
+
+	gasket_nodev_info("Driver registered successfully.");
+	return 0;
+
+fail2:
+	pci_unregister_driver(&internal->pci);
+
+fail1:
+	class_destroy(internal->class);
+
+	g_descs[desc_idx].driver_desc = NULL;
+	return ret;
+}
+EXPORT_SYMBOL(gasket_register_device);
+
+/* See gasket_core.h for description. */
+void gasket_unregister_device(const struct gasket_driver_desc *driver_desc)
+{
+	int i, desc_idx;
+	struct gasket_internal_desc *internal_desc = NULL;
+
+	mutex_lock(&g_mutex);
+	for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
+		if (g_descs[i].driver_desc == driver_desc) {
+			internal_desc = &g_descs[i];
+			desc_idx = i;
+			break;
+		}
+	}
+	mutex_unlock(&g_mutex);
+
+	if (!internal_desc) {
+		gasket_nodev_error(
+			"request to unregister unknown desc: %s, %d:%d",
+			driver_desc->name, driver_desc->major,
+			driver_desc->minor);
+		return;
+	}
+
+	unregister_chrdev_region(
+		MKDEV(driver_desc->major, driver_desc->minor), GASKET_DEV_MAX);
+
+	pci_unregister_driver(&internal_desc->pci);
+
+	class_destroy(internal_desc->class);
+
+	/* Finally, effectively "remove" the driver. */
+	g_descs[desc_idx].driver_desc = NULL;
+
+	gasket_nodev_info("removed %s driver", driver_desc->name);
+}
+EXPORT_SYMBOL(gasket_unregister_device);
+
+/**
+ * Allocate a Gasket device.
+ * @internal_desc: Pointer to the internal data for the device driver.
+ * @pdev: Pointer to the Gasket device pointer, the allocated device.
+ * @kobj_name: PCIe name for the device
+ *
+ * Description: Allocates and initializes a Gasket device structure.
+ *              Adds the device to the device list.
+ *
+ * Returns 0 if successful, a negative error code otherwise.
+ */
+static int gasket_alloc_dev(
+	struct gasket_internal_desc *internal_desc, struct device *parent,
+	struct gasket_dev **pdev, const char *kobj_name)
+{
+	int dev_idx;
+	const struct gasket_driver_desc *driver_desc =
+		internal_desc->driver_desc;
+	struct gasket_dev *gasket_dev;
+	struct gasket_cdev_info *dev_info;
+
+	gasket_nodev_info("Allocating a Gasket device %s.", kobj_name);
+
+	*pdev = NULL;
+
+	dev_idx = gasket_find_dev_slot(internal_desc, kobj_name);
+	if (dev_idx < 0)
+		return dev_idx;
+
+	gasket_dev = *pdev = kzalloc(sizeof(*gasket_dev), GFP_KERNEL);
+	if (!gasket_dev) {
+		gasket_nodev_error("no memory for device");
+		return -ENOMEM;
+	}
+	internal_desc->devs[dev_idx] = gasket_dev;
+
+	mutex_init(&gasket_dev->mutex);
+
+	gasket_dev->internal_desc = internal_desc;
+	gasket_dev->dev_idx = dev_idx;
+	snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", kobj_name);
+	/* gasket_bar_data is uninitialized. */
+	gasket_dev->num_page_tables = driver_desc->num_page_tables;
+	/* max_page_table_size and *page table are uninit'ed */
+	/* interrupt_data is not initialized. */
+	/* status is 0, or GASKET_STATUS_DEAD */
+
+	dev_info = &gasket_dev->dev_info;
+	snprintf(dev_info->name, GASKET_NAME_MAX, "%s_%u", driver_desc->name,
+		 gasket_dev->dev_idx);
+	dev_info->devt =
+		MKDEV(driver_desc->major, driver_desc->minor +
+			gasket_dev->dev_idx);
+	dev_info->device = device_create(internal_desc->class, parent,
+		dev_info->devt, gasket_dev, dev_info->name);
+
+	gasket_nodev_info("Gasket device allocated: %p.", dev_info->device);
+
+	/* cdev has not yet been added; cdev_added is 0 */
+	dev_info->gasket_dev_ptr = gasket_dev;
+	/* ownership is all 0, indicating no owner or opens. */
+
+	return 0;
+}
+
+/*
+ * Free a Gasket device.
+ * @internal_dev: Gasket device pointer; the device to unregister and free.
+ *
+ * Description: Removes the device from the device list and frees
+ *              the Gasket device structure.
+ */
+static void gasket_free_dev(struct gasket_dev *gasket_dev)
+{
+	struct gasket_internal_desc *internal_desc = gasket_dev->internal_desc;
+
+	mutex_lock(&internal_desc->mutex);
+	internal_desc->devs[gasket_dev->dev_idx] = NULL;
+	mutex_unlock(&internal_desc->mutex);
+
+	kfree(gasket_dev);
+}
+
+/*
+ * Finds the next free gasket_internal_dev slot.
+ *
+ * Returns the located slot number on success or a negative number on failure.
+ */
+static int gasket_find_dev_slot(
+	struct gasket_internal_desc *internal_desc, const char *kobj_name)
+{
+	int i;
+
+	mutex_lock(&internal_desc->mutex);
+
+	/* Search for a previous instance of this device. */
+	for (i = 0; i < GASKET_DEV_MAX; i++) {
+		if (internal_desc->devs[i] &&
+		    strcmp(internal_desc->devs[i]->kobj_name, kobj_name) == 0) {
+			gasket_nodev_error("Duplicate device %s", kobj_name);
+			mutex_unlock(&internal_desc->mutex);
+			return -EBUSY;
+		}
+	}
+
+	/* Find a free device slot. */
+	for (i = 0; i < GASKET_DEV_MAX; i++) {
+		if (!internal_desc->devs[i])
+			break;
+	}
+
+	if (i == GASKET_DEV_MAX) {
+		gasket_nodev_info(
+			"Too many registered devices; max %d", GASKET_DEV_MAX);
+		mutex_unlock(&internal_desc->mutex);
+		return -EBUSY;
+	}
+
+	mutex_unlock(&internal_desc->mutex);
+	return i;
+}
+
+/**
+ * PCI subsystem probe function.
+ * @pci_dev: PCI device pointer to the new device.
+ * @id: PCI device id structure pointer, the vendor and device ids.
+ *
+ * Called when a Gasket device is found. Allocates device metadata, maps device
+ * memory, and calls gasket_enable_dev to prepare the device for active use.
+ *
+ * Returns 0 if successful and a negative value otherwise.
+ */
+static int gasket_pci_probe(
+	struct pci_dev *pci_dev, const struct pci_device_id *id)
+{
+	int ret;
+	const char *kobj_name = dev_name(&pci_dev->dev);
+	struct gasket_internal_desc *internal_desc;
+	struct gasket_dev *gasket_dev;
+	const struct gasket_driver_desc *driver_desc;
+	struct device *parent;
+
+	gasket_nodev_info("Add Gasket device %s", kobj_name);
+
+	mutex_lock(&g_mutex);
+	internal_desc = lookup_internal_desc(pci_dev);
+	mutex_unlock(&g_mutex);
+	if (!internal_desc) {
+		gasket_nodev_info("PCI probe called for unknown driver type");
+		return -ENODEV;
+	}
+
+	driver_desc = internal_desc->driver_desc;
+
+	parent = &pci_dev->dev;
+	ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev, kobj_name);
+	if (ret)
+		return ret;
+	if (IS_ERR_OR_NULL(gasket_dev->dev_info.device)) {
+		gasket_nodev_error("Cannot create %s device %s [ret = %ld]",
+				   driver_desc->name, gasket_dev->dev_info.name,
+				   PTR_ERR(gasket_dev->dev_info.device));
+		ret = -ENODEV;
+		goto fail1;
+	}
+	gasket_dev->pci_dev = pci_dev;
+
+	ret = gasket_setup_pci(pci_dev, gasket_dev);
+	if (ret)
+		goto fail2;
+
+	ret = check_and_invoke_callback(gasket_dev, driver_desc->add_dev_cb);
+	if (ret) {
+		gasket_log_error(gasket_dev, "Error in add device cb: %d", ret);
+		goto fail2;
+	}
+
+	ret = gasket_sysfs_create_mapping(
+		gasket_dev->dev_info.device, gasket_dev);
+	if (ret)
+		goto fail3;
+
+	/*
+	 * Once we've created the mapping structures successfully, attempt to
+	 * create a symlink to the pci directory of this object.
+	 */
+	ret = sysfs_create_link(&gasket_dev->dev_info.device->kobj,
+				&pci_dev->dev.kobj, dev_name(&pci_dev->dev));
+	if (ret) {
+		gasket_log_error(
+			gasket_dev, "Cannot create sysfs pci link: %d", ret);
+		goto fail3;
+	}
+	ret = gasket_sysfs_create_entries(
+		gasket_dev->dev_info.device, gasket_sysfs_generic_attrs);
+	if (ret)
+		goto fail4;
+
+	ret = check_and_invoke_callback(
+		gasket_dev, driver_desc->sysfs_setup_cb);
+	if (ret) {
+		gasket_log_error(
+			gasket_dev, "Error in sysfs setup cb: %d", ret);
+		goto fail5;
+	}
+
+	ret = gasket_enable_dev(internal_desc, gasket_dev);
+	if (ret) {
+		gasket_nodev_error("cannot setup %s device", driver_desc->name);
+		gasket_disable_dev(gasket_dev);
+		goto fail5;
+	}
+
+	return 0;
+
+fail5:
+	check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb);
+fail4:
+fail3:
+	gasket_sysfs_remove_mapping(gasket_dev->dev_info.device);
+fail2:
+	gasket_cleanup_pci(gasket_dev);
+	check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb);
+	device_destroy(internal_desc->class, gasket_dev->dev_info.devt);
+fail1:
+	gasket_free_dev(gasket_dev);
+	return ret;
+}
+
+/*
+ * PCI subsystem remove function.
+ * @pci_dev: PCI device pointer; the device to remove.
+ *
+ * Called to remove a Gasket device. Finds the device in the device list and
+ * cleans up metadata.
+ */
+static void gasket_pci_remove(struct pci_dev *pci_dev)
+{
+	int i;
+	struct gasket_internal_desc *internal_desc;
+	struct gasket_dev *gasket_dev = NULL;
+	const struct gasket_driver_desc *driver_desc;
+	/* Find the device desc. */
+	mutex_lock(&g_mutex);
+	internal_desc = lookup_internal_desc(pci_dev);
+	if (!internal_desc) {
+		mutex_unlock(&g_mutex);
+		return;
+	}
+	mutex_unlock(&g_mutex);
+
+	driver_desc = internal_desc->driver_desc;
+
+	/* Now find the specific device */
+	mutex_lock(&internal_desc->mutex);
+	for (i = 0; i < GASKET_DEV_MAX; i++) {
+		if (internal_desc->devs[i] &&
+		    internal_desc->devs[i]->pci_dev == pci_dev) {
+			gasket_dev = internal_desc->devs[i];
+			break;
+		}
+	}
+	mutex_unlock(&internal_desc->mutex);
+
+	if (!gasket_dev)
+		return;
+
+	gasket_nodev_info(
+		"remove %s device %s", internal_desc->driver_desc->name,
+		gasket_dev->kobj_name);
+
+	gasket_disable_dev(gasket_dev);
+	gasket_cleanup_pci(gasket_dev);
+
+	check_and_invoke_callback(gasket_dev, driver_desc->sysfs_cleanup_cb);
+	gasket_sysfs_remove_mapping(gasket_dev->dev_info.device);
+
+	check_and_invoke_callback(gasket_dev, driver_desc->remove_dev_cb);
+
+	device_destroy(internal_desc->class, gasket_dev->dev_info.devt);
+	gasket_free_dev(gasket_dev);
+}
+
+/*
+ * Setup PCI & set up memory mapping for the specified device.
+ * @pci_dev: pointer to the particular PCI device.
+ * @internal_dev: Corresponding Gasket device pointer.
+ *
+ * Enables the PCI device, reads the BAR registers and sets up pointers to the
+ * device's memory mapped IO space.
+ *
+ * Returns 0 on success and a negative value otherwise.
+ */
+static int gasket_setup_pci(
+	struct pci_dev *pci_dev, struct gasket_dev *gasket_dev)
+{
+	int i, mapped_bars, ret;
+
+	gasket_dev->pci_dev = pci_dev;
+	ret = pci_enable_device(pci_dev);
+	if (ret) {
+		gasket_log_error(gasket_dev, "cannot enable PCI device");
+		return ret;
+	}
+
+	pci_set_master(pci_dev);
+
+	for (i = 0; i < GASKET_NUM_BARS; i++) {
+		ret = gasket_map_pci_bar(gasket_dev, i);
+		if (ret) {
+			mapped_bars = i;
+			goto fail;
+		}
+	}
+
+	return 0;
+
+fail:
+	for (i = 0; i < mapped_bars; i++)
+		gasket_unmap_pci_bar(gasket_dev, i);
+
+	pci_disable_device(pci_dev);
+	return -ENOMEM;
+}
+
+/* Unmaps memory and cleans up PCI for the specified device. */
+static void gasket_cleanup_pci(struct gasket_dev *gasket_dev)
+{
+	int i;
+
+	for (i = 0; i < GASKET_NUM_BARS; i++)
+		gasket_unmap_pci_bar(gasket_dev, i);
+
+	pci_disable_device(gasket_dev->pci_dev);
+}
+
+/*
+ * Maps the specified bar into kernel space.
+ * @internal_dev: Device possessing the BAR to map.
+ * @bar_num: The BAR to map.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ * A zero-sized BAR will not be mapped, but is not an error.
+ */
+static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num)
+{
+	struct gasket_internal_desc *internal_desc = gasket_dev->internal_desc;
+	const struct gasket_driver_desc *driver_desc =
+		internal_desc->driver_desc;
+	ulong desc_bytes = driver_desc->bar_descriptions[bar_num].size;
+	int ret;
+
+	if (desc_bytes == 0)
+		return 0;
+
+	if (driver_desc->bar_descriptions[bar_num].type != PCI_BAR) {
+		/* not PCI: skip this entry */
+		return 0;
+	}
+	/*
+	 * pci_resource_start and pci_resource_len return a "resource_size_t",
+	 * which is safely castable to ulong (which itself is the arg to
+	 * request_mem_region).
+	 */
+	gasket_dev->bar_data[bar_num].phys_base =
+		(ulong)pci_resource_start(gasket_dev->pci_dev, bar_num);
+	if (!gasket_dev->bar_data[bar_num].phys_base) {
+		gasket_log_error(gasket_dev, "Cannot get BAR%u base address",
+				 bar_num);
+		return -EINVAL;
+	}
+
+	gasket_dev->bar_data[bar_num].length_bytes =
+		(ulong)pci_resource_len(gasket_dev->pci_dev, bar_num);
+	if (gasket_dev->bar_data[bar_num].length_bytes < desc_bytes) {
+		gasket_log_error(
+			gasket_dev,
+			"PCI BAR %u space is too small: %lu; expected >= %lu",
+			bar_num, gasket_dev->bar_data[bar_num].length_bytes,
+			desc_bytes);
+		return -ENOMEM;
+	}
+
+	if (!request_mem_region(gasket_dev->bar_data[bar_num].phys_base,
+				gasket_dev->bar_data[bar_num].length_bytes,
+				gasket_dev->dev_info.name)) {
+		gasket_log_error(
+			gasket_dev,
+			"Cannot get BAR %d memory region %p",
+			bar_num, &gasket_dev->pci_dev->resource[bar_num]);
+		return -EINVAL;
+	}
+
+	gasket_dev->bar_data[bar_num].virt_base =
+		ioremap_nocache(gasket_dev->bar_data[bar_num].phys_base,
+				gasket_dev->bar_data[bar_num].length_bytes);
+	if (!gasket_dev->bar_data[bar_num].virt_base) {
+		gasket_log_error(
+			gasket_dev,
+			"Cannot remap BAR %d memory region %p",
+			bar_num, &gasket_dev->pci_dev->resource[bar_num]);
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	dma_set_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64));
+	dma_set_coherent_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64));
+
+	return 0;
+
+fail:
+	iounmap(gasket_dev->bar_data[bar_num].virt_base);
+	release_mem_region(gasket_dev->bar_data[bar_num].phys_base,
+			   gasket_dev->bar_data[bar_num].length_bytes);
+	return ret;
+}
+
+/*
+ * Releases PCI BAR mapping.
+ * @internal_dev: Device possessing the BAR to unmap.
+ *
+ * A zero-sized or not-mapped BAR will not be unmapped, but is not an error.
+ */
+static void gasket_unmap_pci_bar(struct gasket_dev *dev, int bar_num)
+{
+	ulong base, bytes;
+	struct gasket_internal_desc *internal_desc = dev->internal_desc;
+	const struct gasket_driver_desc *driver_desc =
+		internal_desc->driver_desc;
+
+	if (driver_desc->bar_descriptions[bar_num].size == 0 ||
+	    !dev->bar_data[bar_num].virt_base)
+		return;
+
+	if (driver_desc->bar_descriptions[bar_num].type != PCI_BAR)
+		return;
+
+	iounmap(dev->bar_data[bar_num].virt_base);
+	dev->bar_data[bar_num].virt_base = NULL;
+
+	base = pci_resource_start(dev->pci_dev, bar_num);
+	if (!base) {
+		gasket_log_error(
+			dev, "cannot get PCI BAR%u base address", bar_num);
+		return;
+	}
+
+	bytes = pci_resource_len(dev->pci_dev, bar_num);
+	release_mem_region(base, bytes);
+}
+
+/*
+ * Handle adding a char device and related info.
+ * @dev_info: Pointer to the dev_info struct for this device.
+ * @file_ops: The file operations for this device.
+ * @owner: The owning module for this device.
+ */
+static int gasket_add_cdev(
+	struct gasket_cdev_info *dev_info,
+	const struct file_operations *file_ops, struct module *owner)
+{
+	int ret;
+
+	cdev_init(&dev_info->cdev, file_ops);
+	dev_info->cdev.owner = owner;
+	ret = cdev_add(&dev_info->cdev, dev_info->devt, 1);
+	if (ret) {
+		gasket_log_error(
+			dev_info->gasket_dev_ptr,
+			"cannot add char device [ret=%d]", ret);
+		return ret;
+	}
+	dev_info->cdev_added = 1;
+
+	return 0;
+}
+
+/*
+ * Performs final init and marks the device as active.
+ * @internal_desc: Pointer to Gasket [internal] driver descriptor structure.
+ * @internal_dev: Pointer to Gasket [internal] device structure.
+ *
+ * Currently forwards all work to device-specific callback; a future phase will
+ * extract elements of character device registration here.
+ */
+static int gasket_enable_dev(
+	struct gasket_internal_desc *internal_desc,
+	struct gasket_dev *gasket_dev)
+{
+	int tbl_idx;
+	int ret;
+	bool has_dma_ops;
+	struct device *ddev;
+	const struct gasket_driver_desc *driver_desc =
+		internal_desc->driver_desc;
+
+	ret = gasket_interrupt_init(
+		gasket_dev, driver_desc->name,
+		driver_desc->interrupt_type, driver_desc->interrupts,
+		driver_desc->num_interrupts, driver_desc->interrupt_pack_width,
+		driver_desc->interrupt_bar_index,
+		driver_desc->wire_interrupt_offsets);
+	if (ret) {
+		gasket_log_error(gasket_dev,
+				 "Critical failure to allocate interrupts: %d",
+				 ret);
+		gasket_interrupt_cleanup(gasket_dev);
+		return ret;
+	}
+
+	has_dma_ops = true;
+
+	for (tbl_idx = 0; tbl_idx < driver_desc->num_page_tables; tbl_idx++) {
+		gasket_log_debug(
+			gasket_dev, "Initializing page table %d.", tbl_idx);
+		if (gasket_dev->pci_dev) {
+			ddev = &gasket_dev->pci_dev->dev;
+		} else {
+			gasket_log_error(
+				gasket_dev,
+				"gasket_enable_dev with no physical device!!");
+			WARN_ON(1);
+			ddev = NULL;
+		}
+		ret = gasket_page_table_init(
+			&gasket_dev->page_table[tbl_idx],
+			&gasket_dev->bar_data[
+				driver_desc->page_table_bar_index],
+			&driver_desc->page_table_configs[tbl_idx],
+			ddev, gasket_dev->pci_dev, has_dma_ops);
+		if (ret) {
+			gasket_log_error(
+				gasket_dev,
+				"Couldn't init page table %d: %d",
+				tbl_idx, ret);
+			return ret;
+		}
+		/*
+		 * Make sure that the page table is clear and set to simple
+		 * addresses.
+		 */
+		gasket_page_table_reset(gasket_dev->page_table[tbl_idx]);
+	}
+
+	/*
+	 * hardware_revision_cb returns a positive integer (the rev) if
+	 * successful.)
+	 */
+	ret = check_and_invoke_callback(
+		gasket_dev, driver_desc->hardware_revision_cb);
+	if (ret < 0) {
+		gasket_log_error(
+			gasket_dev, "Error getting hardware revision: %d", ret);
+		return ret;
+	}
+	gasket_dev->hardware_revision = ret;
+
+	ret = check_and_invoke_callback(gasket_dev, driver_desc->enable_dev_cb);
+	if (ret) {
+		gasket_log_error(
+			gasket_dev, "Error in enable device cb: %d", ret);
+		return ret;
+	}
+
+	/* device_status_cb returns a device status, not an error code. */
+	gasket_dev->status = gasket_get_hw_status(gasket_dev);
+	if (gasket_dev->status == GASKET_STATUS_DEAD)
+		gasket_log_error(gasket_dev, "Device reported as unhealthy.");
+
+	ret = gasket_add_cdev(
+		&gasket_dev->dev_info, &gasket_file_ops, driver_desc->module);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+/*
+ * Disable device operations.
+ * @gasket_dev: Pointer to Gasket device structure.
+ *
+ * Currently forwards all work to device-specific callback; a future phase will
+ * extract elements of character device unregistration here.
+ */
+static void gasket_disable_dev(struct gasket_dev *gasket_dev)
+{
+	const struct gasket_driver_desc *driver_desc =
+		gasket_dev->internal_desc->driver_desc;
+	int i;
+
+	/* Only delete the device if it has been successfully added. */
+	if (gasket_dev->dev_info.cdev_added)
+		cdev_del(&gasket_dev->dev_info.cdev);
+
+	gasket_dev->status = GASKET_STATUS_DEAD;
+
+	gasket_interrupt_cleanup(gasket_dev);
+
+	for (i = 0; i < driver_desc->num_page_tables; ++i) {
+		if (gasket_dev->page_table[i]) {
+			gasket_page_table_reset(gasket_dev->page_table[i]);
+			gasket_page_table_cleanup(gasket_dev->page_table[i]);
+		}
+	}
+
+	check_and_invoke_callback(gasket_dev, driver_desc->disable_dev_cb);
+}
+
+/**
+ * Registered descriptor lookup.
+ *
+ * Precondition: Called with g_mutex held (to avoid a race on return).
+ * Returns NULL if no matching device was found.
+ */
+static struct gasket_internal_desc *lookup_internal_desc(
+	struct pci_dev *pci_dev)
+{
+	int i;
+
+	__must_hold(&g_mutex);
+	for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) {
+		if (g_descs[i].driver_desc &&
+		    g_descs[i].driver_desc->pci_id_table &&
+		    pci_match_id(g_descs[i].driver_desc->pci_id_table, pci_dev))
+			return &g_descs[i];
+	}
+
+	return NULL;
+}
+
+/**
+ * Lookup a name by number in a num_name table.
+ * @num: Number to lookup.
+ * @table: Array of num_name structures, the table for the lookup.
+ *
+ * Description: Searches for num in the table.  If found, the
+ *		corresponding name is returned; otherwise NULL
+ *		is returned.
+ *
+ *		The table must have a NULL name pointer at the end.
+ */
+const char *gasket_num_name_lookup(
+	uint num, const struct gasket_num_name *table)
+{
+	uint i = 0;
+
+	while (table[i].snn_name) {
+		if (num == table[i].snn_num)
+			break;
+		++i;
+	}
+
+	return table[i].snn_name;
+}
+EXPORT_SYMBOL(gasket_num_name_lookup);
+
+/**
+ * Opens the char device file.
+ * @inode: Inode structure pointer of the device file.
+ * @file: File structure pointer.
+ *
+ * Description: Called on an open of the device file.  If the open is for
+ *              writing, and the device is not owned, this process becomes
+ *              the owner.  If the open is for writing and the device is
+ *              already owned by some other process, it is an error.  If
+ *              this process is the owner, increment the open count.
+ *
+ *              Returns 0 if successful, a negative error number otherwise.
+ */
+static int gasket_open(struct inode *inode, struct file *filp)
+{
+	int ret;
+	struct gasket_dev *gasket_dev;
+	const struct gasket_driver_desc *driver_desc;
+	struct gasket_ownership *ownership;
+	char task_name[TASK_COMM_LEN];
+	struct gasket_cdev_info *dev_info = gasket_cdev_get_info(inode->i_cdev);
+
+	if (!dev_info) {
+		gasket_nodev_error("Unable to retrieve device data");
+		return -EINVAL;
+	}
+	gasket_dev = dev_info->gasket_dev_ptr;
+	driver_desc = gasket_dev->internal_desc->driver_desc;
+	ownership = &dev_info->ownership;
+	get_task_comm(task_name, current);
+	filp->private_data = gasket_dev;
+	inode->i_size = 0;
+
+	gasket_log_debug(
+		gasket_dev,
+		"Attempting to open with tgid %u (%s) (f_mode: 0%03o, "
+		"fmode_write: %d is_root: %u)",
+		current->tgid, task_name, filp->f_mode,
+		(filp->f_mode & FMODE_WRITE), capable(CAP_SYS_ADMIN));
+
+	/* Always allow non-writing accesses. */
+	if (!(filp->f_mode & FMODE_WRITE)) {
+		gasket_log_debug(gasket_dev, "Allowing read-only opening.");
+		return 0;
+	}
+
+	mutex_lock(&gasket_dev->mutex);
+
+	gasket_log_debug(
+		gasket_dev, "Current owner open count (owning tgid %u): %d.",
+		ownership->owner, ownership->write_open_count);
+
+	/* Opening a node owned by another TGID is an error (even root.) */
+	if (ownership->is_owned && ownership->owner != current->tgid) {
+		gasket_log_error(
+			gasket_dev,
+			"Process %u is opening a node held by %u.",
+			current->tgid, ownership->owner);
+		mutex_unlock(&gasket_dev->mutex);
+		return -EPERM;
+	}
+
+	/* If the node is not owned, assign it to the current TGID. */
+	if (!ownership->is_owned) {
+		ret = gasket_check_and_invoke_callback_nolock(
+			gasket_dev, driver_desc->device_open_cb);
+		if (ret) {
+			gasket_log_error(
+				gasket_dev, "Error in device open cb: %d", ret);
+			mutex_unlock(&gasket_dev->mutex);
+			return ret;
+		}
+		ownership->is_owned = 1;
+		ownership->owner = current->tgid;
+		gasket_log_debug(gasket_dev, "Device owner is now tgid %u",
+				 ownership->owner);
+	}
+
+	ownership->write_open_count++;
+
+	gasket_log_debug(gasket_dev, "New open count (owning tgid %u): %d",
+			 ownership->owner, ownership->write_open_count);
+
+	mutex_unlock(&gasket_dev->mutex);
+	return 0;
+}
+
+/**
+ * gasket_release - Close of the char device file.
+ * @inode: Inode structure pointer of the device file.
+ * @file: File structure pointer.
+ *
+ * Description: Called on a close of the device file.  If this process
+ *              is the owner, decrement the open count.  On last close
+ *              by the owner, free up buffers and eventfd contexts, and
+ *              release ownership.
+ *
+ *              Returns 0 if successful, a negative error number otherwise.
+ */
+static int gasket_release(struct inode *inode, struct file *file)
+{
+	int i;
+	struct gasket_dev *gasket_dev;
+	struct gasket_ownership *ownership;
+	const struct gasket_driver_desc *driver_desc;
+	char task_name[TASK_COMM_LEN];
+	struct gasket_cdev_info *dev_info =
+		(struct gasket_cdev_info *)gasket_cdev_get_info(inode->i_cdev);
+	if (!dev_info) {
+		gasket_nodev_error("Unable to retrieve device data");
+		return -EINVAL;
+	}
+	gasket_dev = dev_info->gasket_dev_ptr;
+	driver_desc = gasket_dev->internal_desc->driver_desc;
+	ownership = &dev_info->ownership;
+	get_task_comm(task_name, current);
+	mutex_lock(&gasket_dev->mutex);
+
+	gasket_log_debug(
+		gasket_dev,
+		"Releasing device node. Call origin: tgid %u (%s) "
+		"(f_mode: 0%03o, fmode_write: %d, is_root: %u)",
+		current->tgid, task_name, file->f_mode,
+		(file->f_mode & FMODE_WRITE), capable(CAP_SYS_ADMIN));
+	gasket_log_debug(gasket_dev, "Current open count (owning tgid %u): %d",
+			 ownership->owner, ownership->write_open_count);
+
+	if (file->f_mode & FMODE_WRITE) {
+		ownership->write_open_count--;
+		if (ownership->write_open_count == 0) {
+			gasket_log_info(gasket_dev, "Device is now free");
+			ownership->is_owned = 0;
+			ownership->owner = 0;
+
+			/* Forces chip reset before we unmap the page tables. */
+			driver_desc->device_reset_cb(gasket_dev, 0);
+
+			for (i = 0; i < driver_desc->num_page_tables; ++i) {
+				gasket_page_table_unmap_all(
+					gasket_dev->page_table[i]);
+				gasket_page_table_garbage_collect(
+					gasket_dev->page_table[i]);
+				gasket_free_coherent_memory_all(gasket_dev, i);
+			}
+
+			/* Closes device, enters power save. */
+			gasket_check_and_invoke_callback_nolock(
+				gasket_dev, driver_desc->device_close_cb);
+		}
+	}
+
+	gasket_log_info(
+		gasket_dev, "New open count (owning tgid %u): %d",
+		ownership->owner, ownership->write_open_count);
+	mutex_unlock(&gasket_dev->mutex);
+	return 0;
+}
+
+/*
+ * Permission and validity checking for mmap ops.
+ * @gasket_dev: Gasket device information structure.
+ * @vma: Standard virtual memory area descriptor.
+ *
+ * Verifies that the user has permissions to perform the requested mapping and
+ * that the provided descriptor/range is of adequate size to hold the range to
+ * be mapped.
+ */
+static int gasket_mmap_has_permissions(
+	struct gasket_dev *gasket_dev, struct vm_area_struct *vma,
+	int bar_permissions)
+{
+	int requested_permissions;
+	/* Always allow sysadmin to access. */
+	if (capable(CAP_SYS_ADMIN))
+		return 1;
+
+	/* Never allow non-sysadmins to access to a dead device. */
+	if (gasket_dev->status != GASKET_STATUS_ALIVE) {
+		gasket_log_info(gasket_dev, "Device is dead.");
+		return 0;
+	}
+
+	/* Make sure that no wrong flags are set. */
+	requested_permissions =
+		(vma->vm_flags & (VM_WRITE | VM_READ | VM_EXEC));
+	if (requested_permissions & ~(bar_permissions)) {
+		gasket_log_info(
+			gasket_dev,
+			"Attempting to map a region with requested permissions "
+			"0x%x, but region has permissions 0x%x.",
+			requested_permissions, bar_permissions);
+		return 0;
+	}
+
+	/* Do not allow a non-owner to write. */
+	if ((vma->vm_flags & VM_WRITE) &&
+	    !gasket_owned_by_current_tgid(&gasket_dev->dev_info)) {
+		gasket_log_info(
+			gasket_dev,
+			"Attempting to mmap a region for write without owning "
+			"device.");
+		return 0;
+	}
+
+	return 1;
+}
+
+/*
+ * Checks if an address is within the region
+ * allocated for coherent buffer.
+ * @driver_desc: driver description.
+ * @address: offset of address to check.
+ *
+ * Verifies that the input address is within the region allocated to coherent
+ * buffer.
+ */
+static bool gasket_is_coherent_region(
+	const struct gasket_driver_desc *driver_desc, ulong address)
+{
+	struct gasket_coherent_buffer_desc coh_buff_desc =
+		driver_desc->coherent_buffer_description;
+
+	if (coh_buff_desc.permissions != GASKET_NOMAP) {
+		if ((address >= coh_buff_desc.base) &&
+		    (address < coh_buff_desc.base + coh_buff_desc.size)) {
+			return true;
+		}
+	}
+	return false;
+}
+
+static int gasket_get_bar_index(
+	const struct gasket_dev *gasket_dev, ulong phys_addr)
+{
+	int i;
+	const struct gasket_driver_desc *driver_desc;
+
+	driver_desc = gasket_dev->internal_desc->driver_desc;
+	for (i = 0; i < GASKET_NUM_BARS; ++i) {
+		struct gasket_bar_desc bar_desc =
+			driver_desc->bar_descriptions[i];
+
+		if (bar_desc.permissions != GASKET_NOMAP) {
+			if (phys_addr >= bar_desc.base &&
+			    phys_addr < (bar_desc.base + bar_desc.size)) {
+				return i;
+			}
+		}
+	}
+	/* If we haven't found the address by now, it is invalid. */
+	return -EINVAL;
+}
+
+/*
+ * Sets the actual bounds to map, given the device's mappable region.
+ *
+ * Given the device's mappable region, along with the user-requested mapping
+ * start offset and length of the user region, determine how much of this
+ * mappable region can be mapped into the user's region (start/end offsets),
+ * and the physical offset (phys_offset) into the BAR where the mapping should
+ * begin (either the VMA's or region lower bound).
+ *
+ * In other words, this calculates the overlap between the VMA
+ * (bar_offset, requested_length) and the given gasket_mappable_region.
+ *
+ * Returns true if there's anything to map, and false otherwise.
+ */
+static bool gasket_mm_get_mapping_addrs(
+	const struct gasket_mappable_region *region, ulong bar_offset,
+	ulong requested_length, struct gasket_mappable_region *mappable_region,
+	ulong *virt_offset)
+{
+	ulong range_start = region->start;
+	ulong range_length = region->length_bytes;
+	ulong range_end = range_start + range_length;
+
+	*virt_offset = 0;
+	if (bar_offset + requested_length < range_start) {
+		/*
+		* If the requested region is completely below the range,
+		* there is nothing to map.
+		*/
+		return false;
+	} else if (bar_offset <= range_start) {
+		/* If the bar offset is below this range's start
+		 * but the requested length continues into it:
+		 * 1) Only map starting from the beginning of this
+		 *      range's phys. offset, so we don't map unmappable
+		 *	memory.
+		 * 2) The length of the virtual memory to not map is the
+		 *	delta between the bar offset and the
+		 *	mappable start (and since the mappable start is
+		 *	bigger, start - req.)
+		 * 3) The map length is the minimum of the mappable
+		 *	requested length (requested_length - virt_offset)
+		 *	and the actual mappable length of the range.
+		 */
+		mappable_region->start = range_start;
+		*virt_offset = range_start - bar_offset;
+		mappable_region->length_bytes =
+			min(requested_length - *virt_offset, range_length);
+		return true;
+	} else if (bar_offset > range_start &&
+		   bar_offset < range_end) {
+		/*
+		 * If the bar offset is within this range:
+		 * 1) Map starting from the bar offset.
+		 * 2) Because there is no forbidden memory between the
+		 *	bar offset and the range start,
+		 *	virt_offset is 0.
+		 * 3) The map length is the minimum of the requested
+		 *	length and the remaining length in the buffer
+		 *	(range_end - bar_offset)
+		 */
+		mappable_region->start = bar_offset;
+		*virt_offset = 0;
+		mappable_region->length_bytes = min(
+			requested_length, range_end - bar_offset);
+		return true;
+	}
+
+	/*
+	 * If the requested [start] offset is above range_end,
+	 * there's nothing to map.
+	 */
+	return false;
+}
+
+int gasket_mm_unmap_region(
+	const struct gasket_dev *gasket_dev, struct vm_area_struct *vma,
+	const struct gasket_mappable_region *map_region)
+{
+	ulong bar_offset;
+	ulong virt_offset;
+	struct gasket_mappable_region mappable_region;
+	int ret;
+
+	if (map_region->length_bytes == 0)
+		return 0;
+
+	ret = gasket_mm_vma_bar_offset(gasket_dev, vma, &bar_offset);
+	if (ret)
+		return ret;
+
+	if (!gasket_mm_get_mapping_addrs(
+		map_region, bar_offset, vma->vm_end - vma->vm_start,
+		&mappable_region, &virt_offset))
+		return 1;
+
+	/*
+	 * The length passed to zap_vma_ptes MUST BE A MULTIPLE OF
+	 * PAGE_SIZE! Trust me. I have the scars.
+	 *
+	 * Next multiple of y: ceil_div(x, y) * y
+	 */
+	zap_vma_ptes(
+		vma, vma->vm_start + virt_offset,
+		DIV_ROUND_UP(mappable_region.length_bytes, PAGE_SIZE) *
+			PAGE_SIZE);
+	return 0;
+}
+EXPORT_SYMBOL(gasket_mm_unmap_region);
+
+/* Maps a virtual address + range to a physical offset of a BAR. */
+static enum do_map_region_status do_map_region(
+	const struct gasket_dev *gasket_dev, struct vm_area_struct *vma,
+	struct gasket_mappable_region *mappable_region)
+{
+	/* Maximum size of a single call to io_remap_pfn_range. */
+	/* I pulled this number out of thin air. */
+	const ulong max_chunk_size = 64 * 1024 * 1024;
+	ulong chunk_size, mapped_bytes = 0;
+
+	const struct gasket_driver_desc *driver_desc =
+		gasket_dev->internal_desc->driver_desc;
+
+	ulong bar_offset, virt_offset;
+	struct gasket_mappable_region region_to_map;
+	ulong phys_offset, map_length;
+	ulong virt_base, phys_base;
+	int bar_index, ret;
+
+	ret = gasket_mm_vma_bar_offset(gasket_dev, vma, &bar_offset);
+	if (ret)
+		return DO_MAP_REGION_INVALID;
+
+	if (!gasket_mm_get_mapping_addrs(mappable_region, bar_offset,
+					 vma->vm_end - vma->vm_start,
+					 &region_to_map, &virt_offset))
+		return DO_MAP_REGION_INVALID;
+	phys_offset = region_to_map.start;
+	map_length = region_to_map.length_bytes;
+
+	virt_base = vma->vm_start + virt_offset;
+	bar_index =
+		gasket_get_bar_index(
+			gasket_dev,
+			(vma->vm_pgoff << PAGE_SHIFT) +
+				driver_desc->legacy_mmap_address_offset);
+	phys_base = gasket_dev->bar_data[bar_index].phys_base + phys_offset;
+	while (mapped_bytes < map_length) {
+		/*
+		 * io_remap_pfn_range can take a while, so we chunk its
+		 * calls and call cond_resched between each.
+		 */
+		chunk_size = min(max_chunk_size, map_length - mapped_bytes);
+
+		cond_resched();
+		ret = io_remap_pfn_range(
+			vma, virt_base + mapped_bytes,
+			(phys_base + mapped_bytes) >> PAGE_SHIFT,
+			chunk_size, vma->vm_page_prot);
+		if (ret) {
+			gasket_log_error(
+				gasket_dev, "Error remapping PFN range.");
+			goto fail;
+		}
+		mapped_bytes += chunk_size;
+	}
+
+	return DO_MAP_REGION_SUCCESS;
+
+fail:
+	/* Unmap the partial chunk we mapped. */
+	mappable_region->length_bytes = mapped_bytes;
+	if (gasket_mm_unmap_region(gasket_dev, vma, mappable_region))
+		gasket_log_error(
+			gasket_dev,
+			"Error unmapping partial region 0x%lx (0x%lx bytes)",
+			(ulong)virt_offset,
+			(ulong)mapped_bytes);
+
+	return DO_MAP_REGION_FAILURE;
+}
+
+/*
+ * Calculates the offset where the VMA range begins in its containing BAR.
+ * The offset is written into bar_offset on success.
+ * Returns zero on success, anything else on error.
+*/
+static int gasket_mm_vma_bar_offset(
+	const struct gasket_dev *gasket_dev, const struct vm_area_struct *vma,
+	ulong *bar_offset)
+{
+	ulong raw_offset;
+	int bar_index;
+	const struct gasket_driver_desc *driver_desc =
+		gasket_dev->internal_desc->driver_desc;
+
+	raw_offset = (vma->vm_pgoff << PAGE_SHIFT) +
+		driver_desc->legacy_mmap_address_offset;
+	bar_index = gasket_get_bar_index(gasket_dev, raw_offset);
+	if (bar_index < 0) {
+		gasket_log_error(
+			gasket_dev,
+			"Unable to find matching bar for address 0x%lx",
+			raw_offset);
+		trace_gasket_mmap_exit(bar_index);
+		return bar_index;
+	}
+	*bar_offset =
+		raw_offset - driver_desc->bar_descriptions[bar_index].base;
+
+	return 0;
+}
+
+/*
+ * Map a region of coherent memory.
+ * @gasket_dev: Gasket device handle.
+ * @vma: Virtual memory area descriptor with region to map.
+ */
+static int gasket_mmap_coherent(
+	struct gasket_dev *gasket_dev, struct vm_area_struct *vma)
+{
+	const struct gasket_driver_desc *driver_desc =
+		gasket_dev->internal_desc->driver_desc;
+	const ulong requested_length = vma->vm_end - vma->vm_start;
+	int ret;
+	ulong permissions;
+
+	if (requested_length == 0 || requested_length >
+	    gasket_dev->coherent_buffer.length_bytes) {
+		trace_gasket_mmap_exit(-EINVAL);
+		return -EINVAL;
+	}
+
+	permissions = driver_desc->coherent_buffer_description.permissions;
+	if (!gasket_mmap_has_permissions(gasket_dev, vma, permissions)) {
+		gasket_log_error(gasket_dev, "Permission checking failed.");
+		trace_gasket_mmap_exit(-EPERM);
+		return -EPERM;
+	}
+
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+	ret = remap_pfn_range(
+		vma, vma->vm_start,
+		(gasket_dev->coherent_buffer.phys_base) >> PAGE_SHIFT,
+		requested_length, vma->vm_page_prot);
+	if (ret) {
+		gasket_log_error(
+			gasket_dev, "Error remapping PFN range err=%d.", ret);
+		trace_gasket_mmap_exit(ret);
+		return ret;
+	}
+
+	/* Record the user virtual to dma_address mapping that was
+	 * created by the kernel.
+	 */
+	gasket_set_user_virt(
+		gasket_dev, requested_length,
+		gasket_dev->coherent_buffer.phys_base, vma->vm_start);
+	return 0;
+}
+
+/*
+ * Maps a device's BARs into user space.
+ * @filp: File structure pointer describing this node usage session.
+ * @vma: Standard virtual memory area descriptor.
+ *
+ * Maps the entirety of each of the device's BAR ranges into the user memory
+ * range specified by vma.
+ *
+ * Returns 0 on success, a negative errno on error.
+ */
+static int gasket_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	int i, ret;
+	int bar_index;
+	int has_mapped_anything = 0;
+	ulong permissions;
+	ulong raw_offset, vma_size;
+	bool is_coherent_region;
+	const struct gasket_driver_desc *driver_desc;
+	struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data;
+	struct gasket_bar_data *bar_data;
+	const struct gasket_bar_desc *bar_desc;
+	struct gasket_mappable_region *map_regions = NULL;
+	int num_map_regions = 0;
+	enum do_map_region_status map_status;
+
+	if (!gasket_dev) {
+		gasket_nodev_error("Unable to retrieve device data");
+		trace_gasket_mmap_exit(-EINVAL);
+		return -EINVAL;
+	}
+	driver_desc = gasket_dev->internal_desc->driver_desc;
+
+	if (vma->vm_start & (PAGE_SIZE - 1)) {
+		gasket_log_error(
+			gasket_dev, "Base address not page-aligned: 0x%p\n",
+			(void *)vma->vm_start);
+		trace_gasket_mmap_exit(-EINVAL);
+		return -EINVAL;
+	}
+
+	/* Calculate the offset of this range into physical mem. */
+	raw_offset = (vma->vm_pgoff << PAGE_SHIFT) +
+		driver_desc->legacy_mmap_address_offset;
+	vma_size = vma->vm_end - vma->vm_start;
+	trace_gasket_mmap_entry(
+		gasket_dev->dev_info.name, raw_offset, vma_size);
+
+	/*
+	 * Check if the raw offset is within a bar region. If not, check if it
+	 * is a coherent region.
+	 */
+	bar_index = gasket_get_bar_index(gasket_dev, raw_offset);
+	is_coherent_region = gasket_is_coherent_region(driver_desc, raw_offset);
+	if (bar_index < 0 && !is_coherent_region) {
+		gasket_log_error(
+			gasket_dev,
+			"Unable to find matching bar for address 0x%lx",
+			raw_offset);
+		trace_gasket_mmap_exit(bar_index);
+		return bar_index;
+	}
+	if (bar_index > 0 && is_coherent_region) {
+		gasket_log_error(
+			gasket_dev,
+			"double matching bar and coherent buffers for address "
+			"0x%lx",
+			raw_offset);
+		trace_gasket_mmap_exit(bar_index);
+		return bar_index;
+	}
+
+	vma->vm_private_data = gasket_dev;
+
+	if (is_coherent_region)
+		return gasket_mmap_coherent(gasket_dev, vma);
+
+	/* Everything in the rest of this function is for normal BAR mapping. */
+
+	/*
+	 * Subtract the base of the bar from the raw offset to get the
+	 * memory location within the bar to map.
+	 */
+	bar_data = &gasket_dev->bar_data[bar_index];
+
+	bar_desc = &driver_desc->bar_descriptions[bar_index];
+	permissions = bar_desc->permissions;
+	if (!gasket_mmap_has_permissions(gasket_dev, vma, permissions)) {
+		gasket_log_error(gasket_dev, "Permission checking failed.");
+		trace_gasket_mmap_exit(-EPERM);
+		return -EPERM;
+	}
+
+	if (driver_desc->get_mappable_regions_cb) {
+		ret = driver_desc->get_mappable_regions_cb(
+			gasket_dev, bar_index, &map_regions, &num_map_regions);
+		if (ret)
+			return ret;
+	} else {
+		if (!gasket_mmap_has_permissions(gasket_dev, vma,
+						 bar_desc->permissions)) {
+			gasket_log_error(
+				gasket_dev, "Permission checking failed.");
+			trace_gasket_mmap_exit(-EPERM);
+			return -EPERM;
+		}
+		num_map_regions = bar_desc->num_mappable_regions;
+		map_regions = kzalloc(
+			num_map_regions * sizeof(*bar_desc->mappable_regions),
+			GFP_KERNEL);
+		if (map_regions) {
+			memcpy(map_regions, bar_desc->mappable_regions,
+			       num_map_regions *
+					sizeof(*bar_desc->mappable_regions));
+		}
+	}
+
+	if (!map_regions || num_map_regions == 0) {
+		gasket_log_error(gasket_dev, "No mappable regions returned!");
+		return -EINVAL;
+	}
+
+	/* Marks the VMA's pages as uncacheable. */
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	for (i = 0; i < num_map_regions; i++) {
+		map_status = do_map_region(gasket_dev, vma, &map_regions[i]);
+		/* Try the next region if this one was not mappable. */
+		if (map_status == DO_MAP_REGION_INVALID)
+			continue;
+		if (map_status == DO_MAP_REGION_FAILURE)
+			goto fail;
+
+		has_mapped_anything = 1;
+	}
+
+	kfree(map_regions);
+
+	/* If we could not map any memory, the request was invalid. */
+	if (!has_mapped_anything) {
+		gasket_log_error(
+			gasket_dev,
+			"Map request did not contain a valid region.");
+		trace_gasket_mmap_exit(-EINVAL);
+		return -EINVAL;
+	}
+
+	trace_gasket_mmap_exit(0);
+	return 0;
+
+fail:
+	/* Need to unmap any mapped ranges. */
+	num_map_regions = i;
+	for (i = 0; i < num_map_regions; i++)
+		if (gasket_mm_unmap_region(gasket_dev, vma,
+					   &bar_desc->mappable_regions[i]))
+			gasket_log_error(
+				gasket_dev, "Error unmapping range %d.", i);
+	kfree(map_regions);
+
+	return ret;
+}
+
+/*
+ * Determine the health of the Gasket device.
+ * @gasket_dev: Gasket device structure.
+ *
+ * Checks the underlying device health (via the device_status_cb)
+ * and the status of initialized Gasket code systems (currently
+ * only interrupts), then returns a gasket_status appropriately.
+ */
+static int gasket_get_hw_status(struct gasket_dev *gasket_dev)
+{
+	int status;
+	int i;
+	const struct gasket_driver_desc *driver_desc =
+		gasket_dev->internal_desc->driver_desc;
+
+	status = gasket_check_and_invoke_callback_nolock(
+		gasket_dev, driver_desc->device_status_cb);
+	if (status != GASKET_STATUS_ALIVE) {
+		gasket_log_info(gasket_dev, "Hardware reported status %d.",
+				status);
+		return status;
+	}
+
+	status = gasket_interrupt_system_status(gasket_dev);
+	if (status != GASKET_STATUS_ALIVE) {
+		gasket_log_info(gasket_dev,
+				"Interrupt system reported status %d.", status);
+		return status;
+	}
+
+	for (i = 0; i < driver_desc->num_page_tables; ++i) {
+		status = gasket_page_table_system_status(
+			gasket_dev->page_table[i]);
+		if (status != GASKET_STATUS_ALIVE) {
+			gasket_log_info(
+				gasket_dev, "Page table %d reported status %d.",
+				i, status);
+			return status;
+		}
+	}
+
+	return GASKET_STATUS_ALIVE;
+}
+
+/*
+ * Gasket ioctl dispatch function.
+ * @filp: File structure pointer describing this node usage session.
+ * @cmd: ioctl number to handle.
+ * @arg: ioctl-specific data pointer.
+ *
+ * First, checks if the ioctl is a generic ioctl. If not, it passes
+ * the ioctl to the ioctl_handler_cb registered in the driver description.
+ * If the ioctl is a generic ioctl, the function passes it to the
+ * gasket_ioctl_handler in gasket_ioctl.c.
+ */
+static long gasket_ioctl(struct file *filp, uint cmd, ulong arg)
+{
+	struct gasket_dev *gasket_dev;
+	const struct gasket_driver_desc *driver_desc;
+	char path[256];
+
+	if (!filp)
+		return -ENODEV;
+
+	gasket_dev = (struct gasket_dev *)filp->private_data;
+	if (!gasket_dev) {
+		gasket_nodev_error(
+			"Unable to find Gasket structure for file %s",
+			d_path(&filp->f_path, path, 256));
+		return -ENODEV;
+	}
+
+	driver_desc = gasket_dev->internal_desc->driver_desc;
+	if (!driver_desc) {
+		gasket_log_error(
+			gasket_dev,
+			"Unable to find device descriptor for file %s",
+			d_path(&filp->f_path, path, 256));
+		return -ENODEV;
+	}
+
+	if (!gasket_is_supported_ioctl(cmd)) {
+		/*
+		 * The ioctl handler is not a standard Gasket callback, since
+		 * it requires different arguments. This means we can't use
+		 * check_and_invoke_callback.
+		 */
+		if (driver_desc->ioctl_handler_cb)
+			return driver_desc->ioctl_handler_cb(filp, cmd, arg);
+
+		gasket_log_error(
+			gasket_dev, "Received unknown ioctl 0x%x", cmd);
+		return -EINVAL;
+	}
+
+	return gasket_handle_ioctl(filp, cmd, arg);
+}
+
+int gasket_reset(struct gasket_dev *gasket_dev, uint reset_type)
+{
+	int ret;
+
+	mutex_lock(&gasket_dev->mutex);
+	ret = gasket_reset_nolock(gasket_dev, reset_type);
+	mutex_unlock(&gasket_dev->mutex);
+	return ret;
+}
+EXPORT_SYMBOL(gasket_reset);
+
+int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type)
+{
+	int ret;
+	int i;
+	const struct gasket_driver_desc *driver_desc;
+
+	driver_desc = gasket_dev->internal_desc->driver_desc;
+	if (!driver_desc->device_reset_cb) {
+		gasket_log_error(
+			gasket_dev, "No device reset callback was registered.");
+		return -EINVAL;
+	}
+
+	/* Perform a device reset of the requested type. */
+	ret = driver_desc->device_reset_cb(gasket_dev, reset_type);
+	if (ret)
+		gasket_log_error(
+			gasket_dev, "Device reset cb returned %d.", ret);
+
+	/* Reinitialize the page tables and interrupt framework. */
+	for (i = 0; i < driver_desc->num_page_tables; ++i)
+		gasket_page_table_reset(gasket_dev->page_table[i]);
+
+	ret = gasket_interrupt_reinit(gasket_dev);
+	if (ret) {
+		gasket_log_error(
+			gasket_dev, "Unable to reinit interrupts: %d.", ret);
+		return ret;
+	}
+
+	/* Get current device health. */
+	gasket_dev->status = gasket_get_hw_status(gasket_dev);
+	if (gasket_dev->status == GASKET_STATUS_DEAD) {
+		gasket_log_error(gasket_dev, "Device reported as dead.");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(gasket_reset_nolock);
+
+gasket_ioctl_permissions_cb_t gasket_get_ioctl_permissions_cb(
+	struct gasket_dev *gasket_dev) {
+	return gasket_dev->internal_desc->driver_desc->ioctl_permissions_cb;
+}
+EXPORT_SYMBOL(gasket_get_ioctl_permissions_cb);
+
+static ssize_t gasket_write_mappable_regions(
+	char *buf, const struct gasket_driver_desc *driver_desc, int bar_index)
+{
+	int i;
+	ssize_t written;
+	ssize_t total_written = 0;
+	ulong min_addr, max_addr;
+	struct gasket_bar_desc bar_desc =
+		driver_desc->bar_descriptions[bar_index];
+
+	if (bar_desc.permissions == GASKET_NOMAP)
+		return 0;
+	for (i = 0;
+	     (i < bar_desc.num_mappable_regions) && (total_written < PAGE_SIZE);
+	     i++) {
+		min_addr = bar_desc.mappable_regions[i].start -
+			   driver_desc->legacy_mmap_address_offset;
+		max_addr = bar_desc.mappable_regions[i].start -
+			   driver_desc->legacy_mmap_address_offset +
+			   bar_desc.mappable_regions[i].length_bytes;
+		written = scnprintf(buf, PAGE_SIZE - total_written,
+				    "0x%08lx-0x%08lx\n", min_addr, max_addr);
+		total_written += written;
+		buf += written;
+	}
+	return total_written;
+}
+
+static ssize_t gasket_sysfs_data_show(
+	struct device *device, struct device_attribute *attr, char *buf)
+{
+	int i, ret = 0;
+	ssize_t current_written = 0;
+	const struct gasket_driver_desc *driver_desc;
+	struct gasket_dev *gasket_dev;
+	struct gasket_sysfs_attribute *gasket_attr;
+	const struct gasket_bar_desc *bar_desc;
+	enum gasket_sysfs_attribute_type sysfs_type;
+
+	gasket_dev = gasket_sysfs_get_device_data(device);
+	if (!gasket_dev) {
+		gasket_nodev_error(
+			"No sysfs mapping found for device 0x%p", device);
+		return 0;
+	}
+
+	gasket_attr = gasket_sysfs_get_attr(device, attr);
+	if (!gasket_attr) {
+		gasket_nodev_error(
+			"No sysfs attr found for device 0x%p", device);
+		gasket_sysfs_put_device_data(device, gasket_dev);
+		return 0;
+	}
+
+	driver_desc = gasket_dev->internal_desc->driver_desc;
+
+	sysfs_type =
+		(enum gasket_sysfs_attribute_type)gasket_attr->data.attr_type;
+	switch (sysfs_type) {
+	case ATTR_BAR_OFFSETS:
+		for (i = 0; i < GASKET_NUM_BARS; i++) {
+			bar_desc = &driver_desc->bar_descriptions[i];
+			if (bar_desc->size == 0)
+				continue;
+			current_written =
+				snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i,
+					 (ulong)bar_desc->base);
+			buf += current_written;
+			ret += current_written;
+		}
+		break;
+	case ATTR_BAR_SIZES:
+		for (i = 0; i < GASKET_NUM_BARS; i++) {
+			bar_desc = &driver_desc->bar_descriptions[i];
+			if (bar_desc->size == 0)
+				continue;
+			current_written =
+				snprintf(buf, PAGE_SIZE - ret, "%d: 0x%lx\n", i,
+					 (ulong)bar_desc->size);
+			buf += current_written;
+			ret += current_written;
+		}
+		break;
+	case ATTR_DRIVER_VERSION:
+		ret = snprintf(
+			buf, PAGE_SIZE, "%s\n",
+			gasket_dev->internal_desc->driver_desc->driver_version);
+		break;
+	case ATTR_FRAMEWORK_VERSION:
+		ret = snprintf(
+			buf, PAGE_SIZE, "%s\n", GASKET_FRAMEWORK_VERSION);
+		break;
+	case ATTR_DEVICE_TYPE:
+		ret = snprintf(
+			buf, PAGE_SIZE, "%s\n",
+			gasket_dev->internal_desc->driver_desc->name);
+		break;
+	case ATTR_HARDWARE_REVISION:
+		ret = snprintf(
+			buf, PAGE_SIZE, "%d\n", gasket_dev->hardware_revision);
+		break;
+	case ATTR_PCI_ADDRESS:
+		ret = snprintf(buf, PAGE_SIZE, "%s\n", gasket_dev->kobj_name);
+		break;
+	case ATTR_STATUS:
+		ret = snprintf(
+			buf, PAGE_SIZE, "%s\n",
+			gasket_num_name_lookup(
+				gasket_dev->status, gasket_status_name_table));
+		break;
+	case ATTR_IS_DEVICE_OWNED:
+		ret = snprintf(
+			buf, PAGE_SIZE, "%d\n",
+			gasket_dev->dev_info.ownership.is_owned);
+		break;
+	case ATTR_DEVICE_OWNER:
+		ret = snprintf(
+			buf, PAGE_SIZE, "%d\n",
+			gasket_dev->dev_info.ownership.owner);
+		break;
+	case ATTR_WRITE_OPEN_COUNT:
+		ret = snprintf(
+			buf, PAGE_SIZE, "%d\n",
+			gasket_dev->dev_info.ownership.write_open_count);
+		break;
+	case ATTR_RESET_COUNT:
+		ret = snprintf(buf, PAGE_SIZE, "%d\n", gasket_dev->reset_count);
+		break;
+	case ATTR_USER_MEM_RANGES:
+		for (i = 0; i < GASKET_NUM_BARS; ++i) {
+			current_written = gasket_write_mappable_regions(
+				buf, driver_desc, i);
+			buf += current_written;
+			ret += current_written;
+		}
+		break;
+	default:
+		gasket_log_error(
+			gasket_dev, "Unknown attribute: %s", attr->attr.name);
+		ret = 0;
+		break;
+	}
+
+	gasket_sysfs_put_attr(device, gasket_attr);
+	gasket_sysfs_put_device_data(device, gasket_dev);
+	return ret;
+}
+
+/* Get the driver structure for a given gasket_dev.
+ * @dev: pointer to gasket_dev, implementing the requested driver.
+ */
+const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev)
+{
+	return dev->internal_desc->driver_desc;
+}
+
+/* Get the device structure for a given gasket_dev.
+ * @dev: pointer to gasket_dev, implementing the requested driver.
+ */
+struct device *gasket_get_device(struct gasket_dev *dev)
+{
+	if (dev->pci_dev)
+		return &dev->pci_dev->dev;
+	return NULL;
+}
+
+/**
+ * Synchronously waits on device.
+ * @gasket_dev: Device struct.
+ * @bar: Bar
+ * @offset: Register offset
+ * @mask: Register mask
+ * @val: Expected value
+ * @timeout_ns: Timeout in nanoseconds
+ *
+ * Description: Busy waits for a specific combination of bits to be set
+ * on a Gasket register.
+ **/
+int gasket_wait_sync(
+	struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val,
+	u64 timeout_ns)
+{
+	u64 reg;
+	struct timespec start_time, cur_time;
+	u64 diff_nanosec;
+	int count = 0;
+
+	reg = gasket_dev_read_64(gasket_dev, bar, offset);
+	start_time = current_kernel_time();
+	while ((reg & mask) != val) {
+		count++;
+		cur_time = current_kernel_time();
+		diff_nanosec = (u64)(cur_time.tv_sec - start_time.tv_sec) *
+				       1000000000LL +
+			       (u64)(cur_time.tv_nsec) -
+			       (u64)(start_time.tv_nsec);
+		if (diff_nanosec > timeout_ns) {
+			gasket_log_error(
+				gasket_dev,
+				"gasket_wait_sync timeout: reg %llx count %x "
+				"dma %lld ns\n",
+				offset, count, diff_nanosec);
+			return -1;
+		}
+		reg = gasket_dev_read_64(gasket_dev, bar, offset);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(gasket_wait_sync);
+
+/**
+ * Asynchronously waits on device.
+ * @gasket_dev: Device struct.
+ * @bar: Bar
+ * @offset: Register offset
+ * @mask: Register mask
+ * @val: Expected value
+ * @max_retries: number of sleep periods
+ * @delay_ms: Timeout in milliseconds
+ *
+ * Description: Busy waits for a specific combination of bits to be set on a
+ * Gasket register.
+ **/
+int gasket_wait_with_reschedule(
+	struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val,
+	u64 max_retries, u64 delay_ms)
+{
+	u64 retries = 0;
+	u64 tmp;
+
+	while (retries < max_retries) {
+		tmp = gasket_dev_read_64(gasket_dev, bar, offset);
+		if ((tmp & mask) == val)
+			break;
+		schedule_timeout(msecs_to_jiffies(delay_ms));
+		retries++;
+	}
+	if (retries == max_retries) {
+		gasket_log_error(
+			gasket_dev,
+			"gasket_wait_with_reschedule timeout: reg %llx timeout (%llu ms)",
+			offset, max_retries * delay_ms);
+		return -EINVAL;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(gasket_wait_with_reschedule);

drivers/staging/gasket/gasket_core.h

@@ -0,0 +1,719 @@
+/* Gasket generic driver. Defines the set of data types and functions necessary
+ * to define a driver using the Gasket generic driver framework.
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef __GASKET_CORE_H__
+#define __GASKET_CORE_H__
+
+#include <linux/cdev.h>
+#include <linux/compiler.h>
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#include "gasket_constants.h"
+
+/**
+ * struct gasket_num_name - Map numbers to names.
+ * @ein_num: Number.
+ * @ein_name: Name associated with the number, a char pointer.
+ *
+ * This structure maps numbers to names. It is used to provide printable enum
+ * names, e.g {0, "DEAD"} or {1, "ALIVE"}.
+ */
+struct gasket_num_name {
+	uint snn_num;
+	const char *snn_name;
+};
+
+/*
+ * Register location for packed interrupts.
+ * Each value indicates the location of an interrupt field (in units of
+ * gasket_driver_desc->interrupt_pack_width) within the containing register.
+ * In other words, this indicates the shift to use when creating a mask to
+ * extract/set bits within a register for a given interrupt.
+ */
+enum gasket_interrupt_packing {
+	PACK_0 = 0,
+	PACK_1 = 1,
+	PACK_2 = 2,
+	PACK_3 = 3,
+	UNPACKED = 4,
+};
+
+/* Type of the interrupt supported by the device. */
+enum gasket_interrupt_type {
+	PCI_MSIX = 0,
+	PCI_MSI = 1,
+	PLATFORM_WIRE = 2,
+};
+
+/* Used to describe a Gasket interrupt. Contains an interrupt index, a register,
+ * and packing data for that interrupt. The register and packing data
+ * fields is relevant only for PCI_MSIX interrupt type and can be
+ * set to 0 for everything else.
+ */
+struct gasket_interrupt_desc {
+	/* Device-wide interrupt index/number. */
+	int index;
+	/* The register offset controlling this interrupt. */
+	u64 reg;
+	/* The location of this interrupt inside register reg, if packed. */
+	int packing;
+};
+
+/* Offsets to the wire interrupt handling registers */
+struct gasket_wire_interrupt_offsets {
+	u64 pending_bit_array;
+	u64 mask_array;
+};
+
+/*
+ * This enum is used to identify memory regions being part of the physical
+ * memory that belongs to a device.
+ */
+enum mappable_area_type {
+	PCI_BAR = 0, /* Default */
+	BUS_REGION,  /* For SYSBUS devices, i.e. AXI etc... */
+	COHERENT_MEMORY
+};
+
+/*
+ * Metadata for each BAR mapping.
+ * This struct is used so as to track PCI memory, I/O space, AXI and coherent
+ * memory area... i.e. memory objects which can be referenced in the device's
+ * mmap function.
+ */
+struct gasket_bar_data {
+	/* Virtual base address. */
+	u8 __iomem *virt_base;
+
+	/* Physical base address. */
+	ulong phys_base;
+
+	/* Length of the mapping. */
+	ulong length_bytes;
+
+	/* Type of mappable area */
+	enum mappable_area_type type;
+};
+
+/* Maintains device open ownership data. */
+struct gasket_ownership {
+	/* 1 if the device is owned, 0 otherwise. */
+	int is_owned;
+
+	/* TGID of the owner. */
+	pid_t owner;
+
+	/* Count of current device opens in write mode. */
+	int write_open_count;
+};
+
+/* Page table modes of operation. */
+enum gasket_page_table_mode {
+	/* The page table is partitionable as normal, all simple by default. */
+	GASKET_PAGE_TABLE_MODE_NORMAL,
+
+	/* All entries are always simple. */
+	GASKET_PAGE_TABLE_MODE_SIMPLE,
+
+	/* All entries are always extended. No extended bit is used. */
+	GASKET_PAGE_TABLE_MODE_EXTENDED,
+};
+
+/* Page table configuration. One per table. */
+struct gasket_page_table_config {
+	/* The identifier/index of this page table. */
+	int id;
+
+	/* The operation mode of this page table. */
+	enum gasket_page_table_mode mode;
+
+	/* Total (first-level) entries in this page table. */
+	ulong total_entries;
+
+	/* Base register for the page table. */
+	int base_reg;
+
+	/*
+	 * Register containing the extended page table. This value is unused in
+	 * GASKET_PAGE_TABLE_MODE_SIMPLE and GASKET_PAGE_TABLE_MODE_EXTENDED
+	 * modes.
+	 */
+	int extended_reg;
+
+	/* The bit index indicating whether a PT entry is extended. */
+	int extended_bit;
+};
+
+/* Maintains information about a device node. */
+struct gasket_cdev_info {
+	/* The internal name of this device. */
+	char name[GASKET_NAME_MAX];
+
+	/* Device number. */
+	dev_t devt;
+
+	/* Kernel-internal device structure. */
+	struct device *device;
+
+	/* Character device for real. */
+	struct cdev cdev;
+
+	/* Flag indicating if cdev_add has been called for the devices. */
+	int cdev_added;
+
+	/* Pointer to the overall gasket_dev struct for this device. */
+	struct gasket_dev *gasket_dev_ptr;
+
+	/* Ownership data for the device in question. */
+	struct gasket_ownership ownership;
+};
+
+/* Describes the offset and length of mmapable device BAR regions. */
+struct gasket_mappable_region {
+	u64 start;
+	u64 length_bytes;
+};
+
+/* Describe the offset, size, and permissions for a device bar. */
+struct gasket_bar_desc {
+	/*
+	 * The size of each PCI BAR range, in bytes. If a value is 0, that BAR
+	 * will not be mapped into kernel space at all.
+	 * For devices with 64 bit BARs, only elements 0, 2, and 4 should be
+	 * populated, and 1, 3, and 5 should be set to 0.
+	 * For example, for a device mapping 1M in each of the first two 64-bit
+	 * BARs, this field would be set as { 0x100000, 0, 0x100000, 0, 0, 0 }
+	 * (one number per bar_desc struct.)
+	 */
+	u64 size;
+	/* The permissions for this bar. (Should be VM_WRITE/VM_READ/VM_EXEC,
+	 * and can be or'd.) If set to GASKET_NOMAP, the bar will
+	 * not be used for mmapping.
+	 */
+	ulong permissions;
+	/* The memory address corresponding to the base of this bar, if used. */
+	u64 base;
+	/* The number of mappable regions in this bar. */
+	int num_mappable_regions;
+
+	/* The mappable subregions of this bar. */
+	const struct gasket_mappable_region *mappable_regions;
+
+	/* Type of mappable area */
+	enum mappable_area_type type;
+};
+
+/* Describes the offset, size, and permissions for a coherent buffer. */
+struct gasket_coherent_buffer_desc {
+	/* The size of the coherent buffer. */
+	u64 size;
+
+	/* The permissions for this bar. (Should be VM_WRITE/VM_READ/VM_EXEC,
+	 * and can be or'd.) If set to GASKET_NOMAP, the bar will
+	 * not be used for mmaping.
+	 */
+	ulong permissions;
+
+	/* device side address. */
+	u64 base;
+};
+
+/* Coherent buffer structure. */
+struct gasket_coherent_buffer {
+	/* Virtual base address. */
+	u8 __iomem *virt_base;
+
+	/* Physical base address. */
+	ulong phys_base;
+
+	/* Length of the mapping. */
+	ulong length_bytes;
+};
+
+/* Description of Gasket-specific permissions in the mmap field. */
+enum gasket_mapping_options { GASKET_NOMAP = 0 };
+
+/* This struct represents an undefined bar that should never be mapped. */
+#define GASKET_UNUSED_BAR                                                      \
+	{                                                                      \
+		0, GASKET_NOMAP, 0, 0, NULL, 0                                 \
+	}
+
+/* Internal data for a Gasket device. See gasket_core.c for more information. */
+struct gasket_internal_desc;
+
+#define MAX_NUM_COHERENT_PAGES 16
+
+/*
+ * Device data for Gasket device instances.
+ *
+ * This structure contains the data required to manage a Gasket device.
+ */
+struct gasket_dev {
+	/* Pointer to the internal driver description for this device. */
+	struct gasket_internal_desc *internal_desc;
+
+	/* PCI subsystem metadata. */
+	struct pci_dev *pci_dev;
+
+	/* This device's index into internal_desc->devs. */
+	int dev_idx;
+
+	/* The name of this device, as reported by the kernel. */
+	char kobj_name[GASKET_NAME_MAX];
+
+	/* Virtual address of mapped BAR memory range. */
+	struct gasket_bar_data bar_data[GASKET_NUM_BARS];
+
+	/* Coherent buffer. */
+	struct gasket_coherent_buffer coherent_buffer;
+
+	/* Number of page tables for this device. */
+	int num_page_tables;
+
+	/* Address translations. Page tables have a private implementation. */
+	struct gasket_page_table *page_table[GASKET_MAX_NUM_PAGE_TABLES];
+
+	/* Interrupt data for this device. */
+	struct gasket_interrupt_data *interrupt_data;
+
+	/* Status for this device - GASKET_STATUS_ALIVE or _DEAD. */
+	uint status;
+
+	/* Number of times this device has been reset. */
+	uint reset_count;
+
+	/* Dev information for the cdev node. */
+	struct gasket_cdev_info dev_info;
+
+	/* Hardware revision value for this device. */
+	int hardware_revision;
+
+	/*
+	 * Device-specific data; allocated in gasket_driver_desc.add_dev_cb()
+	 * and freed in gasket_driver_desc.remove_dev_cb().
+	 */
+	void *cb_data;
+
+	/* Protects access to per-device data (i.e. this structure). */
+	struct mutex mutex;
+
+	/* cdev hash tracking/membership structure, Accel and legacy. */
+	/* Unused until Accel is upstreamed. */
+	struct hlist_node hlist_node;
+	struct hlist_node legacy_hlist_node;
+};
+
+/* Type of the ioctl permissions check callback. See below. */
+typedef int (*gasket_ioctl_permissions_cb_t)(
+	struct file *filp, uint cmd, ulong arg);
+
+/*
+ * Device type descriptor.
+ *
+ * This structure contains device-specific data needed to identify and address a
+ * type of device to be administered via the Gasket generic driver.
+ *
+ * Device IDs are per-driver. In other words, two drivers using the Gasket
+ * framework will each have a distinct device 0 (for example).
+ */
+struct gasket_driver_desc {
+	/* The name of this device type. */
+	const char *name;
+
+	/* The name of this specific device model. */
+	const char *chip_model;
+
+	/* The version of the chip specified in chip_model. */
+	const char *chip_version;
+
+	/* The version of this driver: "1.0.0", "2.1.3", etc. */
+	const char *driver_version;
+
+	/*
+	 * Non-zero if we should create "legacy" (device and device-class-
+	 * specific) character devices and sysfs nodes.
+	 */
+	/* Unused until Accel is upstreamed. */
+	int legacy_support;
+
+	/* Major and minor numbers identifying the device. */
+	int major, minor;
+
+	/* Module structure for this driver. */
+	struct module *module;
+
+	/* PCI ID table. */
+	const struct pci_device_id *pci_id_table;
+
+	/* The number of page tables handled by this driver. */
+	int num_page_tables;
+
+	/* The index of the bar containing the page tables. */
+	int page_table_bar_index;
+
+	/* Registers used to control each page table. */
+	const struct gasket_page_table_config *page_table_configs;
+
+	/* The bit index indicating whether a PT entry is extended. */
+	int page_table_extended_bit;
+
+	/*
+	 * Legacy mmap address adjusment for legacy devices only. Should be 0
+	 * for any new device.
+	 */
+	ulong legacy_mmap_address_offset;
+
+	/* Set of 6 bar descriptions that describe all PCIe bars.
+	 * Note that BUS/AXI devices (i.e. non PCI devices) use those.
+	 */
+	struct gasket_bar_desc bar_descriptions[GASKET_NUM_BARS];
+
+	/*
+	 * Coherent buffer description.
+	 */
+	struct gasket_coherent_buffer_desc coherent_buffer_description;
+
+	/* Offset of wire interrupt registers. */
+	const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets;
+
+	/* Interrupt type. (One of gasket_interrupt_type). */
+	int interrupt_type;
+
+	/* Index of the bar containing the interrupt registers to program. */
+	int interrupt_bar_index;
+
+	/* Number of interrupts in the gasket_interrupt_desc array */
+	int num_interrupts;
+
+	/* Description of the interrupts for this device. */
+	const struct gasket_interrupt_desc *interrupts;
+
+	/*
+	 * If this device packs multiple interrupt->MSI-X mappings into a
+	 * single register (i.e., "uses packed interrupts"), only a single bit
+	 * width is supported for each interrupt mapping (unpacked/"full-width"
+	 * interrupts are always supported). This value specifies that width. If
+	 * packed interrupts are not used, this value is ignored.
+	 */
+	int interrupt_pack_width;
+
+	/* Driver callback functions - all may be NULL */
+	/*
+	 * add_dev_cb: Callback when a device is found.
+	 * @dev: The gasket_dev struct for this driver instance.
+	 *
+	 * This callback should initialize the device-specific cb_data.
+	 * Called when a device is found by the driver,
+	 * before any BAR ranges have been mapped. If this call fails (returns
+	 * nonzero), remove_dev_cb will be called.
+	 *
+	 */
+	int (*add_dev_cb)(struct gasket_dev *dev);
+
+	/*
+	 * remove_dev_cb: Callback for when a device is removed from the system.
+	 * @dev: The gasket_dev struct for this driver instance.
+	 *
+	 * This callback should free data allocated in add_dev_cb.
+	 * Called immediately before a device is unregistered by the driver.
+	 * All framework-managed resources will have been cleaned up by the time
+	 * this callback is invoked (PCI BARs, character devices, ...).
+	 */
+	int (*remove_dev_cb)(struct gasket_dev *dev);
+
+	/*
+	 * device_open_cb: Callback for when a device node is opened in write
+	 * mode.
+	 * @dev: The gasket_dev struct for this driver instance.
+	 *
+	 * This callback should perform device-specific setup that needs to
+	 * occur only once when a device is first opened.
+	 */
+	int (*device_open_cb)(struct gasket_dev *dev);
+
+	/*
+	 * device_release_cb: Callback when a device is closed.
+	 * @gasket_dev: The gasket_dev struct for this driver instance.
+	 *
+	 * This callback is called whenever a device node fd is closed, as
+	 * opposed to device_close_cb, which is called when the _last_
+	 * descriptor for an open file is closed. This call is intended to
+	 * handle any per-user or per-fd cleanup.
+	 */
+	int (*device_release_cb)(
+		struct gasket_dev *gasket_dev, struct file *file);
+
+	/*
+	 * device_close_cb: Callback for when a device node is closed for the
+	 * last time.
+	 * @dev: The gasket_dev struct for this driver instance.
+	 *
+	 * This callback should perform device-specific cleanup that only
+	 * needs to occur when the last reference to a device node is closed.
+	 *
+	 * This call is intended to handle and device-wide cleanup, as opposed
+	 * to per-fd cleanup (which should be handled by device_release_cb).
+	 */
+	int (*device_close_cb)(struct gasket_dev *dev);
+
+	/*
+	 * enable_dev_cb: Callback immediately before enabling the device.
+	 * @dev: Pointer to the gasket_dev struct for this driver instance.
+	 *
+	 * This callback is invoked after the device has been added and all BAR
+	 * spaces mapped, immediately before registering and enabling the
+	 * [character] device via cdev_add. If this call fails (returns
+	 * nonzero), disable_dev_cb will be called.
+	 *
+	 * Note that cdev are initialized but not active
+	 * (cdev_add has not yet been called) when this callback is invoked.
+	 */
+	int (*enable_dev_cb)(struct gasket_dev *dev);
+
+	/*
+	 * disable_dev_cb: Callback immediately after disabling the device.
+	 * @dev: Pointer to the gasket_dev struct for this driver instance.
+	 *
+	 * Called during device shutdown, immediately after disabling device
+	 * operations via cdev_del.
+	 */
+	int (*disable_dev_cb)(struct gasket_dev *dev);
+
+	/*
+	 * sysfs_setup_cb: Callback to set up driver-specific sysfs nodes.
+	 * @dev: Pointer to the gasket_dev struct for this device.
+	 *
+	 * Called just before enable_dev_cb.
+	 *
+	 */
+	int (*sysfs_setup_cb)(struct gasket_dev *dev);
+
+	/*
+	 * sysfs_cleanup_cb: Callback to clean up driver-specific sysfs nodes.
+	 * @dev: Pointer to the gasket_dev struct for this device.
+	 *
+	 * Called just before disable_dev_cb.
+	 *
+	 */
+	int (*sysfs_cleanup_cb)(struct gasket_dev *dev);
+
+	/*
+	 * get_mappable_regions_cb: Get descriptors of mappable device memory.
+	 * @gasket_dev: Pointer to the struct gasket_dev for this device.
+	 * @bar_index: BAR for which to retrieve memory ranges.
+	 * @mappable_regions: Out-pointer to the list of mappable regions on the
+	 * device/BAR for this process.
+	 * @num_mappable_regions: Out-pointer for the size of mappable_regions.
+	 *
+	 * Called when handling mmap(), this callback is used to determine which
+	 * regions of device memory may be mapped by the current process. This
+	 * information is then compared to mmap request to determine which
+	 * regions to actually map.
+	 */
+	int (*get_mappable_regions_cb)(
+		struct gasket_dev *gasket_dev, int bar_index,
+		struct gasket_mappable_region **mappable_regions,
+		int *num_mappable_regions);
+
+	/*
+	 * ioctl_permissions_cb: Check permissions for generic ioctls.
+	 * @filp: File structure pointer describing this node usage session.
+	 * @cmd: ioctl number to handle.
+	 * @arg: ioctl-specific data pointer.
+	 *
+	 * Returns 1 if the ioctl may be executed, 0 otherwise. If this callback
+	 * isn't specified a default routine will be used, that only allows the
+	 * original device opener (i.e, the "owner") to execute state-affecting
+	 * ioctls.
+	 */
+	gasket_ioctl_permissions_cb_t ioctl_permissions_cb;
+
+	/*
+	 * ioctl_handler_cb: Callback to handle device-specific ioctls.
+	 * @filp: File structure pointer describing this node usage session.
+	 * @cmd: ioctl number to handle.
+	 * @arg: ioctl-specific data pointer.
+	 *
+	 * Invoked whenever an ioctl is called that the generic Gasket
+	 * framework doesn't support. If no cb is registered, unknown ioctls
+	 * return -EINVAL. Should return an error status (either -EINVAL or
+	 * the error result of the ioctl being handled).
+	 */
+	long (*ioctl_handler_cb)(struct file *filp, uint cmd, ulong arg);
+
+	/*
+	 * device_status_cb: Callback to determine device health.
+	 * @dev: Pointer to the gasket_dev struct for this device.
+	 *
+	 * Called to determine if the device is healthy or not. Should return
+	 * a member of the gasket_status_type enum.
+	 *
+	 */
+	int (*device_status_cb)(struct gasket_dev *dev);
+
+	/*
+	 * hardware_revision_cb: Get the device's hardware revision.
+	 * @dev: Pointer to the gasket_dev struct for this device.
+	 *
+	 * Called to determine the reported rev of the physical hardware.
+	 * Revision should be >0. A negative return value is an error.
+	 */
+	int (*hardware_revision_cb)(struct gasket_dev *dev);
+
+	/*
+	 * device_reset_cb: Reset the hardware in question.
+	 * @dev: Pointer to the gasket_dev structure for this device.
+	 * @type: Integer representing reset type. (All
+	 * Gasket resets have an integer representing their type
+	 * defined in (device)_ioctl.h; the specific resets are
+	 * device-dependent, but are handled in the device-specific
+	 * callback anyways.)
+	 *
+	 * Called by reset ioctls. This function should not
+	 * lock the gasket_dev mutex. It should return 0 on success
+	 * and an error on failure.
+	 */
+	int (*device_reset_cb)(struct gasket_dev *dev, uint reset_type);
+};
+
+/*
+ * Register the specified device type with the framework.
+ * @desc: Populated/initialized device type descriptor.
+ *
+ * This function does _not_ take ownership of desc; the underlying struct must
+ * exist until the matching call to gasket_unregister_device.
+ * This function should be called from your driver's module_init function.
+ */
+int gasket_register_device(const struct gasket_driver_desc *desc);
+
+/*
+ * Remove the specified device type from the framework.
+ * @desc: Descriptor for the device type to unregister; it should have been
+ *        passed to gasket_register_device in a previous call.
+ *
+ * This function should be called from your driver's module_exit function.
+ */
+void gasket_unregister_device(const struct gasket_driver_desc *desc);
+
+/*
+ * Reset the Gasket device.
+ * @gasket_dev: Gasket device struct.
+ * @reset_type: Uint representing requested reset type. Should be
+ * valid in the underlying callback.
+ *
+ * Calls device_reset_cb. Returns 0 on success and an error code othewrise.
+ * gasket_reset_nolock will not lock the mutex, gasket_reset will.
+ *
+ */
+int gasket_reset(struct gasket_dev *gasket_dev, uint reset_type);
+int gasket_reset_nolock(struct gasket_dev *gasket_dev, uint reset_type);
+
+/*
+ * Memory management functions. These will likely be spun off into their own
+ * file in the future.
+ */
+
+/* Unmaps the specified mappable region from a VMA. */
+int gasket_mm_unmap_region(
+	const struct gasket_dev *gasket_dev, struct vm_area_struct *vma,
+	const struct gasket_mappable_region *map_region);
+
+/*
+ * Get the ioctl permissions callback.
+ * @gasket_dev: Gasket device structure.
+ */
+gasket_ioctl_permissions_cb_t gasket_get_ioctl_permissions_cb(
+	struct gasket_dev *gasket_dev);
+
+/**
+ * Lookup a name by number in a num_name table.
+ * @num: Number to lookup.
+ * @table: Array of num_name structures, the table for the lookup.
+ *
+ */
+const char *gasket_num_name_lookup(
+	uint num, const struct gasket_num_name *table);
+
+/* Handy inlines */
+static inline ulong gasket_dev_read_64(
+	struct gasket_dev *gasket_dev, int bar, ulong location)
+{
+	return readq(&gasket_dev->bar_data[bar].virt_base[location]);
+}
+
+static inline void gasket_dev_write_64(
+	struct gasket_dev *dev, u64 value, int bar, ulong location)
+{
+	writeq(value, &dev->bar_data[bar].virt_base[location]);
+}
+
+static inline void gasket_dev_write_32(
+	struct gasket_dev *dev, u32 value, int bar, ulong location)
+{
+	writel(value, &dev->bar_data[bar].virt_base[location]);
+}
+
+static inline u32 gasket_dev_read_32(
+	struct gasket_dev *dev, int bar, ulong location)
+{
+	return readl(&dev->bar_data[bar].virt_base[location]);
+}
+
+static inline void gasket_read_modify_write_64(
+	struct gasket_dev *dev, int bar, ulong location, u64 value,
+	u64 mask_width, u64 mask_shift)
+{
+	u64 mask, tmp;
+
+	tmp = gasket_dev_read_64(dev, bar, location);
+	mask = ((1 << mask_width) - 1) << mask_shift;
+	tmp = (tmp & ~mask) | (value << mask_shift);
+	gasket_dev_write_64(dev, tmp, bar, location);
+}
+
+static inline void gasket_read_modify_write_32(
+	struct gasket_dev *dev, int bar, ulong location, u32 value,
+	u32 mask_width, u32 mask_shift)
+{
+	u32 mask, tmp;
+
+	tmp = gasket_dev_read_32(dev, bar, location);
+	mask = ((1 << mask_width) - 1) << mask_shift;
+	tmp = (tmp & ~mask) | (value << mask_shift);
+	gasket_dev_write_32(dev, tmp, bar, location);
+}
+
+/* Get the Gasket driver structure for a given device. */
+const struct gasket_driver_desc *gasket_get_driver_desc(struct gasket_dev *dev);
+
+/* Get the device structure for a given device. */
+struct device *gasket_get_device(struct gasket_dev *dev);
+
+/* Helper function, Synchronous waits on a given set of bits. */
+int gasket_wait_sync(
+	struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val,
+	u64 timeout_ns);
+
+/* Helper function, Asynchronous waits on a given set of bits. */
+int gasket_wait_with_reschedule(
+	struct gasket_dev *gasket_dev, int bar, u64 offset, u64 mask, u64 val,
+	u64 max_retries, u64 delay_ms);
+
+#endif /* __GASKET_CORE_H__ */

drivers/staging/gasket/gasket_interrupt.c

@@ -0,0 +1,638 @@
+/* Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "gasket_interrupt.h"
+
+#include "gasket_constants.h"
+#include "gasket_core.h"
+#include "gasket_logging.h"
+#include "gasket_sysfs.h"
+#include <linux/interrupt.h>
+#include <linux/version.h>
+#ifdef GASKET_KERNEL_TRACE_SUPPORT
+#define CREATE_TRACE_POINTS
+#include <trace/events/gasket_interrupt.h>
+#else
+#define trace_gasket_interrupt_event(x, ...)
+#endif
+/* Retry attempts if the requested number of interrupts aren't available. */
+#define MSIX_RETRY_COUNT 3
+
+/* Instance interrupt management data. */
+struct gasket_interrupt_data {
+	/* The name associated with this interrupt data. */
+	const char *name;
+
+	/* Interrupt type. See gasket_interrupt_type in gasket_core.h */
+	int type;
+
+	/* The PCI device [if any] associated with the owning device. */
+	struct pci_dev *pci_dev;
+
+	/* Set to 1 if MSI-X has successfully been configred, 0 otherwise. */
+	int msix_configured;
+
+	/* The number of interrupts requested by the owning device. */
+	int num_interrupts;
+
+	/* A pointer to the interrupt descriptor struct for this device. */
+	const struct gasket_interrupt_desc *interrupts;
+
+	/* The index of the bar into which interrupts should be mapped. */
+	int interrupt_bar_index;
+
+	/* The width of a single interrupt in a packed interrupt register. */
+	int pack_width;
+
+	/* offset of wire interrupt registers */
+	const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets;
+
+	/*
+	 * Design-wise, these elements should be bundled together, but
+	 * pci_enable_msix's interface requires that they be managed
+	 * individually (requires array of struct msix_entry).
+	 */
+
+	/* The number of successfully configured interrupts. */
+	int num_configured;
+
+	/* The MSI-X data for each requested/configured interrupt. */
+	struct msix_entry *msix_entries;
+
+	/* The eventfd "callback" data for each interrupt. */
+	struct eventfd_ctx **eventfd_ctxs;
+
+	/* The number of times each interrupt has been called. */
+	ulong *interrupt_counts;
+
+	/* Linux IRQ number. */
+	int irq;
+};
+
+/* Function definitions. */
+static ssize_t interrupt_sysfs_show(
+	struct device *device, struct device_attribute *attr, char *buf);
+
+static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id);
+
+/* Structures to display interrupt counts in sysfs. */
+enum interrupt_sysfs_attribute_type {
+	ATTR_INTERRUPT_COUNTS,
+};
+
+static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = {
+	GASKET_SYSFS_RO(
+		interrupt_counts, interrupt_sysfs_show, ATTR_INTERRUPT_COUNTS),
+	GASKET_END_OF_ATTR_ARRAY,
+};
+
+/*
+ * Set up device registers for interrupt handling.
+ * @gasket_dev: The Gasket information structure for this device.
+ *
+ * Sets up the device registers with the correct indices for the relevant
+ * interrupts.
+ */
+static void gasket_interrupt_setup(struct gasket_dev *gasket_dev);
+
+/* MSIX init and cleanup. */
+static int gasket_interrupt_msix_init(
+	struct gasket_interrupt_data *interrupt_data);
+static void gasket_interrupt_msix_cleanup(
+	struct gasket_interrupt_data *interrupt_data);
+static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev);
+
+int gasket_interrupt_init(
+	struct gasket_dev *gasket_dev, const char *name, int type,
+	const struct gasket_interrupt_desc *interrupts,
+	int num_interrupts, int pack_width, int bar_index,
+	const struct gasket_wire_interrupt_offsets *wire_int_offsets)
+{
+	int ret;
+	struct gasket_interrupt_data *interrupt_data;
+
+	interrupt_data = kzalloc(
+		sizeof(struct gasket_interrupt_data), GFP_KERNEL);
+	if (!interrupt_data)
+		return -ENOMEM;
+	gasket_dev->interrupt_data = interrupt_data;
+	interrupt_data->name = name;
+	interrupt_data->type = type;
+	interrupt_data->pci_dev = gasket_dev->pci_dev;
+	interrupt_data->num_interrupts = num_interrupts;
+	interrupt_data->interrupts = interrupts;
+	interrupt_data->interrupt_bar_index = bar_index;
+	interrupt_data->pack_width = pack_width;
+	interrupt_data->num_configured = 0;
+	interrupt_data->wire_interrupt_offsets = wire_int_offsets;
+
+	/* Allocate all dynamic structures. */
+	interrupt_data->msix_entries = kzalloc(
+		sizeof(struct msix_entry) * num_interrupts, GFP_KERNEL);
+	if (!interrupt_data->msix_entries) {
+		kfree(interrupt_data);
+		return -ENOMEM;
+	}
+
+	interrupt_data->eventfd_ctxs = kzalloc(
+		sizeof(struct eventfd_ctx *) * num_interrupts, GFP_KERNEL);
+	if (!interrupt_data->eventfd_ctxs) {
+		kfree(interrupt_data->msix_entries);
+		kfree(interrupt_data);
+		return -ENOMEM;
+	}
+
+	interrupt_data->interrupt_counts = kzalloc(
+		sizeof(ulong) * num_interrupts, GFP_KERNEL);
+	if (!interrupt_data->interrupt_counts) {
+		kfree(interrupt_data->eventfd_ctxs);
+		kfree(interrupt_data->msix_entries);
+		kfree(interrupt_data);
+		return -ENOMEM;
+	}
+
+	switch (interrupt_data->type) {
+	case PCI_MSIX:
+		ret = gasket_interrupt_msix_init(interrupt_data);
+		if (ret)
+			break;
+		force_msix_interrupt_unmasking(gasket_dev);
+		break;
+
+	case PCI_MSI:
+	case PLATFORM_WIRE:
+	default:
+		gasket_nodev_error(
+			"Cannot handle unsupported interrupt type %d.",
+			interrupt_data->type);
+		ret = -EINVAL;
+	};
+
+	if (ret) {
+		/* Failing to setup interrupts will cause the device to report
+		 * GASKET_STATUS_LAMED. But it is not fatal.
+		 */
+		gasket_log_warn(
+			gasket_dev, "Couldn't initialize interrupts: %d", ret);
+		return 0;
+	}
+
+	gasket_interrupt_setup(gasket_dev);
+	gasket_sysfs_create_entries(
+		gasket_dev->dev_info.device, interrupt_sysfs_attrs);
+
+	return 0;
+}
+
+static int gasket_interrupt_msix_init(
+	struct gasket_interrupt_data *interrupt_data)
+{
+	int ret = 1;
+	int i;
+
+	for (i = 0; i < interrupt_data->num_interrupts; i++) {
+		interrupt_data->msix_entries[i].entry = i;
+		interrupt_data->msix_entries[i].vector = 0;
+		interrupt_data->eventfd_ctxs[i] = NULL;
+	}
+
+	/* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */
+	for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++)
+		ret = pci_enable_msix_exact(interrupt_data->pci_dev,
+					    interrupt_data->msix_entries,
+					    interrupt_data->num_interrupts);
+
+	if (ret)
+		return ret > 0 ? -EBUSY : ret;
+	interrupt_data->msix_configured = 1;
+
+	for (i = 0; i < interrupt_data->num_interrupts; i++) {
+		ret = request_irq(
+			interrupt_data->msix_entries[i].vector,
+			gasket_msix_interrupt_handler, 0, interrupt_data->name,
+			interrupt_data);
+
+		if (ret) {
+			gasket_nodev_error(
+				"Cannot get IRQ for interrupt %d, vector %d; "
+				"%d\n",
+				i, interrupt_data->msix_entries[i].vector, ret);
+			return ret;
+		}
+
+		interrupt_data->num_configured++;
+	}
+
+	return 0;
+}
+
+static void gasket_interrupt_msix_cleanup(
+	struct gasket_interrupt_data *interrupt_data)
+{
+	int i;
+
+	for (i = 0; i < interrupt_data->num_configured; i++)
+		free_irq(interrupt_data->msix_entries[i].vector,
+			 interrupt_data);
+	interrupt_data->num_configured = 0;
+
+	if (interrupt_data->msix_configured)
+		pci_disable_msix(interrupt_data->pci_dev);
+	interrupt_data->msix_configured = 0;
+}
+
+/*
+ * On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt
+ * setup code with MSIX vectors masked. This is wrong because nothing else in
+ * the driver will normally touch the MSIX vectors.
+ *
+ * As a temporary hack, force unmasking there.
+ *
+ * TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after
+ * gasket_interrupt_msix_init(), and remove this code.
+ */
+static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev)
+{
+	int i;
+#define MSIX_VECTOR_SIZE 16
+#define MSIX_MASK_BIT_OFFSET 12
+#define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800
+	for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) {
+		/* Check if the MSIX vector is unmasked */
+		ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE +
+				 MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE;
+		u32 mask =
+			gasket_dev_read_32(
+				gasket_dev,
+				gasket_dev->interrupt_data->interrupt_bar_index,
+				location);
+		if (!(mask & 1))
+			continue;
+		/* Unmask the msix vector (clear 32 bits) */
+		gasket_dev_write_32(
+			gasket_dev, 0,
+			gasket_dev->interrupt_data->interrupt_bar_index,
+			location);
+	}
+#undef MSIX_VECTOR_SIZE
+#undef MSIX_MASK_BIT_OFFSET
+#undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE
+}
+
+int gasket_interrupt_reinit(struct gasket_dev *gasket_dev)
+{
+	int ret;
+
+	if (!gasket_dev->interrupt_data) {
+		gasket_log_error(
+			gasket_dev,
+			"Attempted to reinit uninitialized interrupt data.");
+		return -EINVAL;
+	}
+
+	switch (gasket_dev->interrupt_data->type) {
+	case PCI_MSIX:
+		gasket_interrupt_msix_cleanup(gasket_dev->interrupt_data);
+		ret = gasket_interrupt_msix_init(gasket_dev->interrupt_data);
+		if (ret)
+			break;
+		force_msix_interrupt_unmasking(gasket_dev);
+		break;
+
+	case PCI_MSI:
+	case PLATFORM_WIRE:
+	default:
+		gasket_nodev_error(
+			"Cannot handle unsupported interrupt type %d.",
+			gasket_dev->interrupt_data->type);
+		ret = -EINVAL;
+	};
+
+	if (ret) {
+		/* Failing to setup MSIx will cause the device
+		 * to report GASKET_STATUS_LAMED, but is not fatal.
+		 */
+		gasket_log_warn(gasket_dev, "Couldn't init msix: %d", ret);
+		return 0;
+	}
+
+	gasket_interrupt_setup(gasket_dev);
+
+	return 0;
+}
+
+/* See gasket_interrupt.h for description. */
+int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev)
+{
+	gasket_log_debug(gasket_dev, "Clearing interrupt counts.");
+	memset(gasket_dev->interrupt_data->interrupt_counts, 0,
+	       gasket_dev->interrupt_data->num_interrupts *
+			sizeof(*gasket_dev->interrupt_data->interrupt_counts));
+	return 0;
+}
+
+/*
+ * Set up device registers for interrupt handling.
+ * @gasket_dev: The Gasket information structure for this device.
+ *
+ * Sets up the device registers with the correct indices for the relevant
+ * interrupts.
+ */
+static void gasket_interrupt_setup(struct gasket_dev *gasket_dev)
+{
+	int i;
+	int pack_shift;
+	ulong mask;
+	ulong value;
+	struct gasket_interrupt_data *interrupt_data =
+		gasket_dev->interrupt_data;
+
+	if (!interrupt_data) {
+		gasket_log_error(
+			gasket_dev, "Interrupt data is not initialized.");
+		return;
+	}
+
+	gasket_log_debug(gasket_dev, "Running interrupt setup.");
+
+	if (interrupt_data->type == PLATFORM_WIRE ||
+	    interrupt_data->type == PCI_MSI) {
+		/* Nothing needs to be done for platform or PCI devices. */
+		return;
+	}
+
+	if (interrupt_data->type != PCI_MSIX) {
+		gasket_nodev_error(
+			"Cannot handle unsupported interrupt type %d.",
+			interrupt_data->type);
+		return;
+	}
+
+	/* Setup the MSIX table. */
+
+	for (i = 0; i < interrupt_data->num_interrupts; i++) {
+		/*
+		 * If the interrupt is not packed, we can write the index into
+		 * the register directly. If not, we need to deal with a read-
+		 * modify-write and shift based on the packing index.
+		 */
+		gasket_log_debug(
+			gasket_dev,
+			"Setting up interrupt index %d with index 0x%llx and "
+			"packing %d",
+			interrupt_data->interrupts[i].index,
+			interrupt_data->interrupts[i].reg,
+			interrupt_data->interrupts[i].packing);
+		if (interrupt_data->interrupts[i].packing == UNPACKED) {
+			value = interrupt_data->interrupts[i].index;
+		} else {
+			switch (interrupt_data->interrupts[i].packing) {
+			case PACK_0:
+				pack_shift = 0;
+				break;
+			case PACK_1:
+				pack_shift = interrupt_data->pack_width;
+				break;
+			case PACK_2:
+				pack_shift = 2 * interrupt_data->pack_width;
+				break;
+			case PACK_3:
+				pack_shift = 3 * interrupt_data->pack_width;
+				break;
+			default:
+				gasket_nodev_error(
+					"Found interrupt description with "
+					"unknown enum %d.",
+					interrupt_data->interrupts[i].packing);
+				return;
+			}
+
+			mask = ~(0xFFFF << pack_shift);
+			value = gasket_dev_read_64(
+					gasket_dev,
+					interrupt_data->interrupt_bar_index,
+					interrupt_data->interrupts[i].reg) &
+				mask;
+			value |= interrupt_data->interrupts[i].index
+				 << pack_shift;
+		}
+		gasket_dev_write_64(gasket_dev, value,
+				    interrupt_data->interrupt_bar_index,
+				    interrupt_data->interrupts[i].reg);
+	}
+}
+
+/* See gasket_interrupt.h for description. */
+void gasket_interrupt_pause(struct gasket_dev *gasket_dev, int enable_pause)
+{
+	WARN_ON(!gasket_dev);
+
+	if (!gasket_dev->interrupt_data)
+		return; /* nothing to do */
+
+	if (gasket_dev->interrupt_data->type == PCI_MSI ||
+	    gasket_dev->interrupt_data->type == PCI_MSIX) {
+		/* Nothing to be done for MSI/MSIX just yet. */
+	}
+
+	if (gasket_dev->interrupt_data->type == PLATFORM_WIRE) {
+		/* Nothing to be done for PLATFORM_WIRE */
+	}
+}
+EXPORT_SYMBOL(gasket_interrupt_pause);
+
+void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev)
+{
+	struct gasket_interrupt_data *interrupt_data =
+		gasket_dev->interrupt_data;
+	/*
+	 * It is possible to get an error code from gasket_interrupt_init
+	 * before interrupt_data has been allocated, so check it.
+	 */
+	if (!interrupt_data)
+		return;
+
+	switch (interrupt_data->type) {
+	case PCI_MSIX:
+		gasket_interrupt_msix_cleanup(interrupt_data);
+		break;
+
+	case PCI_MSI:
+	case PLATFORM_WIRE:
+	default:
+		gasket_nodev_error(
+			"Cannot handle unsupported interrupt type %d.",
+			interrupt_data->type);
+	};
+
+	kfree(interrupt_data->interrupt_counts);
+	kfree(interrupt_data->eventfd_ctxs);
+	kfree(interrupt_data->msix_entries);
+	kfree(interrupt_data);
+	gasket_dev->interrupt_data = NULL;
+}
+
+int gasket_interrupt_system_status(struct gasket_dev *gasket_dev)
+{
+	if (!gasket_dev->interrupt_data) {
+		gasket_nodev_info("Interrupt data is null.");
+		return GASKET_STATUS_DEAD;
+	}
+
+	if (!gasket_dev->interrupt_data->msix_configured) {
+		gasket_nodev_info("Interrupt not initialized.");
+		return GASKET_STATUS_LAMED;
+	}
+
+	if (gasket_dev->interrupt_data->num_configured !=
+		gasket_dev->interrupt_data->num_interrupts) {
+		gasket_nodev_info("Not all interrupts were configured.");
+		return GASKET_STATUS_LAMED;
+	}
+
+	return GASKET_STATUS_ALIVE;
+}
+
+int gasket_interrupt_set_eventfd(
+	struct gasket_interrupt_data *interrupt_data, int interrupt,
+	int event_fd)
+{
+	struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd);
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	if (interrupt < 0 || interrupt > interrupt_data->num_interrupts)
+		return -EINVAL;
+
+	interrupt_data->eventfd_ctxs[interrupt] = ctx;
+	return 0;
+}
+
+int gasket_interrupt_clear_eventfd(
+	struct gasket_interrupt_data *interrupt_data, int interrupt)
+{
+	if (interrupt < 0 || interrupt > interrupt_data->num_interrupts)
+		return -EINVAL;
+
+	interrupt_data->eventfd_ctxs[interrupt] = NULL;
+	return 0;
+}
+
+int gasket_interrupt_trigger_eventfd(
+	struct gasket_interrupt_data *interrupt_data, int interrupt)
+{
+	struct eventfd_ctx *ctx = interrupt_data->eventfd_ctxs[interrupt];
+
+	if (!ctx)
+		return -EINVAL;
+
+	eventfd_signal(ctx, 1);
+	return 0;
+}
+
+struct msix_entry *gasket_interrupt_get_msix_entries(
+	struct gasket_interrupt_data *interrupt_data)
+{
+	return interrupt_data->msix_entries;
+}
+
+struct eventfd_ctx **gasket_interrupt_get_eventfd_ctxs(
+	struct gasket_interrupt_data *interrupt_data)
+{
+	return interrupt_data->eventfd_ctxs;
+}
+EXPORT_SYMBOL(gasket_interrupt_get_eventfd_ctxs);
+
+static ssize_t interrupt_sysfs_show(
+	struct device *device, struct device_attribute *attr, char *buf)
+{
+	int i, ret;
+	ssize_t written = 0, total_written = 0;
+	struct gasket_interrupt_data *interrupt_data;
+	struct gasket_dev *gasket_dev;
+	struct gasket_sysfs_attribute *gasket_attr;
+	enum interrupt_sysfs_attribute_type sysfs_type;
+
+	gasket_dev = gasket_sysfs_get_device_data(device);
+	if (!gasket_dev) {
+		gasket_nodev_error(
+			"No sysfs mapping found for device 0x%p", device);
+		return 0;
+	}
+
+	gasket_attr = gasket_sysfs_get_attr(device, attr);
+	if (!gasket_attr) {
+		gasket_nodev_error(
+			"No sysfs attr data found for device 0x%p", device);
+		gasket_sysfs_put_device_data(device, gasket_dev);
+		return 0;
+	}
+
+	sysfs_type = (enum interrupt_sysfs_attribute_type)
+		gasket_attr->data.attr_type;
+	interrupt_data = gasket_dev->interrupt_data;
+	switch (sysfs_type) {
+	case ATTR_INTERRUPT_COUNTS:
+		for (i = 0; i < interrupt_data->num_interrupts; ++i) {
+			written =
+				scnprintf(buf, PAGE_SIZE - total_written,
+					  "0x%02x: %ld\n", i,
+					  interrupt_data->interrupt_counts[i]);
+			total_written += written;
+			buf += written;
+		}
+		ret = total_written;
+		break;
+	default:
+		gasket_log_error(
+			gasket_dev, "Unknown attribute: %s", attr->attr.name);
+		ret = 0;
+		break;
+	}
+
+	gasket_sysfs_put_attr(device, gasket_attr);
+	gasket_sysfs_put_device_data(device, gasket_dev);
+	return ret;
+}
+
+/*
+ * MSIX interrupt handler, used with PCI driver.
+ */
+static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id)
+{
+	struct eventfd_ctx *ctx;
+	struct gasket_interrupt_data *interrupt_data = dev_id;
+	int interrupt = -1;
+	int i;
+
+	/* If this linear lookup is a problem, we can maintain a map/hash. */
+	for (i = 0; i < interrupt_data->num_interrupts; i++) {
+		if (interrupt_data->msix_entries[i].vector == irq) {
+			interrupt = interrupt_data->msix_entries[i].entry;
+			break;
+		}
+	}
+	if (interrupt == -1) {
+		gasket_nodev_error("Received unknown irq %d", irq);
+		return IRQ_HANDLED;
+	}
+	trace_gasket_interrupt_event(interrupt_data->name, interrupt);
+
+	ctx = interrupt_data->eventfd_ctxs[interrupt];
+	if (ctx)
+		eventfd_signal(ctx, 1);
+
+	++(interrupt_data->interrupt_counts[interrupt]);
+
+	return IRQ_HANDLED;
+}

drivers/staging/gasket/gasket_interrupt.h

@@ -0,0 +1,172 @@
+/*
+ * Gasket common interrupt module. Defines functions for enabling
+ * eventfd-triggered interrupts between a Gasket device and a host process.
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef __GASKET_INTERRUPT_H__
+#define __GASKET_INTERRUPT_H__
+
+#include <linux/eventfd.h>
+#include <linux/pci.h>
+
+#include "gasket_core.h"
+
+/* Note that this currently assumes that device interrupts are a dense set,
+ * numbered from 0 - (num_interrupts - 1). Should this have to change, these
+ * APIs will have to be updated.
+ */
+
+/* Opaque type used to hold interrupt subsystem data. */
+struct gasket_interrupt_data;
+
+/*
+ * Initialize the interrupt module.
+ * @gasket_dev: The Gasket device structure for the device to be initted.
+ * @type: Type of the interrupt. (See gasket_interrupt_type).
+ * @name: The name to associate with these interrupts.
+ * @interrupts: An array of all interrupt descriptions for this device.
+ * @num_interrupts: The length of the @interrupts array.
+ * @pack_width: The width, in bits, of a single field in a packed interrupt reg.
+ * @bar_index: The bar containing all interrupt registers.
+ *
+ * Allocates and initializes data to track interrupt state for a device.
+ * After this call, no interrupts will be configured/delivered; call
+ * gasket_interrupt_set_vector[_packed] to associate each interrupt with an
+ * __iomem location, then gasket_interrupt_set_eventfd to associate an eventfd
+ * with an interrupt.
+ *
+ * If num_interrupts interrupts are not available, this call will return a
+ * negative error code. In that case, gasket_interrupt_cleanup should still be
+ * called. Returns 0 on success (which can include a device where interrupts
+ * are not possible to set up, but is otherwise OK; that device will report
+ * status LAMED.)
+ */
+int gasket_interrupt_init(
+	struct gasket_dev *gasket_dev, const char *name, int type,
+	const struct gasket_interrupt_desc *interrupts,
+	int num_interrupts, int pack_width, int bar_index,
+	const struct gasket_wire_interrupt_offsets *wire_int_offsets);
+
+/*
+ * Clean up a device's interrupt structure.
+ * @gasket_dev: The Gasket information structure for this device.
+ *
+ * Cleans up the device's interrupts and deallocates data.
+ */
+void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev);
+
+/*
+ * Clean up and re-initialize the MSI-x subsystem.
+ * @gasket_dev: The Gasket information structure for this device.
+ *
+ * Performs a teardown of the MSI-x subsystem and re-initializes it. Does not
+ * free the underlying data structures. Returns 0 on success and an error code
+ * on error.
+ */
+int gasket_interrupt_reinit(struct gasket_dev *gasket_dev);
+
+/*
+ * Reset the counts stored in the interrupt subsystem.
+ * @gasket_dev: The Gasket information structure for this device.
+ *
+ * Sets the counts of all interrupts in the subsystem to 0.
+ */
+int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev);
+
+/*
+ * Associates an eventfd with a device interrupt.
+ * @data: Pointer to device interrupt data.
+ * @interrupt: The device interrupt to configure.
+ * @event_fd: The eventfd to associate with the interrupt.
+ *
+ * Prepares the host to receive notification of device interrupts by associating
+ * event_fd with interrupt. Upon receipt of a device interrupt, event_fd will be
+ * signaled, after successful configuration.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int gasket_interrupt_set_eventfd(
+	struct gasket_interrupt_data *interrupt_data, int interrupt,
+	int event_fd);
+
+/*
+ * Removes an interrupt-eventfd association.
+ * @data: Pointer to device interrupt data.
+ * @interrupt: The device interrupt to de-associate.
+ *
+ * Removes any eventfd associated with the specified interrupt, if any.
+ */
+int gasket_interrupt_clear_eventfd(
+	struct gasket_interrupt_data *interrupt_data, int interrupt);
+
+/*
+ * Signals the eventfd associated with interrupt.
+ * @data: Pointer to device interrupt data.
+ * @interrupt: The device interrupt to signal for.
+ *
+ * Simulates a device interrupt by signaling the eventfd associated with
+ * interrupt, if any.
+ * Returns 0 if the eventfd was successfully triggered, a negative error code
+ * otherwise (if, for example, no eventfd was associated with interrupt).
+ */
+int gasket_interrupt_trigger_eventfd(
+	struct gasket_interrupt_data *interrupt_data, int interrupt);
+
+/*
+ * The below functions exist for backwards compatibility.
+ * No new uses should be written.
+ */
+/*
+ * Retrieve a pointer to data's MSI-X
+ * entries.
+ * @data: The interrupt data from which to extract.
+ *
+ * Returns the internal pointer to data's MSI-X entries.
+ */
+struct msix_entry *gasket_interrupt_get_msix_entries(
+	struct gasket_interrupt_data *interrupt_data);
+
+/*
+ * Get a pointer to data's eventfd contexts.
+ * @data: The interrupt data from which to extract.
+ *
+ * Returns the internal pointer to data's eventfd contexts.
+ *
+ * This function exists for backwards compatibility with older drivers.
+ * No new uses should be written.
+ */
+struct eventfd_ctx **gasket_interrupt_get_eventfd_ctxs(
+	struct gasket_interrupt_data *interrupt_data);
+
+/*
+ * Get the health of the interrupt subsystem.
+ * @gasket_dev: The Gasket device struct.
+ *
+ * Returns DEAD if not set up, LAMED if initialization failed, and ALIVE
+ * otherwise.
+ */
+
+int gasket_interrupt_system_status(struct gasket_dev *gasket_dev);
+
+/*
+ * Masks interrupts and de-register the handler.
+ * After an interrupt pause it is not guaranteed that the chip registers will
+ * be accessible anymore, since the chip may be in a power save mode,
+ * which means that the interrupt handler (if it were to happen) may not
+ * have a way to clear the interrupt condition.
+ * @gasket_dev: The Gasket device struct
+ * @enable_pause: Whether to pause or unpause the interrupts.
+ */
+void gasket_interrupt_pause(struct gasket_dev *gasket_dev, int enable_pause);
+
+#endif

drivers/staging/gasket/gasket_ioctl.c

@@ -0,0 +1,449 @@
+/* Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include "gasket.h"
+#include "gasket_ioctl.h"
+#include "gasket_constants.h"
+#include "gasket_core.h"
+#include "gasket_interrupt.h"
+#include "gasket_logging.h"
+#include "gasket_page_table.h"
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+
+#ifdef GASKET_KERNEL_TRACE_SUPPORT
+#define CREATE_TRACE_POINTS
+#include <trace/events/gasket_ioctl.h>
+#else
+#define trace_gasket_ioctl_entry(x, ...)
+#define trace_gasket_ioctl_exit(x)
+#define trace_gasket_ioctl_integer_data(x)
+#define trace_gasket_ioctl_eventfd_data(x, ...)
+#define trace_gasket_ioctl_page_table_data(x, ...)
+#define trace_gasket_ioctl_config_coherent_allocator(x, ...)
+#endif
+
+static uint gasket_ioctl_check_permissions(struct file *filp, uint cmd);
+static int gasket_set_event_fd(struct gasket_dev *dev, ulong arg);
+static int gasket_read_page_table_size(
+	struct gasket_dev *gasket_dev, ulong arg);
+static int gasket_read_simple_page_table_size(
+	struct gasket_dev *gasket_dev, ulong arg);
+static int gasket_partition_page_table(
+	struct gasket_dev *gasket_dev, ulong arg);
+static int gasket_map_buffers(struct gasket_dev *gasket_dev, ulong arg);
+static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, ulong arg);
+static int gasket_config_coherent_allocator(
+	struct gasket_dev *gasket_dev, ulong arg);
+
+/*
+ * standard ioctl dispatch function.
+ * @filp: File structure pointer describing this node usage session.
+ * @cmd: ioctl number to handle.
+ * @arg: ioctl-specific data pointer.
+ *
+ * Standard ioctl dispatcher; forwards operations to individual handlers.
+ */
+long gasket_handle_ioctl(struct file *filp, uint cmd, ulong arg)
+{
+	struct gasket_dev *gasket_dev;
+	int retval;
+
+	gasket_dev = (struct gasket_dev *)filp->private_data;
+	trace_gasket_ioctl_entry(gasket_dev->dev_info.name, cmd);
+
+	if (gasket_get_ioctl_permissions_cb(gasket_dev)) {
+		retval = gasket_get_ioctl_permissions_cb(gasket_dev)(
+			filp, cmd, arg);
+		if (retval < 0) {
+			trace_gasket_ioctl_exit(-EPERM);
+			return retval;
+		} else if (retval == 0) {
+			trace_gasket_ioctl_exit(-EPERM);
+			return -EPERM;
+		}
+	} else if (!gasket_ioctl_check_permissions(filp, cmd)) {
+		trace_gasket_ioctl_exit(-EPERM);
+		gasket_log_error(gasket_dev, "ioctl cmd=%x noperm.", cmd);
+		return -EPERM;
+	}
+
+	/* Tracing happens in this switch statement for all ioctls with
+	 * an integer argrument, but ioctls with a struct argument
+	 * that needs copying and decoding, that tracing is done within
+	 * the handler call.
+	 */
+	switch (cmd) {
+	case GASKET_IOCTL_RESET:
+		trace_gasket_ioctl_integer_data(arg);
+		retval = gasket_reset(gasket_dev, arg);
+		break;
+	case GASKET_IOCTL_SET_EVENTFD:
+		retval = gasket_set_event_fd(gasket_dev, arg);
+		break;
+	case GASKET_IOCTL_CLEAR_EVENTFD:
+		trace_gasket_ioctl_integer_data(arg);
+		retval = gasket_interrupt_clear_eventfd(
+			gasket_dev->interrupt_data, (int)arg);
+		break;
+	case GASKET_IOCTL_PARTITION_PAGE_TABLE:
+		trace_gasket_ioctl_integer_data(arg);
+		retval = gasket_partition_page_table(gasket_dev, arg);
+		break;
+	case GASKET_IOCTL_NUMBER_PAGE_TABLES:
+		trace_gasket_ioctl_integer_data(gasket_dev->num_page_tables);
+		if (copy_to_user((void __user *)arg,
+				 &gasket_dev->num_page_tables,
+				 sizeof(uint64_t)))
+			retval = -EFAULT;
+		else
+			retval = 0;
+		break;
+	case GASKET_IOCTL_PAGE_TABLE_SIZE:
+		retval = gasket_read_page_table_size(gasket_dev, arg);
+		break;
+	case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE:
+		retval = gasket_read_simple_page_table_size(gasket_dev, arg);
+		break;
+	case GASKET_IOCTL_MAP_BUFFER:
+		retval = gasket_map_buffers(gasket_dev, arg);
+		break;
+	case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR:
+		retval = gasket_config_coherent_allocator(gasket_dev, arg);
+		break;
+	case GASKET_IOCTL_UNMAP_BUFFER:
+		retval = gasket_unmap_buffers(gasket_dev, arg);
+		break;
+	case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS:
+		/* Clear interrupt counts doesn't take an arg, so use 0. */
+		trace_gasket_ioctl_integer_data(0);
+		retval = gasket_interrupt_reset_counts(gasket_dev);
+		break;
+	default:
+		/* If we don't understand the ioctl, the best we can do is trace
+		 * the arg.
+		 */
+		trace_gasket_ioctl_integer_data(arg);
+		gasket_log_warn(
+			gasket_dev,
+			"Unknown ioctl cmd=0x%x not caught by "
+			"gasket_is_supported_ioctl",
+			cmd);
+		retval = -EINVAL;
+		break;
+	}
+
+	trace_gasket_ioctl_exit(retval);
+	return retval;
+}
+
+/*
+ * Determines if an ioctl is part of the standard Gasket framework.
+ * @cmd: The ioctl number to handle.
+ *
+ * Returns 1 if the ioctl is supported and 0 otherwise.
+ */
+long gasket_is_supported_ioctl(uint cmd)
+{
+	switch (cmd) {
+	case GASKET_IOCTL_RESET:
+	case GASKET_IOCTL_SET_EVENTFD:
+	case GASKET_IOCTL_CLEAR_EVENTFD:
+	case GASKET_IOCTL_PARTITION_PAGE_TABLE:
+	case GASKET_IOCTL_NUMBER_PAGE_TABLES:
+	case GASKET_IOCTL_PAGE_TABLE_SIZE:
+	case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE:
+	case GASKET_IOCTL_MAP_BUFFER:
+	case GASKET_IOCTL_UNMAP_BUFFER:
+	case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS:
+	case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR:
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+/*
+ * Permission checker for Gasket ioctls.
+ * @filp: File structure pointer describing this node usage session.
+ * @cmd: ioctl number to handle.
+ *
+ * Standard permissions checker.
+ */
+static uint gasket_ioctl_check_permissions(struct file *filp, uint cmd)
+{
+	uint alive, root, device_owner;
+	fmode_t read, write;
+	struct gasket_dev *gasket_dev = (struct gasket_dev *)filp->private_data;
+
+	alive = (gasket_dev->status == GASKET_STATUS_ALIVE);
+	if (!alive) {
+		gasket_nodev_error(
+			"gasket_ioctl_check_permissions alive %d status %d.",
+			alive, gasket_dev->status);
+	}
+
+	root = capable(CAP_SYS_ADMIN);
+	read = filp->f_mode & FMODE_READ;
+	write = filp->f_mode & FMODE_WRITE;
+	device_owner = (gasket_dev->dev_info.ownership.is_owned &&
+			current->tgid == gasket_dev->dev_info.ownership.owner);
+
+	switch (cmd) {
+	case GASKET_IOCTL_RESET:
+	case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS:
+		return root || (write && device_owner);
+
+	case GASKET_IOCTL_PAGE_TABLE_SIZE:
+	case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE:
+	case GASKET_IOCTL_NUMBER_PAGE_TABLES:
+		return root || read;
+
+	case GASKET_IOCTL_PARTITION_PAGE_TABLE:
+	case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR:
+		return alive && (root || (write && device_owner));
+
+	case GASKET_IOCTL_MAP_BUFFER:
+	case GASKET_IOCTL_UNMAP_BUFFER:
+		return alive && (root || (write && device_owner));
+
+	case GASKET_IOCTL_CLEAR_EVENTFD:
+	case GASKET_IOCTL_SET_EVENTFD:
+		return alive && (root || (write && device_owner));
+	}
+
+	return 0; /* unknown permissions */
+}
+
+/*
+ * Associate an eventfd with an interrupt.
+ * @gasket_dev: Pointer to the current gasket_dev we're using.
+ * @arg: Pointer to gasket_interrupt_eventfd struct in userspace.
+ */
+static int gasket_set_event_fd(struct gasket_dev *gasket_dev, ulong arg)
+{
+	struct gasket_interrupt_eventfd die;
+
+	if (copy_from_user(&die, (void __user *)arg,
+			   sizeof(struct gasket_interrupt_eventfd))) {
+		return -EFAULT;
+	}
+
+	trace_gasket_ioctl_eventfd_data(die.interrupt, die.event_fd);
+
+	return gasket_interrupt_set_eventfd(
+		gasket_dev->interrupt_data, die.interrupt, die.event_fd);
+}
+
+/*
+ * Reads the size of the page table.
+ * @gasket_dev: Pointer to the current gasket_dev we're using.
+ * @arg: Pointer to gasket_page_table_ioctl struct in userspace.
+ */
+static int gasket_read_page_table_size(struct gasket_dev *gasket_dev, ulong arg)
+{
+	int ret = 0;
+	struct gasket_page_table_ioctl ibuf;
+
+	if (copy_from_user(&ibuf, (void __user *)arg,
+			   sizeof(struct gasket_page_table_ioctl)))
+		return -EFAULT;
+
+	if (ibuf.page_table_index >= gasket_dev->num_page_tables)
+		return -EFAULT;
+
+	ibuf.size = gasket_page_table_num_entries(
+		gasket_dev->page_table[ibuf.page_table_index]);
+
+	trace_gasket_ioctl_page_table_data(
+		ibuf.page_table_index, ibuf.size, ibuf.host_address,
+		ibuf.device_address);
+
+	if (copy_to_user((void __user *)arg, &ibuf, sizeof(ibuf)))
+		return -EFAULT;
+
+	return ret;
+}
+
+/*
+ * Reads the size of the simple page table.
+ * @gasket_dev: Pointer to the current gasket_dev we're using.
+ * @arg: Pointer to gasket_page_table_ioctl struct in userspace.
+ */
+static int gasket_read_simple_page_table_size(
+	struct gasket_dev *gasket_dev, ulong arg)
+{
+	int ret = 0;
+	struct gasket_page_table_ioctl ibuf;
+
+	if (copy_from_user(&ibuf, (void __user *)arg,
+			   sizeof(struct gasket_page_table_ioctl)))
+		return -EFAULT;
+
+	if (ibuf.page_table_index >= gasket_dev->num_page_tables)
+		return -EFAULT;
+
+	ibuf.size = gasket_page_table_num_simple_entries(
+		gasket_dev->page_table[ibuf.page_table_index]);
+
+	trace_gasket_ioctl_page_table_data(
+		ibuf.page_table_index, ibuf.size, ibuf.host_address,
+		ibuf.device_address);
+
+	if (copy_to_user((void __user *)arg, &ibuf, sizeof(ibuf)))
+		return -EFAULT;
+
+	return ret;
+}
+
+/*
+ * Sets the boundary between the simple and extended page tables.
+ * @gasket_dev: Pointer to the current gasket_dev we're using.
+ * @arg: Pointer to gasket_page_table_ioctl struct in userspace.
+ */
+static int gasket_partition_page_table(struct gasket_dev *gasket_dev, ulong arg)
+{
+	int ret;
+	struct gasket_page_table_ioctl ibuf;
+	uint max_page_table_size;
+
+	if (copy_from_user(&ibuf, (void __user *)arg,
+			   sizeof(struct gasket_page_table_ioctl)))
+		return -EFAULT;
+
+	trace_gasket_ioctl_page_table_data(
+		ibuf.page_table_index, ibuf.size, ibuf.host_address,
+		ibuf.device_address);
+
+	if (ibuf.page_table_index >= gasket_dev->num_page_tables)
+		return -EFAULT;
+	max_page_table_size = gasket_page_table_max_size(
+		gasket_dev->page_table[ibuf.page_table_index]);
+
+	if (ibuf.size > max_page_table_size) {
+		gasket_log_error(
+			gasket_dev,
+			"Partition request 0x%llx too large, max is 0x%x.",
+			ibuf.size, max_page_table_size);
+		return -EINVAL;
+	}
+
+	mutex_lock(&gasket_dev->mutex);
+
+	ret = gasket_page_table_partition(
+		gasket_dev->page_table[ibuf.page_table_index], ibuf.size);
+	mutex_unlock(&gasket_dev->mutex);
+
+	return ret;
+}
+
+/*
+ * Maps a userspace buffer to a device virtual address.
+ * @gasket_dev: Pointer to the current gasket_dev we're using.
+ * @arg: Pointer to a gasket_page_table_ioctl struct in userspace.
+ */
+static int gasket_map_buffers(struct gasket_dev *gasket_dev, ulong arg)
+{
+	struct gasket_page_table_ioctl ibuf;
+
+	if (copy_from_user(&ibuf, (void __user *)arg,
+			   sizeof(struct gasket_page_table_ioctl)))
+		return -EFAULT;
+
+	trace_gasket_ioctl_page_table_data(
+		ibuf.page_table_index, ibuf.size, ibuf.host_address,
+		ibuf.device_address);
+
+	if (ibuf.page_table_index >= gasket_dev->num_page_tables)
+		return -EFAULT;
+
+	if (gasket_page_table_are_addrs_bad(
+		    gasket_dev->page_table[ibuf.page_table_index],
+		    ibuf.host_address, ibuf.device_address, ibuf.size))
+		return -EINVAL;
+
+	return gasket_page_table_map(
+		gasket_dev->page_table[ibuf.page_table_index],
+		ibuf.host_address, ibuf.device_address, ibuf.size / PAGE_SIZE);
+}
+
+/*
+ * Unmaps a userspace buffer from a device virtual address.
+ * @gasket_dev: Pointer to the current gasket_dev we're using.
+ * @arg: Pointer to a gasket_page_table_ioctl struct in userspace.
+ */
+static int gasket_unmap_buffers(struct gasket_dev *gasket_dev, ulong arg)
+{
+	struct gasket_page_table_ioctl ibuf;
+
+	if (copy_from_user(&ibuf, (void __user *)arg,
+			   sizeof(struct gasket_page_table_ioctl)))
+		return -EFAULT;
+
+	trace_gasket_ioctl_page_table_data(
+		ibuf.page_table_index, ibuf.size, ibuf.host_address,
+		ibuf.device_address);
+
+	if (ibuf.page_table_index >= gasket_dev->num_page_tables)
+		return -EFAULT;
+
+	if (gasket_page_table_is_dev_addr_bad(
+		    gasket_dev->page_table[ibuf.page_table_index],
+		    ibuf.device_address, ibuf.size))
+		return -EINVAL;
+
+	gasket_page_table_unmap(gasket_dev->page_table[ibuf.page_table_index],
+				ibuf.device_address, ibuf.size / PAGE_SIZE);
+
+	return 0;
+}
+
+/*
+ * Tell the driver to reserve structures for coherent allocation, and allocate
+ * or free the corresponding memory.
+ * @dev: Pointer to the current gasket_dev we're using.
+ * @arg: Pointer to a gasket_coherent_alloc_config_ioctl struct in userspace.
+ */
+static int gasket_config_coherent_allocator(
+	struct gasket_dev *gasket_dev, ulong arg)
+{
+	int ret;
+	struct gasket_coherent_alloc_config_ioctl ibuf;
+
+	if (copy_from_user(&ibuf, (void __user *)arg,
+			   sizeof(struct gasket_coherent_alloc_config_ioctl)))
+		return -EFAULT;
+
+	trace_gasket_ioctl_config_coherent_allocator(
+		ibuf.enable, ibuf.size, ibuf.dma_address);
+
+	if (ibuf.page_table_index >= gasket_dev->num_page_tables)
+		return -EFAULT;
+
+	if (ibuf.size > PAGE_SIZE * MAX_NUM_COHERENT_PAGES) {
+		ibuf.size = PAGE_SIZE * MAX_NUM_COHERENT_PAGES;
+		return -ENOMEM;
+	}
+
+	if (ibuf.enable == 0) {
+		ret = gasket_free_coherent_memory(
+			gasket_dev, ibuf.size, ibuf.dma_address,
+			ibuf.page_table_index);
+	} else {
+		ret = gasket_alloc_coherent_memory(
+			gasket_dev, ibuf.size, &ibuf.dma_address,
+			ibuf.page_table_index);
+	}
+	if (copy_to_user((void __user *)arg, &ibuf, sizeof(ibuf)))
+		return -EFAULT;
+
+	return ret;
+}

drivers/staging/gasket/gasket_ioctl.h

@@ -0,0 +1,35 @@
+/* Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef __GASKET_IOCTL_H__
+#define __GASKET_IOCTL_H__
+
+#include "gasket_core.h"
+
+/*
+ * Handle Gasket common ioctls.
+ * @filp: Pointer to the ioctl's file.
+ * @cmd: Ioctl command.
+ * @arg: Ioctl argument pointer.
+ *
+ * Returns 0 on success and nonzero on failure.
+ */
+long gasket_handle_ioctl(struct file *filp, uint cmd, ulong arg);
+
+/*
+ * Determines if an ioctl is part of the standard Gasket framework.
+ * @cmd: The ioctl number to handle.
+ *
+ * Returns 1 if the ioctl is supported and 0 otherwise.
+ */
+long gasket_is_supported_ioctl(uint cmd);
+
+#endif

drivers/staging/gasket/gasket_logging.h

@@ -0,0 +1,71 @@
+/* Common logging utilities for the Gasket driver framework.
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/pci.h>
+#include <linux/printk.h>
+
+#ifndef _GASKET_LOGGING_H_
+#define _GASKET_LOGGING_H_
+
+/* Base macro; other logging can/should be built on top of this. */
+#define gasket_dev_log(level, device, pci_dev, format, arg...)                 \
+	if (false) {                                                           \
+		if (pci_dev) {                                                 \
+			dev_##level(&(pci_dev)->dev, "%s: " format "\n",       \
+				__func__, ##arg);                              \
+		} else {                                                       \
+			gasket_nodev_log(level, format, ##arg);                \
+		}                                                              \
+	}
+
+/* "No-device" logging macros. */
+#define gasket_nodev_log(level, format, arg...)                                \
+	if (false) pr_##level("gasket: %s: " format "\n", __func__, ##arg)
+
+#define gasket_nodev_debug(format, arg...)                                     \
+	if (false) gasket_nodev_log(debug, format, ##arg)
+
+#define gasket_nodev_info(format, arg...)                                      \
+	if (false) gasket_nodev_log(info, format, ##arg)
+
+#define gasket_nodev_warn(format, arg...)                                      \
+	if (false) gasket_nodev_log(warn, format, ##arg)
+
+#define gasket_nodev_error(format, arg...)                                     \
+	if (false) gasket_nodev_log(err, format, ##arg)
+
+/* gasket_dev logging macros */
+#define gasket_log_info(gasket_dev, format, arg...)                            \
+	if (false) gasket_dev_log(info, (gasket_dev)->dev_info.device,         \
+		(gasket_dev)->pci_dev, format, ##arg)
+
+#define gasket_log_warn(gasket_dev, format, arg...)                            \
+	if (false) gasket_dev_log(warn, (gasket_dev)->dev_info.device,         \
+		(gasket_dev)->pci_dev, format, ##arg)
+
+#define gasket_log_error(gasket_dev, format, arg...)                           \
+	if (false) gasket_dev_log(err, (gasket_dev)->dev_info.device,          \
+		(gasket_dev)->pci_dev, format, ##arg)
+
+#define gasket_log_debug(gasket_dev, format, arg...)                           \
+	if (false){                                                            \
+		if ((gasket_dev)->pci_dev) {                                   \
+			dev_dbg(&((gasket_dev)->pci_dev)->dev, "%s: " format   \
+			"\n", __func__, ##arg);                                \
+		} else {                                                       \
+			gasket_nodev_log(debug, format, ##arg);                \
+		}                                                              \
+	}
+
+#endif

drivers/staging/gasket/gasket_page_table.c

@@ -0,0 +1,1771 @@
+/* Implementation of Gasket page table support.
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/*
+ * Implementation of Gasket page table support.
+ *
+ * This file assumes 4kB pages throughout; can be factored out when necessary.
+ *
+ * Address format is as follows:
+ * Simple addresses - those whose containing pages are directly placed in the
+ * device's address translation registers - are laid out as:
+ * [ 63 - 40: Unused | 39 - 28: 0 | 27 - 12: page index | 11 - 0: page offset ]
+ * page index:  The index of the containing page in the device's address
+ *              translation registers.
+ * page offset: The index of the address into the containing page.
+ *
+ * Extended address - those whose containing pages are contained in a second-
+ * level page table whose address is present in the device's address translation
+ * registers - are laid out as:
+ * [ 63 - 40: Unused | 39: flag | 38 - 37: 0 | 36 - 21: dev/level 0 index |
+ *   20 - 12: host/level 1 index | 11 - 0: page offset ]
+ * flag:        Marker indicating that this is an extended address. Always 1.
+ * dev index:   The index of the first-level page in the device's extended
+ *              address translation registers.
+ * host index:  The index of the containing page in the [host-resident] second-
+ *              level page table.
+ * page offset: The index of the address into the containing [second-level]
+ *              page.
+ */
+#include "gasket_page_table.h"
+
+#include <linux/file.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pagemap.h>
+#include <linux/vmalloc.h>
+
+#include "gasket_constants.h"
+#include "gasket_core.h"
+#include "gasket_logging.h"
+
+/* Constants & utility macros */
+/* The number of pages that can be mapped into each second-level page table. */
+#define GASKET_PAGES_PER_SUBTABLE 512
+
+/* The starting position of the page index in a simple virtual address. */
+#define GASKET_SIMPLE_PAGE_SHIFT 12
+
+/* Flag indicating that a [device] slot is valid for use. */
+#define GASKET_VALID_SLOT_FLAG 1
+
+/*
+ * The starting position of the level 0 page index (i.e., the entry in the
+ * device's extended address registers) in an extended address.
+ * Also can be thought of as (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE)),
+ * or (12 + 9).
+ */
+#define GASKET_EXTENDED_LVL0_SHIFT 21
+
+/*
+ * Number of first level pages that Gasket chips support. Equivalent to
+ * log2(NUM_LVL0_PAGE_TABLES)
+ *
+ * At a maximum, allowing for a 34 bits address space (or 16GB)
+ *   = GASKET_EXTENDED_LVL0_WIDTH + (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE)
+ * or, = 13 + 9 + 12
+ */
+#define GASKET_EXTENDED_LVL0_WIDTH 13
+
+/*
+ * The starting position of the level 1 page index (i.e., the entry in the
+ * host second-level/sub- table) in an extended address.
+ */
+#define GASKET_EXTENDED_LVL1_SHIFT 12
+
+/* Page-table specific error logging. */
+#define gasket_pg_tbl_error(pg_tbl, format, arg...)                            \
+	gasket_dev_log(err, (pg_tbl)->device, (struct pci_dev *)NULL, format,  \
+		##arg)
+
+/* Type declarations */
+/* Valid states for a struct gasket_page_table_entry. */
+enum pte_status {
+	PTE_FREE,
+	PTE_INUSE,
+};
+
+/*
+ * Mapping metadata for a single page.
+ *
+ * In this file, host-side page table entries are referred to as that (or PTEs).
+ * Where device vs. host entries are differentiated, device-side or -visible
+ * entries are called "slots". A slot may be either an entry in the device's
+ * address translation table registers or an entry in a second-level page
+ * table ("subtable").
+ *
+ * The full data in this structure is visible on the host [of course]. Only
+ * the address contained in dma_addr is communicated to the device; that points
+ * to the actual page mapped and described by this structure.
+ */
+struct gasket_page_table_entry {
+	/* The status of this entry/slot: free or in use. */
+	enum pte_status status;
+
+	/* Address of the page in DMA space. */
+	dma_addr_t dma_addr;
+
+	/* Linux page descriptor for the page described by this structure. */
+	struct page *page;
+
+	/*
+	 * Index for alignment into host vaddrs.
+	 * When a user specifies a host address for a mapping, that address may
+	 * not be page-aligned. Offset is the index into the containing page of
+	 * the host address (i.e., host_vaddr & (PAGE_SIZE - 1)).
+	 * This is necessary for translating between user-specified addresses
+	 * and page-aligned addresses.
+	 */
+	int offset;
+
+	/*
+	 * If this is an extended and first-level entry, sublevel points
+	 * to the second-level entries underneath this entry.
+	 */
+	struct gasket_page_table_entry *sublevel;
+};
+
+/*
+ * Maintains virtual to physical address mapping for a coherent page that is
+ * allocated by this module for a given device.
+ * Note that coherent pages mappings virt mapping cannot be tracked by the
+ * Linux kernel, and coherent pages don't have a struct page associated,
+ * hence Linux kernel cannot perform a get_user_page_xx() on a phys address
+ * that was allocated coherent.
+ * This structure trivially implements this mechanism.
+ */
+struct gasket_coherent_page_entry {
+	/* Phys address, dma'able by the owner device */
+	dma_addr_t paddr;
+
+	/* Kernel virtual address */
+	u64 user_virt;
+
+	/* User virtual address that was mapped by the mmap kernel subsystem */
+	u64 kernel_virt;
+
+	/*
+	 * Whether this page has been mapped into a user land process virtual
+	 * space
+	 */
+	u32 in_use;
+};
+
+/*
+ * [Host-side] page table descriptor.
+ *
+ * This structure tracks the metadata necessary to manage both simple and
+ * extended page tables.
+ */
+struct gasket_page_table {
+	/* The config used to create this page table. */
+	struct gasket_page_table_config config;
+
+	/* The number of simple (single-level) entries in the page table. */
+	uint num_simple_entries;
+
+	/* The number of extended (two-level) entries in the page table. */
+	uint num_extended_entries;
+
+	/* Array of [host-side] page table entries. */
+	struct gasket_page_table_entry *entries;
+
+	/* Number of actively mapped kernel pages in this table. */
+	uint num_active_pages;
+
+	/* Device register: base of/first slot in the page table. */
+	u64 __iomem *base_slot;
+
+	/* Device register: holds the offset indicating the start of the
+	 * extended address region of the device's address translation table.
+	 */
+	u64 __iomem *extended_offset_reg;
+
+	/* Device structure for the underlying device. Only used for logging. */
+	struct device *device;
+
+	/* PCI system descriptor for the underlying device. */
+	struct pci_dev *pci_dev;
+
+	/* Location of the extended address bit for this Gasket device. */
+	u64 extended_flag;
+
+	/* Mutex to protect page table internals. */
+	struct mutex mutex;
+
+	/* Number of coherent pages accessible thru by this page table */
+	int num_coherent_pages;
+
+	/*
+	 * List of coherent memory (physical) allocated for a device.
+	 *
+	 * This structure also remembers the user virtual mapping, this is
+	 * hacky, but we need to do this because the kernel doesn't keep track
+	 * of the user coherent pages (pfn pages), and virt to coherent page
+	 * mapping.
+	 * TODO: use find_vma() APIs to convert host address to vm_area, to
+	 * dma_addr_t instead of storing user virtu address in
+	 * gasket_coherent_page_entry
+	 *
+	 * Note that the user virtual mapping is created by the driver, in
+	 * gasket_mmap function, so user_virt belongs in the driver anyhow.
+	 */
+	struct gasket_coherent_page_entry *coherent_pages;
+
+	/*
+	 * Whether the page table uses arch specific dma_ops or
+	 * whether the driver is supplying its own.
+	 */
+	bool dma_ops;
+};
+
+/* Mapping declarations */
+static int gasket_map_simple_pages(
+	struct gasket_page_table *pg_tbl, ulong host_addr,
+	ulong dev_addr, uint num_pages);
+static int gasket_map_extended_pages(
+	struct gasket_page_table *pg_tbl, ulong host_addr,
+	ulong dev_addr, uint num_pages);
+static int gasket_perform_mapping(
+	struct gasket_page_table *pg_tbl,
+	struct gasket_page_table_entry *pte_base, u64 __iomem *att_base,
+	ulong host_addr, uint num_pages, int is_simple_mapping);
+
+static int gasket_alloc_simple_entries(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages);
+static int gasket_alloc_extended_entries(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_entries);
+static int gasket_alloc_extended_subtable(
+	struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte,
+	u64 __iomem *att_reg);
+
+/* Unmapping declarations */
+static void gasket_page_table_unmap_nolock(
+	struct gasket_page_table *pg_tbl, ulong start_addr, uint num_pages);
+static void gasket_page_table_unmap_all_nolock(
+	struct gasket_page_table *pg_tbl);
+static void gasket_unmap_simple_pages(
+	struct gasket_page_table *pg_tbl, ulong start_addr, uint num_pages);
+static void gasket_unmap_extended_pages(
+	struct gasket_page_table *pg_tbl, ulong start_addr, uint num_pages);
+static void gasket_perform_unmapping(
+	struct gasket_page_table *pg_tbl,
+	struct gasket_page_table_entry *pte_base, u64 __iomem *att_base,
+	uint num_pages, int is_simple_mapping);
+
+static void gasket_free_extended_subtable(
+	struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte,
+	u64 __iomem *att_reg);
+static int gasket_release_page(struct page *page);
+
+/* Other/utility declarations */
+static inline int gasket_addr_is_simple(
+	struct gasket_page_table *pg_tbl, ulong addr);
+static int gasket_is_simple_dev_addr_bad(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages);
+static int gasket_is_extended_dev_addr_bad(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages);
+static int gasket_is_pte_range_free(
+	struct gasket_page_table_entry *pte, uint num_entries);
+static void gasket_page_table_garbage_collect_nolock(
+	struct gasket_page_table *pg_tbl);
+
+/* Address format declarations */
+static ulong gasket_components_to_dev_address(
+	struct gasket_page_table *pg_tbl, int is_simple, uint page_index,
+	uint offset);
+static int gasket_simple_page_idx(
+	struct gasket_page_table *pg_tbl, ulong dev_addr);
+static ulong gasket_extended_lvl0_page_idx(
+	struct gasket_page_table *pg_tbl, ulong dev_addr);
+static ulong gasket_extended_lvl1_page_idx(
+	struct gasket_page_table *pg_tbl, ulong dev_addr);
+
+static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr);
+
+/* Public/exported functions */
+/* See gasket_page_table.h for description. */
+int gasket_page_table_init(
+	struct gasket_page_table **ppg_tbl,
+	const struct gasket_bar_data *bar_data,
+	const struct gasket_page_table_config *page_table_config,
+	struct device *device, struct pci_dev *pci_dev, bool has_dma_ops)
+{
+	ulong bytes;
+	struct gasket_page_table *pg_tbl;
+	ulong total_entries = page_table_config->total_entries;
+
+	/*
+	 * TODO: Verify config->total_entries against value read from the
+	 * hardware register that contains the page table size.
+	 */
+	if (total_entries == ULONG_MAX) {
+		gasket_nodev_error(
+			"Error reading page table size. "
+			"Initializing page table with size 0.");
+		total_entries = 0;
+	}
+
+	gasket_nodev_debug(
+		"Attempting to initialize page table of size 0x%lx.",
+		total_entries);
+
+	gasket_nodev_debug(
+		"Table has base reg 0x%x, extended offset reg 0x%x.",
+		page_table_config->base_reg,
+		page_table_config->extended_reg);
+
+	*ppg_tbl = kzalloc(sizeof(**ppg_tbl), GFP_KERNEL);
+	if (!*ppg_tbl) {
+		gasket_nodev_error("No memory for page table.");
+		return -ENOMEM;
+	}
+
+	pg_tbl = *ppg_tbl;
+	bytes = total_entries * sizeof(struct gasket_page_table_entry);
+	if (bytes != 0) {
+		pg_tbl->entries = vmalloc(bytes);
+		if (!pg_tbl->entries) {
+			gasket_nodev_error(
+				"No memory for address translation metadata.");
+			kfree(pg_tbl);
+			*ppg_tbl = NULL;
+			return -ENOMEM;
+		}
+		memset(pg_tbl->entries, 0, bytes);
+	}
+
+	mutex_init(&pg_tbl->mutex);
+	memcpy(&pg_tbl->config, page_table_config, sizeof(*page_table_config));
+	if (pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_NORMAL ||
+	    pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_SIMPLE) {
+		pg_tbl->num_simple_entries = total_entries;
+		pg_tbl->num_extended_entries = 0;
+		pg_tbl->extended_flag =	1ull << page_table_config->extended_bit;
+	} else {
+		pg_tbl->num_simple_entries = 0;
+		pg_tbl->num_extended_entries = total_entries;
+		pg_tbl->extended_flag = 0;
+	}
+	pg_tbl->num_active_pages = 0;
+	pg_tbl->base_slot = (u64 __iomem *)&(
+		bar_data->virt_base[page_table_config->base_reg]);
+	pg_tbl->extended_offset_reg = (u64 __iomem *)&(
+		bar_data->virt_base[page_table_config->extended_reg]);
+	pg_tbl->device = device;
+	pg_tbl->pci_dev = pci_dev;
+	pg_tbl->dma_ops = has_dma_ops;
+
+	gasket_nodev_debug("Page table initialized successfully.");
+
+	return 0;
+}
+
+/* See gasket_page_table.h for description. */
+void gasket_page_table_cleanup(struct gasket_page_table *pg_tbl)
+{
+	/* Deallocate free second-level tables. */
+	gasket_page_table_garbage_collect(pg_tbl);
+
+	/* TODO: Check that all PTEs have been freed? */
+
+	vfree(pg_tbl->entries);
+	pg_tbl->entries = NULL;
+
+	kfree(pg_tbl);
+}
+
+/* See gasket_page_table.h for description. */
+int gasket_page_table_partition(
+	struct gasket_page_table *pg_tbl, uint num_simple_entries)
+{
+	int i, start;
+
+	mutex_lock(&pg_tbl->mutex);
+	if (num_simple_entries > pg_tbl->config.total_entries) {
+		mutex_unlock(&pg_tbl->mutex);
+		return -EINVAL;
+	}
+
+	gasket_page_table_garbage_collect_nolock(pg_tbl);
+
+	start = min(pg_tbl->num_simple_entries, num_simple_entries);
+
+	for (i = start; i < pg_tbl->config.total_entries; i++) {
+		if (pg_tbl->entries[i].status != PTE_FREE) {
+			gasket_pg_tbl_error(pg_tbl, "entry %d is not free", i);
+			mutex_unlock(&pg_tbl->mutex);
+			return -EBUSY;
+		}
+	}
+
+	pg_tbl->num_simple_entries = num_simple_entries;
+	pg_tbl->num_extended_entries =
+		pg_tbl->config.total_entries - num_simple_entries;
+	writeq(num_simple_entries, pg_tbl->extended_offset_reg);
+
+	mutex_unlock(&pg_tbl->mutex);
+	return 0;
+}
+EXPORT_SYMBOL(gasket_page_table_partition);
+
+/*
+ * See gasket_page_table.h for general description.
+ *
+ * gasket_page_table_map calls either gasket_map_simple_pages() or
+ * gasket_map_extended_pages() to actually perform the mapping.
+ *
+ * The page table mutex is held for the entire operation.
+ */
+int gasket_page_table_map(
+	struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr,
+	uint num_pages)
+{
+	int ret;
+
+	if (!num_pages)
+		return 0;
+
+	mutex_lock(&pg_tbl->mutex);
+
+	if (gasket_addr_is_simple(pg_tbl, dev_addr)) {
+		ret = gasket_map_simple_pages(
+			pg_tbl, host_addr, dev_addr, num_pages);
+	} else {
+		ret = gasket_map_extended_pages(
+			pg_tbl, host_addr, dev_addr, num_pages);
+	}
+
+	mutex_unlock(&pg_tbl->mutex);
+
+	gasket_nodev_debug(
+		"gasket_page_table_map done: ha %llx daddr %llx num %d, "
+		"ret %d\n",
+		(unsigned long long)host_addr,
+		(unsigned long long)dev_addr, num_pages, ret);
+	return ret;
+}
+EXPORT_SYMBOL(gasket_page_table_map);
+
+/*
+ * See gasket_page_table.h for general description.
+ *
+ * gasket_page_table_unmap takes the page table lock and calls either
+ * gasket_unmap_simple_pages() or gasket_unmap_extended_pages() to
+ * actually unmap the pages from device space.
+ *
+ * The page table mutex is held for the entire operation.
+ */
+void gasket_page_table_unmap(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages)
+{
+	if (!num_pages)
+		return;
+
+	mutex_lock(&pg_tbl->mutex);
+	gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages);
+	mutex_unlock(&pg_tbl->mutex);
+}
+EXPORT_SYMBOL(gasket_page_table_unmap);
+
+static void gasket_page_table_unmap_all_nolock(struct gasket_page_table *pg_tbl)
+{
+	gasket_unmap_simple_pages(
+		pg_tbl, gasket_components_to_dev_address(pg_tbl, 1, 0, 0),
+		pg_tbl->num_simple_entries);
+	gasket_unmap_extended_pages(
+		pg_tbl, gasket_components_to_dev_address(pg_tbl, 0, 0, 0),
+		pg_tbl->num_extended_entries * GASKET_PAGES_PER_SUBTABLE);
+}
+
+/* See gasket_page_table.h for description. */
+void gasket_page_table_unmap_all(struct gasket_page_table *pg_tbl)
+{
+	mutex_lock(&pg_tbl->mutex);
+	gasket_page_table_unmap_all_nolock(pg_tbl);
+	mutex_unlock(&pg_tbl->mutex);
+}
+EXPORT_SYMBOL(gasket_page_table_unmap_all);
+
+/* See gasket_page_table.h for description. */
+void gasket_page_table_reset(struct gasket_page_table *pg_tbl)
+{
+	mutex_lock(&pg_tbl->mutex);
+	gasket_page_table_unmap_all_nolock(pg_tbl);
+	writeq(pg_tbl->config.total_entries, pg_tbl->extended_offset_reg);
+	mutex_unlock(&pg_tbl->mutex);
+}
+
+/* See gasket_page_table.h for description. */
+void gasket_page_table_garbage_collect(struct gasket_page_table *pg_tbl)
+{
+	mutex_lock(&pg_tbl->mutex);
+	gasket_page_table_garbage_collect_nolock(pg_tbl);
+	mutex_unlock(&pg_tbl->mutex);
+}
+
+/* See gasket_page_table.h for description. */
+int gasket_page_table_lookup_page(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, struct page **ppage,
+	ulong *poffset)
+{
+	uint page_num;
+	struct gasket_page_table_entry *pte;
+
+	mutex_lock(&pg_tbl->mutex);
+	if (gasket_addr_is_simple(pg_tbl, dev_addr)) {
+		page_num = gasket_simple_page_idx(pg_tbl, dev_addr);
+		if (page_num >= pg_tbl->num_simple_entries)
+			goto fail;
+
+		pte = pg_tbl->entries + page_num;
+		if (pte->status != PTE_INUSE)
+			goto fail;
+	} else {
+		/* Find the level 0 entry, */
+		page_num = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
+		if (page_num >= pg_tbl->num_extended_entries)
+			goto fail;
+
+		pte = pg_tbl->entries + pg_tbl->num_simple_entries + page_num;
+		if (pte->status != PTE_INUSE)
+			goto fail;
+
+		/* and its contained level 1 entry. */
+		page_num = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
+		pte = pte->sublevel + page_num;
+		if (pte->status != PTE_INUSE)
+			goto fail;
+	}
+
+	*ppage = pte->page;
+	*poffset = pte->offset;
+	mutex_unlock(&pg_tbl->mutex);
+	return 0;
+
+fail:
+	*ppage = NULL;
+	*poffset = 0;
+	mutex_unlock(&pg_tbl->mutex);
+	return -1;
+}
+
+/* See gasket_page_table.h for description. */
+int gasket_page_table_are_addrs_bad(
+	struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr,
+	ulong bytes)
+{
+	if (host_addr & (PAGE_SIZE - 1)) {
+		gasket_pg_tbl_error(
+			pg_tbl,
+			"host mapping address 0x%lx must be page aligned",
+			host_addr);
+		return 1;
+	}
+
+	return gasket_page_table_is_dev_addr_bad(pg_tbl, dev_addr, bytes);
+}
+EXPORT_SYMBOL(gasket_page_table_are_addrs_bad);
+
+/* See gasket_page_table.h for description. */
+int gasket_page_table_is_dev_addr_bad(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, ulong bytes)
+{
+	uint num_pages = bytes / PAGE_SIZE;
+
+	if (bytes & (PAGE_SIZE - 1)) {
+		gasket_pg_tbl_error(
+			pg_tbl,
+			"mapping size 0x%lX must be page aligned", bytes);
+		return 1;
+	}
+
+	if (num_pages == 0) {
+		gasket_pg_tbl_error(
+			pg_tbl,
+			"requested mapping is less than one page: %lu / %lu",
+			bytes, PAGE_SIZE);
+		return 1;
+	}
+
+	if (gasket_addr_is_simple(pg_tbl, dev_addr))
+		return gasket_is_simple_dev_addr_bad(
+			pg_tbl, dev_addr, num_pages);
+	else
+		return gasket_is_extended_dev_addr_bad(
+			pg_tbl, dev_addr, num_pages);
+}
+EXPORT_SYMBOL(gasket_page_table_is_dev_addr_bad);
+
+/* See gasket_page_table.h for description. */
+uint gasket_page_table_max_size(struct gasket_page_table *page_table)
+{
+	if (!page_table) {
+		gasket_nodev_error("Passed a null page table.");
+		return 0;
+	}
+	return page_table->config.total_entries;
+}
+EXPORT_SYMBOL(gasket_page_table_max_size);
+
+/* See gasket_page_table.h for description. */
+uint gasket_page_table_num_entries(struct gasket_page_table *pg_tbl)
+{
+	if (!pg_tbl) {
+		gasket_nodev_error("Passed a null page table.");
+		return 0;
+	}
+
+	return pg_tbl->num_simple_entries + pg_tbl->num_extended_entries;
+}
+EXPORT_SYMBOL(gasket_page_table_num_entries);
+
+/* See gasket_page_table.h for description. */
+uint gasket_page_table_num_simple_entries(struct gasket_page_table *pg_tbl)
+{
+	if (!pg_tbl) {
+		gasket_nodev_error("Passed a null page table.");
+		return 0;
+	}
+
+	return pg_tbl->num_simple_entries;
+}
+EXPORT_SYMBOL(gasket_page_table_num_simple_entries);
+
+/* See gasket_page_table.h for description. */
+uint gasket_page_table_num_extended_entries(struct gasket_page_table *pg_tbl)
+{
+	if (!pg_tbl) {
+		gasket_nodev_error("Passed a null page table.");
+		return 0;
+	}
+
+	return pg_tbl->num_extended_entries;
+}
+EXPORT_SYMBOL(gasket_page_table_num_extended_entries);
+
+uint gasket_page_table_num_active_pages(struct gasket_page_table *pg_tbl)
+{
+	if (!pg_tbl) {
+		gasket_nodev_error("Passed a null page table.");
+		return 0;
+	}
+
+	return pg_tbl->num_active_pages;
+}
+EXPORT_SYMBOL(gasket_page_table_num_active_pages);
+
+/* See gasket_page_table.h */
+int gasket_page_table_system_status(struct gasket_page_table *page_table)
+{
+	if (!page_table) {
+		gasket_nodev_error("Passed a null page table.");
+		return GASKET_STATUS_LAMED;
+	}
+
+	if (gasket_page_table_num_entries(page_table) == 0) {
+		gasket_nodev_error("Page table size is 0.");
+		return GASKET_STATUS_LAMED;
+	}
+
+	return GASKET_STATUS_ALIVE;
+}
+
+/* Internal functions */
+
+/* Mapping functions */
+/*
+ * Allocate and map pages to simple addresses.
+ * @pg_tbl: Gasket page table pointer.
+ * @host_addr: Starting host virtual memory address of the pages.
+ * @dev_addr: Starting device address of the pages.
+ * @cnt: Count of the number of device pages to map.
+ *
+ * Description: gasket_map_simple_pages calls gasket_simple_alloc_pages() to
+ *		allocate the page table slots, then calls
+ *		gasket_perform_mapping() to actually do the work of mapping the
+ *		pages into the the simple page table (device translation table
+ *		registers).
+ *
+ *		The sd_mutex must be held when gasket_map_simple_pages() is
+ *		called.
+ *
+ *		Returns 0 if successful or a non-zero error number otherwise.
+ *		If there is an error, no pages are mapped.
+ */
+static int gasket_map_simple_pages(
+	struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr,
+	uint num_pages)
+{
+	int ret;
+	uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr);
+
+	ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages);
+	if (ret) {
+		gasket_pg_tbl_error(
+			pg_tbl,
+			"page table slots %u (@ 0x%lx) to %u are not available",
+			slot_idx, dev_addr, slot_idx + num_pages - 1);
+		return ret;
+	}
+
+	ret = gasket_perform_mapping(
+		pg_tbl, pg_tbl->entries + slot_idx,
+		pg_tbl->base_slot + slot_idx, host_addr, num_pages, 1);
+
+	if (ret) {
+		gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages);
+		gasket_pg_tbl_error(pg_tbl, "gasket_perform_mapping %d.", ret);
+	}
+	return ret;
+}
+
+/*
+ * gasket_map_extended_pages - Get and map buffers to extended addresses.
+ * @pg_tbl: Gasket page table pointer.
+ * @host_addr: Starting host virtual memory address of the pages.
+ * @dev_addr: Starting device address of the pages.
+ * @num_pages: The number of device pages to map.
+ *
+ * Description: gasket_map_extended_buffers calls
+ *		gasket_alloc_extended_entries() to allocate the page table
+ *		slots, then loops over the level 0 page table entries, and for
+ *		each calls gasket_perform_mapping() to map the buffers into the
+ *		level 1 page table for that level 0 entry.
+ *
+ *		The page table mutex must be held when
+ *		gasket_map_extended_pages() is called.
+ *
+ *		Returns 0 if successful or a non-zero error number otherwise.
+ *		If there is an error, no pages are mapped.
+ */
+static int gasket_map_extended_pages(
+	struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr,
+	uint num_pages)
+{
+	int ret;
+	ulong dev_addr_end;
+	uint slot_idx, remain, len;
+	struct gasket_page_table_entry *pte;
+	u64 __iomem *slot_base;
+
+	ret = gasket_alloc_extended_entries(pg_tbl, dev_addr, num_pages);
+	if (ret) {
+		dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1;
+		gasket_pg_tbl_error(
+			pg_tbl,
+			"page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are "
+			"not available",
+			gasket_extended_lvl0_page_idx(pg_tbl, dev_addr),
+			dev_addr,
+			gasket_extended_lvl1_page_idx(pg_tbl, dev_addr),
+			gasket_extended_lvl0_page_idx(pg_tbl, dev_addr_end),
+			gasket_extended_lvl1_page_idx(pg_tbl, dev_addr_end));
+		return ret;
+	}
+
+	remain = num_pages;
+	slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
+	pte = pg_tbl->entries + pg_tbl->num_simple_entries +
+	      gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
+
+	while (remain > 0) {
+		len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx);
+
+		slot_base =
+			(u64 __iomem *)(page_address(pte->page) + pte->offset);
+		ret = gasket_perform_mapping(
+			pg_tbl, pte->sublevel + slot_idx, slot_base + slot_idx,
+			host_addr, len, 0);
+		if (ret) {
+			gasket_page_table_unmap_nolock(
+				pg_tbl, dev_addr, num_pages);
+			return ret;
+		}
+
+		remain -= len;
+		slot_idx = 0;
+		pte++;
+		host_addr += len * PAGE_SIZE;
+	}
+
+	return 0;
+}
+
+/*
+ * TODO: dma_map_page() is not plugged properly when running under qemu. i.e.
+ * dma_ops are not set properly, which causes the kernel to assert.
+ *
+ * This temporary hack allows the driver to work on qemu, but need to be fixed:
+ * - either manually set the dma_ops for the architecture (which incidentally
+ * can't be done in an out-of-tree module) - or get qemu to fill the device tree
+ * properly so as linux plug the proper dma_ops or so as the driver can detect
+ * that it is runnig on qemu
+ */
+static inline dma_addr_t _no_op_dma_map_page(
+	struct device *dev, struct page *page, size_t offset, size_t size,
+	enum dma_data_direction dir)
+{
+	/*
+	 * struct dma_map_ops *ops = get_dma_ops(dev);
+	 * dma_addr_t addr;
+	 *
+	 * kmemcheck_mark_initialized(page_address(page) + offset, size);
+	 * BUG_ON(!valid_dma_direction(dir));
+	 * addr = ops->map_page(dev, page, offset, size, dir, NULL);
+	 * debug_dma_map_page(dev, page, offset, size, dir, addr, false);
+	 */
+
+	return page_to_phys(page);
+}
+
+/*
+ * Get and map last level page table buffers.
+ * @pg_tbl: Gasket page table pointer.
+ * @ptes: Array of page table entries to describe this mapping, one per
+ *        page to map.
+ * @slots: Location(s) to write device-mapped page address. If this is a simple
+ *	   mapping, these will be address translation registers. If this is
+ *	   an extended mapping, these will be within a second-level page table
+ *	   allocated by the host and so must have their __iomem attribute
+ *	   casted away.
+ * @host_addr: Starting [host] virtual memory address of the buffers.
+ * @num_pages: The number of device pages to map.
+ * @is_simple_mapping: 1 if this is a simple mapping, 0 otherwise.
+ *
+ * Description: gasket_perform_mapping calls get_user_pages() to get pages
+ *		of user memory and pin them.  It then calls dma_map_page() to
+ *		map them for DMA.  Finally, the mapped DMA addresses are written
+ *		into the page table.
+ *
+ *		This function expects that the page table entries are
+ *		already allocated.  The level argument determines how the
+ *		final page table entries are written: either into PCIe memory
+ *		mapped space for a level 0 page table or into kernel memory
+ *		for a level 1 page table.
+ *
+ *		The page pointers are saved for later releasing the pages.
+ *
+ *		Returns 0 if successful or a non-zero error number otherwise.
+ */
+static int gasket_perform_mapping(
+	struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes,
+	u64 __iomem *slots, ulong host_addr, uint num_pages,
+	int is_simple_mapping)
+{
+	int ret;
+	ulong offset;
+	struct page *page;
+	dma_addr_t dma_addr;
+	ulong page_addr;
+	int i;
+
+	for (i = 0; i < num_pages; i++) {
+		page_addr = host_addr + i * PAGE_SIZE;
+		offset = page_addr & (PAGE_SIZE - 1);
+		gasket_nodev_debug("gasket_perform_mapping i %d\n", i);
+		if (is_coherent(pg_tbl, host_addr)) {
+			u64 off =
+				(u64)host_addr -
+				(u64)pg_tbl->coherent_pages[0].user_virt;
+			ptes[i].page = 0;
+			ptes[i].offset = offset;
+			ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr +
+					   off + i * PAGE_SIZE;
+		} else {
+			ret = get_user_pages_fast(
+				page_addr - offset, 1, 1, &page);
+
+			if (ret <= 0) {
+				gasket_pg_tbl_error(
+					pg_tbl,
+					"get user pages failed for addr=0x%lx, "
+					"offset=0x%lx [ret=%d]",
+					page_addr, offset, ret);
+				return ret ? ret : -ENOMEM;
+			}
+			++pg_tbl->num_active_pages;
+
+			ptes[i].page = page;
+			ptes[i].offset = offset;
+
+			/* Map the page into DMA space. */
+			if (pg_tbl->dma_ops) {
+				/* hook in to kernel map functions */
+				ptes[i].dma_addr = dma_map_page(pg_tbl->device,
+					page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+			} else {
+				ptes[i].dma_addr = _no_op_dma_map_page(
+					pg_tbl->device, page, 0, PAGE_SIZE,
+					DMA_BIDIRECTIONAL);
+			}
+
+			gasket_nodev_debug(
+				"    gasket_perform_mapping dev %p "
+				"i %d pte %p pfn %p -> mapped %llx\n",
+				pg_tbl->device, i, &ptes[i],
+				(void *)page_to_pfn(page),
+				(unsigned long long)ptes[i].dma_addr);
+
+			if (ptes[i].dma_addr == -1) {
+				gasket_nodev_error(
+					"gasket_perform_mapping i %d"
+					" -> fail to map page %llx "
+					"[pfn %p ohys %p]\n",
+					i,
+					(unsigned long long)ptes[i].dma_addr,
+					(void *)page_to_pfn(page),
+					(void *)page_to_phys(page));
+				return -1;
+			}
+			/* Wait until the page is mapped. */
+			mb();
+		}
+
+		/* Make the DMA-space address available to the device. */
+		dma_addr = (ptes[i].dma_addr + offset) | GASKET_VALID_SLOT_FLAG;
+
+		if (is_simple_mapping) {
+			writeq(dma_addr, &slots[i]);
+		} else {
+			((u64 __force *)slots)[i] = dma_addr;
+			/* Extended page table vectors are in DRAM,
+			 * and so need to be synced each time they are updated.
+			 */
+			dma_map_single(pg_tbl->device,
+				       (void *)&((u64 __force *)slots)[i],
+				       sizeof(u64), DMA_TO_DEVICE);
+		}
+		ptes[i].status = PTE_INUSE;
+	}
+	return 0;
+}
+
+/**
+ * Allocate page table entries in a simple table.
+ * @pg_tbl: Gasket page table pointer.
+ * @dev_addr: Starting device address for the (eventual) mappings.
+ * @num_pages: Count of pages to be mapped.
+ *
+ * Description: gasket_alloc_simple_entries checks to see if a range of page
+ *		table slots are available.  As long as the sd_mutex is
+ *		held, the slots will be available.
+ *
+ *		The page table mutex must be held when
+ *		gasket_alloc_simple entries() is called.
+ *
+ *		Returns 0 if successful, or non-zero if the requested device
+ *		addresses are not available.
+ */
+static int gasket_alloc_simple_entries(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages)
+{
+	if (!gasket_is_pte_range_free(
+		    pg_tbl->entries + gasket_simple_page_idx(pg_tbl, dev_addr),
+		    num_pages))
+		return -EBUSY;
+
+	return 0;
+}
+
+/**
+ * Allocate slots in an extended page table.
+ * @pg_tbl: Gasket page table pointer.
+ * @dev_addr: Starting device address for the (eventual) mappings.
+ * @num_pages: Count of pages to be mapped.
+ *
+ * Description: gasket_alloc_extended_entries checks to see if a range of page
+ *		table slots are available. If necessary, memory is allocated for
+ *		second level page tables.
+ *
+ *		Note that memory for second level page tables is allocated
+ *		as needed, but that memory is only freed on the final close
+ *		of the device file, when the page tables are repartitioned,
+ *		or the the device is removed.  If there is an error or if
+ *		the full range of slots is not available, any memory
+ *		allocated for second level page tables remains allocated
+ *		until final close, repartition, or device removal.
+ *
+ *		The page table mutex must be held when
+ *		gasket_alloc_extended_entries() is called.
+ *
+ *		Returns 0 if successful, or non-zero if the slots are
+ *		not available.
+ */
+static int gasket_alloc_extended_entries(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_entries)
+{
+	int ret = 0;
+	uint remain, subtable_slot_idx, len;
+	struct gasket_page_table_entry *pte;
+	u64 __iomem *slot;
+
+	remain = num_entries;
+	subtable_slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
+	pte = pg_tbl->entries + pg_tbl->num_simple_entries +
+	      gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
+	slot = pg_tbl->base_slot + pg_tbl->num_simple_entries +
+	       gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
+
+	while (remain > 0) {
+		len = min(remain,
+			  GASKET_PAGES_PER_SUBTABLE - subtable_slot_idx);
+
+		if (pte->status == PTE_FREE) {
+			ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot);
+			if (ret) {
+				gasket_pg_tbl_error(
+					pg_tbl,
+					"no memory for extended addr subtable");
+				return ret;
+			}
+		} else {
+			if (!gasket_is_pte_range_free(
+				    pte->sublevel + subtable_slot_idx, len))
+				return -EBUSY;
+		}
+
+		remain -= len;
+		subtable_slot_idx = 0;
+		pte++;
+		slot++;
+	}
+
+	return 0;
+}
+
+/**
+ * Allocate a second level page table.
+ * @pg_tbl: Gasket page table pointer.
+ * @pte: Extended page table entry under/for which to allocate a second level.
+ * @slot: [Device] slot corresponding to pte.
+ *
+ * Description: Allocate the memory for a second level page table (subtable) at
+ *	        the given level 0 entry.  Then call dma_map_page() to map the
+ *		second level page table for DMA.  Finally, write the
+ *		mapped DMA address into the device page table.
+ *
+ *		The page table mutex must be held when
+ *		gasket_alloc_extended_subtable() is called.
+ *
+ *		Returns 0 if successful, or a non-zero error otherwise.
+ */
+static int gasket_alloc_extended_subtable(
+	struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte,
+	u64 __iomem *slot)
+{
+	ulong page_addr, subtable_bytes;
+	dma_addr_t dma_addr;
+
+	/* XXX FIX ME XXX this is inefficient for non-4K page sizes */
+
+	/* GFP_DMA flag must be passed to architectures for which
+	 * part of the memory range is not considered DMA'able.
+	 * This seems to be the case for Juno board with 4.5.0 Linaro kernel
+	 */
+	page_addr = get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	if (!page_addr)
+		return -ENOMEM;
+	pte->page = virt_to_page((void *)page_addr);
+	pte->offset = 0;
+
+	subtable_bytes = sizeof(struct gasket_page_table_entry) *
+		GASKET_PAGES_PER_SUBTABLE;
+	pte->sublevel = vmalloc(subtable_bytes);
+	if (!pte->sublevel) {
+		free_page(page_addr);
+		memset(pte, 0, sizeof(struct gasket_page_table_entry));
+		return -ENOMEM;
+	}
+	memset(pte->sublevel, 0, subtable_bytes);
+
+	/* Map the page into DMA space. */
+	if (pg_tbl->dma_ops) {
+		pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0,
+			PAGE_SIZE, DMA_BIDIRECTIONAL);
+	} else {
+		pte->dma_addr = _no_op_dma_map_page(pg_tbl->device, pte->page,
+			0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+	}
+	/* Wait until the page is mapped. */
+	mb();
+
+	/* make the addresses available to the device */
+	dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG;
+	writeq(dma_addr, slot);
+
+	pte->status = PTE_INUSE;
+
+	return 0;
+}
+
+/* Unmapping functions */
+/*
+ * Non-locking entry to unmapping routines.
+ * @pg_tbl: Gasket page table structure.
+ * @dev_addr: Starting device address of the pages to unmap.
+ * @num_pages: The number of device pages to unmap.
+ *
+ * Description: Version of gasket_unmap_pages that assumes the page table lock
+ *              is held.
+ */
+static void gasket_page_table_unmap_nolock(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages)
+{
+	if (!num_pages)
+		return;
+
+	if (gasket_addr_is_simple(pg_tbl, dev_addr))
+		gasket_unmap_simple_pages(pg_tbl, dev_addr, num_pages);
+	else
+		gasket_unmap_extended_pages(pg_tbl, dev_addr, num_pages);
+}
+
+/*
+ * Unmap and release pages mapped to simple addresses.
+ * @pg_tbl: Gasket page table pointer.
+ * @dev_addr: Starting device address of the buffers.
+ * @num_pages: The number of device pages to unmap.
+ *
+ * Description: gasket_simple_unmap_pages calls gasket_perform_unmapping() to
+ * unmap and release the buffers in the level 0 page table.
+ *
+ * The sd_mutex must be held when gasket_unmap_simple_pages() is called.
+ */
+static void gasket_unmap_simple_pages(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages)
+{
+	uint slot = gasket_simple_page_idx(pg_tbl, dev_addr);
+
+	gasket_perform_unmapping(pg_tbl, pg_tbl->entries + slot,
+				 pg_tbl->base_slot + slot, num_pages, 1);
+}
+
+/**
+ * Unmap and release buffers to extended addresses.
+ * @pg_tbl: Gasket page table pointer.
+ * @dev_addr: Starting device address of the pages to unmap.
+ * @addr: Starting device address of the buffers.
+ * @num_pages: The number of device pages to unmap.
+ *
+ * Description: gasket_extended_unmap_pages loops over the level 0 page table
+ *		entries, and for each calls gasket_perform_unmapping() to unmap
+ *		the buffers from the level 1 page [sub]table for that level 0
+ *		entry.
+ *
+ *		The page table mutex must be held when
+ *		gasket_unmap_extended_pages() is called.
+ */
+static void gasket_unmap_extended_pages(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages)
+{
+	uint slot_idx, remain, len;
+	struct gasket_page_table_entry *pte;
+	u64 __iomem *slot_base;
+
+	remain = num_pages;
+	slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
+	pte = pg_tbl->entries + pg_tbl->num_simple_entries +
+	      gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
+
+	while (remain > 0) {
+		/* TODO: Add check to ensure pte remains valid? */
+		len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx);
+
+		if (pte->status == PTE_INUSE) {
+			slot_base = (u64 __iomem *)(page_address(pte->page) +
+						    pte->offset);
+			gasket_perform_unmapping(
+				pg_tbl, pte->sublevel + slot_idx,
+				slot_base + slot_idx, len, 0);
+		}
+
+		remain -= len;
+		slot_idx = 0;
+		pte++;
+	}
+}
+
+/*
+ * Unmap and release mapped pages.
+ * @pg_tbl: Gasket page table pointer.
+ * @ptes: Array of page table entries to describe the mapped range, one per
+ *        page to unmap.
+ * @slots: Device slots corresponding to the mappings described by "ptes".
+ *         As with ptes, one element per page to unmap.
+ *         If these are simple mappings, these will be address translation
+ *         registers. If these are extended mappings, these will be witin a
+ *         second-level page table allocated on the host, and so must have
+ *	   their __iomem attribute casted away.
+ * @num_pages: Number of pages to unmap.
+ * @is_simple_mapping: 1 if this is a simple mapping, 0 otherwise.
+ *
+ * Description: gasket_perform_unmapping() loops through the metadata entries
+ *		in a last level page table (simple table or extended subtable),
+ *		and for each page:
+ *		 - Unmaps the page from DMA space (dma_unmap_page),
+ *		 - Returns the page to the OS (gasket_release_page),
+ *		The entry in the page table is written to 0. The metadata
+ *		type is set to PTE_FREE and the metadata is all reset
+ *		to 0.
+ *
+ *		The page table mutex must be held when this function is called.
+ */
+static void gasket_perform_unmapping(
+	struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes,
+	u64 __iomem *slots, uint num_pages, int is_simple_mapping)
+{
+	int i;
+	/*
+	 * For each page table entry and corresponding entry in the device's
+	 * address translation table:
+	 */
+	for (i = 0; i < num_pages; i++) {
+		/* release the address from the device, */
+		if (is_simple_mapping || ptes[i].status == PTE_INUSE)
+			writeq(0, &slots[i]);
+		else
+			((u64 __force *)slots)[i] = 0;
+		/* Force sync around the address release. */
+		mb();
+
+		/* release the address from the driver, */
+		if (ptes[i].status == PTE_INUSE) {
+			if (ptes[i].dma_addr) {
+				dma_unmap_page(pg_tbl->device, ptes[i].dma_addr,
+					       PAGE_SIZE, DMA_FROM_DEVICE);
+			}
+			if (gasket_release_page(ptes[i].page))
+				--pg_tbl->num_active_pages;
+		}
+		ptes[i].status = PTE_FREE;
+
+		/* and clear the PTE. */
+		memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry));
+	}
+}
+
+/*
+ * Free a second level page [sub]table.
+ * @pg_tbl: Gasket page table pointer.
+ * @pte: Page table entry _pointing_to_ the subtable to free.
+ * @slot: Device slot holding a pointer to the sublevel's contents.
+ *
+ * Description: Safely deallocates a second-level [sub]table by:
+ *  - Marking the containing first-level PTE as free
+ *  - Setting the corresponding [extended] device slot as NULL
+ *  - Unmapping the PTE from DMA space.
+ *  - Freeing the subtable's memory.
+ *  - Deallocating the page and clearing out the PTE.
+ *
+ * The page table mutex must be held before this call.
+ */
+static void gasket_free_extended_subtable(
+	struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *pte,
+	u64 __iomem *slot)
+{
+	/* Release the page table from the driver */
+	pte->status = PTE_FREE;
+
+	/* Release the page table from the device */
+	writeq(0, slot);
+	/* Force sync around the address release. */
+	mb();
+
+	if (pte->dma_addr)
+		dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE,
+			       DMA_BIDIRECTIONAL);
+
+	vfree(pte->sublevel);
+
+	if (pte->page)
+		free_page((ulong)page_address(pte->page));
+
+	memset(pte, 0, sizeof(struct gasket_page_table_entry));
+}
+
+/*
+ * Safely return a page to the OS.
+ * @page: The page to return to the OS.
+ * Returns 1 if the page was released, 0 if it was
+ * ignored.
+ */
+static int gasket_release_page(struct page *page)
+{
+	if (!page)
+		return 0;
+
+	if (!PageReserved(page))
+		SetPageDirty(page);
+	put_page(page);
+
+	return 1;
+}
+
+/* Evaluates to nonzero if the specified virtual address is simple. */
+static inline int gasket_addr_is_simple(
+	struct gasket_page_table *pg_tbl, ulong addr)
+{
+	return !((addr) & (pg_tbl)->extended_flag);
+}
+
+/*
+ * Validity checking for simple addresses.
+ * @pg_tbl: Gasket page table pointer.
+ * @dev_addr: The device address to which the pages will be mapped.
+ * @num_pages: The number of pages in the range to consider.
+ *
+ * Description: This call verifies that address translation commutes (from
+ * address to/from page + offset) and that the requested page range starts and
+ * ends within the set of currently-partitioned simple pages.
+ */
+static int gasket_is_simple_dev_addr_bad(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages)
+{
+	ulong page_offset = dev_addr & (PAGE_SIZE - 1);
+	ulong page_index =
+		(dev_addr / PAGE_SIZE) & (pg_tbl->config.total_entries - 1);
+
+	if (gasket_components_to_dev_address(
+		pg_tbl, 1, page_index, page_offset) != dev_addr) {
+		gasket_pg_tbl_error(
+			pg_tbl, "address is invalid, 0x%lX", dev_addr);
+		return 1;
+	}
+
+	if (page_index >= pg_tbl->num_simple_entries) {
+		gasket_pg_tbl_error(
+			pg_tbl,
+			"starting slot at %lu is too large, max is < %u",
+			page_index, pg_tbl->num_simple_entries);
+		return 1;
+	}
+
+	if (page_index + num_pages > pg_tbl->num_simple_entries) {
+		gasket_pg_tbl_error(
+			pg_tbl,
+			"ending slot at %lu is too large, max is <= %u",
+			page_index + num_pages, pg_tbl->num_simple_entries);
+		return 1;
+	}
+
+	return 0;
+}
+
+/*
+ * Verifies that address translation commutes (from address to/from page +
+ * offset) and that the requested page range starts and ends within the set of
+ * currently-partitioned simple pages.
+ *
+ * @pg_tbl: Gasket page table pointer.
+ * @dev_addr: The device address to which the pages will be mapped.
+ * @num_pages: The number of second-level/sub pages in the range to consider.
+ */
+static int gasket_is_extended_dev_addr_bad(
+	struct gasket_page_table *pg_tbl, ulong dev_addr, uint num_pages)
+{
+	/* Starting byte index of dev_addr into the first mapped page */
+	ulong page_offset = dev_addr & (PAGE_SIZE - 1);
+	ulong page_global_idx, page_lvl0_idx;
+	ulong num_lvl0_pages;
+	ulong addr;
+
+	/* check if the device address is out of bound */
+	addr = dev_addr & ~((pg_tbl)->extended_flag);
+	if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) {
+		gasket_pg_tbl_error(pg_tbl, "device address out of bound, 0x%p",
+				    (void *)dev_addr);
+		return 1;
+	}
+
+	/* Find the starting sub-page index in the space of all sub-pages. */
+	page_global_idx = (dev_addr / PAGE_SIZE) &
+		(pg_tbl->config.total_entries * GASKET_PAGES_PER_SUBTABLE - 1);
+
+	/* Find the starting level 0 index. */
+	page_lvl0_idx = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
+
+	/* Get the count of affected level 0 pages. */
+	num_lvl0_pages = (num_pages + GASKET_PAGES_PER_SUBTABLE - 1) /
+		GASKET_PAGES_PER_SUBTABLE;
+
+	if (gasket_components_to_dev_address(
+		pg_tbl, 0, page_global_idx, page_offset) != dev_addr) {
+		gasket_pg_tbl_error(
+			pg_tbl, "address is invalid, 0x%p", (void *)dev_addr);
+		return 1;
+	}
+
+	if (page_lvl0_idx >= pg_tbl->num_extended_entries) {
+		gasket_pg_tbl_error(
+			pg_tbl,
+			"starting level 0 slot at %lu is too large, max is < "
+			"%u", page_lvl0_idx, pg_tbl->num_extended_entries);
+		return 1;
+	}
+
+	if (page_lvl0_idx + num_lvl0_pages > pg_tbl->num_extended_entries) {
+		gasket_pg_tbl_error(
+			pg_tbl,
+			"ending level 0 slot at %lu is too large, max is <= %u",
+			page_lvl0_idx + num_lvl0_pages,
+			pg_tbl->num_extended_entries);
+		return 1;
+	}
+
+	return 0;
+}
+
+/*
+ * Checks if a range of PTEs is free.
+ * @ptes: The set of PTEs to check.
+ * @num_entries: The number of PTEs to check.
+ *
+ * Description: Iterates over the input PTEs to determine if all have been
+ * marked as FREE or if any are INUSE. In the former case, 1/true is returned.
+ * Otherwise, 0/false is returned.
+ *
+ * The page table mutex must be held before this call.
+ */
+static int gasket_is_pte_range_free(
+	struct gasket_page_table_entry *ptes, uint num_entries)
+{
+	int i;
+
+	for (i = 0; i < num_entries; i++) {
+		if (ptes[i].status != PTE_FREE)
+			return 0;
+	}
+
+	return 1;
+}
+
+/*
+ * Actually perform collection.
+ * @pg_tbl: Gasket page table structure.
+ *
+ * Description: Version of gasket_page_table_garbage_collect that assumes the
+ *		page table lock is held.
+ */
+static void gasket_page_table_garbage_collect_nolock(
+	struct gasket_page_table *pg_tbl)
+{
+	struct gasket_page_table_entry *pte;
+	u64 __iomem *slot;
+
+	/* XXX FIX ME XXX -- more efficient to keep a usage count */
+	/* rather than scanning the second level page tables */
+
+	for (pte = pg_tbl->entries + pg_tbl->num_simple_entries,
+	     slot = pg_tbl->base_slot + pg_tbl->num_simple_entries;
+	     pte < pg_tbl->entries + pg_tbl->config.total_entries;
+	     pte++, slot++) {
+		if (pte->status == PTE_INUSE) {
+			if (gasket_is_pte_range_free(
+				    pte->sublevel, GASKET_PAGES_PER_SUBTABLE))
+				gasket_free_extended_subtable(
+					pg_tbl, pte, slot);
+		}
+	}
+}
+
+/*
+ * Converts components to a device address.
+ * @pg_tbl: Gasket page table structure.
+ * @is_simple: nonzero if this should be a simple entry, zero otherwise.
+ * @page_index: The page index into the respective table.
+ * @offset: The offset within the requested page.
+ *
+ * Simple utility function to convert (simple, page, offset) into a device
+ * address.
+ * Examples:
+ * Simple page 0, offset 32:
+ *  Input (0, 0, 32), Output 0x20
+ * Simple page 1000, offset 511:
+ *  Input (0, 1000, 512), Output 0x3E81FF
+ * Extended page 0, offset 32:
+ *  Input (0, 0, 32), Output 0x8000000020
+ * Extended page 1000, offset 511:
+ *  Input (1, 1000, 512), Output 0x8003E81FF
+ */
+static ulong gasket_components_to_dev_address(
+	struct gasket_page_table *pg_tbl, int is_simple, uint page_index,
+	uint offset)
+{
+	ulong lvl0_index, lvl1_index;
+
+	if (is_simple) {
+		/* Return simple addresses directly. */
+		lvl0_index = page_index & (pg_tbl->config.total_entries - 1);
+		return (lvl0_index << GASKET_SIMPLE_PAGE_SHIFT) | offset;
+	}
+
+	/*
+	 * This could be compressed into fewer statements, but
+	 * A) the compiler should optimize it
+	 * B) this is not slow
+	 * C) this is an uncommon operation
+	 * D) this is actually readable this way.
+	 */
+	lvl0_index = page_index / GASKET_PAGES_PER_SUBTABLE;
+	lvl1_index = page_index & (GASKET_PAGES_PER_SUBTABLE - 1);
+	return (pg_tbl)->extended_flag |
+	       (lvl0_index << GASKET_EXTENDED_LVL0_SHIFT) |
+	       (lvl1_index << GASKET_EXTENDED_LVL1_SHIFT) | offset;
+}
+
+/*
+ * Gets the index of the address' page in the simple table.
+ * @pg_tbl: Gasket page table structure.
+ * @dev_addr: The address whose page index to retrieve.
+ *
+ * Description: Treats the input address as a simple address and determines the
+ * index of its underlying page in the simple page table (i.e., device address
+ * translation registers.
+ *
+ * Does not perform validity checking.
+ */
+static int gasket_simple_page_idx(
+	struct gasket_page_table *pg_tbl, ulong dev_addr)
+{
+	return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) &
+		(pg_tbl->config.total_entries - 1);
+}
+
+/*
+ * Gets the level 0 page index for the given address.
+ * @pg_tbl: Gasket page table structure.
+ * @dev_addr: The address whose page index to retrieve.
+ *
+ * Description: Treats the input address as an extended address and determines
+ * the index of its underlying page in the first-level extended page table
+ * (i.e., device extended address translation registers).
+ *
+ * Does not perform validity checking.
+ */
+static ulong gasket_extended_lvl0_page_idx(
+	struct gasket_page_table *pg_tbl, ulong dev_addr)
+{
+	return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) &
+	       ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1);
+}
+
+/*
+ * Gets the level 1 page index for the given address.
+ * @pg_tbl: Gasket page table structure.
+ * @dev_addr: The address whose page index to retrieve.
+ *
+ * Description: Treats the input address as an extended address and determines
+ * the index of its underlying page in the second-level extended page table
+ * (i.e., host memory pointed to by a first-level page table entry).
+ *
+ * Does not perform validity checking.
+ */
+static ulong gasket_extended_lvl1_page_idx(
+	struct gasket_page_table *pg_tbl, ulong dev_addr)
+{
+	return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) &
+	       (GASKET_PAGES_PER_SUBTABLE - 1);
+}
+
+/*
+ * Determines whether a host buffer was mapped as coherent memory.
+ * @pg_tbl: gasket_page_table structure tracking the host buffer mapping
+ * @host_addr: user virtual address within a host buffer
+ *
+ * Description: A Gasket page_table currently support one contiguous
+ * dma range, mapped to one contiguous virtual memory range. Check if the
+ * host_addr is within start of page 0, and end of last page, for that range.
+ */
+static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr)
+{
+	u64 min, max;
+
+	/* whether the host address is within user virt range */
+	if (!pg_tbl->coherent_pages)
+		return 0;
+
+	min = (u64)pg_tbl->coherent_pages[0].user_virt;
+	max = min + PAGE_SIZE * pg_tbl->num_coherent_pages;
+
+	return min <= host_addr && host_addr < max;
+}
+
+/*
+ * Records the host_addr to coherent dma memory mapping.
+ * @gasket_dev: Gasket Device.
+ * @size: Size of the virtual address range to map.
+ * @dma_address: Dma address within the coherent memory range.
+ * @vma: Virtual address we wish to map to coherent memory.
+ *
+ * Description: For each page in the virtual address range, record the
+ * coherent page mgasket_pretapping.
+ */
+int gasket_set_user_virt(
+	struct gasket_dev *gasket_dev, u64 size, dma_addr_t dma_address,
+	ulong vma)
+{
+	int j;
+	struct gasket_page_table *pg_tbl;
+
+	unsigned int num_pages = size / PAGE_SIZE;
+
+	/*
+	 * TODO: for future chipset, better handling of the case where multiple
+	 * page tables are supported on a given device
+	 */
+	pg_tbl = gasket_dev->page_table[0];
+	if (!pg_tbl) {
+		gasket_nodev_error(
+			"gasket_set_user_virt: invalid page table index");
+		return 0;
+	}
+	for (j = 0; j < num_pages; j++) {
+		pg_tbl->coherent_pages[j].user_virt =
+			(u64)vma + j * PAGE_SIZE;
+	}
+	return 0;
+}
+
+/*
+ * Allocate a block of coherent memory.
+ * @gasket_dev: Gasket Device.
+ * @size: Size of the memory block.
+ * @dma_address: Dma address allocated by the kernel.
+ * @index: Index of the gasket_page_table within this Gasket device
+ *
+ * Description: Allocate a contiguous coherent memory block, DMA'ble
+ * by this device.
+ */
+int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size,
+				 dma_addr_t *dma_address, u64 index)
+{
+	dma_addr_t handle;
+	void *mem;
+	int j;
+	unsigned int num_pages = (size + PAGE_SIZE - 1) / (PAGE_SIZE);
+	const struct gasket_driver_desc *driver_desc =
+		gasket_get_driver_desc(gasket_dev);
+
+	if (!gasket_dev->page_table[index])
+		return -EFAULT;
+
+	if (num_pages == 0)
+		return -EINVAL;
+
+	mem = dma_alloc_coherent(gasket_get_device(gasket_dev),
+				 num_pages * PAGE_SIZE, &handle, 0);
+	if (!mem)
+		goto nomem;
+
+	gasket_dev->page_table[index]->num_coherent_pages = num_pages;
+
+	/* allocate the physical memory block */
+	gasket_dev->page_table[index]->coherent_pages = kzalloc(
+		num_pages * sizeof(struct gasket_coherent_page_entry),
+		GFP_KERNEL);
+	if (!gasket_dev->page_table[index]->coherent_pages)
+		goto nomem;
+	*dma_address = 0;
+
+	gasket_dev->coherent_buffer.length_bytes =
+		PAGE_SIZE * (num_pages);
+	gasket_dev->coherent_buffer.phys_base = handle;
+	gasket_dev->coherent_buffer.virt_base = mem;
+
+	*dma_address = driver_desc->coherent_buffer_description.base;
+		for (j = 0; j < num_pages; j++) {
+		gasket_dev->page_table[index]->coherent_pages[j].paddr =
+			handle + j * PAGE_SIZE;
+		gasket_dev->page_table[index]->coherent_pages[j].kernel_virt =
+			(u64)mem + j * PAGE_SIZE;
+	}
+
+	if (*dma_address == 0)
+		goto nomem;
+	return 0;
+
+nomem:
+	if (mem) {
+		dma_free_coherent(gasket_get_device(gasket_dev),
+				  num_pages * PAGE_SIZE, mem, handle);
+	}
+
+	if (gasket_dev->page_table[index]->coherent_pages) {
+		kfree(gasket_dev->page_table[index]->coherent_pages);
+		gasket_dev->page_table[index]->coherent_pages = 0;
+	}
+	gasket_dev->page_table[index]->num_coherent_pages = 0;
+	return -ENOMEM;
+}
+
+/*
+ * Free a block of coherent memory.
+ * @gasket_dev: Gasket Device.
+ * @size: Size of the memory block.
+ * @dma_address: Dma address allocated by the kernel.
+ * @index: Index of the gasket_page_table within this Gasket device
+ *
+ * Description: Release memory allocated thru gasket_alloc_coherent_memory.
+ */
+int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size,
+				dma_addr_t dma_address, u64 index)
+{
+	const struct gasket_driver_desc *driver_desc;
+
+	if (!gasket_dev->page_table[index])
+		return -EFAULT;
+
+	driver_desc = gasket_get_driver_desc(gasket_dev);
+
+	if (driver_desc->coherent_buffer_description.base != dma_address)
+		return -EADDRNOTAVAIL;
+
+	if (gasket_dev->coherent_buffer.length_bytes) {
+		dma_free_coherent(gasket_get_device(gasket_dev),
+				  gasket_dev->coherent_buffer.length_bytes,
+				  gasket_dev->coherent_buffer.virt_base,
+				  gasket_dev->coherent_buffer.phys_base);
+		gasket_dev->coherent_buffer.length_bytes = 0;
+		gasket_dev->coherent_buffer.virt_base = 0;
+		gasket_dev->coherent_buffer.phys_base = 0;
+	}
+	return 0;
+}
+
+/*
+ * Release all coherent memory.
+ * @gasket_dev: Gasket Device.
+ * @index: Index of the gasket_page_table within this Gasket device
+ *
+ * Description: Release all memory allocated thru gasket_alloc_coherent_memory.
+ */
+void gasket_free_coherent_memory_all(
+	struct gasket_dev *gasket_dev, u64 index)
+{
+	if (!gasket_dev->page_table[index])
+		return;
+
+	if (gasket_dev->coherent_buffer.length_bytes) {
+		dma_free_coherent(gasket_get_device(gasket_dev),
+				  gasket_dev->coherent_buffer.length_bytes,
+				  gasket_dev->coherent_buffer.virt_base,
+				  gasket_dev->coherent_buffer.phys_base);
+		gasket_dev->coherent_buffer.length_bytes = 0;
+		gasket_dev->coherent_buffer.virt_base = 0;
+		gasket_dev->coherent_buffer.phys_base = 0;
+	}
+}

drivers/staging/gasket/gasket_page_table.h

@@ -0,0 +1,265 @@
+/* Gasket Page Table functionality. This file describes the address
+ * translation/paging functionality supported by the Gasket driver framework.
+ * As much as possible, internal details are hidden to simplify use -
+ * all calls are thread-safe (protected by an internal mutex) except where
+ * indicated otherwise.
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __GASKET_ADDR_TRNSL_H__
+#define __GASKET_ADDR_TRNSL_H__
+
+#include <linux/pci.h>
+#include <linux/types.h>
+
+#include "gasket_constants.h"
+#include "gasket_core.h"
+
+/*
+ * Structure used for managing address translation on a device. All details are
+ * internal to the implementation.
+ */
+struct gasket_page_table;
+
+/*
+ * Allocate and init address translation data.
+ * @ppage_table: Pointer to Gasket page table pointer. Set by this call.
+ * @att_base_reg: [Mapped] pointer to the first entry in the device's address
+ *                translation table.
+ * @extended_offset_reg: [Mapped] pointer to the device's register containing
+ *                       the starting index of the extended translation table.
+ * @extended_bit_location: The index of the bit indicating whether an address
+ *                         is extended.
+ * @total_entries: The total number of entries in the device's address
+ *                 translation table.
+ * @device: Device structure for the underlying device. Only used for logging.
+ * @pci_dev: PCI system descriptor for the underlying device.
+ * @bool has_dma_ops: Whether the page table uses arch specific dma_ops or
+ * whether the driver will supply its own.
+ *
+ * Description: Allocates and initializes data to track address translation -
+ * simple and extended page table metadata. Initially, the page table is
+ * partitioned such that all addresses are "simple" (single-level lookup).
+ * gasket_partition_page_table can be called to change this paritioning.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int gasket_page_table_init(
+	struct gasket_page_table **ppg_tbl,
+	const struct gasket_bar_data *bar_data,
+	const struct gasket_page_table_config *page_table_config,
+	struct device *device, struct pci_dev *pci_dev, bool dma_ops);
+
+/*
+ * Deallocate and cleanup page table data.
+ * @page_table: Gasket page table pointer.
+ *
+ * Description: The inverse of gasket_init; frees page_table and its contained
+ *              elements.
+ *
+ *	        Because this call destroys the page table, it cannot be
+ *	        thread-safe (mutex-protected)!
+ */
+void gasket_page_table_cleanup(struct gasket_page_table *page_table);
+
+/*
+ * Sets the size of the simple page table.
+ * @page_table: Gasket page table pointer.
+ * @num_simple_entries: Desired size of the simple page table (in entries).
+ *
+ * Description: gasket_partition_page_table checks to see if the simple page
+ *              size can be changed (i.e., if there are no active extended
+ *              mappings in the new simple size range), and, if so,
+ *              sets the new simple and extended page table sizes.
+ *
+ *              Returns 0 if successful, or non-zero if the page table entries
+ *              are not free.
+ */
+int gasket_page_table_partition(
+	struct gasket_page_table *page_table, uint num_simple_entries);
+
+/*
+ * Get and map [host] user space pages into device memory.
+ * @page_table: Gasket page table pointer.
+ * @host_addr: Starting host virtual memory address of the pages.
+ * @dev_addr: Starting device address of the pages.
+ * @num_pages: Number of [4kB] pages to map.
+ *
+ * Description: Maps the "num_pages" pages of host memory pointed to by
+ *              host_addr to the address "dev_addr" in device memory.
+ *
+ *              The caller is responsible for checking the addresses ranges.
+ *
+ *              Returns 0 if successful or a non-zero error number otherwise.
+ *              If there is an error, no pages are mapped.
+ */
+int gasket_page_table_map(struct gasket_page_table *page_table, ulong host_addr,
+			  ulong dev_addr, uint num_pages);
+
+/*
+ * Un-map host pages from device memory.
+ * @page_table: Gasket page table pointer.
+ * @dev_addr: Starting device address of the pages to unmap.
+ * @num_pages: The number of [4kB] pages to unmap.
+ *
+ * Description: The inverse of gasket_map_pages. Unmaps pages from the device.
+ */
+void gasket_page_table_unmap(
+	struct gasket_page_table *page_table, ulong dev_addr, uint num_pages);
+
+/*
+ * Unmap ALL host pages from device memory.
+ * @page_table: Gasket page table pointer.
+ */
+void gasket_page_table_unmap_all(struct gasket_page_table *page_table);
+
+/*
+ * Unmap all host pages from device memory and reset the table to fully simple
+ * addressing.
+ * @page_table: Gasket page table pointer.
+ */
+void gasket_page_table_reset(struct gasket_page_table *page_table);
+
+/*
+ * Reclaims unused page table memory.
+ * @page_table: Gasket page table pointer.
+ *
+ * Description: Examines the page table and frees any currently-unused
+ *              allocations. Called internally on gasket_cleanup().
+ */
+void gasket_page_table_garbage_collect(struct gasket_page_table *page_table);
+
+/*
+ * Retrieve the backing page for a device address.
+ * @page_table: Gasket page table pointer.
+ * @dev_addr: Gasket device address.
+ * @ppage: Pointer to a page pointer for the returned page.
+ * @poffset: Pointer to an unsigned long for the returned offset.
+ *
+ * Description: Interprets the address and looks up the corresponding page
+ *              in the page table and the offset in that page.  (We need an
+ *              offset because the host page may be larger than the Gasket chip
+ *              page it contains.)
+ *
+ *              Returns 0 if successful, -1 for an error.  The page pointer
+ *              and offset are returned through the pointers, if successful.
+ */
+int gasket_page_table_lookup_page(
+	struct gasket_page_table *page_table, ulong dev_addr,
+	struct page **page, ulong *poffset);
+
+/*
+ * Checks validity for input addrs and size.
+ * @page_table: Gasket page table pointer.
+ * @host_addr: Host address to check.
+ * @dev_addr: Gasket device address.
+ * @bytes: Size of the range to check (in bytes).
+ *
+ * Description: This call performs a number of checks to verify that the ranges
+ * specified by both addresses and the size are valid for mapping pages into
+ * device memory.
+ *
+ * Returns 1 if true - if the mapping is bad, 0 otherwise.
+ */
+int gasket_page_table_are_addrs_bad(
+	struct gasket_page_table *page_table, ulong host_addr, ulong dev_addr,
+	ulong bytes);
+
+/*
+ * Checks validity for input dev addr and size.
+ * @page_table: Gasket page table pointer.
+ * @dev_addr: Gasket device address.
+ * @bytes: Size of the range to check (in bytes).
+ *
+ * Description: This call performs a number of checks to verify that the range
+ * specified by the device address and the size is valid for mapping pages into
+ * device memory.
+ *
+ * Returns 1 if true - if the address is bad, 0 otherwise.
+ */
+int gasket_page_table_is_dev_addr_bad(
+	struct gasket_page_table *page_table, ulong dev_addr, ulong bytes);
+
+/*
+ * Gets maximum size for the given page table.
+ * @page_table: Gasket page table pointer.
+ */
+uint gasket_page_table_max_size(struct gasket_page_table *page_table);
+
+/*
+ * Gets the total number of entries in the arg.
+ * @page_table: Gasket page table pointer.
+ */
+uint gasket_page_table_num_entries(struct gasket_page_table *page_table);
+
+/*
+ * Gets the number of simple entries.
+ * @page_table: Gasket page table pointer.
+ */
+uint gasket_page_table_num_simple_entries(struct gasket_page_table *page_table);
+
+/*
+ * Gets the number of extended entries.
+ * @page_table: Gasket page table pointer.
+ */
+uint gasket_page_table_num_extended_entries(
+	struct gasket_page_table *page_table);
+
+/*
+ * Gets the number of actively pinned pages.
+ * @page_table: Gasket page table pointer.
+ */
+uint gasket_page_table_num_active_pages(struct gasket_page_table *page_table);
+
+/*
+ * Get status of page table managed by @page_table.
+ * @page_table: Gasket page table pointer.
+ */
+int gasket_page_table_system_status(struct gasket_page_table *page_table);
+
+/*
+ * Allocate a block of coherent memory.
+ * @gasket_dev: Gasket Device.
+ * @size: Size of the memory block.
+ * @dma_address: Dma address allocated by the kernel.
+ * @index: Index of the gasket_page_table within this Gasket device
+ *
+ * Description: Allocate a contiguous coherent memory block, DMA'ble
+ * by this device.
+ */
+int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, uint64_t size,
+				 dma_addr_t *dma_address, uint64_t index);
+/* Release a block of contiguous coherent memory, in use by a device. */
+int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, uint64_t size,
+				dma_addr_t dma_address, uint64_t index);
+
+/* Release all coherent memory. */
+void gasket_free_coherent_memory_all(struct gasket_dev *gasket_dev,
+				     uint64_t index);
+
+/*
+ * Records the host_addr to coherent dma memory mapping.
+ * @gasket_dev: Gasket Device.
+ * @size: Size of the virtual address range to map.
+ * @dma_address: Dma address within the coherent memory range.
+ * @vma: Virtual address we wish to map to coherent memory.
+ *
+ * Description: For each page in the virtual address range, record the
+ * coherent page mapping.
+ *
+ * Does not perform validity checking.
+ */
+int gasket_set_user_virt(struct gasket_dev *gasket_dev, uint64_t size,
+			 dma_addr_t dma_address, ulong vma);
+
+#endif

drivers/staging/gasket/gasket_sysfs.c

@@ -0,0 +1,497 @@
+/* Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include "gasket_sysfs.h"
+
+#include "gasket_core.h"
+#include "gasket_logging.h"
+
+/*
+ * Pair of kernel device and user-specified pointer. Used in lookups in sysfs
+ * "show" functions to return user data.
+ */
+
+struct gasket_sysfs_mapping {
+	/*
+	 * The device bound to this mapping. If this is NULL, then this mapping
+	 * is free.
+	 */
+	struct device *device;
+
+	/* Legacy device struct, if used by this mapping's driver. */
+	struct device *legacy_device;
+
+	/* The Gasket descriptor for this device. */
+	struct gasket_dev *gasket_dev;
+
+	/* This device's set of sysfs attributes/nodes. */
+	struct gasket_sysfs_attribute *attributes;
+
+	/* The number of live elements in "attributes". */
+	int attribute_count;
+
+	/* Protects structure from simultaneous access. */
+	struct mutex mutex;
+
+	/* Tracks active users of this mapping. */
+	struct kref refcount;
+};
+
+/*
+ * Data needed to manage users of this sysfs utility.
+ * Currently has a fixed size; if space is a concern, this can be dynamically
+ * allocated.
+ */
+/*
+ * 'Global' (file-scoped) list of mappings between devices and gasket_data
+ * pointers. This removes the requirement to have a gasket_sysfs_data
+ * handle in all files.
+ */
+static struct gasket_sysfs_mapping dev_mappings[GASKET_SYSFS_NUM_MAPPINGS];
+
+/*
+ * Callback when a mapping's refcount goes to zero.
+ * @ref: The reference count of the containing sysfs mapping.
+ */
+static void release_entry(struct kref *ref)
+{
+	/* All work is done after the return from kref_put. */
+}
+
+/*
+ * Looks up mapping information for the given device.
+ * @device: The device whose mapping to look for.
+ *
+ * Looks up the requested device and takes a reference and returns it if found,
+ * and returns NULL otherwise.
+ */
+static struct gasket_sysfs_mapping *get_mapping(struct device *device)
+{
+	int i;
+
+	if (!device) {
+		gasket_nodev_error("Received NULL device!");
+		return NULL;
+	}
+
+	for (i = 0; i < GASKET_SYSFS_NUM_MAPPINGS; i++) {
+		mutex_lock(&dev_mappings[i].mutex);
+		if (dev_mappings[i].device == device ||
+		    dev_mappings[i].legacy_device == device) {
+			kref_get(&dev_mappings[i].refcount);
+			mutex_unlock(&dev_mappings[i].mutex);
+			return &dev_mappings[i];
+		}
+		mutex_unlock(&dev_mappings[i].mutex);
+	}
+
+	gasket_nodev_info("Mapping to device %s not found.", device->kobj.name);
+	return NULL;
+}
+
+/*
+ * Returns a reference to a mapping.
+ * @mapping: The mapping we're returning.
+ *
+ * Decrements the refcount for the given mapping (if valid). If the refcount is
+ * zero, then it cleans up the mapping - in this function as opposed to the
+ * kref_put callback, due to a potential deadlock.
+ *
+ * Although put_mapping_n exists, this function is left here (as an implicit
+ * put_mapping_n(..., 1) for convenience.
+ */
+static void put_mapping(struct gasket_sysfs_mapping *mapping)
+{
+	int i;
+	int num_files_to_remove = 0;
+	struct device_attribute *files_to_remove;
+	struct device *device;
+	struct device *legacy_device;
+
+	if (!mapping) {
+		gasket_nodev_info("Mapping should not be NULL.");
+		return;
+	}
+
+	mutex_lock(&mapping->mutex);
+	if (mapping->refcount.refcount.refs.counter == 0)
+		gasket_nodev_error("Refcount is already 0!");
+	if (kref_put(&mapping->refcount, release_entry)) {
+		gasket_nodev_info("Removing Gasket sysfs mapping, device %s",
+				  mapping->device->kobj.name);
+		/*
+		 * We can't remove the sysfs nodes in the kref callback, since
+		 * device_remove_file() blocks until the node is free.
+		 * Readers/writers of sysfs nodes, though, will be blocked on
+		 * the mapping mutex, resulting in deadlock. To fix this, the
+		 * sysfs nodes are removed outside the lock.
+		 */
+		device = mapping->device;
+		legacy_device = mapping->legacy_device;
+		num_files_to_remove = mapping->attribute_count;
+		files_to_remove = kzalloc(
+			num_files_to_remove * sizeof(*files_to_remove),
+			GFP_KERNEL);
+		for (i = 0; i < num_files_to_remove; i++)
+			files_to_remove[i] = mapping->attributes[i].attr;
+
+		kfree(mapping->attributes);
+		mapping->attributes = NULL;
+		mapping->attribute_count = 0;
+		mapping->device = NULL;
+		mapping->gasket_dev = NULL;
+	}
+	mutex_unlock(&mapping->mutex);
+
+	if (num_files_to_remove != 0) {
+		for (i = 0; i < num_files_to_remove; ++i) {
+			device_remove_file(device, &files_to_remove[i]);
+			if (legacy_device)
+				device_remove_file(
+					legacy_device, &files_to_remove[i]);
+		}
+		kfree(files_to_remove);
+	}
+}
+
+/*
+ * Returns a reference N times.
+ * @mapping: The mapping to return.
+ *
+ * In higher-level resource acquire/release function pairs, the release function
+ * will need to release a mapping 2x - once for the refcount taken in the
+ * release function itself, and once for the count taken in the acquire call.
+ */
+static void put_mapping_n(struct gasket_sysfs_mapping *mapping, int times)
+{
+	int i;
+
+	for (i = 0; i < times; i++)
+		put_mapping(mapping);
+}
+
+void gasket_sysfs_init(void)
+{
+	int i;
+
+	for (i = 0; i < GASKET_SYSFS_NUM_MAPPINGS; i++) {
+		dev_mappings[i].device = NULL;
+		mutex_init(&dev_mappings[i].mutex);
+	}
+}
+
+int gasket_sysfs_create_mapping(
+	struct device *device, struct gasket_dev *gasket_dev)
+{
+	struct gasket_sysfs_mapping *mapping;
+	int map_idx = -1;
+
+	/*
+	 * We need a function-level mutex to protect against the same device
+	 * being added [multiple times] simultaneously.
+	 */
+	static DEFINE_MUTEX(function_mutex);
+
+	mutex_lock(&function_mutex);
+
+	gasket_nodev_info(
+		"Creating sysfs entries for device pointer 0x%p.", device);
+
+	/* Check that the device we're adding hasn't already been added. */
+	mapping = get_mapping(device);
+	if (mapping) {
+		gasket_nodev_error(
+			"Attempting to re-initialize sysfs mapping for device "
+			"0x%p.", device);
+		put_mapping(mapping);
+		mutex_unlock(&function_mutex);
+		return -EINVAL;
+	}
+
+	/* Find the first empty entry in the array. */
+	for (map_idx = 0; map_idx < GASKET_SYSFS_NUM_MAPPINGS; ++map_idx) {
+		mutex_lock(&dev_mappings[map_idx].mutex);
+		if (!dev_mappings[map_idx].device)
+			/* Break with the mutex held! */
+			break;
+		mutex_unlock(&dev_mappings[map_idx].mutex);
+	}
+
+	if (map_idx == GASKET_SYSFS_NUM_MAPPINGS) {
+		gasket_nodev_error("All mappings have been exhausted!");
+		mutex_unlock(&function_mutex);
+		return -ENOMEM;
+	}
+
+	gasket_nodev_info(
+		"Creating sysfs mapping for device %s.", device->kobj.name);
+
+	mapping = &dev_mappings[map_idx];
+	kref_init(&mapping->refcount);
+	mapping->device = device;
+	mapping->gasket_dev = gasket_dev;
+	mapping->attributes = kzalloc(
+		GASKET_SYSFS_MAX_NODES * sizeof(*mapping->attributes),
+		GFP_KERNEL);
+	mapping->attribute_count = 0;
+	if (!mapping->attributes) {
+		gasket_nodev_error("Unable to allocate sysfs attribute array.");
+		mutex_unlock(&mapping->mutex);
+		mutex_unlock(&function_mutex);
+		return -ENOMEM;
+	}
+
+	mutex_unlock(&mapping->mutex);
+	mutex_unlock(&function_mutex);
+
+	/* Don't decrement the refcount here! One open count keeps it alive! */
+	return 0;
+}
+
+int gasket_sysfs_create_entries(
+	struct device *device, const struct gasket_sysfs_attribute *attrs)
+{
+	int i;
+	int ret;
+	struct gasket_sysfs_mapping *mapping = get_mapping(device);
+
+	if (!mapping) {
+		gasket_nodev_error(
+			"Creating entries for device 0x%p without first "
+			"initializing mapping.",
+			device);
+		return -EINVAL;
+	}
+
+	mutex_lock(&mapping->mutex);
+	for (i = 0; strcmp(attrs[i].attr.attr.name, GASKET_ARRAY_END_MARKER);
+		i++) {
+		if (mapping->attribute_count == GASKET_SYSFS_MAX_NODES) {
+			gasket_nodev_error(
+				"Maximum number of sysfs nodes reached for "
+				"device.");
+			mutex_unlock(&mapping->mutex);
+			put_mapping(mapping);
+			return -ENOMEM;
+		}
+
+		ret = device_create_file(device, &attrs[i].attr);
+		if (ret) {
+			gasket_nodev_error("Unable to create device entries");
+			mutex_unlock(&mapping->mutex);
+			put_mapping(mapping);
+			return ret;
+		}
+
+		if (mapping->legacy_device) {
+			ret = device_create_file(mapping->legacy_device,
+						 &attrs[i].attr);
+			if (ret) {
+				gasket_log_error(
+					mapping->gasket_dev,
+					"Unable to create legacy sysfs entries;"
+					" rc: %d",
+					ret);
+				mutex_unlock(&mapping->mutex);
+				put_mapping(mapping);
+				return ret;
+			}
+		}
+
+		mapping->attributes[mapping->attribute_count] = attrs[i];
+		++mapping->attribute_count;
+	}
+
+	mutex_unlock(&mapping->mutex);
+	put_mapping(mapping);
+	return 0;
+}
+EXPORT_SYMBOL(gasket_sysfs_create_entries);
+
+void gasket_sysfs_remove_mapping(struct device *device)
+{
+	struct gasket_sysfs_mapping *mapping = get_mapping(device);
+
+	if (!mapping) {
+		gasket_nodev_error(
+			"Attempted to remove non-existent sysfs mapping to "
+			"device 0x%p",
+			device);
+		return;
+	}
+
+	put_mapping_n(mapping, 2);
+}
+
+struct gasket_dev *gasket_sysfs_get_device_data(struct device *device)
+{
+	struct gasket_sysfs_mapping *mapping = get_mapping(device);
+
+	if (!mapping) {
+		gasket_nodev_error("device %p not registered.", device);
+		return NULL;
+	}
+
+	return mapping->gasket_dev;
+}
+EXPORT_SYMBOL(gasket_sysfs_get_device_data);
+
+void gasket_sysfs_put_device_data(struct device *device, struct gasket_dev *dev)
+{
+	struct gasket_sysfs_mapping *mapping = get_mapping(device);
+
+	if (!mapping)
+		return;
+
+	/* See comment of put_mapping_n() for why the '2' is necessary. */
+	put_mapping_n(mapping, 2);
+}
+EXPORT_SYMBOL(gasket_sysfs_put_device_data);
+
+struct gasket_sysfs_attribute *gasket_sysfs_get_attr(
+	struct device *device, struct device_attribute *attr)
+{
+	int i;
+	int num_attrs;
+	struct gasket_sysfs_mapping *mapping = get_mapping(device);
+	struct gasket_sysfs_attribute *attrs = NULL;
+
+	if (!mapping)
+		return NULL;
+
+	attrs = mapping->attributes;
+	num_attrs = mapping->attribute_count;
+	for (i = 0; i < num_attrs; ++i) {
+		if (!strcmp(attrs[i].attr.attr.name, attr->attr.name))
+			return &attrs[i];
+	}
+
+	gasket_nodev_error("Unable to find match for device_attribute %s",
+			   attr->attr.name);
+	return NULL;
+}
+EXPORT_SYMBOL(gasket_sysfs_get_attr);
+
+void gasket_sysfs_put_attr(
+	struct device *device, struct gasket_sysfs_attribute *attr)
+{
+	int i;
+	int num_attrs;
+	struct gasket_sysfs_mapping *mapping = get_mapping(device);
+	struct gasket_sysfs_attribute *attrs = NULL;
+
+	if (!mapping)
+		return;
+
+	attrs = mapping->attributes;
+	num_attrs = mapping->attribute_count;
+	for (i = 0; i < num_attrs; ++i) {
+		if (&attrs[i] == attr) {
+			put_mapping_n(mapping, 2);
+			return;
+		}
+	}
+
+	gasket_nodev_error(
+		"Unable to put unknown attribute: %s", attr->attr.attr.name);
+}
+EXPORT_SYMBOL(gasket_sysfs_put_attr);
+
+ssize_t gasket_sysfs_register_show(
+	struct device *device, struct device_attribute *attr, char *buf)
+{
+	ulong reg_address, reg_bar, reg_value;
+	struct gasket_sysfs_mapping *mapping;
+	struct gasket_dev *gasket_dev;
+	struct gasket_sysfs_attribute *gasket_attr;
+
+	mapping = get_mapping(device);
+	if (!mapping) {
+		gasket_nodev_info("Device driver may have been removed.");
+		return 0;
+	}
+
+	gasket_dev = mapping->gasket_dev;
+	if (!gasket_dev) {
+		gasket_nodev_error(
+			"No sysfs mapping found for device 0x%p", device);
+		put_mapping(mapping);
+		return 0;
+	}
+
+	gasket_attr = gasket_sysfs_get_attr(device, attr);
+	if (!gasket_attr) {
+		put_mapping(mapping);
+		return 0;
+	}
+
+	reg_address = gasket_attr->data.bar_address.offset;
+	reg_bar = gasket_attr->data.bar_address.bar;
+	reg_value = gasket_dev_read_64(gasket_dev, reg_bar, reg_address);
+
+	gasket_sysfs_put_attr(device, gasket_attr);
+	put_mapping(mapping);
+	return snprintf(buf, PAGE_SIZE, "0x%lX\n", reg_value);
+}
+EXPORT_SYMBOL(gasket_sysfs_register_show);
+
+ssize_t gasket_sysfs_register_store(
+	struct device *device, struct device_attribute *attr, const char *buf,
+	size_t count)
+{
+	ulong parsed_value = 0;
+	struct gasket_sysfs_mapping *mapping;
+	struct gasket_dev *gasket_dev;
+	struct gasket_sysfs_attribute *gasket_attr;
+
+	if (count < 3 || buf[0] != '0' || buf[1] != 'x') {
+		gasket_nodev_error(
+			"sysfs register write format: \"0x<hex value>\".");
+		return -EINVAL;
+	}
+
+	if (kstrtoul(buf, 16, &parsed_value) != 0) {
+		gasket_nodev_error(
+			"Unable to parse input as 64-bit hex value: %s.", buf);
+		return -EINVAL;
+	}
+
+	mapping = get_mapping(device);
+	if (!mapping) {
+		gasket_nodev_info("Device driver may have been removed.");
+		return 0;
+	}
+
+	gasket_dev = mapping->gasket_dev;
+	if (!gasket_dev) {
+		gasket_nodev_info("Device driver may have been removed.");
+		return 0;
+	}
+
+	gasket_attr = gasket_sysfs_get_attr(device, attr);
+	if (!gasket_attr) {
+		put_mapping(mapping);
+		return count;
+	}
+
+	gasket_dev_write_64(gasket_dev, parsed_value,
+			    gasket_attr->data.bar_address.bar,
+			    gasket_attr->data.bar_address.offset);
+
+	if (gasket_attr->write_callback)
+		gasket_attr->write_callback(
+			gasket_dev, gasket_attr, parsed_value);
+
+	gasket_sysfs_put_attr(device, gasket_attr);
+	put_mapping(mapping);
+	return count;
+}
+EXPORT_SYMBOL(gasket_sysfs_register_store);

drivers/staging/gasket/gasket_sysfs.h

@@ -0,0 +1,199 @@
+/* Set of common sysfs utilities.
+ *
+ * Copyright (C) 2018 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/* The functions described here are a set of utilities to allow each file in the
+ * Gasket driver framework to manage their own set of sysfs entries, instead of
+ * centralizing all that work in one file.
+ *
+ * The goal of these utilities is to allow for sysfs entries to be easily
+ * created without causing a proliferation of sysfs "show" functions. This
+ * requires O(N) string lookups during show function execution, but as reading
+ * sysfs entries is rarely performance-critical, this is likely acceptible.
+ */
+#ifndef __GASKET_SYSFS_H__
+#define __GASKET_SYSFS_H__
+
+#include "gasket_constants.h"
+#include "gasket_core.h"
+#include <linux/device.h>
+#include <linux/stringify.h>
+#include <linux/sysfs.h>
+
+/* The maximum number of mappings/devices a driver needs to support. */
+#define GASKET_SYSFS_NUM_MAPPINGS (GASKET_FRAMEWORK_DESC_MAX * GASKET_DEV_MAX)
+
+/* The maximum number of sysfs nodes in a directory.
+ */
+#define GASKET_SYSFS_MAX_NODES 196
+
+/* End markers for sysfs struct arrays. */
+#define GASKET_ARRAY_END_TOKEN GASKET_RESERVED_ARRAY_END
+#define GASKET_ARRAY_END_MARKER __stringify(GASKET_ARRAY_END_TOKEN)
+
+/*
+ * Terminator struct for a gasket_sysfs_attr array. Must be at the end of
+ * all gasket_sysfs_attribute arrays.
+ */
+#define GASKET_END_OF_ATTR_ARRAY                                               \
+	{                                                                      \
+		.attr = __ATTR(GASKET_ARRAY_END_TOKEN, S_IRUGO, NULL, NULL),   \
+		.data.attr_type = 0,                                           \
+	}
+
+/*
+ * Pairing of sysfs attribute and user data.
+ * Used in lookups in sysfs "show" functions to return attribute metadata.
+ */
+struct gasket_sysfs_attribute {
+	/* The underlying sysfs device attribute associated with this data. */
+	struct device_attribute attr;
+
+	/* User-specified data to associate with the attribute. */
+	union {
+		struct bar_address_ {
+			ulong bar;
+			ulong offset;
+		} bar_address;
+		uint attr_type;
+	} data;
+
+	/*
+	 * Function pointer to a callback to be invoked when this attribute is
+	 * written (if so configured). The arguments are to the Gasket device
+	 * pointer, the enclosing gasket_attr structure, and the value written.
+	 * The callback should perform any logging necessary, as errors cannot
+	 * be returned from the callback.
+	 */
+	void (*write_callback)(
+		struct gasket_dev *dev, struct gasket_sysfs_attribute *attr,
+		ulong value);
+};
+
+#define GASKET_SYSFS_RO(_name, _show_function, _attr_type)                     \
+	{                                                                      \
+		.attr = __ATTR(_name, S_IRUGO, _show_function, NULL),          \
+		.data.attr_type = _attr_type                                   \
+	}
+#define GASKET_SYSFS_REG(_name, _offset, _bar)                                 \
+	{                                                                      \
+		.attr = __ATTR(_name, S_IRUGO, gasket_sysfs_register_show,     \
+			NULL),                                                 \
+		.data.bar_address = {                                          \
+			.bar = _bar,                                           \
+			.offset = _offset                                      \
+		}                                                              \
+	}
+
+/* Initializes the Gasket sysfs subsystem.
+ *
+ * Description: Performs one-time initialization. Must be called before usage
+ * at [Gasket] module load time.
+ */
+void gasket_sysfs_init(void);
+
+/*
+ * Create an entry in mapping_data between a device and a Gasket device.
+ * @device: Device struct to map to.
+ * @gasket_dev: The dev struct associated with the driver controlling @device.
+ *
+ * Description: This function maps a gasket_dev* to a device*. This mapping can
+ * be used in sysfs_show functions to get a handle to the gasket_dev struct
+ * controlling the device node.
+ *
+ * If this function is not called before gasket_sysfs_create_entries, a warning
+ * will be logged.
+ */
+int gasket_sysfs_create_mapping(
+	struct device *device, struct gasket_dev *gasket_dev);
+
+/*
+ * Creates bulk entries in sysfs.
+ * @device: Kernel device structure.
+ * @attrs: List of attributes/sysfs entries to create.
+ *
+ * Description: Creates each sysfs entry described in "attrs". Can be called
+ * multiple times for a given @device. If the gasket_dev specified in
+ * gasket_sysfs_create_mapping had a legacy device, the entries will be created
+ * for it, as well.
+ */
+int gasket_sysfs_create_entries(
+	struct device *device, const struct gasket_sysfs_attribute *attrs);
+
+/*
+ * Removes a device mapping from the global table.
+ * @device: Device to unmap.
+ *
+ * Description: Removes the device->Gasket device mapping from the internal
+ * table.
+ */
+void gasket_sysfs_remove_mapping(struct device *device);
+
+/*
+ * User data lookup based on kernel device structure.
+ * @device: Kernel device structure.
+ *
+ * Description: Returns the user data associated with "device" in a prior call
+ * to gasket_sysfs_create_entries. Returns NULL if no mapping can be found.
+ * Upon success, this call take a reference to internal sysfs data that must be
+ * released with gasket_sysfs_put_device_data. While this reference is held, the
+ * underlying device sysfs information/structure will remain valid/will not be
+ * deleted.
+ */
+struct gasket_dev *gasket_sysfs_get_device_data(struct device *device);
+
+/*
+ * Releases a references to internal data.
+ * @device: Kernel device structure.
+ * @dev: Gasket device descriptor (returned by gasket_sysfs_get_device_data).
+ */
+void gasket_sysfs_put_device_data(
+	struct device *device, struct gasket_dev *gasket_dev);
+
+/*
+ * Gasket-specific attribute lookup.
+ * @device: Kernel device structure.
+ * @attr: Device attribute to look up.
+ *
+ * Returns the Gasket sysfs attribute associated with the kernel device
+ * attribute and device structure itself. Upon success, this call will take a
+ * reference to internal sysfs data that must be released with a call to
+ * gasket_sysfs_get_device_data. While this reference is held, the underlying
+ * device sysfs information/structure will remain valid/will not be deleted.
+ */
+struct gasket_sysfs_attribute *gasket_sysfs_get_attr(
+	struct device *device, struct device_attribute *attr);
+
+/*
+ * Releases a references to internal data.
+ * @device: Kernel device structure.
+ * @attr: Gasket sysfs attribute descriptor (returned by
+ *        gasket_sysfs_get_attr).
+ */
+void gasket_sysfs_put_attr(
+	struct device *device, struct gasket_sysfs_attribute *attr);
+
+/* Display a register as a sysfs node. */
+ssize_t gasket_sysfs_register_show(
+	struct device *device, struct device_attribute *attr, char *buf);
+
+/*
+ * Write to a register sysfs node.
+ * @buf: NULL-terminated data being written.
+ * @count: number of bytes in the "buf" argument.
+ */
+ssize_t gasket_sysfs_register_store(
+	struct device *device, struct device_attribute *attr, const char *buf,
+	size_t count);
+
+#endif /* __GASKET_SYSFS_H__ */