diff options
Diffstat (limited to 'drivers/staging/gasket/apex_driver.c')
| -rw-r--r-- | drivers/staging/gasket/apex_driver.c | 741 |
1 files changed, 741 insertions, 0 deletions
diff --git a/drivers/staging/gasket/apex_driver.c b/drivers/staging/gasket/apex_driver.c new file mode 100644 index 000000000000..c747e9ca4518 --- /dev/null +++ b/drivers/staging/gasket/apex_driver.c @@ -0,0 +1,741 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for the Apex chip. + * + * Copyright (C) 2018 Google, Inc. + */ + +#include <linux/compiler.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/init.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/pci.h> +#include <linux/printk.h> +#include <linux/sched.h> +#include <linux/uaccess.h> + +#include "apex.h" + +#include "gasket_core.h" +#include "gasket_interrupt.h" +#include "gasket_page_table.h" +#include "gasket_sysfs.h" + +/* Constants */ +#define APEX_DEVICE_NAME "Apex" +#define APEX_DRIVER_VERSION "1.0" + +/* CSRs are in BAR 2. */ +#define APEX_BAR_INDEX 2 + +#define APEX_PCI_VENDOR_ID 0x1ac1 +#define APEX_PCI_DEVICE_ID 0x089a + +/* Bar Offsets. */ +#define APEX_BAR_OFFSET 0 +#define APEX_CM_OFFSET 0x1000000 + +/* The sizes of each Apex BAR 2. */ +#define APEX_BAR_BYTES 0x100000 +#define APEX_CH_MEM_BYTES (PAGE_SIZE * MAX_NUM_COHERENT_PAGES) + +/* The number of user-mappable memory ranges in BAR2 of a Apex chip. */ +#define NUM_REGIONS 3 + +/* The number of nodes in a Apex chip. */ +#define NUM_NODES 1 + +/* + * The total number of entries in the page table. Should match the value read + * from the register APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE. + */ +#define APEX_PAGE_TABLE_TOTAL_ENTRIES 8192 + +#define APEX_EXTENDED_SHIFT 63 /* Extended address bit position. */ + +/* Check reset 120 times */ +#define APEX_RESET_RETRY 120 +/* Wait 100 ms between checks. Total 12 sec wait maximum. */ +#define APEX_RESET_DELAY 100 + +/* Enumeration of the supported sysfs entries. */ +enum sysfs_attribute_type { + ATTR_KERNEL_HIB_PAGE_TABLE_SIZE, + ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE, + ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES, +}; + +/* + * Register offsets into BAR2 memory. + * Only values necessary for driver implementation are defined. + */ +enum apex_bar2_regs { + APEX_BAR2_REG_SCU_BASE = 0x1A300, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_SIZE = 0x46000, + APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE = 0x46008, + APEX_BAR2_REG_KERNEL_HIB_TRANSLATION_ENABLE = 0x46010, + APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL = 0x46018, + APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL = 0x46020, + APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL = 0x46028, + APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL = 0x46030, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL = 0x46038, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL = 0x46040, + APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL = 0x46048, + APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE = 0x46050, + APEX_BAR2_REG_KERNEL_HIB_DMA_PAUSE_MASK = 0x46058, + APEX_BAR2_REG_KERNEL_HIB_STATUS_BLOCK_DELAY = 0x46060, + APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY0 = 0x46068, + APEX_BAR2_REG_KERNEL_HIB_MSIX_PENDING_BIT_ARRAY1 = 0x46070, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT = 0x46078, + APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT = 0x46080, + APEX_BAR2_REG_KERNEL_WIRE_INT_PENDING_BIT_ARRAY = 0x48778, + APEX_BAR2_REG_KERNEL_WIRE_INT_MASK_ARRAY = 0x48780, + APEX_BAR2_REG_USER_HIB_DMA_PAUSE = 0x486D8, + APEX_BAR2_REG_USER_HIB_DMA_PAUSED = 0x486E0, + APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER = 0x4A000, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE = 0x50000, + + /* Error registers - Used mostly for debug */ + APEX_BAR2_REG_USER_HIB_ERROR_STATUS = 0x86f0, + APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS = 0x41a0, +}; + +/* Addresses for packed registers. */ +#define APEX_BAR2_REG_AXI_QUIESCE (APEX_BAR2_REG_SCU_BASE + 0x2C) +#define APEX_BAR2_REG_GCB_CLOCK_GATE (APEX_BAR2_REG_SCU_BASE + 0x14) +#define APEX_BAR2_REG_SCU_0 (APEX_BAR2_REG_SCU_BASE + 0xc) +#define APEX_BAR2_REG_SCU_1 (APEX_BAR2_REG_SCU_BASE + 0x10) +#define APEX_BAR2_REG_SCU_2 (APEX_BAR2_REG_SCU_BASE + 0x14) +#define APEX_BAR2_REG_SCU_3 (APEX_BAR2_REG_SCU_BASE + 0x18) +#define APEX_BAR2_REG_SCU_4 (APEX_BAR2_REG_SCU_BASE + 0x1c) +#define APEX_BAR2_REG_SCU_5 (APEX_BAR2_REG_SCU_BASE + 0x20) + +#define SCU3_RG_PWR_STATE_OVR_BIT_OFFSET 26 +#define SCU3_RG_PWR_STATE_OVR_MASK_WIDTH 2 +#define SCU3_CUR_RST_GCB_BIT_MASK 0x10 +#define SCU2_RG_RST_GCB_BIT_MASK 0xc + +/* Configuration for page table. */ +static struct gasket_page_table_config apex_page_table_configs[NUM_NODES] = { + { + .id = 0, + .mode = GASKET_PAGE_TABLE_MODE_NORMAL, + .total_entries = APEX_PAGE_TABLE_TOTAL_ENTRIES, + .base_reg = APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE, + .extended_reg = APEX_BAR2_REG_KERNEL_HIB_EXTENDED_TABLE, + .extended_bit = APEX_EXTENDED_SHIFT, + }, +}; + +/* The regions in the BAR2 space that can be mapped into user space. */ +static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = { + { 0x40000, 0x1000 }, + { 0x44000, 0x1000 }, + { 0x48000, 0x1000 }, +}; + +static const struct gasket_mappable_region cm_mappable_regions[1] = { { 0x0, + APEX_CH_MEM_BYTES } }; + +/* Gasket device interrupts enums must be dense (i.e., no empty slots). */ +enum apex_interrupt { + APEX_INTERRUPT_INSTR_QUEUE = 0, + APEX_INTERRUPT_INPUT_ACTV_QUEUE = 1, + APEX_INTERRUPT_PARAM_QUEUE = 2, + APEX_INTERRUPT_OUTPUT_ACTV_QUEUE = 3, + APEX_INTERRUPT_SC_HOST_0 = 4, + APEX_INTERRUPT_SC_HOST_1 = 5, + APEX_INTERRUPT_SC_HOST_2 = 6, + APEX_INTERRUPT_SC_HOST_3 = 7, + APEX_INTERRUPT_TOP_LEVEL_0 = 8, + APEX_INTERRUPT_TOP_LEVEL_1 = 9, + APEX_INTERRUPT_TOP_LEVEL_2 = 10, + APEX_INTERRUPT_TOP_LEVEL_3 = 11, + APEX_INTERRUPT_FATAL_ERR = 12, + APEX_INTERRUPT_COUNT = 13, +}; + +/* Interrupt descriptors for Apex */ +static struct gasket_interrupt_desc apex_interrupts[] = { + { + APEX_INTERRUPT_INSTR_QUEUE, + APEX_BAR2_REG_KERNEL_HIB_INSTR_QUEUE_INTVECCTL, + UNPACKED, + }, + { + APEX_INTERRUPT_INPUT_ACTV_QUEUE, + APEX_BAR2_REG_KERNEL_HIB_INPUT_ACTV_QUEUE_INTVECCTL, + UNPACKED + }, + { + APEX_INTERRUPT_PARAM_QUEUE, + APEX_BAR2_REG_KERNEL_HIB_PARAM_QUEUE_INTVECCTL, + UNPACKED + }, + { + APEX_INTERRUPT_OUTPUT_ACTV_QUEUE, + APEX_BAR2_REG_KERNEL_HIB_OUTPUT_ACTV_QUEUE_INTVECCTL, + UNPACKED + }, + { + APEX_INTERRUPT_SC_HOST_0, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL, + PACK_0 + }, + { + APEX_INTERRUPT_SC_HOST_1, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL, + PACK_1 + }, + { + APEX_INTERRUPT_SC_HOST_2, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL, + PACK_2 + }, + { + APEX_INTERRUPT_SC_HOST_3, + APEX_BAR2_REG_KERNEL_HIB_SC_HOST_INTVECCTL, + PACK_3 + }, + { + APEX_INTERRUPT_TOP_LEVEL_0, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL, + PACK_0 + }, + { + APEX_INTERRUPT_TOP_LEVEL_1, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL, + PACK_1 + }, + { + APEX_INTERRUPT_TOP_LEVEL_2, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL, + PACK_2 + }, + { + APEX_INTERRUPT_TOP_LEVEL_3, + APEX_BAR2_REG_KERNEL_HIB_TOP_LEVEL_INTVECCTL, + PACK_3 + }, + { + APEX_INTERRUPT_FATAL_ERR, + APEX_BAR2_REG_KERNEL_HIB_FATAL_ERR_INTVECCTL, + UNPACKED + }, +}; + + +/* Allows device to enter power save upon driver close(). */ +static int allow_power_save = 1; + +/* Allows SW based clock gating. */ +static int allow_sw_clock_gating; + +/* Allows HW based clock gating. */ +/* Note: this is not mutual exclusive with SW clock gating. */ +static int allow_hw_clock_gating = 1; + +/* Act as if only GCB is instantiated. */ +static int bypass_top_level; + +module_param(allow_power_save, int, 0644); +module_param(allow_sw_clock_gating, int, 0644); +module_param(allow_hw_clock_gating, int, 0644); +module_param(bypass_top_level, int, 0644); + +/* Check the device status registers and return device status ALIVE or DEAD. */ +static int apex_get_status(struct gasket_dev *gasket_dev) +{ + /* TODO: Check device status. */ + return GASKET_STATUS_ALIVE; +} + +/* Enter GCB reset state. */ +static int apex_enter_reset(struct gasket_dev *gasket_dev) +{ + if (bypass_top_level) + return 0; + + /* + * Software reset: + * Enable sleep mode + * - Software force GCB idle + * - Enable GCB idle + */ + gasket_read_modify_write_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER, + 0x0, 1, 32); + + /* - Initiate DMA pause */ + gasket_dev_write_64(gasket_dev, 1, APEX_BAR_INDEX, + APEX_BAR2_REG_USER_HIB_DMA_PAUSE); + + /* - Wait for DMA pause complete. */ + if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_USER_HIB_DMA_PAUSED, 1, 1, + APEX_RESET_DELAY, APEX_RESET_RETRY)) { + dev_err(gasket_dev->dev, + "DMAs did not quiesce within timeout (%d ms)\n", + APEX_RESET_RETRY * APEX_RESET_DELAY); + return -ETIMEDOUT; + } + + /* - Enable GCB reset (0x1 to rg_rst_gcb) */ + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_2, 0x1, 2, 2); + + /* - Enable GCB clock Gate (0x1 to rg_gated_gcb) */ + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_2, 0x1, 2, 18); + + /* - Enable GCB memory shut down (0x3 to rg_force_ram_sd) */ + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 0x3, 2, 14); + + /* - Wait for RAM shutdown. */ + if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 1 << 6, 1 << 6, + APEX_RESET_DELAY, APEX_RESET_RETRY)) { + dev_err(gasket_dev->dev, + "RAM did not shut down within timeout (%d ms)\n", + APEX_RESET_RETRY * APEX_RESET_DELAY); + return -ETIMEDOUT; + } + + return 0; +} + +/* Quit GCB reset state. */ +static int apex_quit_reset(struct gasket_dev *gasket_dev) +{ + u32 val0, val1; + + if (bypass_top_level) + return 0; + + /* + * Disable sleep mode: + * - Disable GCB memory shut down: + * - b00: Not forced (HW controlled) + * - b1x: Force disable + */ + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 0x0, 2, 14); + + /* + * - Disable software clock gate: + * - b00: Not forced (HW controlled) + * - b1x: Force disable + */ + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_2, 0x0, 2, 18); + + /* + * - Disable GCB reset (rg_rst_gcb): + * - b00: Not forced (HW controlled) + * - b1x: Force disable = Force not Reset + */ + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_2, 0x2, 2, 2); + + /* - Wait for RAM enable. */ + if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 1 << 6, 0, + APEX_RESET_DELAY, APEX_RESET_RETRY)) { + dev_err(gasket_dev->dev, + "RAM did not enable within timeout (%d ms)\n", + APEX_RESET_RETRY * APEX_RESET_DELAY); + return -ETIMEDOUT; + } + + /* - Wait for Reset complete. */ + if (gasket_wait_with_reschedule(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, + SCU3_CUR_RST_GCB_BIT_MASK, 0, + APEX_RESET_DELAY, APEX_RESET_RETRY)) { + dev_err(gasket_dev->dev, + "GCB did not leave reset within timeout (%d ms)\n", + APEX_RESET_RETRY * APEX_RESET_DELAY); + return -ETIMEDOUT; + } + + if (!allow_hw_clock_gating) { + val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); + /* Inactive and Sleep mode are disabled. */ + gasket_read_modify_write_32(gasket_dev, + APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 0x3, + SCU3_RG_PWR_STATE_OVR_MASK_WIDTH, + SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); + val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); + dev_dbg(gasket_dev->dev, + "Disallow HW clock gating 0x%x -> 0x%x\n", val0, val1); + } else { + val0 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); + /* Inactive mode enabled - Sleep mode disabled. */ + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3, 2, + SCU3_RG_PWR_STATE_OVR_MASK_WIDTH, + SCU3_RG_PWR_STATE_OVR_BIT_OFFSET); + val1 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); + dev_dbg(gasket_dev->dev, "Allow HW clock gating 0x%x -> 0x%x\n", + val0, val1); + } + + return 0; +} + +/* Reset the Apex hardware. Called on final close via device_close_cb. */ +static int apex_device_cleanup(struct gasket_dev *gasket_dev) +{ + u64 scalar_error; + u64 hib_error; + int ret = 0; + + hib_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_USER_HIB_ERROR_STATUS); + scalar_error = gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCALAR_CORE_ERROR_STATUS); + + dev_dbg(gasket_dev->dev, + "%s 0x%p hib_error 0x%llx scalar_error 0x%llx\n", + __func__, gasket_dev, hib_error, scalar_error); + + if (allow_power_save) + ret = apex_enter_reset(gasket_dev); + + return ret; +} + +/* Determine if GCB is in reset state. */ +static bool is_gcb_in_reset(struct gasket_dev *gasket_dev) +{ + u32 val = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); + + /* Masks rg_rst_gcb bit of SCU_CTRL_2 */ + return (val & SCU3_CUR_RST_GCB_BIT_MASK); +} + +/* Reset the hardware, then quit reset. Called on device open. */ +static int apex_reset(struct gasket_dev *gasket_dev) +{ + int ret; + + if (bypass_top_level) + return 0; + + if (!is_gcb_in_reset(gasket_dev)) { + /* We are not in reset - toggle the reset bit so as to force + * re-init of custom block + */ + dev_dbg(gasket_dev->dev, "%s: toggle reset\n", __func__); + + ret = apex_enter_reset(gasket_dev); + if (ret) + return ret; + } + ret = apex_quit_reset(gasket_dev); + + return ret; +} + +/* + * Check permissions for Apex ioctls. + * Returns true if the current user may execute this ioctl, and false otherwise. + */ +static bool apex_ioctl_check_permissions(struct file *filp, uint cmd) +{ + return !!(filp->f_mode & FMODE_WRITE); +} + +/* Gates or un-gates Apex clock. */ +static long apex_clock_gating(struct gasket_dev *gasket_dev, + struct apex_gate_clock_ioctl __user *argp) +{ + struct apex_gate_clock_ioctl ibuf; + + if (bypass_top_level || !allow_sw_clock_gating) + return 0; + + if (copy_from_user(&ibuf, argp, sizeof(ibuf))) + return -EFAULT; + + dev_dbg(gasket_dev->dev, "%s %llu\n", __func__, ibuf.enable); + + if (ibuf.enable) { + /* Quiesce AXI, gate GCB clock. */ + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_AXI_QUIESCE, 0x1, 1, + 16); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_GCB_CLOCK_GATE, 0x1, + 2, 18); + } else { + /* Un-gate GCB clock, un-quiesce AXI. */ + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_GCB_CLOCK_GATE, 0x0, + 2, 18); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_AXI_QUIESCE, 0x0, 1, + 16); + } + return 0; +} + +/* Apex-specific ioctl handler. */ +static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) +{ + struct gasket_dev *gasket_dev = filp->private_data; + + if (!apex_ioctl_check_permissions(filp, cmd)) + return -EPERM; + + switch (cmd) { + case APEX_IOCTL_GATE_CLOCK: + return apex_clock_gating(gasket_dev, argp); + default: + return -ENOTTY; /* unknown command */ + } +} + +/* Display driver sysfs entries. */ +static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, + char *buf) +{ + int ret; + struct gasket_dev *gasket_dev; + struct gasket_sysfs_attribute *gasket_attr; + enum sysfs_attribute_type type; + + gasket_dev = gasket_sysfs_get_device_data(device); + if (!gasket_dev) { + dev_err(device, "No Apex device sysfs mapping found\n"); + return -ENODEV; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + dev_err(device, "No Apex device sysfs attr data found\n"); + gasket_sysfs_put_device_data(device, gasket_dev); + return -ENODEV; + } + + type = (enum sysfs_attribute_type)gasket_sysfs_get_attr(device, attr); + switch (type) { + case ATTR_KERNEL_HIB_PAGE_TABLE_SIZE: + ret = scnprintf(buf, PAGE_SIZE, "%u\n", + gasket_page_table_num_entries( + gasket_dev->page_table[0])); + break; + case ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE: + ret = scnprintf(buf, PAGE_SIZE, "%u\n", + gasket_page_table_num_entries( + gasket_dev->page_table[0])); + break; + case ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES: + ret = scnprintf(buf, PAGE_SIZE, "%u\n", + gasket_page_table_num_active_pages( + gasket_dev->page_table[0])); + break; + default: + dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", + attr->attr.name); + ret = 0; + break; + } + + gasket_sysfs_put_attr(device, gasket_attr); + gasket_sysfs_put_device_data(device, gasket_dev); + return ret; +} + +static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { + GASKET_SYSFS_RO(node_0_page_table_entries, sysfs_show, + ATTR_KERNEL_HIB_PAGE_TABLE_SIZE), + GASKET_SYSFS_RO(node_0_simple_page_table_entries, sysfs_show, + ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE), + GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show, + ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES), + GASKET_END_OF_ATTR_ARRAY +}; + +/* On device open, perform a core reinit reset. */ +static int apex_device_open_cb(struct gasket_dev *gasket_dev) +{ + return gasket_reset_nolock(gasket_dev); +} + +static const struct pci_device_id apex_pci_ids[] = { + { PCI_DEVICE(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID) }, { 0 } +}; + +static void apex_pci_fixup_class(struct pci_dev *pdev) +{ + pdev->class = (PCI_CLASS_SYSTEM_OTHER << 8) | pdev->class; +} +DECLARE_PCI_FIXUP_CLASS_HEADER(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID, + PCI_CLASS_NOT_DEFINED, 8, apex_pci_fixup_class); + +static int apex_pci_probe(struct pci_dev *pci_dev, + const struct pci_device_id *id) +{ + int ret; + ulong page_table_ready, msix_table_ready; + int retries = 0; + struct gasket_dev *gasket_dev; + + ret = pci_enable_device(pci_dev); + if (ret) { + dev_err(&pci_dev->dev, "error enabling PCI device\n"); + return ret; + } + + pci_set_master(pci_dev); + + ret = gasket_pci_add_device(pci_dev, &gasket_dev); + if (ret) { + dev_err(&pci_dev->dev, "error adding gasket device\n"); + pci_disable_device(pci_dev); + return ret; + } + + pci_set_drvdata(pci_dev, gasket_dev); + apex_reset(gasket_dev); + + while (retries < APEX_RESET_RETRY) { + page_table_ready = + gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE_INIT); + msix_table_ready = + gasket_dev_read_64(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE_INIT); + if (page_table_ready && msix_table_ready) + break; + schedule_timeout(msecs_to_jiffies(APEX_RESET_DELAY)); + retries++; + } + + if (retries == APEX_RESET_RETRY) { + if (!page_table_ready) + dev_err(gasket_dev->dev, "Page table init timed out\n"); + if (!msix_table_ready) + dev_err(gasket_dev->dev, "MSI-X table init timed out\n"); + ret = -ETIMEDOUT; + goto remove_device; + } + + ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, + apex_sysfs_attrs); + if (ret) + dev_err(&pci_dev->dev, "error creating device sysfs entries\n"); + + ret = gasket_enable_device(gasket_dev); + if (ret) { + dev_err(&pci_dev->dev, "error enabling gasket device\n"); + goto remove_device; + } + + /* Place device in low power mode until opened */ + if (allow_power_save) + apex_enter_reset(gasket_dev); + + return 0; + +remove_device: + gasket_pci_remove_device(pci_dev); + pci_disable_device(pci_dev); + return ret; +} + +static void apex_pci_remove(struct pci_dev *pci_dev) +{ + struct gasket_dev *gasket_dev = pci_get_drvdata(pci_dev); + + gasket_disable_device(gasket_dev); + gasket_pci_remove_device(pci_dev); + pci_disable_device(pci_dev); +} + +static struct gasket_driver_desc apex_desc = { + .name = "apex", + .driver_version = APEX_DRIVER_VERSION, + .major = 120, + .minor = 0, + .module = THIS_MODULE, + .pci_id_table = apex_pci_ids, + + .num_page_tables = NUM_NODES, + .page_table_bar_index = APEX_BAR_INDEX, + .page_table_configs = apex_page_table_configs, + .page_table_extended_bit = APEX_EXTENDED_SHIFT, + + .bar_descriptions = { + GASKET_UNUSED_BAR, + GASKET_UNUSED_BAR, + { APEX_BAR_BYTES, (VM_WRITE | VM_READ), APEX_BAR_OFFSET, + NUM_REGIONS, mappable_regions, PCI_BAR }, + GASKET_UNUSED_BAR, + GASKET_UNUSED_BAR, + GASKET_UNUSED_BAR, + }, + .coherent_buffer_description = { + APEX_CH_MEM_BYTES, + (VM_WRITE | VM_READ), + APEX_CM_OFFSET, + }, + .interrupt_type = PCI_MSIX, + .interrupt_bar_index = APEX_BAR_INDEX, + .num_interrupts = APEX_INTERRUPT_COUNT, + .interrupts = apex_interrupts, + .interrupt_pack_width = 7, + + .device_open_cb = apex_device_open_cb, + .device_close_cb = apex_device_cleanup, + + .ioctl_handler_cb = apex_ioctl, + .device_status_cb = apex_get_status, + .hardware_revision_cb = NULL, + .device_reset_cb = apex_reset, +}; + +static struct pci_driver apex_pci_driver = { + .name = "apex", + .probe = apex_pci_probe, + .remove = apex_pci_remove, + .id_table = apex_pci_ids, +}; + +static int __init apex_init(void) +{ + int ret; + + ret = gasket_register_device(&apex_desc); + if (ret) + return ret; + ret = pci_register_driver(&apex_pci_driver); + if (ret) + gasket_unregister_device(&apex_desc); + return ret; +} + +static void apex_exit(void) +{ + pci_unregister_driver(&apex_pci_driver); + gasket_unregister_device(&apex_desc); +} +MODULE_DESCRIPTION("Google Apex driver"); +MODULE_VERSION(APEX_DRIVER_VERSION); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("John Joseph <jnjoseph@google.com>"); +MODULE_DEVICE_TABLE(pci, apex_pci_ids); +module_init(apex_init); +module_exit(apex_exit); |