/* * Copyright 2012 Tilera Corporation. All Rights Reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation, version 2. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or * NON INFRINGEMENT. See the GNU General Public License for * more details. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * This file contains the routines to search for PCI buses, * enumerate the buses, and configure any attached devices. */ #define DEBUG_PCI_CFG 0 #if DEBUG_PCI_CFG #define TRACE_CFG_WR(size, val, bus, dev, func, offset) \ pr_info("CFG WR %d-byte VAL %#x to bus %d dev %d func %d addr %u\n", \ size, val, bus, dev, func, offset & 0xFFF); #define TRACE_CFG_RD(size, val, bus, dev, func, offset) \ pr_info("CFG RD %d-byte VAL %#x from bus %d dev %d func %d addr %u\n", \ size, val, bus, dev, func, offset & 0xFFF); #else #define TRACE_CFG_WR(...) #define TRACE_CFG_RD(...) #endif static int pci_probe = 1; /* Information on the PCIe RC ports configuration. */ static int pcie_rc[TILEGX_NUM_TRIO][TILEGX_TRIO_PCIES]; /* * On some platforms with one or more Gx endpoint ports, we need to * delay the PCIe RC port probe for a few seconds to work around * a HW PCIe link-training bug. The exact delay is specified with * a kernel boot argument in the form of "pcie_rc_delay=T,P,S", * where T is the TRIO instance number, P is the port number and S is * the delay in seconds. If the argument is specified, but the delay is * not provided, the value will be DEFAULT_RC_DELAY. */ static int rc_delay[TILEGX_NUM_TRIO][TILEGX_TRIO_PCIES]; /* Default number of seconds that the PCIe RC port probe can be delayed. */ #define DEFAULT_RC_DELAY 10 /* The PCI I/O space size in each PCI domain. */ #define IO_SPACE_SIZE 0x10000 /* Provide shorter versions of some very long constant names. */ #define AUTO_CONFIG_RC \ TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_RC #define AUTO_CONFIG_RC_G1 \ TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_RC_G1 #define AUTO_CONFIG_EP \ TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_ENDPOINT #define AUTO_CONFIG_EP_G1 \ TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_ENDPOINT_G1 /* Array of the PCIe ports configuration info obtained from the BIB. */ struct pcie_trio_ports_property pcie_ports[TILEGX_NUM_TRIO]; /* Number of configured TRIO instances. */ int num_trio_shims; /* All drivers share the TRIO contexts defined here. */ gxio_trio_context_t trio_contexts[TILEGX_NUM_TRIO]; /* Pointer to an array of PCIe RC controllers. */ struct pci_controller pci_controllers[TILEGX_NUM_TRIO * TILEGX_TRIO_PCIES]; int num_rc_controllers; static struct pci_ops tile_cfg_ops; /* Mask of CPUs that should receive PCIe interrupts. */ static struct cpumask intr_cpus_map; /* * Pick a CPU to receive and handle the PCIe interrupts, based on the IRQ #. * For now, we simply send interrupts to non-dataplane CPUs. * We may implement methods to allow user to specify the target CPUs, * e.g. via boot arguments. */ static int tile_irq_cpu(int irq) { unsigned int count; int i = 0; int cpu; count = cpumask_weight(&intr_cpus_map); if (unlikely(count == 0)) { pr_warn("intr_cpus_map empty, interrupts will be delivered to dataplane tiles\n"); return irq % (smp_height * smp_width); } count = irq % count; for_each_cpu(cpu, &intr_cpus_map) { if (i++ == count) break; } return cpu; } /* Open a file descriptor to the TRIO shim. */ static int tile_pcie_open(int trio_index) { gxio_trio_context_t *context = &trio_contexts[trio_index]; int ret; int mac; /* This opens a file descriptor to the TRIO shim. */ ret = gxio_trio_init(context, trio_index); if (ret < 0) goto gxio_trio_init_failure; /* Allocate an ASID for the kernel. */ ret = gxio_trio_alloc_asids(context, 1, 0, 0); if (ret < 0) { pr_err("PCI: ASID alloc failure on TRIO %d, give up\n", trio_index); goto asid_alloc_failure; } context->asid = ret; #ifdef USE_SHARED_PCIE_CONFIG_REGION /* * Alloc a PIO region for config access, shared by all MACs per TRIO. * This shouldn't fail since the kernel is supposed to the first * client of the TRIO's PIO regions. */ ret = gxio_trio_alloc_pio_regions(context, 1, 0, 0); if (ret < 0) { pr_err("PCI: CFG PIO alloc failure on TRIO %d, give up\n", trio_index); goto pio_alloc_failure; } context->pio_cfg_index = ret; /* * For PIO CFG, the bus_address_hi parameter is 0. The mac parameter * is also 0 because it is specified in PIO_REGION_SETUP_CFG_ADDR. */ ret = gxio_trio_init_pio_region_aux(context, context->pio_cfg_index, 0, 0, HV_TRIO_PIO_FLAG_CONFIG_SPACE); if (ret < 0) { pr_err("PCI: CFG PIO init failure on TRIO %d, give up\n", trio_index); goto pio_alloc_failure; } #endif /* Get the properties of the PCIe ports on this TRIO instance. */ ret = gxio_trio_get_port_property(context, &pcie_ports[trio_index]); if (ret < 0) { pr_err("PCI: PCIE_GET_PORT_PROPERTY failure, error %d, on TRIO %d\n", ret, trio_index); goto get_port_property_failure; } context->mmio_base_mac = iorpc_ioremap(context->fd, 0, HV_TRIO_CONFIG_IOREMAP_SIZE); if (context->mmio_base_mac == NULL) { pr_err("PCI: TRIO config space mapping failure, error %d, on TRIO %d\n", ret, trio_index); ret = -ENOMEM; goto trio_mmio_mapping_failure; } /* Check the port strap state which will override the BIB setting. */ for (mac = 0; mac < TILEGX_TRIO_PCIES; mac++) { TRIO_PCIE_INTFC_PORT_CONFIG_t port_config; unsigned int reg_offset; /* Ignore ports that are not specified in the BIB. */ if (!pcie_ports[trio_index].ports[mac].allow_rc && !pcie_ports[trio_index].ports[mac].allow_ep) continue; reg_offset = (TRIO_PCIE_INTFC_PORT_CONFIG << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE << TRIO_CFG_REGION_ADDR__INTFC_SHIFT) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); port_config.word = __gxio_mmio_read(context->mmio_base_mac + reg_offset); if (port_config.strap_state != AUTO_CONFIG_RC && port_config.strap_state != AUTO_CONFIG_RC_G1) { /* * If this is really intended to be an EP port, record * it so that the endpoint driver will know about it. */ if (port_config.strap_state == AUTO_CONFIG_EP || port_config.strap_state == AUTO_CONFIG_EP_G1) pcie_ports[trio_index].ports[mac].allow_ep = 1; } } return ret; trio_mmio_mapping_failure: get_port_property_failure: asid_alloc_failure: #ifdef USE_SHARED_PCIE_CONFIG_REGION pio_alloc_failure: #endif hv_dev_close(context->fd); gxio_trio_init_failure: context->fd = -1; return ret; } static int __init tile_trio_init(void) { int i; /* We loop over all the TRIO shims. */ for (i = 0; i < TILEGX_NUM_TRIO; i++) { if (tile_pcie_open(i) < 0) continue; num_trio_shims++; } return 0; } postcore_initcall(tile_trio_init); static void tilegx_legacy_irq_ack(struct irq_data *d) { __insn_mtspr(SPR_IPI_EVENT_RESET_K, 1UL << d->irq); } static void tilegx_legacy_irq_mask(struct irq_data *d) { __insn_mtspr(SPR_IPI_MASK_SET_K, 1UL << d->irq); } static void tilegx_legacy_irq_unmask(struct irq_data *d) { __insn_mtspr(SPR_IPI_MASK_RESET_K, 1UL << d->irq); } static struct irq_chip tilegx_legacy_irq_chip = { .name = "tilegx_legacy_irq", .irq_ack = tilegx_legacy_irq_ack, .irq_mask = tilegx_legacy_irq_mask, .irq_unmask = tilegx_legacy_irq_unmask, /* TBD: support set_affinity. */ }; /* * This is a wrapper function of the kernel level-trigger interrupt * handler handle_level_irq() for PCI legacy interrupts. The TRIO * is configured such that only INTx Assert interrupts are proxied * to Linux which just calls handle_level_irq() after clearing the * MAC INTx Assert status bit associated with this interrupt. */ static void trio_handle_level_irq(struct irq_desc *desc) { struct pci_controller *controller = irq_desc_get_handler_data(desc); gxio_trio_context_t *trio_context = controller->trio; uint64_t intx = (uint64_t)irq_desc_get_chip_data(desc); int mac = controller->mac; unsigned int reg_offset; uint64_t level_mask; handle_level_irq(desc); /* * Clear the INTx Level status, otherwise future interrupts are * not sent. */ reg_offset = (TRIO_PCIE_INTFC_MAC_INT_STS << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); level_mask = TRIO_PCIE_INTFC_MAC_INT_STS__INT_LEVEL_MASK << intx; __gxio_mmio_write(trio_context->mmio_base_mac + reg_offset, level_mask); } /* * Create kernel irqs and set up the handlers for the legacy interrupts. * Also some minimum initialization for the MSI support. */ static int tile_init_irqs(struct pci_controller *controller) { int i; int j; int irq; int result; cpumask_copy(&intr_cpus_map, cpu_online_mask); for (i = 0; i < 4; i++) { gxio_trio_context_t *context = controller->trio; int cpu; /* Ask the kernel to allocate an IRQ. */ irq = irq_alloc_hwirq(-1); if (!irq) { pr_err("PCI: no free irq vectors, failed for %d\n", i); goto free_irqs; } controller->irq_intx_table[i] = irq; /* Distribute the 4 IRQs to different tiles. */ cpu = tile_irq_cpu(irq); /* Configure the TRIO intr binding for this IRQ. */ result = gxio_trio_config_legacy_intr(context, cpu_x(cpu), cpu_y(cpu), KERNEL_PL, irq, controller->mac, i); if (result < 0) { pr_err("PCI: MAC intx config failed for %d\n", i); goto free_irqs; } /* Register the IRQ handler with the kernel. */ irq_set_chip_and_handler(irq, &tilegx_legacy_irq_chip, trio_handle_level_irq); irq_set_chip_data(irq, (void *)(uint64_t)i); irq_set_handler_data(irq, controller); } return 0; free_irqs: for (j = 0; j < i; j++) irq_free_hwirq(controller->irq_intx_table[j]); return -1; } /* * Return 1 if the port is strapped to operate in RC mode. */ static int strapped_for_rc(gxio_trio_context_t *trio_context, int mac) { TRIO_PCIE_INTFC_PORT_CONFIG_t port_config; unsigned int reg_offset; /* Check the port configuration. */ reg_offset = (TRIO_PCIE_INTFC_PORT_CONFIG << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE << TRIO_CFG_REGION_ADDR__INTFC_SHIFT) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); port_config.word = __gxio_mmio_read(trio_context->mmio_base_mac + reg_offset); if (port_config.strap_state == AUTO_CONFIG_RC || port_config.strap_state == AUTO_CONFIG_RC_G1) return 1; else return 0; } /* * Find valid controllers and fill in pci_controller structs for each * of them. * * Return the number of controllers discovered. */ int __init tile_pci_init(void) { int ctl_index = 0; int i, j; if (!pci_probe) { pr_info("PCI: disabled by boot argument\n"); return 0; } pr_info("PCI: Searching for controllers...\n"); if (num_trio_shims == 0 || sim_is_simulator()) return 0; /* * Now determine which PCIe ports are configured to operate in RC * mode. There is a difference in the port configuration capability * between the Gx36 and Gx72 devices. * * The Gx36 has configuration capability for each of the 3 PCIe * interfaces (disable, auto endpoint, auto RC, etc.). * On the Gx72, you can only select one of the 3 PCIe interfaces per * TRIO to train automatically. Further, the allowable training modes * are reduced to four options (auto endpoint, auto RC, stream x1, * stream x4). * * For Gx36 ports, it must be allowed to be in RC mode by the * Board Information Block, and the hardware strapping pins must be * set to RC mode. * * For Gx72 ports, the port will operate in RC mode if either of the * following is true: * 1. It is allowed to be in RC mode by the Board Information Block, * and the BIB doesn't allow the EP mode. * 2. It is allowed to be in either the RC or the EP mode by the BIB, * and the hardware strapping pin is set to RC mode. */ for (i = 0; i < TILEGX_NUM_TRIO; i++) { gxio_trio_context_t *context = &trio_contexts[i]; if (context->fd < 0) continue; for (j = 0; j < TILEGX_TRIO_PCIES; j++) { int is_rc = 0; if (pcie_ports[i].is_gx72 && pcie_ports[i].ports[j].allow_rc) { if (!pcie_ports[i].ports[j].allow_ep || strapped_for_rc(context, j)) is_rc = 1; } else if (pcie_ports[i].ports[j].allow_rc && strapped_for_rc(context, j)) { is_rc = 1; } if (is_rc) { pcie_rc[i][j] = 1; num_rc_controllers++; } } } /* Return if no PCIe ports are configured to operate in RC mode. */ if (num_rc_controllers == 0) return 0; /* Set the TRIO pointer and MAC index for each PCIe RC port. */ for (i = 0; i < TILEGX_NUM_TRIO; i++) { for (j = 0; j < TILEGX_TRIO_PCIES; j++) { if (pcie_rc[i][j]) { pci_controllers[ctl_index].trio = &trio_contexts[i]; pci_controllers[ctl_index].mac = j; pci_controllers[ctl_index].trio_index = i; ctl_index++; if (ctl_index == num_rc_controllers) goto out; } } } out: /* Configure each PCIe RC port. */ for (i = 0; i < num_rc_controllers; i++) { /* Configure the PCIe MAC to run in RC mode. */ struct pci_controller *controller = &pci_controllers[i]; controller->index = i; controller->ops = &tile_cfg_ops; controller->io_space.start = PCIBIOS_MIN_IO + (i * IO_SPACE_SIZE); controller->io_space.end = controller->io_space.start + IO_SPACE_SIZE - 1; BUG_ON(controller->io_space.end > IO_SPACE_LIMIT); controller->io_space.flags = IORESOURCE_IO; snprintf(controller->io_space_name, sizeof(controller->io_space_name), "PCI I/O domain %d", i); controller->io_space.name = controller->io_space_name; /* * The PCI memory resource is located above the PA space. * For every host bridge, the BAR window or the MMIO aperture * is in range [3GB, 4GB - 1] of a 4GB space beyond the * PA space. */ controller->mem_offset = TILE_PCI_MEM_START + (i * TILE_PCI_BAR_WINDOW_TOP); controller->mem_space.start = controller->mem_offset + TILE_PCI_BAR_WINDOW_TOP - TILE_PCI_BAR_WINDOW_SIZE; controller->mem_space.end = controller->mem_offset + TILE_PCI_BAR_WINDOW_TOP - 1; controller->mem_space.flags = IORESOURCE_MEM; snprintf(controller->mem_space_name, sizeof(controller->mem_space_name), "PCI mem domain %d", i); controller->mem_space.name = controller->mem_space_name; } return num_rc_controllers; } /* * (pin - 1) converts from the PCI standard's [1:4] convention to * a normal [0:3] range. */ static int tile_map_irq(const struct pci_dev *dev, u8 device, u8 pin) { struct pci_controller *controller = (struct pci_controller *)dev->sysdata; return controller->irq_intx_table[pin - 1]; } static void fixup_read_and_payload_sizes(struct pci_controller *controller) { gxio_trio_context_t *trio_context = controller->trio; struct pci_bus *root_bus = controller->root_bus; TRIO_PCIE_RC_DEVICE_CONTROL_t dev_control; TRIO_PCIE_RC_DEVICE_CAP_t rc_dev_cap; unsigned int reg_offset; struct pci_bus *child; int mac; int err; mac = controller->mac; /* Set our max read request size to be 4KB. */ reg_offset = (TRIO_PCIE_RC_DEVICE_CONTROL << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); dev_control.word = __gxio_mmio_read32(trio_context->mmio_base_mac + reg_offset); dev_control.max_read_req_sz = 5; __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset, dev_control.word); /* * Set the max payload size supported by this Gx PCIe MAC. * Though Gx PCIe supports Max Payload Size of up to 1024 bytes, * experiments have shown that setting MPS to 256 yields the * best performance. */ reg_offset = (TRIO_PCIE_RC_DEVICE_CAP << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); rc_dev_cap.word = __gxio_mmio_read32(trio_context->mmio_base_mac + reg_offset); rc_dev_cap.mps_sup = 1; __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset, rc_dev_cap.word); /* Configure PCI Express MPS setting. */ list_for_each_entry(child, &root_bus->children, node) pcie_bus_configure_settings(child); /* * Set the mac_config register in trio based on the MPS/MRS of the link. */ reg_offset = (TRIO_PCIE_RC_DEVICE_CONTROL << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); dev_control.word = __gxio_mmio_read32(trio_context->mmio_base_mac + reg_offset); err = gxio_trio_set_mps_mrs(trio_context, dev_control.max_payload_size, dev_control.max_read_req_sz, mac); if (err < 0) { pr_err("PCI: PCIE_CONFIGURE_MAC_MPS_MRS failure, MAC %d on TRIO %d\n", mac, controller->trio_index); } } static int setup_pcie_rc_delay(char *str) { unsigned long delay = 0; unsigned long trio_index; unsigned long mac; if (str == NULL || !isdigit(*str)) return -EINVAL; trio_index = simple_strtoul(str, (char **)&str, 10); if (trio_index >= TILEGX_NUM_TRIO) return -EINVAL; if (*str != ',') return -EINVAL; str++; if (!isdigit(*str)) return -EINVAL; mac = simple_strtoul(str, (char **)&str, 10); if (mac >= TILEGX_TRIO_PCIES) return -EINVAL; if (*str != '\0') { if (*str != ',') return -EINVAL; str++; if (!isdigit(*str)) return -EINVAL; delay = simple_strtoul(str, (char **)&str, 10); } rc_delay[trio_index][mac] = delay ? : DEFAULT_RC_DELAY; return 0; } early_param("pcie_rc_delay", setup_pcie_rc_delay); /* PCI initialization entry point, called by subsys_initcall. */ int __init pcibios_init(void) { resource_size_t offset; LIST_HEAD(resources); int next_busno; struct pci_host_bridge *bridge; int i; tile_pci_init(); if (num_rc_controllers == 0) return 0; /* * Delay a bit in case devices aren't ready. Some devices are * known to require at least 20ms here, but we use a more * conservative value. */ msleep(250); /* Scan all of the recorded PCI controllers. */ for (next_busno = 0, i = 0; i < num_rc_controllers; i++) { struct pci_controller *controller = &pci_controllers[i]; gxio_trio_context_t *trio_context = controller->trio; TRIO_PCIE_INTFC_PORT_STATUS_t port_status; TRIO_PCIE_INTFC_TX_FIFO_CTL_t tx_fifo_ctl; struct pci_bus *bus; unsigned int reg_offset; unsigned int class_code_revision; int trio_index; int mac; int ret; if (trio_context->fd < 0) continue; trio_index = controller->trio_index; mac = controller->mac; /* * Check for PCIe link-up status to decide if we need * to force the link to come up. */ reg_offset = (TRIO_PCIE_INTFC_PORT_STATUS << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE << TRIO_CFG_REGION_ADDR__INTFC_SHIFT) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); port_status.word = __gxio_mmio_read(trio_context->mmio_base_mac + reg_offset); if (!port_status.dl_up) { if (rc_delay[trio_index][mac]) { pr_info("Delaying PCIe RC TRIO init %d sec on MAC %d on TRIO %d\n", rc_delay[trio_index][mac], mac, trio_index); msleep(rc_delay[trio_index][mac] * 1000); } ret = gxio_trio_force_rc_link_up(trio_context, mac); if (ret < 0) pr_err("PCI: PCIE_FORCE_LINK_UP failure, MAC %d on TRIO %d\n", mac, trio_index); } pr_info("PCI: Found PCI controller #%d on TRIO %d MAC %d\n", i, trio_index, controller->mac); /* Delay the bus probe if needed. */ if (rc_delay[trio_index][mac]) { pr_info("Delaying PCIe RC bus enumerating %d sec on MAC %d on TRIO %d\n", rc_delay[trio_index][mac], mac, trio_index); msleep(rc_delay[trio_index][mac] * 1000); } else { /* * Wait a bit here because some EP devices * take longer to come up. */ msleep(1000); } /* Check for PCIe link-up status again. */ port_status.word = __gxio_mmio_read(trio_context->mmio_base_mac + reg_offset); if (!port_status.dl_up) { if (pcie_ports[trio_index].ports[mac].removable) { pr_info("PCI: link is down, MAC %d on TRIO %d\n", mac, trio_index); pr_info("This is expected if no PCIe card is connected to this link\n"); } else pr_err("PCI: link is down, MAC %d on TRIO %d\n", mac, trio_index); continue; } /* * Ensure that the link can come out of L1 power down state. * Strictly speaking, this is needed only in the case of * heavy RC-initiated DMAs. */ reg_offset = (TRIO_PCIE_INTFC_TX_FIFO_CTL << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); tx_fifo_ctl.word = __gxio_mmio_read(trio_context->mmio_base_mac + reg_offset); tx_fifo_ctl.min_p_credits = 0; __gxio_mmio_write(trio_context->mmio_base_mac + reg_offset, tx_fifo_ctl.word); /* * Change the device ID so that Linux bus crawl doesn't confuse * the internal bridge with any Tilera endpoints. */ reg_offset = (TRIO_PCIE_RC_DEVICE_ID_VEN_ID << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset, (TILERA_GX36_RC_DEV_ID << TRIO_PCIE_RC_DEVICE_ID_VEN_ID__DEV_ID_SHIFT) | TILERA_VENDOR_ID); /* Set the internal P2P bridge class code. */ reg_offset = (TRIO_PCIE_RC_REVISION_ID << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) | (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); class_code_revision = __gxio_mmio_read32(trio_context->mmio_base_mac + reg_offset); class_code_revision = (class_code_revision & 0xff) | (PCI_CLASS_BRIDGE_PCI << 16); __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset, class_code_revision); #ifdef USE_SHARED_PCIE_CONFIG_REGION /* Map in the MMIO space for the PIO region. */ offset = HV_TRIO_PIO_OFFSET(trio_context->pio_cfg_index) | (((unsigned long long)mac) << TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT); #else /* Alloc a PIO region for PCI config access per MAC. */ ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0); if (ret < 0) { pr_err("PCI: PCI CFG PIO alloc failure for mac %d on TRIO %d, give up\n", mac, trio_index); continue; } trio_context->pio_cfg_index[mac] = ret; /* For PIO CFG, the bus_address_hi parameter is 0. */ ret = gxio_trio_init_pio_region_aux(trio_context, trio_context->pio_cfg_index[mac], mac, 0, HV_TRIO_PIO_FLAG_CONFIG_SPACE); if (ret < 0) { pr_err("PCI: PCI CFG PIO init failure for mac %d on TRIO %d, give up\n", mac, trio_index); continue; } offset = HV_TRIO_PIO_OFFSET(trio_context->pio_cfg_index[mac]) | (((unsigned long long)mac) << TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT); #endif /* * To save VMALLOC space, we take advantage of the fact that * bit 29 in the PIO CFG address format is reserved 0. With * TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT being 30, * this cuts VMALLOC space usage from 1GB to 512MB per mac. */ trio_context->mmio_base_pio_cfg[mac] = iorpc_ioremap(trio_context->fd, offset, (1UL << (TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT - 1))); if (trio_context->mmio_base_pio_cfg[mac] == NULL) { pr_err("PCI: PIO map failure for mac %d on TRIO %d\n", mac, trio_index); continue; } /* Initialize the PCIe interrupts. */ if (tile_init_irqs(controller)) { pr_err("PCI: IRQs init failure for mac %d on TRIO %d\n", mac, trio_index); continue; } /* * The PCI memory resource is located above the PA space. * The memory range for the PCI root bus should not overlap * with the physical RAM. */ pci_add_resource_offset(&resources, &controller->mem_space, controller->mem_offset); pci_add_resource(&resources, &controller->io_space); controller->first_busno = next_busno; bridge = pci_alloc_host_bridge(0); if (!bridge) break; list_splice_init(&resources, &bridge->windows); bridge->dev.parent = NULL; bridge->sysdata = controller; bridge->busnr = next_busno; bridge->ops = controller->ops; bridge->swizzle_irq = pci_common_swizzle; bridge->map_irq = tile_map_irq; pci_scan_root_bus_bridge(bridge); bus = bridge->bus; controller->root_bus = bus; next_busno = bus->busn_res.end + 1; } /* * This comes from the generic Linux PCI driver. * * It allocates all of the resources (I/O memory, etc) * associated with the devices read in above. */ pci_assign_unassigned_resources(); /* Record the I/O resources in the PCI controller structure. */ for (i = 0; i < num_rc_controllers; i++) { struct pci_controller *controller = &pci_controllers[i]; gxio_trio_context_t *trio_context = controller->trio; struct pci_bus *root_bus = pci_controllers[i].root_bus; int ret; int j; /* * Skip controllers that are not properly initialized or * have down links. */ if (root_bus == NULL) continue; /* Configure the max_payload_size values for this domain. */ fixup_read_and_payload_sizes(controller); /* Alloc a PIO region for PCI memory access for each RC port. */ ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0); if (ret < 0) { pr_err("PCI: MEM PIO alloc failure on TRIO %d mac %d, give up\n", controller->trio_index, controller->mac); continue; } controller->pio_mem_index = ret; /* * For PIO MEM, the bus_address_hi parameter is hard-coded 0 * because we always assign 32-bit PCI bus BAR ranges. */ ret = gxio_trio_init_pio_region_aux(trio_context, controller->pio_mem_index, controller->mac, 0, 0); if (ret < 0) { pr_err("PCI: MEM PIO init failure on TRIO %d mac %d, give up\n", controller->trio_index, controller->mac); continue; } #ifdef CONFIG_TILE_PCI_IO /* * Alloc a PIO region for PCI I/O space access for each RC port. */ ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0); if (ret < 0) { pr_err("PCI: I/O PIO alloc failure on TRIO %d mac %d, give up\n", controller->trio_index, controller->mac); continue; } controller->pio_io_index = ret; /* * For PIO IO, the bus_address_hi parameter is hard-coded 0 * because PCI I/O address space is 32-bit. */ ret = gxio_trio_init_pio_region_aux(trio_context, controller->pio_io_index, controller->mac, 0, HV_TRIO_PIO_FLAG_IO_SPACE); if (ret < 0) { pr_err("PCI: I/O PIO init failure on TRIO %d mac %d, give up\n", controller->trio_index, controller->mac); continue; } #endif /* * Configure a Mem-Map region for each memory controller so * that Linux can map all of its PA space to the PCI bus. * Use the IOMMU to handle hash-for-home memory. */ for_each_online_node(j) { unsigned long start_pfn = node_start_pfn[j]; unsigned long end_pfn = node_end_pfn[j]; unsigned long nr_pages = end_pfn - start_pfn; ret = gxio_trio_alloc_memory_maps(trio_context, 1, 0, 0); if (ret < 0) { pr_err("PCI: Mem-Map alloc failure on TRIO %d mac %d for MC %d, give up\n", controller->trio_index, controller->mac, j); goto alloc_mem_map_failed; } controller->mem_maps[j] = ret; /* * Initialize the Mem-Map and the I/O MMU so that all * the physical memory can be accessed by the endpoint * devices. The base bus address is set to the base CPA * of this memory controller plus an offset (see pci.h). * The region's base VA is set to the base CPA. The * I/O MMU table essentially translates the CPA to * the real PA. Implicitly, for node 0, we create * a separate Mem-Map region that serves as the inbound * window for legacy 32-bit devices. This is a direct * map of the low 4GB CPA space. */ ret = gxio_trio_init_memory_map_mmu_aux(trio_context, controller->mem_maps[j], start_pfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT, trio_context->asid, controller->mac, (start_pfn << PAGE_SHIFT) + TILE_PCI_MEM_MAP_BASE_OFFSET, j, GXIO_TRIO_ORDER_MODE_UNORDERED); if (ret < 0) { pr_err("PCI: Mem-Map init failure on TRIO %d mac %d for MC %d, give up\n", controller->trio_index, controller->mac, j); goto alloc_mem_map_failed; } continue; alloc_mem_map_failed: break; } pci_bus_add_devices(root_bus); } return 0; } subsys_initcall(pcibios_init); /* Process any "pci=" kernel boot arguments. */ char *__init pcibios_setup(char *str) { if (!strcmp(str, "off")) { pci_probe = 0; return NULL; } return str; } /* * Called for each device after PCI setup is done. * We initialize the PCI device capabilities conservatively, assuming that * all devices can only address the 32-bit DMA space. The exception here is * that the device dma_offset is set to the value that matches the 64-bit * capable devices. This is OK because dma_offset is not used by legacy * dma_ops, nor by the hybrid dma_ops's streaming DMAs, which are 64-bit ops. * This implementation matches the kernel design of setting PCI devices' * coherent_dma_mask to 0xffffffffull by default, allowing the device drivers * to skip calling pci_set_consistent_dma_mask(DMA_BIT_MASK(32)). */ static void pcibios_fixup_final(struct pci_dev *pdev) { set_dma_ops(&pdev->dev, gx_legacy_pci_dma_map_ops); set_dma_offset(&pdev->dev, TILE_PCI_MEM_MAP_BASE_OFFSET); pdev->dev.archdata.max_direct_dma_addr = TILE_PCI_MAX_DIRECT_DMA_ADDRESS; pdev->dev.coherent_dma_mask = TILE_PCI_MAX_DIRECT_DMA_ADDRESS; } DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, pcibios_fixup_final); /* Map a PCI MMIO bus address into VA space. */ void __iomem *ioremap(resource_size_t phys_addr, unsigned long size) { struct pci_controller *controller = NULL; resource_size_t bar_start; resource_size_t bar_end; resource_size_t offset; resource_size_t start; resource_size_t end; int trio_fd; int i; start = phys_addr; end = phys_addr + size - 1; /* * By searching phys_addr in each controller's mem_space, we can * determine the controller that should accept the PCI memory access. */ for (i = 0; i < num_rc_controllers; i++) { /* * Skip controllers that are not properly initialized or * have down links. */ if (pci_controllers[i].root_bus == NULL) continue; bar_start = pci_controllers[i].mem_space.start; bar_end = pci_controllers[i].mem_space.end; if ((start >= bar_start) && (end <= bar_end)) { controller = &pci_controllers[i]; break; } } if (controller == NULL) return NULL; trio_fd = controller->trio->fd; /* Convert the resource start to the bus address offset. */ start = phys_addr - controller->mem_offset; offset = HV_TRIO_PIO_OFFSET(controller->pio_mem_index) + start; /* We need to keep the PCI bus address's in-page offset in the VA. */ return iorpc_ioremap(trio_fd, offset, size) + (start & (PAGE_SIZE - 1)); } EXPORT_SYMBOL(ioremap); #ifdef CONFIG_TILE_PCI_IO /* Map a PCI I/O address into VA space. */ void __iomem *ioport_map(unsigned long port, unsigned int size) { struct pci_controller *controller = NULL; resource_size_t bar_start; resource_size_t bar_end; resource_size_t offset; resource_size_t start; resource_size_t end; int trio_fd; int i; start = port; end = port + size - 1; /* * By searching the port in each controller's io_space, we can * determine the controller that should accept the PCI I/O access. */ for (i = 0; i < num_rc_controllers; i++) { /* * Skip controllers that are not properly initialized or * have down links. */ if (pci_controllers[i].root_bus == NULL) continue; bar_start = pci_controllers[i].io_space.start; bar_end = pci_controllers[i].io_space.end; if ((start >= bar_start) && (end <= bar_end)) { controller = &pci_controllers[i]; break; } } if (controller == NULL) return NULL; trio_fd = controller->trio->fd; /* Convert the resource start to the bus address offset. */ port -= controller->io_space.start; offset = HV_TRIO_PIO_OFFSET(controller->pio_io_index) + port; /* We need to keep the PCI bus address's in-page offset in the VA. */ return iorpc_ioremap(trio_fd, offset, size) + (port & (PAGE_SIZE - 1)); } EXPORT_SYMBOL(ioport_map); void ioport_unmap(void __iomem *addr) { iounmap(addr); } EXPORT_SYMBOL(ioport_unmap); #endif void pci_iounmap(struct pci_dev *dev, void __iomem *addr) { iounmap(addr); } EXPORT_SYMBOL(pci_iounmap); /**************************************************************** * * Tile PCI config space read/write routines * ****************************************************************/ /* * These are the normal read and write ops * These are expanded with macros from pci_bus_read_config_byte() etc. * * devfn is the combined PCI device & function. * * offset is in bytes, from the start of config space for the * specified bus & device. */ static int tile_cfg_read(struct pci_bus *bus, unsigned int devfn, int offset, int size, u32 *val) { struct pci_controller *controller = bus->sysdata; gxio_trio_context_t *trio_context = controller->trio; int busnum = bus->number & 0xff; int device = PCI_SLOT(devfn); int function = PCI_FUNC(devfn); int config_type = 1; TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR_t cfg_addr; void *mmio_addr; /* * Map all accesses to the local device on root bus into the * MMIO space of the MAC. Accesses to the downstream devices * go to the PIO space. */ if (pci_is_root_bus(bus)) { if (device == 0) { /* * This is the internal downstream P2P bridge, * access directly. */ unsigned int reg_offset; reg_offset = ((offset & 0xFFF) << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_PROTECTED << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) | (controller->mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); mmio_addr = trio_context->mmio_base_mac + reg_offset; goto valid_device; } else { /* * We fake an empty device for (device > 0), * since there is only one device on bus 0. */ goto invalid_device; } } /* * Accesses to the directly attached device have to be * sent as type-0 configs. */ if (busnum == (controller->first_busno + 1)) { /* * There is only one device off of our built-in P2P bridge. */ if (device != 0) goto invalid_device; config_type = 0; } cfg_addr.word = 0; cfg_addr.reg_addr = (offset & 0xFFF); cfg_addr.fn = function; cfg_addr.dev = device; cfg_addr.bus = busnum; cfg_addr.type = config_type; /* * Note that we don't set the mac field in cfg_addr because the * mapping is per port. */ mmio_addr = trio_context->mmio_base_pio_cfg[controller->mac] + cfg_addr.word; valid_device: switch (size) { case 4: *val = __gxio_mmio_read32(mmio_addr); break; case 2: *val = __gxio_mmio_read16(mmio_addr); break; case 1: *val = __gxio_mmio_read8(mmio_addr); break; default: return PCIBIOS_FUNC_NOT_SUPPORTED; } TRACE_CFG_RD(size, *val, busnum, device, function, offset); return 0; invalid_device: switch (size) { case 4: *val = 0xFFFFFFFF; break; case 2: *val = 0xFFFF; break; case 1: *val = 0xFF; break; default: return PCIBIOS_FUNC_NOT_SUPPORTED; } return 0; } /* * See tile_cfg_read() for relevant comments. * Note that "val" is the value to write, not a pointer to that value. */ static int tile_cfg_write(struct pci_bus *bus, unsigned int devfn, int offset, int size, u32 val) { struct pci_controller *controller = bus->sysdata; gxio_trio_context_t *trio_context = controller->trio; int busnum = bus->number & 0xff; int device = PCI_SLOT(devfn); int function = PCI_FUNC(devfn); int config_type = 1; TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR_t cfg_addr; void *mmio_addr; u32 val_32 = (u32)val; u16 val_16 = (u16)val; u8 val_8 = (u8)val; /* * Map all accesses to the local device on root bus into the * MMIO space of the MAC. Accesses to the downstream devices * go to the PIO space. */ if (pci_is_root_bus(bus)) { if (device == 0) { /* * This is the internal downstream P2P bridge, * access directly. */ unsigned int reg_offset; reg_offset = ((offset & 0xFFF) << TRIO_CFG_REGION_ADDR__REG_SHIFT) | (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_PROTECTED << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) | (controller->mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT); mmio_addr = trio_context->mmio_base_mac + reg_offset; goto valid_device; } else { /* * We fake an empty device for (device > 0), * since there is only one device on bus 0. */ goto invalid_device; } } /* * Accesses to the directly attached device have to be * sent as type-0 configs. */ if (busnum == (controller->first_busno + 1)) { /* * There is only one device off of our built-in P2P bridge. */ if (device != 0) goto invalid_device; config_type = 0; } cfg_addr.word = 0; cfg_addr.reg_addr = (offset & 0xFFF); cfg_addr.fn = function; cfg_addr.dev = device; cfg_addr.bus = busnum; cfg_addr.type = config_type; /* * Note that we don't set the mac field in cfg_addr because the * mapping is per port. */ mmio_addr = trio_context->mmio_base_pio_cfg[controller->mac] + cfg_addr.word; valid_device: switch (size) { case 4: __gxio_mmio_write32(mmio_addr, val_32); TRACE_CFG_WR(size, val_32, busnum, device, function, offset); break; case 2: __gxio_mmio_write16(mmio_addr, val_16); TRACE_CFG_WR(size, val_16, busnum, device, function, offset); break; case 1: __gxio_mmio_write8(mmio_addr, val_8); TRACE_CFG_WR(size, val_8, busnum, device, function, offset); break; default: return PCIBIOS_FUNC_NOT_SUPPORTED; } invalid_device: return 0; } static struct pci_ops tile_cfg_ops = { .read = tile_cfg_read, .write = tile_cfg_write, }; /* MSI support starts here. */ static unsigned int tilegx_msi_startup(struct irq_data *d) { if (irq_data_get_msi_desc(d)) pci_msi_unmask_irq(d); return 0; } static void tilegx_msi_ack(struct irq_data *d) { __insn_mtspr(SPR_IPI_EVENT_RESET_K, 1UL << d->irq); } static void tilegx_msi_mask(struct irq_data *d) { pci_msi_mask_irq(d); __insn_mtspr(SPR_IPI_MASK_SET_K, 1UL << d->irq); } static void tilegx_msi_unmask(struct irq_data *d) { __insn_mtspr(SPR_IPI_MASK_RESET_K, 1UL << d->irq); pci_msi_unmask_irq(d); } static struct irq_chip tilegx_msi_chip = { .name = "tilegx_msi", .irq_startup = tilegx_msi_startup, .irq_ack = tilegx_msi_ack, .irq_mask = tilegx_msi_mask, .irq_unmask = tilegx_msi_unmask, /* TBD: support set_affinity. */ }; int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) { struct pci_controller *controller; gxio_trio_context_t *trio_context; struct msi_msg msg; int default_irq; uint64_t mem_map_base; uint64_t mem_map_limit; u64 msi_addr; int mem_map; int cpu; int irq; int ret; irq = irq_alloc_hwirq(-1); if (!irq) return -ENOSPC; /* * Since we use a 64-bit Mem-Map to accept the MSI write, we fail * devices that are not capable of generating a 64-bit message address. * These devices will fall back to using the legacy interrupts. * Most PCIe endpoint devices do support 64-bit message addressing. */ if (desc->msi_attrib.is_64 == 0) { dev_info(&pdev->dev, "64-bit MSI message address not supported, falling back to legacy interrupts\n"); ret = -ENOMEM; goto is_64_failure; } default_irq = desc->msi_attrib.default_irq; controller = irq_get_handler_data(default_irq); BUG_ON(!controller); trio_context = controller->trio; /* * Allocate a scatter-queue that will accept the MSI write and * trigger the TILE-side interrupts. We use the scatter-queue regions * before the mem map regions, because the latter are needed by more * applications. */ mem_map = gxio_trio_alloc_scatter_queues(trio_context, 1, 0, 0); if (mem_map >= 0) { TRIO_MAP_SQ_DOORBELL_FMT_t doorbell_template = {{ .pop = 0, .doorbell = 1, }}; mem_map += TRIO_NUM_MAP_MEM_REGIONS; mem_map_base = MEM_MAP_INTR_REGIONS_BASE + mem_map * MEM_MAP_INTR_REGION_SIZE; mem_map_limit = mem_map_base + MEM_MAP_INTR_REGION_SIZE - 1; msi_addr = mem_map_base + MEM_MAP_INTR_REGION_SIZE - 8; msg.data = (unsigned int)doorbell_template.word; } else { /* SQ regions are out, allocate from map mem regions. */ mem_map = gxio_trio_alloc_memory_maps(trio_context, 1, 0, 0); if (mem_map < 0) { dev_info(&pdev->dev, "%s Mem-Map alloc failure - failed to initialize MSI interrupts - falling back to legacy interrupts\n", desc->msi_attrib.is_msix ? "MSI-X" : "MSI"); ret = -ENOMEM; goto msi_mem_map_alloc_failure; } mem_map_base = MEM_MAP_INTR_REGIONS_BASE + mem_map * MEM_MAP_INTR_REGION_SIZE; mem_map_limit = mem_map_base + MEM_MAP_INTR_REGION_SIZE - 1; msi_addr = mem_map_base + TRIO_MAP_MEM_REG_INT3 - TRIO_MAP_MEM_REG_INT0; msg.data = mem_map; } /* We try to distribute different IRQs to different tiles. */ cpu = tile_irq_cpu(irq); /* * Now call up to the HV to configure the MSI interrupt and * set up the IPI binding. */ ret = gxio_trio_config_msi_intr(trio_context, cpu_x(cpu), cpu_y(cpu), KERNEL_PL, irq, controller->mac, mem_map, mem_map_base, mem_map_limit, trio_context->asid); if (ret < 0) { dev_info(&pdev->dev, "HV MSI config failed\n"); goto hv_msi_config_failure; } irq_set_msi_desc(irq, desc); msg.address_hi = msi_addr >> 32; msg.address_lo = msi_addr & 0xffffffff; pci_write_msi_msg(irq, &msg); irq_set_chip_and_handler(irq, &tilegx_msi_chip, handle_level_irq); irq_set_handler_data(irq, controller); return 0; hv_msi_config_failure: /* Free mem-map */ msi_mem_map_alloc_failure: is_64_failure: irq_free_hwirq(irq); return ret; } void arch_teardown_msi_irq(unsigned int irq) { irq_free_hwirq(irq); }