diff options
Diffstat (limited to 'drivers/pci/probe.c')
| -rw-r--r-- | drivers/pci/probe.c | 1061 |
1 files changed, 692 insertions, 369 deletions
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 2f66988cea25..41183aed8f5d 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -3,13 +3,15 @@ * PCI detection and setup code */ +#include <linux/array_size.h> #include <linux/kernel.h> #include <linux/delay.h> #include <linux/init.h> #include <linux/pci.h> #include <linux/msi.h> -#include <linux/of_device.h> #include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/pci_hotplug.h> #include <linux/slab.h> #include <linux/module.h> @@ -19,6 +21,7 @@ #include <linux/hypervisor.h> #include <linux/irqdomain.h> #include <linux/pm_runtime.h> +#include <linux/bitfield.h> #include "pci.h" #define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */ @@ -95,7 +98,7 @@ static void release_pcibus_dev(struct device *dev) kfree(pci_bus); } -static struct class pcibus_class = { +static const struct class pcibus_class = { .name = "pci_bus", .dev_release = &release_pcibus_dev, .dev_groups = pcibus_groups, @@ -165,39 +168,66 @@ static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar) #define PCI_COMMAND_DECODE_ENABLE (PCI_COMMAND_MEMORY | PCI_COMMAND_IO) /** - * pci_read_base - Read a PCI BAR + * __pci_size_bars - Read the raw BAR mask for a range of PCI BARs + * @dev: the PCI device + * @count: number of BARs to size + * @pos: starting config space position + * @sizes: array to store mask values + * @rom: indicate whether to use ROM mask, which avoids enabling ROM BARs + * + * Provided @sizes array must be sufficiently sized to store results for + * @count u32 BARs. Caller is responsible for disabling decode to specified + * BAR range around calling this function. This function is intended to avoid + * disabling decode around sizing each BAR individually, which can result in + * non-trivial overhead in virtualized environments with very large PCI BARs. + */ +static void __pci_size_bars(struct pci_dev *dev, int count, + unsigned int pos, u32 *sizes, bool rom) +{ + u32 orig, mask = rom ? PCI_ROM_ADDRESS_MASK : ~0; + int i; + + for (i = 0; i < count; i++, pos += 4, sizes++) { + pci_read_config_dword(dev, pos, &orig); + pci_write_config_dword(dev, pos, mask); + pci_read_config_dword(dev, pos, sizes); + pci_write_config_dword(dev, pos, orig); + } +} + +void __pci_size_stdbars(struct pci_dev *dev, int count, + unsigned int pos, u32 *sizes) +{ + __pci_size_bars(dev, count, pos, sizes, false); +} + +static void __pci_size_rom(struct pci_dev *dev, unsigned int pos, u32 *sizes) +{ + __pci_size_bars(dev, 1, pos, sizes, true); +} + +/** + * __pci_read_base - Read a PCI BAR * @dev: the PCI device * @type: type of the BAR * @res: resource buffer to be filled in * @pos: BAR position in the config space + * @sizes: array of one or more pre-read BAR masks * * Returns 1 if the BAR is 64-bit, or 0 if 32-bit. */ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, - struct resource *res, unsigned int pos) + struct resource *res, unsigned int pos, u32 *sizes) { - u32 l = 0, sz = 0, mask; + u32 l = 0, sz; u64 l64, sz64, mask64; - u16 orig_cmd; struct pci_bus_region region, inverted_region; - - mask = type ? PCI_ROM_ADDRESS_MASK : ~0; - - /* No printks while decoding is disabled! */ - if (!dev->mmio_always_on) { - pci_read_config_word(dev, PCI_COMMAND, &orig_cmd); - if (orig_cmd & PCI_COMMAND_DECODE_ENABLE) { - pci_write_config_word(dev, PCI_COMMAND, - orig_cmd & ~PCI_COMMAND_DECODE_ENABLE); - } - } + const char *res_name = pci_resource_name(dev, res - dev->resource); res->name = pci_name(dev); pci_read_config_dword(dev, pos, &l); - pci_write_config_dword(dev, pos, l | mask); - pci_read_config_dword(dev, pos, &sz); - pci_write_config_dword(dev, pos, l); + sz = sizes[0]; /* * All bits set in sz means the device isn't working properly. @@ -205,14 +235,14 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, * memory BAR or a ROM, bit 0 must be clear; if it's an io BAR, bit * 1 must be clear. */ - if (sz == 0xffffffff) + if (PCI_POSSIBLE_ERROR(sz)) sz = 0; /* * I don't know how l can have all bits set. Copied from old code. * Maybe it fixes a bug on some ancient platform. */ - if (l == 0xffffffff) + if (PCI_POSSIBLE_ERROR(l)) l = 0; if (type == pci_bar_unknown) { @@ -237,25 +267,19 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, if (res->flags & IORESOURCE_MEM_64) { pci_read_config_dword(dev, pos + 4, &l); - pci_write_config_dword(dev, pos + 4, ~0); - pci_read_config_dword(dev, pos + 4, &sz); - pci_write_config_dword(dev, pos + 4, l); + sz = sizes[1]; l64 |= ((u64)l << 32); sz64 |= ((u64)sz << 32); mask64 |= ((u64)~0 << 32); } - if (!dev->mmio_always_on && (orig_cmd & PCI_COMMAND_DECODE_ENABLE)) - pci_write_config_word(dev, PCI_COMMAND, orig_cmd); - if (!sz64) goto fail; sz64 = pci_size(l64, sz64, mask64); if (!sz64) { - pci_info(dev, FW_BUG "reg 0x%x: invalid BAR (can't size)\n", - pos); + pci_info(dev, FW_BUG "%s: invalid; can't size\n", res_name); goto fail; } @@ -265,8 +289,8 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED; res->start = 0; res->end = 0; - pci_err(dev, "reg 0x%x: can't handle BAR larger than 4GB (size %#010llx)\n", - pos, (unsigned long long)sz64); + pci_err(dev, "%s: can't handle BAR larger than 4GB (size %#010llx)\n", + res_name, (unsigned long long)sz64); goto out; } @@ -275,8 +299,8 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, res->flags |= IORESOURCE_UNSET; res->start = 0; res->end = sz64 - 1; - pci_info(dev, "reg 0x%x: can't handle BAR above 4GB (bus address %#010llx)\n", - pos, (unsigned long long)l64); + pci_info(dev, "%s: can't handle BAR above 4GB (bus address %#010llx)\n", + res_name, (unsigned long long)l64); goto out; } } @@ -302,8 +326,8 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, res->flags |= IORESOURCE_UNSET; res->start = 0; res->end = region.end - region.start; - pci_info(dev, "reg 0x%x: initial BAR value %#010llx invalid\n", - pos, (unsigned long long)region.start); + pci_info(dev, "%s: initial BAR value %#010llx invalid\n", + res_name, (unsigned long long)region.start); } goto out; @@ -313,14 +337,19 @@ fail: res->flags = 0; out: if (res->flags) - pci_info(dev, "reg 0x%x: %pR\n", pos, res); + pci_info(dev, "%s %pR\n", res_name, res); return (res->flags & IORESOURCE_MEM_64) ? 1 : 0; } -static void pci_read_bases(struct pci_dev *dev, unsigned int howmany, int rom) +static __always_inline void pci_read_bases(struct pci_dev *dev, + unsigned int howmany, int rom) { + u32 rombar, stdbars[PCI_STD_NUM_BARS]; unsigned int pos, reg; + u16 orig_cmd; + + BUILD_BUG_ON(statically_true(howmany > PCI_STD_NUM_BARS)); if (dev->non_compliant_bars) return; @@ -329,10 +358,28 @@ static void pci_read_bases(struct pci_dev *dev, unsigned int howmany, int rom) if (dev->is_virtfn) return; + /* No printks while decoding is disabled! */ + if (!dev->mmio_always_on) { + pci_read_config_word(dev, PCI_COMMAND, &orig_cmd); + if (orig_cmd & PCI_COMMAND_DECODE_ENABLE) { + pci_write_config_word(dev, PCI_COMMAND, + orig_cmd & ~PCI_COMMAND_DECODE_ENABLE); + } + } + + __pci_size_stdbars(dev, howmany, PCI_BASE_ADDRESS_0, stdbars); + if (rom) + __pci_size_rom(dev, rom, &rombar); + + if (!dev->mmio_always_on && + (orig_cmd & PCI_COMMAND_DECODE_ENABLE)) + pci_write_config_word(dev, PCI_COMMAND, orig_cmd); + for (pos = 0; pos < howmany; pos++) { struct resource *res = &dev->resource[pos]; reg = PCI_BASE_ADDRESS_0 + (pos << 2); - pos += __pci_read_base(dev, pci_bar_unknown, res, reg); + pos += __pci_read_base(dev, pci_bar_unknown, + res, reg, &stdbars[pos]); } if (rom) { @@ -340,68 +387,16 @@ static void pci_read_bases(struct pci_dev *dev, unsigned int howmany, int rom) dev->rom_base_reg = rom; res->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH | IORESOURCE_READONLY | IORESOURCE_SIZEALIGN; - __pci_read_base(dev, pci_bar_mem32, res, rom); - } -} - -static void pci_read_bridge_windows(struct pci_dev *bridge) -{ - u16 io; - u32 pmem, tmp; - - pci_read_config_word(bridge, PCI_IO_BASE, &io); - if (!io) { - pci_write_config_word(bridge, PCI_IO_BASE, 0xe0f0); - pci_read_config_word(bridge, PCI_IO_BASE, &io); - pci_write_config_word(bridge, PCI_IO_BASE, 0x0); - } - if (io) - bridge->io_window = 1; - - /* - * DECchip 21050 pass 2 errata: the bridge may miss an address - * disconnect boundary by one PCI data phase. Workaround: do not - * use prefetching on this device. - */ - if (bridge->vendor == PCI_VENDOR_ID_DEC && bridge->device == 0x0001) - return; - - pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem); - if (!pmem) { - pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE, - 0xffe0fff0); - pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem); - pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE, 0x0); - } - if (!pmem) - return; - - bridge->pref_window = 1; - - if ((pmem & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) { - - /* - * Bridge claims to have a 64-bit prefetchable memory - * window; verify that the upper bits are actually - * writable. - */ - pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &pmem); - pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, - 0xffffffff); - pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &tmp); - pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, pmem); - if (tmp) - bridge->pref_64_window = 1; + __pci_read_base(dev, pci_bar_mem32, res, rom, &rombar); } } -static void pci_read_bridge_io(struct pci_bus *child) +static void pci_read_bridge_io(struct pci_dev *dev, struct resource *res, + bool log) { - struct pci_dev *dev = child->self; u8 io_base_lo, io_limit_lo; unsigned long io_mask, io_granularity, base, limit; struct pci_bus_region region; - struct resource *res; io_mask = PCI_IO_RANGE_MASK; io_granularity = 0x1000; @@ -411,7 +406,6 @@ static void pci_read_bridge_io(struct pci_bus *child) io_granularity = 0x400; } - res = child->resource[0]; pci_read_config_byte(dev, PCI_IO_BASE, &io_base_lo); pci_read_config_byte(dev, PCI_IO_LIMIT, &io_limit_lo); base = (io_base_lo & io_mask) << 8; @@ -426,47 +420,54 @@ static void pci_read_bridge_io(struct pci_bus *child) limit |= ((unsigned long) io_limit_hi << 16); } + res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO; + if (base <= limit) { - res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO; region.start = base; region.end = limit + io_granularity - 1; pcibios_bus_to_resource(dev->bus, res, ®ion); - pci_info(dev, " bridge window %pR\n", res); + if (log) + pci_info(dev, " bridge window %pR\n", res); + } else { + resource_set_range(res, 0, 0); + res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED; } } -static void pci_read_bridge_mmio(struct pci_bus *child) +static void pci_read_bridge_mmio(struct pci_dev *dev, struct resource *res, + bool log) { - struct pci_dev *dev = child->self; u16 mem_base_lo, mem_limit_lo; unsigned long base, limit; struct pci_bus_region region; - struct resource *res; - res = child->resource[1]; pci_read_config_word(dev, PCI_MEMORY_BASE, &mem_base_lo); pci_read_config_word(dev, PCI_MEMORY_LIMIT, &mem_limit_lo); base = ((unsigned long) mem_base_lo & PCI_MEMORY_RANGE_MASK) << 16; limit = ((unsigned long) mem_limit_lo & PCI_MEMORY_RANGE_MASK) << 16; + + res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | IORESOURCE_MEM; + if (base <= limit) { - res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | IORESOURCE_MEM; region.start = base; region.end = limit + 0xfffff; pcibios_bus_to_resource(dev->bus, res, ®ion); - pci_info(dev, " bridge window %pR\n", res); + if (log) + pci_info(dev, " bridge window %pR\n", res); + } else { + resource_set_range(res, 0, 0); + res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED; } } -static void pci_read_bridge_mmio_pref(struct pci_bus *child) +static void pci_read_bridge_mmio_pref(struct pci_dev *dev, struct resource *res, + bool log) { - struct pci_dev *dev = child->self; u16 mem_base_lo, mem_limit_lo; u64 base64, limit64; pci_bus_addr_t base, limit; struct pci_bus_region region; - struct resource *res; - res = child->resource[2]; pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, &mem_base_lo); pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, &mem_limit_lo); base64 = (mem_base_lo & PCI_PREF_RANGE_MASK) << 16; @@ -498,16 +499,87 @@ static void pci_read_bridge_mmio_pref(struct pci_bus *child) return; } + res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) | IORESOURCE_MEM | + IORESOURCE_PREFETCH; + if (res->flags & PCI_PREF_RANGE_TYPE_64) + res->flags |= IORESOURCE_MEM_64; + if (base <= limit) { - res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) | - IORESOURCE_MEM | IORESOURCE_PREFETCH; - if (res->flags & PCI_PREF_RANGE_TYPE_64) - res->flags |= IORESOURCE_MEM_64; region.start = base; region.end = limit + 0xfffff; pcibios_bus_to_resource(dev->bus, res, ®ion); - pci_info(dev, " bridge window %pR\n", res); + if (log) + pci_info(dev, " bridge window %pR\n", res); + } else { + resource_set_range(res, 0, 0); + res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED; + } +} + +static void pci_read_bridge_windows(struct pci_dev *bridge) +{ + u32 buses; + u16 io; + u32 pmem, tmp; + struct resource res; + + pci_read_config_dword(bridge, PCI_PRIMARY_BUS, &buses); + res.flags = IORESOURCE_BUS; + res.start = (buses >> 8) & 0xff; + res.end = (buses >> 16) & 0xff; + pci_info(bridge, "PCI bridge to %pR%s\n", &res, + bridge->transparent ? " (subtractive decode)" : ""); + + pci_read_config_word(bridge, PCI_IO_BASE, &io); + if (!io) { + pci_write_config_word(bridge, PCI_IO_BASE, 0xe0f0); + pci_read_config_word(bridge, PCI_IO_BASE, &io); + pci_write_config_word(bridge, PCI_IO_BASE, 0x0); + } + if (io) { + bridge->io_window = 1; + pci_read_bridge_io(bridge, &res, true); + } + + pci_read_bridge_mmio(bridge, &res, true); + + /* + * DECchip 21050 pass 2 errata: the bridge may miss an address + * disconnect boundary by one PCI data phase. Workaround: do not + * use prefetching on this device. + */ + if (bridge->vendor == PCI_VENDOR_ID_DEC && bridge->device == 0x0001) + return; + + pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem); + if (!pmem) { + pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE, + 0xffe0fff0); + pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem); + pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE, 0x0); + } + if (!pmem) + return; + + bridge->pref_window = 1; + + if ((pmem & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) { + + /* + * Bridge claims to have a 64-bit prefetchable memory + * window; verify that the upper bits are actually + * writable. + */ + pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &pmem); + pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, + 0xffffffff); + pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &tmp); + pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, pmem); + if (tmp) + bridge->pref_64_window = 1; } + + pci_read_bridge_mmio_pref(bridge, &res, true); } void pci_read_bridge_bases(struct pci_bus *child) @@ -527,19 +599,23 @@ void pci_read_bridge_bases(struct pci_bus *child) for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) child->resource[i] = &dev->resource[PCI_BRIDGE_RESOURCES+i]; - pci_read_bridge_io(child); - pci_read_bridge_mmio(child); - pci_read_bridge_mmio_pref(child); - - if (dev->transparent) { - pci_bus_for_each_resource(child->parent, res, i) { - if (res && res->flags) { - pci_bus_add_resource(child, res, - PCI_SUBTRACTIVE_DECODE); - pci_info(dev, " bridge window %pR (subtractive decode)\n", - res); - } - } + pci_read_bridge_io(child->self, + child->resource[PCI_BUS_BRIDGE_IO_WINDOW], false); + pci_read_bridge_mmio(child->self, + child->resource[PCI_BUS_BRIDGE_MEM_WINDOW], false); + pci_read_bridge_mmio_pref(child->self, + child->resource[PCI_BUS_BRIDGE_PREF_MEM_WINDOW], + false); + + if (!dev->transparent) + return; + + pci_bus_for_each_resource(child->parent, res) { + if (!res || !res->flags) + continue; + + pci_bus_add_resource(child, res); + pci_info(dev, " bridge window %pR (subtractive decode)\n", res); } } @@ -574,9 +650,26 @@ static void pci_release_host_bridge_dev(struct device *dev) pci_free_resource_list(&bridge->windows); pci_free_resource_list(&bridge->dma_ranges); + + /* Host bridges only have domain_nr set in the emulation case */ + if (bridge->domain_nr != PCI_DOMAIN_NR_NOT_SET) + pci_bus_release_emul_domain_nr(bridge->domain_nr); + kfree(bridge); } +static const struct attribute_group *pci_host_bridge_groups[] = { +#ifdef CONFIG_PCI_IDE + &pci_ide_attr_group, +#endif + NULL +}; + +static const struct device_type pci_host_bridge_type = { + .groups = pci_host_bridge_groups, + .release = pci_release_host_bridge_dev, +}; + static void pci_init_host_bridge(struct pci_host_bridge *bridge) { INIT_LIST_HEAD(&bridge->windows); @@ -594,6 +687,10 @@ static void pci_init_host_bridge(struct pci_host_bridge *bridge) bridge->native_pme = 1; bridge->native_ltr = 1; bridge->native_dpc = 1; + bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET; + bridge->native_cxl_error = 1; + bridge->dev.type = &pci_host_bridge_type; + pci_ide_init_host_bridge(bridge); device_initialize(&bridge->dev); } @@ -607,7 +704,6 @@ struct pci_host_bridge *pci_alloc_host_bridge(size_t priv) return NULL; pci_init_host_bridge(bridge); - bridge->dev.release = pci_release_host_bridge_dev; return bridge; } @@ -628,11 +724,17 @@ struct pci_host_bridge *devm_pci_alloc_host_bridge(struct device *dev, if (!bridge) return NULL; + bridge->dev.parent = dev; + ret = devm_add_action_or_reset(dev, devm_pci_alloc_host_bridge_release, bridge); if (ret) return NULL; + ret = devm_of_pci_bridge_init(dev, bridge); + if (ret) + return NULL; + return bridge; } EXPORT_SYMBOL(devm_pci_alloc_host_bridge); @@ -671,7 +773,7 @@ const unsigned char pcie_link_speed[] = { PCIE_SPEED_8_0GT, /* 3 */ PCIE_SPEED_16_0GT, /* 4 */ PCIE_SPEED_32_0GT, /* 5 */ - PCI_SPEED_UNKNOWN, /* 6 */ + PCIE_SPEED_64_0GT, /* 6 */ PCI_SPEED_UNKNOWN, /* 7 */ PCI_SPEED_UNKNOWN, /* 8 */ PCI_SPEED_UNKNOWN, /* 9 */ @@ -713,6 +815,7 @@ const char *pci_speed_string(enum pci_bus_speed speed) "8.0 GT/s PCIe", /* 0x16 */ "16.0 GT/s PCIe", /* 0x17 */ "32.0 GT/s PCIe", /* 0x18 */ + "64.0 GT/s PCIe", /* 0x19 */ }; if (speed < ARRAY_SIZE(speed_strings)) @@ -721,9 +824,14 @@ const char *pci_speed_string(enum pci_bus_speed speed) } EXPORT_SYMBOL_GPL(pci_speed_string); -void pcie_update_link_speed(struct pci_bus *bus, u16 linksta) +void pcie_update_link_speed(struct pci_bus *bus) { - bus->cur_bus_speed = pcie_link_speed[linksta & PCI_EXP_LNKSTA_CLS]; + struct pci_dev *bridge = bus->self; + u16 linksta, linksta2; + + pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &linksta); + pcie_capability_read_word(bridge, PCI_EXP_LNKSTA2, &linksta2); + __pcie_update_link_speed(bus, linksta, linksta2); } EXPORT_SYMBOL_GPL(pcie_update_link_speed); @@ -798,22 +906,19 @@ static void pci_set_bus_speed(struct pci_bus *bus) } bus->max_bus_speed = max; - bus->cur_bus_speed = pcix_bus_speed[ - (status & PCI_X_SSTATUS_FREQ) >> 6]; + bus->cur_bus_speed = + pcix_bus_speed[FIELD_GET(PCI_X_SSTATUS_FREQ, status)]; return; } if (pci_is_pcie(bridge)) { u32 linkcap; - u16 linksta; pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &linkcap); bus->max_bus_speed = pcie_link_speed[linkcap & PCI_EXP_LNKCAP_SLS]; - bridge->link_active_reporting = !!(linkcap & PCI_EXP_LNKCAP_DLLLARC); - pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &linksta); - pcie_update_link_speed(bus, linksta); + pcie_update_link_speed(bus); } } @@ -821,15 +926,18 @@ static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus) { struct irq_domain *d; + /* If the host bridge driver sets a MSI domain of the bridge, use it */ + d = dev_get_msi_domain(bus->bridge); + /* * Any firmware interface that can resolve the msi_domain * should be called from here. */ - d = pci_host_bridge_of_msi_domain(bus); + if (!d) + d = pci_host_bridge_of_msi_domain(bus); if (!d) d = pci_host_bridge_acpi_msi_domain(bus); -#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN /* * If no IRQ domain was found via the OF tree, try looking it up * directly through the fwnode_handle. @@ -841,7 +949,6 @@ static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus) d = irq_find_matching_fwnode(fwnode, DOMAIN_BUS_PCI_MSI); } -#endif return d; } @@ -867,14 +974,26 @@ static void pci_set_bus_msi_domain(struct pci_bus *bus) dev_set_msi_domain(&bus->dev, d); } +static bool pci_preserve_config(struct pci_host_bridge *host_bridge) +{ + if (pci_acpi_preserve_config(host_bridge)) + return true; + + if (host_bridge->dev.parent && host_bridge->dev.parent->of_node) + return of_pci_preserve_config(host_bridge->dev.parent->of_node); + + return false; +} + static int pci_register_host_bridge(struct pci_host_bridge *bridge) { struct device *parent = bridge->dev.parent; - struct resource_entry *window, *n; + struct resource_entry *window, *next, *n; struct pci_bus *bus, *b; - resource_size_t offset; + resource_size_t offset, next_offset; LIST_HEAD(resources); - struct resource *res; + struct resource *res, *next_res; + bool bus_registered = false; char addr[64], *fmt; const char *name; int err; @@ -885,14 +1004,18 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) bridge->bus = bus; - /* Temporarily move resources off the list */ - list_splice_init(&bridge->windows, &resources); bus->sysdata = bridge->sysdata; - bus->msi = bridge->msi; bus->ops = bridge->ops; bus->number = bus->busn_res.start = bridge->busnr; #ifdef CONFIG_PCI_DOMAINS_GENERIC - bus->domain_nr = pci_bus_find_domain_nr(bus, parent); + if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET) + bus->domain_nr = pci_bus_find_domain_nr(bus, parent); + else + bus->domain_nr = bridge->domain_nr; + if (bus->domain_nr < 0) { + err = bus->domain_nr; + goto free; + } #endif b = pci_find_bus(pci_domain_nr(bus), bridge->busnr); @@ -910,15 +1033,19 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) if (err) goto free; + /* Temporarily move resources off the list */ + list_splice_init(&bridge->windows, &resources); err = device_add(&bridge->dev); - if (err) { - put_device(&bridge->dev); + if (err) goto free; - } + bus->bridge = get_device(&bridge->dev); device_enable_async_suspend(bus->bridge); pci_set_bus_of_node(bus); pci_set_bus_msi_domain(bus); + if (bridge->msi_domain && !dev_get_msi_domain(&bus->dev) && + !pci_host_of_has_msi_map(parent)) + bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI; if (!parent) set_dev_node(bus->bridge, pcibus_to_node(bus)); @@ -930,11 +1057,18 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) name = dev_name(&bus->dev); err = device_register(&bus->dev); + bus_registered = true; if (err) goto unregister; pcibios_add_bus(bus); + if (bus->ops->add_bus) { + err = bus->ops->add_bus(bus); + if (WARN_ON(err < 0)) + dev_err(&bus->dev, "failed to add bus: %d\n", err); + } + /* Create legacy_io and legacy_mem files for this bus */ pci_create_legacy_files(bus); @@ -946,16 +1080,45 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE) dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n"); + /* Check if the boot configuration by FW needs to be preserved */ + bridge->preserve_config = pci_preserve_config(bridge); + + /* Coalesce contiguous windows */ + resource_list_for_each_entry_safe(window, n, &resources) { + if (list_is_last(&window->node, &resources)) + break; + + next = list_next_entry(window, node); + offset = window->offset; + res = window->res; + next_offset = next->offset; + next_res = next->res; + + if (res->flags != next_res->flags || offset != next_offset) + continue; + + if (res->end + 1 == next_res->start) { + next_res->start = res->start; + res->flags = res->start = res->end = 0; + } + } + /* Add initial resources to the bus */ resource_list_for_each_entry_safe(window, n, &resources) { - list_move_tail(&window->node, &bridge->windows); offset = window->offset; res = window->res; + if (!res->flags && !res->start && !res->end) { + release_resource(res); + resource_list_destroy_entry(window); + continue; + } + + list_move_tail(&window->node, &bridge->windows); if (res->flags & IORESOURCE_BUS) pci_bus_insert_busn_res(bus, bus->number, res->end); else - pci_bus_add_resource(bus, res, 0); + pci_bus_add_resource(bus, res); if (offset) { if (resource_type(res) == IORESOURCE_IO) @@ -972,6 +1135,8 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr); } + of_pci_make_host_bridge_node(bridge); + down_write(&pci_bus_sem); list_add_tail(&bus->node, &pci_root_buses); up_write(&pci_bus_sem); @@ -981,9 +1146,16 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) unregister: put_device(&bridge->dev); device_del(&bridge->dev); - free: - kfree(bus); +#ifdef CONFIG_PCI_DOMAINS_GENERIC + if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET) + pci_bus_release_domain_nr(parent, bus->domain_nr); +#endif + if (bus_registered) + put_device(&bus->dev); + else + kfree(bus); + return err; } @@ -1030,6 +1202,7 @@ static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent, struct pci_dev *bridge, int busnr) { struct pci_bus *child; + struct pci_host_bridge *host; int i; int ret; @@ -1039,11 +1212,15 @@ static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent, return NULL; child->parent = parent; - child->ops = parent->ops; - child->msi = parent->msi; child->sysdata = parent->sysdata; child->bus_flags = parent->bus_flags; + host = pci_find_host_bridge(parent); + if (host->child_ops) + child->ops = host->child_ops; + else + child->ops = parent->ops; + /* * Initialize some portions of the bus device, but don't register * it now as the parent is not properly set up yet. @@ -1087,7 +1264,10 @@ static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent, add_dev: pci_set_bus_msi_domain(child); ret = device_register(&child->dev); - WARN_ON(ret < 0); + if (WARN_ON(ret < 0)) { + put_device(&child->dev); + return NULL; + } pcibios_add_bus(child); @@ -1118,15 +1298,17 @@ struct pci_bus *pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev, } EXPORT_SYMBOL(pci_add_new_bus); -static void pci_enable_crs(struct pci_dev *pdev) +static void pci_enable_rrs_sv(struct pci_dev *pdev) { u16 root_cap = 0; - /* Enable CRS Software Visibility if supported */ + /* Enable Configuration RRS Software Visibility if supported */ pcie_capability_read_word(pdev, PCI_EXP_RTCAP, &root_cap); - if (root_cap & PCI_EXP_RTCAP_CRSVIS) + if (root_cap & PCI_EXP_RTCAP_RRS_SV) { pcie_capability_set_word(pdev, PCI_EXP_RTCTL, - PCI_EXP_RTCTL_CRSSVE); + PCI_EXP_RTCTL_RRS_SVE); + pdev->config_rrs_sv = 1; + } } static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, @@ -1158,8 +1340,8 @@ static bool pci_ea_fixed_busnrs(struct pci_dev *dev, u8 *sec, u8 *sub) offset = ea + PCI_EA_FIRST_ENT; pci_read_config_dword(dev, offset, &dw); - ea_sec = dw & PCI_EA_SEC_BUS_MASK; - ea_sub = (dw & PCI_EA_SUB_BUS_MASK) >> PCI_EA_SUB_BUS_SHIFT; + ea_sec = FIELD_GET(PCI_EA_SEC_BUS_MASK, dw); + ea_sub = FIELD_GET(PCI_EA_SUB_BUS_MASK, dw); if (ea_sec == 0 || ea_sub < ea_sec) return false; @@ -1241,11 +1423,9 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT); - pci_enable_crs(dev); - if ((secondary || subordinate) && !pcibios_assign_all_busses() && !is_cardbus && !broken) { - unsigned int cmax; + unsigned int cmax, buses; /* * Bus already configured by firmware, process it in the @@ -1270,7 +1450,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, child->bridge_ctl = bctl; } - cmax = pci_scan_child_bus(child); + buses = subordinate - secondary; + cmax = pci_scan_child_bus_extend(child, buses); if (cmax > subordinate) pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n", subordinate, cmax); @@ -1410,6 +1591,9 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, } out: + /* Clear errors in the Secondary Status Register */ + pci_write_config_word(dev, PCI_SEC_STATUS, 0xffff); + pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl); pm_runtime_put(&dev->dev); @@ -1468,6 +1652,7 @@ void set_pcie_port_type(struct pci_dev *pdev) { int pos; u16 reg16; + u32 reg32; int type; struct pci_dev *parent; @@ -1478,8 +1663,24 @@ void set_pcie_port_type(struct pci_dev *pdev) pdev->pcie_cap = pos; pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16); pdev->pcie_flags_reg = reg16; - pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, ®16); - pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD; + + type = pci_pcie_type(pdev); + if (type == PCI_EXP_TYPE_ROOT_PORT) + pci_enable_rrs_sv(pdev); + + pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, &pdev->devcap); + pdev->pcie_mpss = FIELD_GET(PCI_EXP_DEVCAP_PAYLOAD, pdev->devcap); + + pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); + if (reg32 & PCI_EXP_LNKCAP_DLLLARC) + pdev->link_active_reporting = 1; + +#ifdef CONFIG_PCIEASPM + if (reg32 & PCI_EXP_LNKCAP_ASPM_L0S) + pdev->aspm_l0s_support = 1; + if (reg32 & PCI_EXP_LNKCAP_ASPM_L1) + pdev->aspm_l1_support = 1; +#endif parent = pci_upstream_bridge(pdev); if (!parent) @@ -1490,7 +1691,6 @@ void set_pcie_port_type(struct pci_dev *pdev) * correctly so detect impossible configurations here and correct * the port type accordingly. */ - type = pci_pcie_type(pdev); if (type == PCI_EXP_TYPE_DOWNSTREAM) { /* * If pdev claims to be downstream port but the parent @@ -1522,38 +1722,66 @@ void set_pcie_hotplug_bridge(struct pci_dev *pdev) pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, ®32); if (reg32 & PCI_EXP_SLTCAP_HPC) - pdev->is_hotplug_bridge = 1; + pdev->is_hotplug_bridge = pdev->is_pciehp = 1; } static void set_pcie_thunderbolt(struct pci_dev *dev) { - int vsec = 0; - u32 header; - - while ((vsec = pci_find_next_ext_capability(dev, vsec, - PCI_EXT_CAP_ID_VNDR))) { - pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER, &header); + u16 vsec; - /* Is the device part of a Thunderbolt controller? */ - if (dev->vendor == PCI_VENDOR_ID_INTEL && - PCI_VNDR_HEADER_ID(header) == PCI_VSEC_ID_INTEL_TBT) { - dev->is_thunderbolt = 1; - return; - } - } + /* Is the device part of a Thunderbolt controller? */ + vsec = pci_find_vsec_capability(dev, PCI_VENDOR_ID_INTEL, PCI_VSEC_ID_INTEL_TBT); + if (vsec) + dev->is_thunderbolt = 1; } static void set_pcie_untrusted(struct pci_dev *dev) { - struct pci_dev *parent; + struct pci_dev *parent = pci_upstream_bridge(dev); + if (!parent) + return; /* - * If the upstream bridge is untrusted we treat this device + * If the upstream bridge is untrusted we treat this device as * untrusted as well. */ - parent = pci_upstream_bridge(dev); - if (parent && parent->untrusted) + if (parent->untrusted) { + dev->untrusted = true; + return; + } + + if (arch_pci_dev_is_removable(dev)) { + pci_dbg(dev, "marking as untrusted\n"); dev->untrusted = true; + } +} + +static void pci_set_removable(struct pci_dev *dev) +{ + struct pci_dev *parent = pci_upstream_bridge(dev); + + if (!parent) + return; + /* + * We (only) consider everything tunneled below an external_facing + * device to be removable by the user. We're mainly concerned with + * consumer platforms with user accessible thunderbolt ports that are + * vulnerable to DMA attacks, and we expect those ports to be marked by + * the firmware as external_facing. Devices in traditional hotplug + * slots can technically be removed, but the expectation is that unless + * the port is marked with external_facing, such devices are less + * accessible to user / may not be removed by end user, and thus not + * exposed as "removable" to userspace. + */ + if (dev_is_removable(&parent->dev)) { + dev_set_removable(&dev->dev, DEVICE_REMOVABLE); + return; + } + + if (arch_pci_dev_is_removable(dev)) { + pci_dbg(dev, "marking as removable\n"); + dev_set_removable(&dev->dev, DEVICE_REMOVABLE); + } } /** @@ -1575,15 +1803,15 @@ static void set_pcie_untrusted(struct pci_dev *dev) static bool pci_ext_cfg_is_aliased(struct pci_dev *dev) { #ifdef CONFIG_PCI_QUIRKS - int pos; + int pos, ret; u32 header, tmp; pci_read_config_dword(dev, PCI_VENDOR_ID, &header); for (pos = PCI_CFG_SPACE_SIZE; pos < PCI_CFG_SPACE_EXP_SIZE; pos += PCI_CFG_SPACE_SIZE) { - if (pci_read_config_dword(dev, pos, &tmp) != PCIBIOS_SUCCESSFUL - || header != tmp) + ret = pci_read_config_dword(dev, pos, &tmp); + if ((ret != PCIBIOS_SUCCESSFUL) || (header != tmp)) return false; } @@ -1594,7 +1822,7 @@ static bool pci_ext_cfg_is_aliased(struct pci_dev *dev) } /** - * pci_cfg_space_size - Get the configuration space size of the PCI device + * pci_cfg_space_size_ext - Get the configuration space size of the PCI device * @dev: PCI device * * Regular PCI devices have 256 bytes, but PCI-X 2 and PCI Express devices @@ -1611,7 +1839,7 @@ static int pci_cfg_space_size_ext(struct pci_dev *dev) if (pci_read_config_dword(dev, pos, &status) != PCIBIOS_SUCCESSFUL) return PCI_CFG_SPACE_SIZE; - if (status == 0xffffffff || pci_ext_cfg_is_aliased(dev)) + if (PCI_POSSIBLE_ERROR(status) || pci_ext_cfg_is_aliased(dev)) return PCI_CFG_SPACE_SIZE; return PCI_CFG_SPACE_EXP_SIZE; @@ -1698,22 +1926,6 @@ static u8 pci_hdr_type(struct pci_dev *dev) #define LEGACY_IO_RESOURCE (IORESOURCE_IO | IORESOURCE_PCI_FIXED) -static void pci_msi_setup_pci_dev(struct pci_dev *dev) -{ - /* - * Disable the MSI hardware to avoid screaming interrupts - * during boot. This is the power on reset default so - * usually this should be a noop. - */ - dev->msi_cap = pci_find_capability(dev, PCI_CAP_ID_MSI); - if (dev->msi_cap) - pci_msi_set_enable(dev, 0); - - dev->msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX); - if (dev->msix_cap) - pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); -} - /** * pci_intx_mask_broken - Test PCI_COMMAND_INTX_DISABLE writability * @dev: PCI device @@ -1744,16 +1956,53 @@ static int pci_intx_mask_broken(struct pci_dev *dev) static void early_dump_pci_device(struct pci_dev *pdev) { - u32 value[256 / 4]; + u32 value[PCI_CFG_SPACE_SIZE / sizeof(u32)]; int i; pci_info(pdev, "config space:\n"); - for (i = 0; i < 256; i += 4) - pci_read_config_dword(pdev, i, &value[i / 4]); + for (i = 0; i < ARRAY_SIZE(value); i++) + pci_read_config_dword(pdev, i * sizeof(u32), &value[i]); print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, - value, 256, false); + value, ARRAY_SIZE(value) * sizeof(u32), false); +} + +static const char *pci_type_str(struct pci_dev *dev) +{ + static const char * const str[] = { + "PCIe Endpoint", + "PCIe Legacy Endpoint", + "PCIe unknown", + "PCIe unknown", + "PCIe Root Port", + "PCIe Switch Upstream Port", + "PCIe Switch Downstream Port", + "PCIe to PCI/PCI-X bridge", + "PCI/PCI-X to PCIe bridge", + "PCIe Root Complex Integrated Endpoint", + "PCIe Root Complex Event Collector", + }; + int type; + + if (pci_is_pcie(dev)) { + type = pci_pcie_type(dev); + if (type < ARRAY_SIZE(str)) + return str[type]; + + return "PCIe unknown"; + } + + switch (dev->hdr_type) { + case PCI_HEADER_TYPE_NORMAL: + return "conventional PCI endpoint"; + case PCI_HEADER_TYPE_BRIDGE: + return "conventional PCI bridge"; + case PCI_HEADER_TYPE_CARDBUS: + return "CardBus bridge"; + default: + return "conventional PCI"; + } } /** @@ -1771,7 +2020,7 @@ int pci_setup_device(struct pci_dev *dev) u32 class; u16 cmd; u8 hdr_type; - int pos = 0; + int err, pos = 0; struct pci_bus_region region; struct resource *res; @@ -1780,11 +2029,16 @@ int pci_setup_device(struct pci_dev *dev) dev->sysdata = dev->bus->sysdata; dev->dev.parent = dev->bus->bridge; dev->dev.bus = &pci_bus_type; - dev->hdr_type = hdr_type & 0x7f; - dev->multifunction = !!(hdr_type & 0x80); + dev->hdr_type = FIELD_GET(PCI_HEADER_TYPE_MASK, hdr_type); + dev->multifunction = FIELD_GET(PCI_HEADER_TYPE_MFD, hdr_type); dev->error_state = pci_channel_io_normal; set_pcie_port_type(dev); + err = pci_set_of_node(dev); + if (err) + return err; + pci_set_acpi_fwnode(dev); + pci_dev_assign_slot(dev); /* @@ -1802,9 +2056,6 @@ int pci_setup_device(struct pci_dev *dev) dev->revision = class & 0xff; dev->class = class >> 8; /* upper 3 bytes */ - pci_info(dev, "[%04x:%04x] type %02x class %#08x\n", - dev->vendor, dev->device, dev->hdr_type, dev->class); - if (pci_early_dump) early_dump_pci_device(dev); @@ -1816,12 +2067,21 @@ int pci_setup_device(struct pci_dev *dev) set_pcie_untrusted(dev); + if (pci_is_pcie(dev)) + dev->supported_speeds = pcie_get_supported_speeds(dev); + /* "Unknown power state" */ dev->current_state = PCI_UNKNOWN; /* Early fixups, before probing the BARs */ pci_fixup_device(pci_fixup_early, dev); + pci_set_removable(dev); + + pci_info(dev, "[%04x:%04x] type %02x class %#08x %s\n", + dev->vendor, dev->device, dev->hdr_type, dev->class, + pci_type_str(dev)); + /* Device class may be changed after fixup */ class = dev->class >> 8; @@ -1842,7 +2102,7 @@ int pci_setup_device(struct pci_dev *dev) if (class == PCI_CLASS_BRIDGE_PCI) goto bad; pci_read_irq(dev); - pci_read_bases(dev, 6, PCI_ROM_ADDRESS); + pci_read_bases(dev, PCI_STD_NUM_BARS, PCI_ROM_ADDRESS); pci_subsystem_ids(dev, &dev->subsystem_vendor, &dev->subsystem_device); @@ -1861,14 +2121,14 @@ int pci_setup_device(struct pci_dev *dev) res = &dev->resource[0]; res->flags = LEGACY_IO_RESOURCE; pcibios_bus_to_resource(dev->bus, res, ®ion); - pci_info(dev, "legacy IDE quirk: reg 0x10: %pR\n", + pci_info(dev, "BAR 0 %pR: legacy IDE quirk\n", res); region.start = 0x3F6; region.end = 0x3F6; res = &dev->resource[1]; res->flags = LEGACY_IO_RESOURCE; pcibios_bus_to_resource(dev->bus, res, ®ion); - pci_info(dev, "legacy IDE quirk: reg 0x14: %pR\n", + pci_info(dev, "BAR 1 %pR: legacy IDE quirk\n", res); } if ((progif & 4) == 0) { @@ -1877,14 +2137,14 @@ int pci_setup_device(struct pci_dev *dev) res = &dev->resource[2]; res->flags = LEGACY_IO_RESOURCE; pcibios_bus_to_resource(dev->bus, res, ®ion); - pci_info(dev, "legacy IDE quirk: reg 0x18: %pR\n", + pci_info(dev, "BAR 2 %pR: legacy IDE quirk\n", res); region.start = 0x376; region.end = 0x376; res = &dev->resource[3]; res->flags = LEGACY_IO_RESOURCE; pcibios_bus_to_resource(dev->bus, res, ®ion); - pci_info(dev, "legacy IDE quirk: reg 0x1c: %pR\n", + pci_info(dev, "BAR 3 %pR: legacy IDE quirk\n", res); } } @@ -1920,6 +2180,7 @@ int pci_setup_device(struct pci_dev *dev) default: /* unknown header */ pci_err(dev, "unknown header type %02x, ignoring device\n", dev->hdr_type); + pci_release_of_node(dev); return -EIO; bad: @@ -2048,6 +2309,17 @@ int pci_configure_extended_tags(struct pci_dev *dev, void *ign) return 0; } +static void pci_dev3_init(struct pci_dev *pdev) +{ + u16 cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DEV3); + u32 val = 0; + + if (!cap) + return; + pci_read_config_dword(pdev, cap + PCI_DEV3_STA, &val); + pdev->fm_enabled = !!(val & PCI_DEV3_STA_SEGMENT); +} + /** * pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable * @dev: PCI device to query @@ -2068,7 +2340,7 @@ static void pci_configure_relaxed_ordering(struct pci_dev *dev) { struct pci_dev *root; - /* PCI_EXP_DEVICE_RELAX_EN is RsvdP in VFs */ + /* PCI_EXP_DEVCTL_RELAX_EN is RsvdP in VFs */ if (dev->is_virtfn) return; @@ -2090,56 +2362,10 @@ static void pci_configure_relaxed_ordering(struct pci_dev *dev) } } -static void pci_configure_ltr(struct pci_dev *dev) -{ -#ifdef CONFIG_PCIEASPM - struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); - struct pci_dev *bridge; - u32 cap, ctl; - - if (!pci_is_pcie(dev)) - return; - - pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &cap); - if (!(cap & PCI_EXP_DEVCAP2_LTR)) - return; - - pcie_capability_read_dword(dev, PCI_EXP_DEVCTL2, &ctl); - if (ctl & PCI_EXP_DEVCTL2_LTR_EN) { - if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) { - dev->ltr_path = 1; - return; - } - - bridge = pci_upstream_bridge(dev); - if (bridge && bridge->ltr_path) - dev->ltr_path = 1; - - return; - } - - if (!host->native_ltr) - return; - - /* - * Software must not enable LTR in an Endpoint unless the Root - * Complex and all intermediate Switches indicate support for LTR. - * PCIe r4.0, sec 6.18. - */ - if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || - ((bridge = pci_upstream_bridge(dev)) && - bridge->ltr_path)) { - pcie_capability_set_word(dev, PCI_EXP_DEVCTL2, - PCI_EXP_DEVCTL2_LTR_EN); - dev->ltr_path = 1; - } -#endif -} - static void pci_configure_eetlp_prefix(struct pci_dev *dev) { -#ifdef CONFIG_PCI_PASID struct pci_dev *bridge; + unsigned int eetlp_max; int pcie_type; u32 cap; @@ -2151,15 +2377,19 @@ static void pci_configure_eetlp_prefix(struct pci_dev *dev) return; pcie_type = pci_pcie_type(dev); + + eetlp_max = FIELD_GET(PCI_EXP_DEVCAP2_EE_PREFIX_MAX, cap); + /* 00b means 4 */ + eetlp_max = eetlp_max ?: 4; + if (pcie_type == PCI_EXP_TYPE_ROOT_PORT || pcie_type == PCI_EXP_TYPE_RC_END) - dev->eetlp_prefix_path = 1; + dev->eetlp_prefix_max = eetlp_max; else { bridge = pci_upstream_bridge(dev); - if (bridge && bridge->eetlp_prefix_path) - dev->eetlp_prefix_path = 1; + if (bridge && bridge->eetlp_prefix_max) + dev->eetlp_prefix_max = eetlp_max; } -#endif } static void pci_configure_serr(struct pci_dev *dev) @@ -2186,6 +2416,7 @@ static void pci_configure_device(struct pci_dev *dev) pci_configure_extended_tags(dev, NULL); pci_configure_relaxed_ordering(dev); pci_configure_ltr(dev); + pci_configure_aspm_l1ss(dev); pci_configure_eetlp_prefix(dev); pci_configure_serr(dev); @@ -2195,7 +2426,7 @@ static void pci_configure_device(struct pci_dev *dev) static void pci_release_capabilities(struct pci_dev *dev) { pci_aer_exit(dev); - pci_vpd_release(dev); + pci_rcec_exit(dev); pci_iov_release(dev); pci_free_cap_save_buffers(dev); } @@ -2219,9 +2450,14 @@ static void pci_release_dev(struct device *dev) pci_bus_put(pci_dev->bus); kfree(pci_dev->driver_override); bitmap_free(pci_dev->dma_alias_mask); + dev_dbg(dev, "device released\n"); kfree(pci_dev); } +static const struct device_type pci_dev_type = { + .groups = pci_dev_attr_groups, +}; + struct pci_dev *pci_alloc_dev(struct pci_bus *bus) { struct pci_dev *dev; @@ -2233,33 +2469,37 @@ struct pci_dev *pci_alloc_dev(struct pci_bus *bus) INIT_LIST_HEAD(&dev->bus_list); dev->dev.type = &pci_dev_type; dev->bus = pci_bus_get(bus); + dev->driver_exclusive_resource = (struct resource) { + .name = "PCI Exclusive", + .start = 0, + .end = -1, + }; + spin_lock_init(&dev->pcie_cap_lock); +#ifdef CONFIG_PCI_MSI + raw_spin_lock_init(&dev->msi_lock); +#endif return dev; } EXPORT_SYMBOL(pci_alloc_dev); -static bool pci_bus_crs_vendor_id(u32 l) -{ - return (l & 0xffff) == 0x0001; -} - -static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l, +static bool pci_bus_wait_rrs(struct pci_bus *bus, int devfn, u32 *l, int timeout) { int delay = 1; - if (!pci_bus_crs_vendor_id(*l)) - return true; /* not a CRS completion */ + if (!pci_bus_rrs_vendor_id(*l)) + return true; /* not a Configuration RRS completion */ if (!timeout) - return false; /* CRS, but caller doesn't want to wait */ + return false; /* RRS, but caller doesn't want to wait */ /* * We got the reserved Vendor ID that indicates a completion with - * Configuration Request Retry Status (CRS). Retry until we get a + * Configuration Request Retry Status (RRS). Retry until we get a * valid Vendor ID or we time out. */ - while (pci_bus_crs_vendor_id(*l)) { + while (pci_bus_rrs_vendor_id(*l)) { if (delay > timeout) { pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n", pci_domain_nr(bus), bus->number, @@ -2293,13 +2533,13 @@ bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l, if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l)) return false; - /* Some broken boards return 0 or ~0 if a slot is empty: */ - if (*l == 0xffffffff || *l == 0x00000000 || + /* Some broken boards return 0 or ~0 (PCI_ERROR_RESPONSE) if a slot is empty: */ + if (PCI_POSSIBLE_ERROR(*l) || *l == 0x00000000 || *l == 0x0000ffff || *l == 0xffff0000) return false; - if (pci_bus_crs_vendor_id(*l)) - return pci_bus_wait_crs(bus, devfn, l, timeout); + if (pci_bus_rrs_vendor_id(*l)) + return pci_bus_wait_rrs(bus, devfn, l, timeout); return true; } @@ -2323,6 +2563,56 @@ bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l, } EXPORT_SYMBOL(pci_bus_read_dev_vendor_id); +#if IS_ENABLED(CONFIG_PCI_PWRCTRL) +static struct platform_device *pci_pwrctrl_create_device(struct pci_bus *bus, int devfn) +{ + struct pci_host_bridge *host = pci_find_host_bridge(bus); + struct platform_device *pdev; + struct device_node *np; + + np = of_pci_find_child_device(dev_of_node(&bus->dev), devfn); + if (!np) + return NULL; + + pdev = of_find_device_by_node(np); + if (pdev) { + put_device(&pdev->dev); + goto err_put_of_node; + } + + /* + * First check whether the pwrctrl device really needs to be created or + * not. This is decided based on at least one of the power supplies + * being defined in the devicetree node of the device. + */ + if (!of_pci_supply_present(np)) { + pr_debug("PCI/pwrctrl: Skipping OF node: %s\n", np->name); + goto err_put_of_node; + } + + /* Now create the pwrctrl device */ + pdev = of_platform_device_create(np, NULL, &host->dev); + if (!pdev) { + pr_err("PCI/pwrctrl: Failed to create pwrctrl device for node: %s\n", np->name); + goto err_put_of_node; + } + + of_node_put(np); + + return pdev; + +err_put_of_node: + of_node_put(np); + + return NULL; +} +#else +static struct platform_device *pci_pwrctrl_create_device(struct pci_bus *bus, int devfn) +{ + return NULL; +} +#endif + /* * Read the config data for a PCI device, sanity-check it, * and fill in the dev structure. @@ -2332,6 +2622,15 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) struct pci_dev *dev; u32 l; + /* + * Create pwrctrl device (if required) for the PCI device to handle the + * power state. If the pwrctrl device is created, then skip scanning + * further as the pwrctrl core will rescan the bus after powering on + * the device. + */ + if (pci_pwrctrl_create_device(bus, devfn)) + return NULL; + if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000)) return NULL; @@ -2343,8 +2642,6 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) dev->vendor = l & 0xffff; dev->device = (l >> 16) & 0xffff; - pci_set_of_node(dev); - if (pci_setup_device(dev)) { pci_bus_put(dev->bus); kfree(dev); @@ -2373,16 +2670,25 @@ void pcie_report_downtraining(struct pci_dev *dev) __pcie_print_link_status(dev, false); } +static void pci_imm_ready_init(struct pci_dev *dev) +{ + u16 status; + + pci_read_config_word(dev, PCI_STATUS, &status); + if (status & PCI_STATUS_IMM_READY) + dev->imm_ready = 1; +} + static void pci_init_capabilities(struct pci_dev *dev) { pci_ea_init(dev); /* Enhanced Allocation */ - - /* Setup MSI caps & disable MSI/MSI-X interrupts */ - pci_msi_setup_pci_dev(dev); + pci_msi_init(dev); /* Disable MSI */ + pci_msix_init(dev); /* Disable MSI-X */ /* Buffers for saving PCIe and PCI-X capabilities */ pci_allocate_cap_save_buffers(dev); + pci_imm_ready_init(dev); /* Immediate Readiness */ pci_pm_init(dev); /* Power Management */ pci_vpd_init(dev); /* Vital Product Data */ pci_configure_ari(dev); /* Alternative Routing-ID Forwarding */ @@ -2390,15 +2696,19 @@ static void pci_init_capabilities(struct pci_dev *dev) pci_ats_init(dev); /* Address Translation Services */ pci_pri_init(dev); /* Page Request Interface */ pci_pasid_init(dev); /* Process Address Space ID */ - pci_enable_acs(dev); /* Enable ACS P2P upstream forwarding */ + pci_acs_init(dev); /* Access Control Services */ pci_ptm_init(dev); /* Precision Time Measurement */ pci_aer_init(dev); /* Advanced Error Reporting */ pci_dpc_init(dev); /* Downstream Port Containment */ + pci_rcec_init(dev); /* Root Complex Event Collector */ + pci_doe_init(dev); /* Data Object Exchange */ + pci_tph_init(dev); /* TLP Processing Hints */ + pci_rebar_init(dev); /* Resizable BAR */ + pci_dev3_init(dev); /* Device 3 capabilities */ + pci_ide_init(dev); /* Link Integrity and Data Encryption */ pcie_report_downtraining(dev); - - if (pci_probe_reset_function(dev) == 0) - dev->reset_fn = 1; + pci_init_reset_methods(dev); } /* @@ -2411,7 +2721,7 @@ static struct irq_domain *pci_dev_msi_domain(struct pci_dev *dev) struct irq_domain *d; /* - * If a domain has been set through the pcibios_add_device() + * If a domain has been set through the pcibios_device_add() * callback, then this is the one (platform code knows best). */ d = dev_get_msi_domain(&dev->dev); @@ -2462,13 +2772,13 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus) dma_set_max_seg_size(&dev->dev, 65536); dma_set_seg_boundary(&dev->dev, 0xffffffff); + pcie_failed_link_retrain(dev); + /* Fix up broken headers */ pci_fixup_device(pci_fixup_header, dev); pci_reassigndev_resource_alignment(dev); - dev->state_saved = false; - pci_init_capabilities(dev); /* @@ -2479,16 +2789,22 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus) list_add_tail(&dev->bus_list, &bus->devices); up_write(&pci_bus_sem); - ret = pcibios_add_device(dev); + ret = pcibios_device_add(dev); WARN_ON(ret < 0); /* Set up MSI IRQ domain */ pci_set_msi_domain(dev); /* Notifier could use PCI capabilities */ - dev->match_driver = false; ret = device_add(&dev->dev); WARN_ON(ret < 0); + + /* Establish pdev->tsm for newly added (e.g. new SR-IOV VFs) */ + pci_tsm_init(dev); + + pci_npem_create(dev); + + pci_doe_sysfs_init(dev); } struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn) @@ -2511,32 +2827,39 @@ struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn) } EXPORT_SYMBOL(pci_scan_single_device); -static unsigned next_fn(struct pci_bus *bus, struct pci_dev *dev, unsigned fn) +static int next_ari_fn(struct pci_bus *bus, struct pci_dev *dev, int fn) { int pos; u16 cap = 0; - unsigned next_fn; + unsigned int next_fn; - if (pci_ari_enabled(bus)) { - if (!dev) - return 0; - pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI); - if (!pos) - return 0; + if (!dev) + return -ENODEV; - pci_read_config_word(dev, pos + PCI_ARI_CAP, &cap); - next_fn = PCI_ARI_CAP_NFN(cap); - if (next_fn <= fn) - return 0; /* protect against malformed list */ + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI); + if (!pos) + return -ENODEV; - return next_fn; - } + pci_read_config_word(dev, pos + PCI_ARI_CAP, &cap); + next_fn = PCI_ARI_CAP_NFN(cap); + if (next_fn <= fn) + return -ENODEV; /* protect against malformed list */ - /* dev may be NULL for non-contiguous multifunction devices */ - if (!dev || dev->multifunction) - return (fn + 1) % 8; + return next_fn; +} - return 0; +static int next_fn(struct pci_bus *bus, struct pci_dev *dev, int fn) +{ + if (pci_ari_enabled(bus)) + return next_ari_fn(bus, dev, fn); + + if (fn >= 7) + return -ENODEV; + /* only multifunction devices may have more functions */ + if (dev && !dev->multifunction) + return -ENODEV; + + return fn + 1; } static int only_one_child(struct pci_bus *bus) @@ -2574,26 +2897,30 @@ static int only_one_child(struct pci_bus *bus) */ int pci_scan_slot(struct pci_bus *bus, int devfn) { - unsigned fn, nr = 0; struct pci_dev *dev; + int fn = 0, nr = 0; if (only_one_child(bus) && (devfn > 0)) return 0; /* Already scanned the entire slot */ - dev = pci_scan_single_device(bus, devfn); - if (!dev) - return 0; - if (!pci_dev_is_added(dev)) - nr++; - - for (fn = next_fn(bus, dev, 0); fn > 0; fn = next_fn(bus, dev, fn)) { + do { dev = pci_scan_single_device(bus, devfn + fn); if (dev) { if (!pci_dev_is_added(dev)) nr++; - dev->multifunction = 1; + if (fn > 0) + dev->multifunction = 1; + } else if (fn == 0) { + /* + * Function 0 is required unless we are running on + * a hypervisor that passes through individual PCI + * functions. + */ + if (!hypervisor_isolated_pci_functions()) + break; } - } + fn = next_fn(bus, dev, fn); + } while (fn >= 0); /* Only one slot has PCIe device */ if (bus->self && nr) @@ -2789,29 +3116,14 @@ static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, { unsigned int used_buses, normal_bridges = 0, hotplug_bridges = 0; unsigned int start = bus->busn_res.start; - unsigned int devfn, fn, cmax, max = start; + unsigned int devnr, cmax, max = start; struct pci_dev *dev; - int nr_devs; dev_dbg(&bus->dev, "scanning bus\n"); /* Go find them, Rover! */ - for (devfn = 0; devfn < 256; devfn += 8) { - nr_devs = pci_scan_slot(bus, devfn); - - /* - * The Jailhouse hypervisor may pass individual functions of a - * multi-function device to a guest without passing function 0. - * Look for them as well. - */ - if (jailhouse_paravirt() && nr_devs == 0) { - for (fn = 1; fn < 8; fn++) { - dev = pci_scan_single_device(bus, devfn + fn); - if (dev) - dev->multifunction = 1; - } - } - } + for (devnr = 0; devnr < PCI_MAX_NR_DEVS; devnr++) + pci_scan_slot(bus, PCI_DEVFN(devnr, 0)); /* Reserve buses for SR-IOV capability */ used_buses = pci_iov_bus_range(bus); @@ -2853,8 +3165,8 @@ static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, * hotplug bridges too much during the second scan below. */ used_buses++; - if (cmax - max > 1) - used_buses += cmax - max - 1; + if (max - cmax > 1) + used_buses += max - cmax - 1; } /* Scan bridges that need to be reconfigured */ @@ -2862,7 +3174,6 @@ static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, unsigned int buses = 0; if (!hotplug_bridges && normal_bridges == 1) { - /* * There is only one bridge on the bus (upstream * port) so it gets all available buses which it @@ -2871,7 +3182,6 @@ static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, */ buses = available_buses; } else if (dev->is_hotplug_bridge) { - /* * Distribute the extra buses between hotplug * bridges if any. @@ -2890,11 +3200,10 @@ static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, /* * Make sure a hotplug bridge has at least the minimum requested * number of buses but allow it to grow up to the maximum available - * bus number of there is room. + * bus number if there is room. */ if (bus->self && bus->self->is_hotplug_bridge) { - used_buses = max_t(unsigned int, available_buses, - pci_hotplug_bus_size - 1); + used_buses = max(available_buses, pci_hotplug_bus_size - 1); if (max - start < used_buses) { max = start + used_buses; @@ -2985,7 +3294,9 @@ int pci_host_probe(struct pci_host_bridge *bridge) struct pci_bus *bus, *child; int ret; + pci_lock_rescan_remove(); ret = pci_scan_root_bus_bridge(bridge); + pci_unlock_rescan_remove(); if (ret < 0) { dev_err(bridge->dev.parent, "Scanning root bridge failed"); return ret; @@ -2993,22 +3304,33 @@ int pci_host_probe(struct pci_host_bridge *bridge) bus = bridge->bus; + /* If we must preserve the resource configuration, claim now */ + if (bridge->preserve_config) + pci_bus_claim_resources(bus); + /* - * We insert PCI resources into the iomem_resource and - * ioport_resource trees in either pci_bus_claim_resources() - * or pci_bus_assign_resources(). + * Assign whatever was left unassigned. If we didn't claim above, + * this will reassign everything. */ - if (pci_has_flag(PCI_PROBE_ONLY)) { - pci_bus_claim_resources(bus); - } else { - pci_bus_size_bridges(bus); - pci_bus_assign_resources(bus); + pci_assign_unassigned_root_bus_resources(bus); - list_for_each_entry(child, &bus->children, node) - pcie_bus_configure_settings(child); - } + list_for_each_entry(child, &bus->children, node) + pcie_bus_configure_settings(child); + pci_lock_rescan_remove(); pci_bus_add_devices(bus); + pci_unlock_rescan_remove(); + + /* + * Ensure pm_runtime_enable() is called for the controller drivers + * before calling pci_host_probe(). The PM framework expects that + * if the parent device supports runtime PM, it will be enabled + * before child runtime PM is enabled. + */ + pm_runtime_set_active(&bridge->dev); + pm_runtime_no_callbacks(&bridge->dev); + devm_pm_runtime_enable(&bridge->dev); + return 0; } EXPORT_SYMBOL_GPL(pci_host_probe); @@ -3086,6 +3408,7 @@ int pci_scan_root_bus_bridge(struct pci_host_bridge *bridge) resource_list_for_each_entry(window, &bridge->windows) if (window->res->flags & IORESOURCE_BUS) { + bridge->busnr = window->res->start; found = true; break; } @@ -3216,7 +3539,7 @@ EXPORT_SYMBOL_GPL(pci_rescan_bus); * pci_rescan_bus(), pci_rescan_bus_bridge_resize() and PCI device removal * routines should always be executed under this mutex. */ -static DEFINE_MUTEX(pci_rescan_remove_lock); +DEFINE_MUTEX(pci_rescan_remove_lock); void pci_lock_rescan_remove(void) { |