diff options
Diffstat (limited to 'drivers/cxl/core/port.c')
| -rw-r--r-- | drivers/cxl/core/port.c | 476 |
1 files changed, 321 insertions, 155 deletions
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index af92c67bc954..fef3aa0c6680 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -30,18 +30,21 @@ * instantiated by the core. */ -/* - * All changes to the interleave configuration occur with this lock held - * for write. - */ -DECLARE_RWSEM(cxl_region_rwsem); - static DEFINE_IDA(cxl_port_ida); static DEFINE_XARRAY(cxl_root_buses); +/* + * The terminal device in PCI is NULL and @platform_bus + * for platform devices (for cxl_test) + */ +static bool is_cxl_host_bridge(struct device *dev) +{ + return (!dev || dev == &platform_bus); +} + int cxl_num_decoders_committed(struct cxl_port *port) { - lockdep_assert_held(&cxl_region_rwsem); + lockdep_assert_held(&cxl_rwsem.region); return port->commit_end + 1; } @@ -176,7 +179,7 @@ static ssize_t target_list_show(struct device *dev, ssize_t offset; int rc; - guard(rwsem_read)(&cxl_region_rwsem); + guard(rwsem_read)(&cxl_rwsem.region); rc = emit_target_list(cxlsd, buf); if (rc < 0) return rc; @@ -194,25 +197,35 @@ static ssize_t mode_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev); + struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + /* without @cxl_rwsem.dpa, make sure @part is not reloaded */ + int part = READ_ONCE(cxled->part); + const char *desc; + + if (part < 0) + desc = "none"; + else + desc = cxlds->part[part].res.name; - return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxled->mode)); + return sysfs_emit(buf, "%s\n", desc); } static ssize_t mode_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev); - enum cxl_decoder_mode mode; + enum cxl_partition_mode mode; ssize_t rc; if (sysfs_streq(buf, "pmem")) - mode = CXL_DECODER_PMEM; + mode = CXL_PARTMODE_PMEM; else if (sysfs_streq(buf, "ram")) - mode = CXL_DECODER_RAM; + mode = CXL_PARTMODE_RAM; else return -EINVAL; - rc = cxl_dpa_set_mode(cxled, mode); + rc = cxl_dpa_set_part(cxled, mode); if (rc) return rc; @@ -225,7 +238,7 @@ static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *at { struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev); - guard(rwsem_read)(&cxl_dpa_rwsem); + guard(rwsem_read)(&cxl_rwsem.dpa); return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled)); } static DEVICE_ATTR_RO(dpa_resource); @@ -437,7 +450,7 @@ struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev) return NULL; return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev); } -EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL); +EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, "CXL"); static void cxl_root_decoder_release(struct device *dev) { @@ -471,19 +484,19 @@ bool is_endpoint_decoder(struct device *dev) { return dev->type == &cxl_decoder_endpoint_type; } -EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, CXL); +EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, "CXL"); bool is_root_decoder(struct device *dev) { return dev->type == &cxl_decoder_root_type; } -EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL); +EXPORT_SYMBOL_NS_GPL(is_root_decoder, "CXL"); bool is_switch_decoder(struct device *dev) { return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type; } -EXPORT_SYMBOL_NS_GPL(is_switch_decoder, CXL); +EXPORT_SYMBOL_NS_GPL(is_switch_decoder, "CXL"); struct cxl_decoder *to_cxl_decoder(struct device *dev) { @@ -493,7 +506,7 @@ struct cxl_decoder *to_cxl_decoder(struct device *dev) return NULL; return container_of(dev, struct cxl_decoder, dev); } -EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL); +EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, "CXL"); struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev) { @@ -502,7 +515,7 @@ struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev) return NULL; return container_of(dev, struct cxl_endpoint_decoder, cxld.dev); } -EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL); +EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, "CXL"); struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev) { @@ -511,7 +524,7 @@ struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev) return NULL; return container_of(dev, struct cxl_switch_decoder, cxld.dev); } -EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, CXL); +EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, "CXL"); static void cxl_ep_release(struct cxl_ep *ep) { @@ -549,13 +562,9 @@ static ssize_t decoders_committed_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_port *port = to_cxl_port(dev); - int rc; - down_read(&cxl_region_rwsem); - rc = sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port)); - up_read(&cxl_region_rwsem); - - return rc; + guard(rwsem_read)(&cxl_rwsem.region); + return sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port)); } static DEVICE_ATTR_RO(decoders_committed); @@ -585,7 +594,7 @@ bool is_cxl_port(const struct device *dev) { return dev->type == &cxl_port_type; } -EXPORT_SYMBOL_NS_GPL(is_cxl_port, CXL); +EXPORT_SYMBOL_NS_GPL(is_cxl_port, "CXL"); struct cxl_port *to_cxl_port(const struct device *dev) { @@ -594,19 +603,21 @@ struct cxl_port *to_cxl_port(const struct device *dev) return NULL; return container_of(dev, struct cxl_port, dev); } -EXPORT_SYMBOL_NS_GPL(to_cxl_port, CXL); +EXPORT_SYMBOL_NS_GPL(to_cxl_port, "CXL"); + +struct cxl_port *parent_port_of(struct cxl_port *port) +{ + if (!port || !port->parent_dport) + return NULL; + return port->parent_dport->port; +} static void unregister_port(void *_port) { struct cxl_port *port = _port; - struct cxl_port *parent; + struct cxl_port *parent = parent_port_of(port); struct device *lock_dev; - if (is_cxl_root(port)) - parent = NULL; - else - parent = to_cxl_port(port->dev.parent); - /* * CXL root port's and the first level of ports are unregistered * under the platform firmware device lock, all other ports are @@ -738,6 +749,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev, xa_init(&port->dports); xa_init(&port->endpoints); xa_init(&port->regions); + port->component_reg_phys = CXL_RESOURCE_NONE; device_initialize(dev); lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth); @@ -856,9 +868,7 @@ static int cxl_port_add(struct cxl_port *port, if (rc) return rc; - rc = cxl_port_setup_regs(port, component_reg_phys); - if (rc) - return rc; + port->component_reg_phys = component_reg_phys; } else { rc = dev_set_name(dev, "root%d", port->id); if (rc) @@ -942,7 +952,7 @@ struct cxl_port *devm_cxl_add_port(struct device *host, return port; } -EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL); +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, "CXL"); struct cxl_root *devm_cxl_add_root(struct device *host, const struct cxl_root_ops *ops) @@ -958,7 +968,7 @@ struct cxl_root *devm_cxl_add_root(struct device *host, cxl_root->ops = ops; return cxl_root; } -EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, CXL); +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, "CXL"); struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port) { @@ -974,7 +984,7 @@ struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port) return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev); } -EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, "CXL"); static void unregister_pci_bus(void *uport_dev) { @@ -995,7 +1005,7 @@ int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev, return rc; return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev); } -EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL); +EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, "CXL"); static bool dev_is_cxl_root_child(struct device *dev) { @@ -1027,16 +1037,7 @@ struct cxl_root *find_cxl_root(struct cxl_port *port) get_device(&iter->dev); return to_cxl_root(iter); } -EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL); - -void put_cxl_root(struct cxl_root *cxl_root) -{ - if (!cxl_root) - return; - - put_device(&cxl_root->port.dev); -} -EXPORT_SYMBOL_NS_GPL(put_cxl_root, CXL); +EXPORT_SYMBOL_NS_GPL(find_cxl_root, "CXL"); static struct cxl_dport *find_dport(struct cxl_port *port, int id) { @@ -1180,6 +1181,20 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, if (rc) return ERR_PTR(rc); + /* + * Setup port register if this is the first dport showed up. Having + * a dport also means that there is at least 1 active link. + */ + if (port->nr_dports == 1 && + port->component_reg_phys != CXL_RESOURCE_NONE) { + rc = cxl_port_setup_regs(port, port->component_reg_phys); + if (rc) { + xa_erase(&port->dports, (unsigned long)dport->dport_dev); + return ERR_PTR(rc); + } + port->component_reg_phys = CXL_RESOURCE_NONE; + } + get_device(dport_dev); rc = devm_add_action_or_reset(host, cxl_dport_remove, dport); if (rc) @@ -1230,7 +1245,7 @@ struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port, return dport; } -EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL); +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, "CXL"); /** * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port @@ -1264,7 +1279,7 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, return dport; } -EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, CXL); +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, "CXL"); static int add_ep(struct cxl_ep *new) { @@ -1355,21 +1370,6 @@ static struct cxl_port *find_cxl_port(struct device *dport_dev, return port; } -static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port, - struct device *dport_dev, - struct cxl_dport **dport) -{ - struct cxl_find_port_ctx ctx = { - .dport_dev = dport_dev, - .parent_port = parent_port, - .dport = dport, - }; - struct cxl_port *port; - - port = __find_cxl_port(&ctx); - return port; -} - /* * All users of grandparent() are using it to walk PCIe-like switch port * hierarchy. A PCIe switch is comprised of a bridge device representing the @@ -1421,7 +1421,7 @@ int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint) cxlmd->depth = endpoint->depth; return devm_add_action_or_reset(dev, delete_endpoint, cxlmd); } -EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, "CXL"); /* * The natural end of life of a non-root 'cxl_port' is when its parent port goes @@ -1430,7 +1430,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL); * through ->remove(). This "bottom-up" removal selectively removes individual * child ports manually. This depends on devm_cxl_add_port() to not change is * devm action registration order, and for dports to have already been - * destroyed by reap_dports(). + * destroyed by del_dports(). */ static void delete_switch_port(struct cxl_port *port) { @@ -1439,18 +1439,24 @@ static void delete_switch_port(struct cxl_port *port) devm_release_action(port->dev.parent, unregister_port, port); } -static void reap_dports(struct cxl_port *port) +static void del_dport(struct cxl_dport *dport) +{ + struct cxl_port *port = dport->port; + + devm_release_action(&port->dev, cxl_dport_unlink, dport); + devm_release_action(&port->dev, cxl_dport_remove, dport); + devm_kfree(&port->dev, dport); +} + +static void del_dports(struct cxl_port *port) { struct cxl_dport *dport; unsigned long index; device_lock_assert(&port->dev); - xa_for_each(&port->dports, index, dport) { - devm_release_action(&port->dev, cxl_dport_unlink, dport); - devm_release_action(&port->dev, cxl_dport_remove, dport); - devm_kfree(&port->dev, dport); - } + xa_for_each(&port->dports, index, dport) + del_dport(dport); } struct detach_ctx { @@ -1508,7 +1514,7 @@ static void cxl_detach_ep(void *data) */ died = true; port->dead = true; - reap_dports(port); + del_dports(port); } device_unlock(&port->dev); @@ -1539,16 +1545,157 @@ static resource_size_t find_component_registers(struct device *dev) return map.resource; } +static int match_port_by_uport(struct device *dev, const void *data) +{ + const struct device *uport_dev = data; + struct cxl_port *port; + + if (!is_cxl_port(dev)) + return 0; + + port = to_cxl_port(dev); + return uport_dev == port->uport_dev; +} + +/* + * Function takes a device reference on the port device. Caller should do a + * put_device() when done. + */ +static struct cxl_port *find_cxl_port_by_uport(struct device *uport_dev) +{ + struct device *dev; + + dev = bus_find_device(&cxl_bus_type, NULL, uport_dev, match_port_by_uport); + if (dev) + return to_cxl_port(dev); + return NULL; +} + +static int update_decoder_targets(struct device *dev, void *data) +{ + struct cxl_dport *dport = data; + struct cxl_switch_decoder *cxlsd; + struct cxl_decoder *cxld; + int i; + + if (!is_switch_decoder(dev)) + return 0; + + cxlsd = to_cxl_switch_decoder(dev); + cxld = &cxlsd->cxld; + guard(rwsem_write)(&cxl_rwsem.region); + + for (i = 0; i < cxld->interleave_ways; i++) { + if (cxld->target_map[i] == dport->port_id) { + cxlsd->target[i] = dport; + dev_dbg(dev, "dport%d found in target list, index %d\n", + dport->port_id, i); + return 1; + } + } + + return 0; +} + +DEFINE_FREE(del_cxl_dport, struct cxl_dport *, if (!IS_ERR_OR_NULL(_T)) del_dport(_T)) +static struct cxl_dport *cxl_port_add_dport(struct cxl_port *port, + struct device *dport_dev) +{ + struct cxl_dport *dport; + int rc; + + device_lock_assert(&port->dev); + if (!port->dev.driver) + return ERR_PTR(-ENXIO); + + dport = cxl_find_dport_by_dev(port, dport_dev); + if (dport) { + dev_dbg(&port->dev, "dport%d:%s already exists\n", + dport->port_id, dev_name(dport_dev)); + return ERR_PTR(-EBUSY); + } + + struct cxl_dport *new_dport __free(del_cxl_dport) = + devm_cxl_add_dport_by_dev(port, dport_dev); + if (IS_ERR(new_dport)) + return new_dport; + + cxl_switch_parse_cdat(new_dport); + + if (ida_is_empty(&port->decoder_ida)) { + rc = devm_cxl_switch_port_decoders_setup(port); + if (rc) + return ERR_PTR(rc); + dev_dbg(&port->dev, "first dport%d:%s added with decoders\n", + new_dport->port_id, dev_name(dport_dev)); + return no_free_ptr(new_dport); + } + + /* New dport added, update the decoder targets */ + device_for_each_child(&port->dev, new_dport, update_decoder_targets); + + dev_dbg(&port->dev, "dport%d:%s added\n", new_dport->port_id, + dev_name(dport_dev)); + + return no_free_ptr(new_dport); +} + +static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev, + struct cxl_port *parent_port, + struct cxl_dport *parent_dport, + struct device *uport_dev, + struct device *dport_dev) +{ + resource_size_t component_reg_phys; + + device_lock_assert(&parent_port->dev); + if (!parent_port->dev.driver) { + dev_warn(ep_dev, + "port %s:%s:%s disabled, failed to enumerate CXL.mem\n", + dev_name(&parent_port->dev), dev_name(uport_dev), + dev_name(dport_dev)); + } + + struct cxl_port *port __free(put_cxl_port) = + find_cxl_port_by_uport(uport_dev); + if (!port) { + component_reg_phys = find_component_registers(uport_dev); + port = devm_cxl_add_port(&parent_port->dev, uport_dev, + component_reg_phys, parent_dport); + if (IS_ERR(port)) + return ERR_CAST(port); + + /* + * retry to make sure a port is found. a port device + * reference is taken. + */ + port = find_cxl_port_by_uport(uport_dev); + if (!port) + return ERR_PTR(-ENODEV); + + dev_dbg(ep_dev, "created port %s:%s\n", + dev_name(&port->dev), dev_name(port->uport_dev)); + } else { + /* + * Port was created before right before this function is + * called. Signal the caller to deal with it. + */ + return ERR_PTR(-EAGAIN); + } + + guard(device)(&port->dev); + return cxl_port_add_dport(port, dport_dev); +} + static int add_port_attach_ep(struct cxl_memdev *cxlmd, struct device *uport_dev, struct device *dport_dev) { struct device *dparent = grandparent(dport_dev); struct cxl_dport *dport, *parent_dport; - resource_size_t component_reg_phys; int rc; - if (!dparent) { + if (is_cxl_host_bridge(dparent)) { /* * The iteration reached the topology root without finding the * CXL-root 'cxl_port' on a previous iteration, fail for now to @@ -1560,42 +1707,31 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, } struct cxl_port *parent_port __free(put_cxl_port) = - find_cxl_port(dparent, &parent_dport); + find_cxl_port_by_uport(dparent->parent); if (!parent_port) { /* iterate to create this parent_port */ return -EAGAIN; } - /* - * Definition with __free() here to keep the sequence of - * dereferencing the device of the port before the parent_port releasing. - */ - struct cxl_port *port __free(put_cxl_port) = NULL; scoped_guard(device, &parent_port->dev) { - if (!parent_port->dev.driver) { - dev_warn(&cxlmd->dev, - "port %s:%s disabled, failed to enumerate CXL.mem\n", - dev_name(&parent_port->dev), dev_name(uport_dev)); - return -ENXIO; + parent_dport = cxl_find_dport_by_dev(parent_port, dparent); + if (!parent_dport) { + parent_dport = cxl_port_add_dport(parent_port, dparent); + if (IS_ERR(parent_dport)) + return PTR_ERR(parent_dport); } - port = find_cxl_port_at(parent_port, dport_dev, &dport); - if (!port) { - component_reg_phys = find_component_registers(uport_dev); - port = devm_cxl_add_port(&parent_port->dev, uport_dev, - component_reg_phys, parent_dport); - if (IS_ERR(port)) - return PTR_ERR(port); - - /* retry find to pick up the new dport information */ - port = find_cxl_port_at(parent_port, dport_dev, &dport); - if (!port) - return -ENXIO; + dport = devm_cxl_create_port(&cxlmd->dev, parent_port, + parent_dport, uport_dev, + dport_dev); + if (IS_ERR(dport)) { + /* Port already exists, restart iteration */ + if (PTR_ERR(dport) == -EAGAIN) + return 0; + return PTR_ERR(dport); } } - dev_dbg(&cxlmd->dev, "add to new port %s:%s\n", - dev_name(&port->dev), dev_name(port->uport_dev)); rc = cxl_add_ep(dport, &cxlmd->dev); if (rc == -EBUSY) { /* @@ -1608,6 +1744,25 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, return rc; } +static struct cxl_dport *find_or_add_dport(struct cxl_port *port, + struct device *dport_dev) +{ + struct cxl_dport *dport; + + device_lock_assert(&port->dev); + dport = cxl_find_dport_by_dev(port, dport_dev); + if (!dport) { + dport = cxl_port_add_dport(port, dport_dev); + if (IS_ERR(dport)) + return dport; + + /* New dport added, restart iteration */ + return ERR_PTR(-EAGAIN); + } + + return dport; +} + int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd) { struct device *dev = &cxlmd->dev; @@ -1636,11 +1791,7 @@ retry: struct device *uport_dev; struct cxl_dport *dport; - /* - * The terminal "grandparent" in PCI is NULL and @platform_bus - * for platform devices - */ - if (!dport_dev || dport_dev == &platform_bus) + if (is_cxl_host_bridge(dport_dev)) return 0; uport_dev = dport_dev->parent; @@ -1654,12 +1805,26 @@ retry: dev_name(iter), dev_name(dport_dev), dev_name(uport_dev)); struct cxl_port *port __free(put_cxl_port) = - find_cxl_port(dport_dev, &dport); + find_cxl_port_by_uport(uport_dev); if (port) { dev_dbg(&cxlmd->dev, "found already registered port %s:%s\n", dev_name(&port->dev), dev_name(port->uport_dev)); + + /* + * RP port enumerated by cxl_acpi without dport will + * have the dport added here. + */ + scoped_guard(device, &port->dev) { + dport = find_or_add_dport(port, dport_dev); + if (IS_ERR(dport)) { + if (PTR_ERR(dport) == -EAGAIN) + goto retry; + return PTR_ERR(dport); + } + } + rc = cxl_add_ep(dport, &cxlmd->dev); /* @@ -1672,6 +1837,8 @@ retry: if (rc && rc != -EBUSY) return rc; + cxl_gpf_port_setup(dport); + /* Any more ports to add between this one and the root? */ if (!dev_is_cxl_root_child(&port->dev)) continue; @@ -1692,41 +1859,41 @@ retry: return 0; } -EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL); +EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, "CXL"); struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev, struct cxl_dport **dport) { return find_cxl_port(pdev->dev.parent, dport); } -EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, "CXL"); struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd, struct cxl_dport **dport) { return find_cxl_port(grandparent(&cxlmd->dev), dport); } -EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, "CXL"); static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd, - struct cxl_port *port, int *target_map) + struct cxl_port *port) { + struct cxl_decoder *cxld = &cxlsd->cxld; int i; - if (!target_map) - return 0; - device_lock_assert(&port->dev); if (xa_empty(&port->dports)) - return -EINVAL; + return 0; - guard(rwsem_write)(&cxl_region_rwsem); + guard(rwsem_write)(&cxl_rwsem.region); for (i = 0; i < cxlsd->cxld.interleave_ways; i++) { - struct cxl_dport *dport = find_dport(port, target_map[i]); + struct cxl_dport *dport = find_dport(port, cxld->target_map[i]); - if (!dport) - return -ENXIO; + if (!dport) { + /* dport may be activated later */ + continue; + } cxlsd->target[i] = dport; } @@ -1840,7 +2007,7 @@ struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port, cxlrd->qos_class = CXL_QOS_CLASS_INVALID; return cxlrd; } -EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, "CXL"); /** * cxl_switch_decoder_alloc - Allocate a switch level decoder @@ -1877,7 +2044,7 @@ struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port, cxld->dev.type = &cxl_decoder_switch_type; return cxlsd; } -EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, "CXL"); /** * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder @@ -1899,6 +2066,7 @@ struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port) return ERR_PTR(-ENOMEM); cxled->pos = -1; + cxled->part = -1; cxld = &cxled->cxld; rc = cxl_decoder_init(port, cxld); if (rc) { @@ -1909,14 +2077,11 @@ struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port) cxld->dev.type = &cxl_decoder_endpoint_type; return cxled; } -EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, "CXL"); /** * cxl_decoder_add_locked - Add a decoder with targets * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc() - * @target_map: A list of downstream ports that this decoder can direct memory - * traffic to. These numbers should correspond with the port number - * in the PCIe Link Capabilities structure. * * Certain types of decoders may not have any targets. The main example of this * is an endpoint device. A more awkward example is a hostbridge whose root @@ -1930,7 +2095,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL); * Return: Negative error code if the decoder wasn't properly configured; else * returns 0. */ -int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map) +int cxl_decoder_add_locked(struct cxl_decoder *cxld) { struct cxl_port *port; struct device *dev; @@ -1951,7 +2116,7 @@ int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map) if (!is_endpoint_decoder(dev)) { struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev); - rc = decoder_populate_targets(cxlsd, port, target_map); + rc = decoder_populate_targets(cxlsd, port); if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) { dev_err(&port->dev, "Failed to populate active decoder targets\n"); @@ -1965,14 +2130,11 @@ int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map) return device_add(dev); } -EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, "CXL"); /** * cxl_decoder_add - Add a decoder with targets * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc() - * @target_map: A list of downstream ports that this decoder can direct memory - * traffic to. These numbers should correspond with the port number - * in the PCIe Link Capabilities structure. * * This is the unlocked variant of cxl_decoder_add_locked(). * See cxl_decoder_add_locked(). @@ -1980,7 +2142,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL); * Context: Process context. Takes and releases the device lock of the port that * owns the @cxld. */ -int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map) +int cxl_decoder_add(struct cxl_decoder *cxld) { struct cxl_port *port; @@ -1993,18 +2155,15 @@ int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map) port = to_cxl_port(cxld->dev.parent); guard(device)(&port->dev); - return cxl_decoder_add_locked(cxld, target_map); + return cxl_decoder_add_locked(cxld); } -EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, "CXL"); static void cxld_unregister(void *dev) { - struct cxl_endpoint_decoder *cxled; - - if (is_endpoint_decoder(dev)) { - cxled = to_cxl_endpoint_decoder(dev); - cxl_decoder_kill_region(cxled); - } + if (is_endpoint_decoder(dev)) + cxl_decoder_detach(NULL, to_cxl_endpoint_decoder(dev), -1, + DETACH_INVALIDATE); device_unregister(dev); } @@ -2013,7 +2172,7 @@ int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld) { return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev); } -EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, "CXL"); /** * __cxl_driver_register - register a driver for the cxl bus @@ -2046,13 +2205,13 @@ int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner, return driver_register(&cxl_drv->drv); } -EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL); +EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, "CXL"); void cxl_driver_unregister(struct cxl_driver *cxl_drv) { driver_unregister(&cxl_drv->drv); } -EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, "CXL"); static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env) { @@ -2104,19 +2263,19 @@ void cxl_bus_rescan(void) queue_work(cxl_bus_wq, &rescan_work); } -EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, "CXL"); void cxl_bus_drain(void) { drain_workqueue(cxl_bus_wq); } -EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, "CXL"); bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd) { return queue_work(cxl_bus_wq, &cxlmd->detach_work); } -EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL); +EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL"); static void add_latency(struct access_coordinate *c, long latency) { @@ -2242,7 +2401,7 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, return 0; } -EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, "CXL"); int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port, struct access_coordinate *c) @@ -2291,7 +2450,7 @@ static const struct attribute_group *cxl_bus_attribute_groups[] = { NULL, }; -struct bus_type cxl_bus_type = { +const struct bus_type cxl_bus_type = { .name = "cxl", .uevent = cxl_bus_uevent, .match = cxl_bus_match, @@ -2299,7 +2458,7 @@ struct bus_type cxl_bus_type = { .remove = cxl_bus_remove, .bus_groups = cxl_bus_attribute_groups, }; -EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_bus_type, "CXL"); static struct dentry *cxl_debugfs; @@ -2307,7 +2466,7 @@ struct dentry *cxl_debugfs_create_dir(const char *dir) { return debugfs_create_dir(dir, cxl_debugfs); } -EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, "CXL"); static __init int cxl_core_init(void) { @@ -2339,8 +2498,14 @@ static __init int cxl_core_init(void) if (rc) goto err_region; + rc = cxl_ras_init(); + if (rc) + goto err_ras; + return 0; +err_ras: + cxl_region_exit(); err_region: bus_unregister(&cxl_bus_type); err_bus: @@ -2352,6 +2517,7 @@ err_wq: static void cxl_core_exit(void) { + cxl_ras_exit(); cxl_region_exit(); bus_unregister(&cxl_bus_type); destroy_workqueue(cxl_bus_wq); @@ -2363,4 +2529,4 @@ subsys_initcall(cxl_core_init); module_exit(cxl_core_exit); MODULE_DESCRIPTION("CXL: Core Compute Express Link support"); MODULE_LICENSE("GPL v2"); -MODULE_IMPORT_NS(CXL); +MODULE_IMPORT_NS("CXL"); |